[EDI]: developing edi class

bzr revid: aar@tinyerp.com-20110603133826-ol8rzeey17hvxlar
This commit is contained in:
Amit Parmar (OpenERP)
2011-06-03 19:08:26 +05:30
+352 -270
View File
@@ -25,12 +25,27 @@ import hashlib
import time
import base64
import urllib2
import release
def safe_unique_id(model, database_id):
"""Generate a unique string to represent a (model,database id) pair
without being too long, without revealing the database id, and
with a very low probability of collisions.
Each EDI record and each relationship value are represented using a unique
database identifier. These database identifiers include the database unique
ID, as a way to uniquely refer to any record within any OpenERP instance,
without conflict.
For OpenERP records that have an existing "XML ID" (i.e. an entry in
ir.model.data), the EDI unique identifier for this record will be made of
"%s:%s" % (the database's UUID, the XML ID). The database's UUID MUST
NOT contain a colon characters (this is guaranteed by the UUID algorithm).
For OpenERP records that have no existing "XML ID", a new one should be
created during the EDI export. It is recommended that the generated XML ID
contains a readable reference to the record model, plus a unique value that
hides the database ID.
"""
msg = "%s-%s-%s" % (model, time.time(), database_id)
digest = hashlib.sha1(msg).digest()
@@ -42,99 +57,127 @@ def safe_unique_id(model, database_id):
class ir_edi_document(osv.osv):
_name = 'ir.edi.document'
_description = 'ir_edi_document'
_description = 'To represent the EDI export of any OpenERP record, it could be an invoice, a project task, etc.'
_columns = {
'name': fields.char('EDI token', size = 128),
'document': fields.text()
'name': fields.char("EDI token", size = 128, help="EDI Token is a unique identifier for the EDI document."),
'document': fields.text("Document", help="hold the serialization of the EDI document.")
}
_sql_constraints = [
('name_uniq', 'unique (name)', 'The EDI Token must be unique!')
]
def new_edi_token(self,browse_records,database_id):
"""Generate new unique tokent value to uniquely identify each edi_document"""
db_uuid = safe_unique_id(browse_records, database_id)
edi_token = hashlib.sha256('%s-%s-%s' % (time.time(),db_uuid,time.time())).hexdigest()
def new_edi_token(self, record):
"""
Return a new, random unique token to identify an edi.document
:param record: It's a object of browse_record of any model
"""
db_uuid = safe_unique_id(record._name, record.id)
edi_token = hashlib.sha256('%s-%s-%s' % (time.time(), db_uuid, time.time())).hexdigest()
return edi_token
def serialize(self,edi_list_of_dicts):
"""Serialize the list of dictionaries using json dumps method"""
serialized_list = json.dumps(edi_list_of_dicts)
def serialize(self, edi_documents):
"""Serialize the list of dictionaries using json dumps method
perform a JSON serialization of a list of dicts prepared by generate_edi() and return a UTF-8 encoded string that could be passed to deserialize()
:param edi_dicts: it's list of edi_dict
"""
serialized_list = json.dumps(edi_dicts)
return serialized_list
def generate_edi(self,browse_record_list):
def generate_edi(self, cr, uid, records, context=None):
"""
Generate the list of dictionaries using edi_export method of edi class
:param records: it's a object of browse_record_list of any model
"""
""" Generate the list of dictionaries using edi_export method of edi class """
edi_list = []
for record in browse_record_list:
edi_struct = record.edi_export(record)
edi_list.append(edi_struct)
for record in records:
record_model_obj = self.pool.get(record._name)
edi_list += record_model_obj.edi_export(cr, uid, [record], context=context)
return self.serialize(edi_list)
def get_document(self,cr,uid,edi_token,context=None):
def get_document(self, cr, uid, edi_token, context=None):
"""
Get the edi document from database using given edi token
returns the string serialization that is in the database (column: document) for the given edi_token or raise.
"""
"""Get the edi document from database using given edi token """
records = self.search(cr,uid,[('name','=',edi_token)],context=context)
records = self.name_search(cr, uid, edi_token, context=context)
if records:
doc_string = self.read(cr,uid,records,['document'],context=context)
edi = self.browse(cr, uid, records[0], context=context)
return edi.document
else:
raise osv.except_osv(_('Error !'),
_('Desired EDI Document does not Exist'))
return doc_string
def load_edi(self,cr,uid,ids,edi_list_of_dicts,context=None):
"""loads the values from list of dictionaries to the corresponding OpenERP records
using the edi_import method of edi class"""
edi_list = []
for record in edi_list_of_dicts:
edi_record = record.edi_import(record)
return True
def deserialize(self,cr,uid,edi_document_string,context=None):
""" Deserialized the edi document string"""
deserialize_string = json.loads(edi_document_string)
return self.load_edi(cr,uid,deserialize_string,context=context)
def export_doc(self, cr, uid, browse_records, database_id, context=None):
"""the method handles the flow of the edi document generation and store it in
the database and return the edi_token of the particular document"""
edi_document = {
'name': self.new_edi_token(browse_records,database_id),
'document': self.generate_edi(browse_records)
}
unique_id = self.create(cr,uid,edi_document,context=context)
return edi_document['name']
def import_doc(self, cr, uid, ids, edi_document=None,edi_url=None,context=None):
"""the method handles the flow of importing particular edi document and
updates the database values on the basis of the edi document using
edi_loads method """
if edi_document is not None:
self.deserialize(cr,uid,edi_document,context=context)
elif edi_url is not None:
edi_document = urllib2.urlopen(edi_url).read()
self.deserialize(cr,uid,edi_document,context=context)
def load_edi(self, cr, uid, edi_documents, context=None):
"""
loads the values from list of dictionaries to the corresponding OpenERP records
using the edi_import method of edi class
For each edi record (dict) in the list, call the corresponding osv.edi_import() method, based on the __model attribute (see section 2.4 of for spec of
osv.edi_import)
:param edi_dicts: list of edi_dict
"""
for edi_document in edi_documents:
model = edi_document.get('__model')
assert model, _('model should be provided in EDI Dict')
model_obj = self.pool.get(model)
model_obj.edi_import(cr, uid, edi_document, context=context)
return True
def deserialize(self, edi_document_string):
""" Deserialized the edi document string
perform JSON deserialization from an edi document string, and returns a list of dicts
"""
edi_document = json.loads(edi_document_string)
return edi_document
def export(self, cr, uid, records, context=None):
"""
The method handles the flow of the edi document generation and store it in
the database and return the edi_token of the particular document
Steps:
* call generate_edi() to get a serialization and new_edi_token() to get a unique ID
* serialize the list returned by generate_edi() using serialize(), and save it in database with unique ID.
* return the unique ID
: param records: list of browse_record of any model
"""
exported_ids = []
for record in records:
document = self.generate_edi(cr, uid, [record], context)
token = self.new_edi_token(record)
self.create(cr, uid, {
'name': token,
'document': document
}, context=context)
exported_ids.append(token)
return exported_ids
def import(self, cr, uid, edi_document=None, edi_url=None, context=None):
"""
The method handles the flow of importing particular edi document and
updates the database values on the basis of the edi document using
edi_loads method
* N: a serialized edi.document or the URL to download a serialized document
* If a URL is provided, download it first to get the document
* Calls deserialize() to get the resulting list of dicts from the document
* Call load_edi() with the list of dicts, to create or update the corresponding OpenERP records based on the edi.document.
"""
if edi_url and not edi_document:
edi_document = urllib2.urlopen(edi_url).read()
assert edi_document, _('EDI Document should be provided')
edi_documents = self.deserialize(edi_document)
return self.load_edi(cr, uid, edi_documents, context=context)
ir_edi_document()
@@ -148,102 +191,104 @@ class edi(object):
specific behavior, based on the primitives provided by this superclass."""
def edi_metadata(self, cr, uid,browse_rec_list, context=None):
def edi_metadata(self, cr, uid, records, context=None):
"""Return a list representing the boilerplate EDI structure for
exporting the record in the given browse_rec_list, including
the metadata fields"""
# generic implementation!
data_object = self.pool.get('ir.model.data')
the metadata fields
if self.edi_document['__model'] is not None:
model_ids = data_object.search(cr,uid,[('model','=',self.edi_document['__model'])],context=context)
for data in data_object.browse(cr,uid,model_ids):
self.edi_document['__module'] = data.module
if data.name is not None:
self.edi_document['__id'] = context['active_id']+':'+data.name
else:
self.edi_document['__id'] = context['active_id']+':'+safe_unique_id(data.model,)
self.edi_document['__last_updata'] = context['__last_update'].values()[0]
The metadata fields MUST always include:
- __model': the OpenERP model name
- __module': the OpenERP module containing the model
- __id': the unique (cross-DB) identifier for this record
- __last_update': last update date of record, ISO date string in UTC
- __version': a list of components for the version
- __attachments': a list (possibly empty) of dicts describing the files attached to this record.
"""
# generic implementation!
attachment_object = self.pool.get('ir.attachment')
edi_dict_list = []
for record in records:
attachment_ids = attachment_object.search(cr, uid, [('res_model','=', record._name), ('res_id', '=', record.id)])
attachment_dict_list = []
for attachment in attachment_object.browse(cr, uid, attachment_ids, context=context):
attachment_dict_list.append({
'name' : attachment.name,
'content': base64.encodestring(attachment.datas),
'file_name': attachment.datas_fname,
})
version = []
for ver in release.version.split('.'):
version.append(int(ver))
edi_dict = {
'__model' : record._name,
'__module' : record._module,
'__id': record.id,
'__last_update': record.write_date, #TODO: convert into UTC
'__version': version,
'__attachments': attachment_dict_list
}
edi_dict_list.append(edi_dict)
return True
return edi_dict_list
def edi_m2o(self, cr, uid,browse_rec):
def edi_m2o(self, cr, uid, record, context=None):
"""Return a list representing a M2O EDI value for
the given browse_record."""
the given browse_record.
M2O are passed as pair (ID, Name)
Exmaple: ['db-uuid:xml-id', 'Partner Name']
"""
# generic implementation!
model = browse_rec._name
res = []
data_object = self.pool.get('ir.model.data')
uu_id = data_model.search(cr,uid,[('model','=',model)])
for xml in data_model.browse(cr,uid,uu_id):
if xml:
xml_ID = xml.name
else:
xml_ID = None
for data in browse_rec:
if data.name is not None:
if xml_ID is not None:
res.append(safe_unique_id(model,data.id)+':'+xml_ID)
else:
res.append(safe_unique_id(model,data.id)+':'+safe_unique_id(model,data.id))
res.append(data.name)
return res
db_uuid = safe_unique_id(record._name, record.id)
xml_ids = data_object.search(cr, uid, [('model','=',record._name),('res_id','=',record.id)])
if xml_ids:
xml_id = data_object.browse(cr, uid, xml_ids[0], context=context)
xml_id = xml_id.name
db_uuid = '%s:%s' %(db_uuid, xml_id)
name = record.name
return [db_uuid, name]
def edi_o2m(self, cr, uid,line = None,browse_rec_list, context=None):
def edi_o2m(self, cr, uid, records, context=None):
"""Return a list representing a O2M EDI value for
the browse_records from the given browse_record_list."""
# generic implementation!
res = []
init_data = {}
model = browse_rec_list._name
data_object = self.pool.get('ir.model.data')
uu_id = data_model.search(cr,uid,[('model','=',model)])
for xml in data_model.browse(cr,uid,uu_id):
if xml:
xml_ID = xml.name
else:
xml_ID = None
data_model = self.pool.get(model)
for record in browse_rec_list:
if xml_ID is not None:
init_data['__id'] = safe_unique_id(model,record.id)+':'+xml_ID
else:
init_data['__id'] = safe_unique_id(model,record.id)+':'+model+'-'+safe_unique_id(model,record.id)
init_data['__last_update'] =record.write_date
init_data.update(self.determine_type(cr,uid,ids,line,record,context=context))
res.append(init_data)
return res
the browse_records from the given browse_record_list.
def edi_m2m(self, cr, uid, line = None, browse_rec_list, context=None):
"""Return a list representing a M2M EDI value for
the browse_records from the given browse_record_list."""
Example:
[ # O2M fields would be a list of dicts, with their
{ '__id': 'db-uuid:xml-id', # own __id.
'__last_update': 'iso date', # The update date, just in case...
'name': 'some name',
...
}],
"""
# generic implementation!
model = browse_rec_list[0]._name
res = []
init_data = []
data_object = self.pool.get('ir.model.data')
uu_id = data_model.search(cr,uid,[('model','=',model)])
for xml in data_model.browse(cr,uid,uu_id):
if xml:
xml_ID = xml.name
else:
xml_ID = None
data_model = self.pool.get(model)
for record in browse_rec_list:
if xml_ID is not None:
init_data.append(safe_unique_id(model,record.id)+':'+xml_ID)
else:
init_data.append(safe_unique_id(model,record.id)+':'+model+'-'+safe_unique_id(model,record.id))
if record.name is not None:
init_data.append('name:'+record.name)
res.append(init_data)
# blob:
return res
dict_list = []
for record in records:
model_obj = self.pool.get(record._name)
dict_list += model_obj.edi_export(cr, uid, [record.id], context=context)
return dict_list
def edi_m2m(self, cr, uid, records, context=None):
"""Return a list representing a M2M EDI value for
the browse_records from the given browse_record_list.
Example:
'related_tasks': [ # M2M fields would exported as a list of pairs,
['db-uuid:xml-id1', # similar to a list of M2O values.
'Task 01: bla bla'],
['db-uuid:xml-id2',
'Task 02: bla bla']
]
"""
# generic implementation!
dict_list = []
for record in records:
dict_list.append(self.edi_o2m(cr, uid, record, context=None))
return dict_list
def edi_export(self, cr, uid, ids, edi_struct=None, context=None):
"""Returns a list of dicts representing an edi.document containing the
@@ -265,135 +310,172 @@ class edi(object):
be included in the exported data.
"""
# generic implementation!
self.edi_document = {
'__model':'',
'__module':'',
'__id':'',
'__last_update':'',
'__version':'',
'__attachments':'',
}
self.edi_document['__model'] = self._name
if context is None:
context = {}
_fields = self.fields_get(cr, uid, context=context)
fields_object = self.pool.get('ir.model.fields')
model_object = self.pool.get(self.edi_document['__model'])
edi_metadata(cr, uid, edi_struct, context=context)
record = model_object.read(cr,uid,ids,[],context=context)
for field in edi_struct.keys():
f_ids = fields_object.search(cr,uid,[('name','=',field),('model','=',self.edi_document['__model'])],context=context)
for fname in fields_object.browse(cr,uid,f_ids):
if fname.ttype == 'many2one':
self.edi_document[field] = self.edi_m2o(cr, uid, browse_rec = record[0][field])
elif fname.ttype == 'one2many':
self.edi_ducument[field] = self.edi_o2m(cr,uid,line = edi_struct[field],browse_rec_list = record[0][field])
elif fname.ttype == 'many2many':
self.edi_document[field] = self.edi_m2m(cr,uid,line = edi_struct[field],browse_rec_list = record[0][field])
fields_to_export = edi_struct and edi_struct.keys() or _fields.key()
edi_dict_list = []
for row in self.browse(cr, uid, ids, fields_to_export, context):
edi_dict = {}
edi_dict.update(self.edi_metadata(cr, uid, [row.id], context=context)[0])
for field in fields_to_export:
cols = self._fields[field]
if cols['type'] == 'many2one':
m2o_record = row.getattrs(field)
value = self.m2o(cr, uid, m2o_record, context=context)
elif cols['type'] == 'many2many':
m2m_records = row.getattrs(field)
value = self.m2m(cr, uid, m2m_records, context=context)
elif cols['type'] == 'one2many':
o2m_records = row.getattrs(field)
value = self.o2m(cr, uid, o2m_records, context=context)
else:
self.edi_document[field] = record[0][field]
value = row.getattrs(field)
edi_dict[field] = value
edi_dict_list.append(edi_dict)
return edi_dict_list
return self.edi_document
def determine_type(self,cr,uid,ids,record_line,browse_record=None,context=None):
model = browse_record._name
fields_object = self.pool.get('ir.model.fields')
obj = self.pool.get(model)
field_data = obj.read(cr,uid,browse_record.id,record_line.keys())[0]
for field in field_data.keys():
field_ids = field_object.search(cr,uid,[field])
for field_type in field_object.browse(cr,uid,field_ids):
if field_type.ttype == 'many2one':
record_line.update(field,self.edi_m2o(cr,uid,browse_rec = field_data[field]))
elif field_type.ttype == 'one2many':
pass
elif field_type.ttype == 'many2many':
pass
else:
record_line.update(field,field_data[field])
return record_line
def edi_import(self, cr, uid, edi_document, context=None):
"""Imports a list of dicts representing an edi.document, using the
generic algorithm.
All relationship fields are exported in a special way, and provide their own
unique identifier, so that we can avoid duplication of records when importing.
Note: for all ir.model.data entries, the "module" value to use for read/write
should always be "edi_import", and the "name" value should be the full
db_id provided in the EDI.
1: Many2One
M2O fields are always exported as a pair [db_id, name], where db_id
is in the form "db_uuid:xml_id", both being values that come directly
from the original database.
The import should behave like this:
a. Look in ir.model.data for a record that matches the db_id.
If found, replace the m2o value with the correct database ID and stop.
If not found, continue to next step.
b. Perform name_search(name) to look for a record that matches the
given m2o name. If only one record is found, create the missing
ir.model.data record to link it to the db_id, and the replace the m2o
value with the correct database ID, then stop. If zero result or
multiple results are found, go to next step.
c. Create the new record using the only field value that is known: the
name, and create the ir.model.data entry to map to it.
This should work for many models, and if not, the module should
provide a custom EDI import logic to care for it.
2: One2Many
O2M fields are always exported as a list of dicts, where each dict corresponds
to a full EDI record. The import should not update existing records
if they already exist, it should only link them to the parent object.
a. Look for a record that matches the db_id provided in the __id field. If
found, keep the corresponding database id, and connect it to the parent
using a write value like (4,db_id).
b. If not found via db_id, create a new entry using the same method that
imports a full EDI record (recursive call!), grab the resulting db id,
and use it to connect to the parent via a write value like (4, db_id).
3: Many2Many
M2M fields are always exported as a list of pairs similar to M2O.
For each pair in the M2M:
a. Perform the same steps as for a Many2One (see 1.2.1.1)
b. After finding the database ID of the final record in the database,
connect it to the parent record via a write value like (4, db_id).
"""
# generic implementation!
new = True
resource = {}
model = edi_document['__model']
model_object = self.pool.get(model)
data_model = self.pool.get('ir.model.data')
field_model = self.pool.get('ir.model.fields')
fields = edi_document.keys()
fields_to_import = []
data_line = []
model_data = self.pool.get('ir.model.data')
_fields = self.fields_get(cr, uid, context=context)
current_module = 'edi_import'
values = {}
for field in edi_document.keys():
if field == '__id':
db_id , xml_ID = edi_document[field].split(':')
self.data_ids = data_model.search(cr,uid,[('name','=',xml_ID)])
f_ids = fields_object.search(cr,uid,[('name','=',field),('model','=',model)],context=context)
for fname in fields_object.browse(cr,uid,f_ids):
if fname.ttype == 'many2one':
field_ids,field_xml = edi_document[field][0].split(':')
field_ids = int(field_ids)
model_ids = data_model.search(cr,uid,[('name','=',field_xml)])
if not len(model_ids):
obj = self.pool.get(fname.relation)
model1_ids = obj.search(cr,uid,[('name','=',edi_document[field][1])])
if len(model1_ids) == 1:
resourse[field] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':model1_ids})[0]
elif not len(model1_ids) or len(model1_ids) > 1:
edi_document[field][1] = obj.create(cr,uid,{'name':edi_document[field][1]})
resource[field] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':edi_document[field][0]})[0]
if not field.startwith('__'):
field_to_import.append(field)
edi_field_value = edi_document[field]
if _fields[field]['type'] in ('many2one', 'many2many'):
if _fields[field]['type'] == 'many2one':
edi_parent_documents = [edi_field_value]
else:
resource[field] = model_ids[0]
elif fname.ttype == 'one2many':
for records in edi_document[field]:
res = {}
field_ids,field_xml = records['__id'].split(':')
field_ids = int(field_ids)
model3_ids = data_model.search(cr,uid,[('name','=',field_xml)])
for fields in records.keys():
if fields != '__id' or fields != '__last_update':
res[fields] = records[fields]
if not len(model3_ids):
resource[field][records] = (4,self.edi_import(cr,uid,res,context = context)[0])
edi_parent_documents = edi_field_value
parent_lines = []
for edi_parent_document in edi_parent_documents:
#Look in ir.model.data for a record that matches the db_id.
#If found, replace the m2o value with the correct database ID and stop.
#If not found, continue to next step
db_uuid, xml_id = tuple(edi_parent_document[0].split(':'))
data_ids = model_data.name_search(cr, uid, xml_id)
if data_ids:
d = model_data.browse(cr, uid, data_ids[0], context=context)
parent_lines.append(d.res_id)
else:
resource[field][records] = (4,model3_ids[0])obj = self.pool.get(model)
record_ids = obj.create(cr,uid,resource,context=context)
return record_ids
#Perform name_search(name) to look for a record that matches the
#given m2o name. If only one record is found, create the missing
#ir.model.data record to link it to the db_id, and the replace the m2o
#value with the correct database ID, then stop. If zero result or
#multiple results are found, go to next step.
#Create the new record using the only field value that is known: the
#name, and create the ir.model.data entry to map to it.
relation_object = self.get.pool(_fields[field]['relation'])
relation_ids = relation_object.name_search(cr, uid, edi_parent_document[1])
if relation_ids and len(relation_ids) == 1:
relation_id = relation_ids[0]
else:
relation_id = relation_object.create(cr, uid, {'name': edi_parent_document[1]}, context=context)
model_data.create(cr, uid, {
'name': xml_id,
'model': relation_object._name,
#'module':relation_object._module,
'res_id':relation_id,
},context=context)
parent_lines.append(relation_id)
if _fields[field]['type'] == 'many2one':
values[field] = parent_lines[0]
else:
many2many_ids = []
for m2m_id in parent_lines:
many2many_ids.append((4,m2m_id))
values[field] = many2many_ids
elif fname.ttype == 'many2many':
for records in edi_document[field]:
field_ids,field_xml = records[0].split(':')
field_ids = int(field_ids)
model4_ids = data_model.search(cr,uid,[('name','=',field_xml)])
if not len(model4_ids):
obj = self.pool.get(fname.relation)
name,value = records[1].split(':')
model5_ids = obj.search(cr,uid,[(name,'=',value)])
if len(model5_ids) == 1:
resource[field][records] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':model5_ids})[0]
elif not len(model5_ids) or len(model5_ids) > 1:
edi_document[field][records][1] = obj.create(cr,uid,{'name':edi_document[field][1]})
resource[field] = (4,data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':edi_document[field][records][1]})[0])
elif _fields[field]['type'] == 'one2many':
#Look for a record that matches the db_id provided in the __id field. If
#found, keep the corresponding database id, and connect it to the parent
#using a write value like (4,db_id).
#If not found via db_id, create a new entry using the same method that
#imports a full EDI record (recursive call!), grab the resulting db id,
#and use it to connect to the parent via a write value like (4, db_id).
relations = []
relation_object = self.get.pool(_fields[field]['relation'])
for edi_relation_document in edi_field_value:
db_uuid, xml_id = tuple(edi_relation_document.get('__id').split(':'))
data_ids = model_data.name_search(cr, uid, xml_id)
if data_ids:
d = model_data.browse(cr, uid, data_ids[0], context=context)
relations.append(d.res_id)
else:
resource[field] = (4,model4_ids[0])
r = relation_object.edi_import(cr, uid, edi_relation_document, context=context)
relations.append(r)
one2many_ids = []
for o2m_id in relations:
one2many_ids.append((4,o2m_id))
values[field] = one2many_ids
else:
resource[field] = edi_document[field]
if len(self.data_ids):
obj = self.pool.get(model)
record_ids = obj.write(cr,uid,resource,context=context)
return record_ids
else:
obj = self.pool.get(model)
record_ids = obj.create(cr,uid,resource,context=context)
return record_ids
values[field] = edi_field_value
return model_data._update(cr, uid, self._name, current_module, values, context=context)