diff --git a/openerp/addons/base/ir/ir_edi.py b/openerp/addons/base/ir/ir_edi.py index d5146b4c655..4ed7b0a4d3a 100644 --- a/openerp/addons/base/ir/ir_edi.py +++ b/openerp/addons/base/ir/ir_edi.py @@ -25,12 +25,27 @@ import hashlib import time import base64 import urllib2 +import release def safe_unique_id(model, database_id): - """Generate a unique string to represent a (model,database id) pair without being too long, without revealing the database id, and with a very low probability of collisions. + + Each EDI record and each relationship value are represented using a unique + database identifier. These database identifiers include the database unique + ID, as a way to uniquely refer to any record within any OpenERP instance, + without conflict. + + For OpenERP records that have an existing "XML ID" (i.e. an entry in + ir.model.data), the EDI unique identifier for this record will be made of + "%s:%s" % (the database's UUID, the XML ID). The database's UUID MUST + NOT contain a colon characters (this is guaranteed by the UUID algorithm). + + For OpenERP records that have no existing "XML ID", a new one should be + created during the EDI export. It is recommended that the generated XML ID + contains a readable reference to the record model, plus a unique value that + hides the database ID. """ msg = "%s-%s-%s" % (model, time.time(), database_id) digest = hashlib.sha1(msg).digest() @@ -42,99 +57,127 @@ def safe_unique_id(model, database_id): class ir_edi_document(osv.osv): _name = 'ir.edi.document' - _description = 'ir_edi_document' + _description = 'To represent the EDI export of any OpenERP record, it could be an invoice, a project task, etc.' _columns = { - 'name': fields.char('EDI token', size = 128), - 'document': fields.text() + 'name': fields.char("EDI token", size = 128, help="EDI Token is a unique identifier for the EDI document."), + 'document': fields.text("Document", help="hold the serialization of the EDI document.") } + _sql_constraints = [ + ('name_uniq', 'unique (name)', 'The EDI Token must be unique!') + ] - - def new_edi_token(self,browse_records,database_id): - - """Generate new unique tokent value to uniquely identify each edi_document""" - - db_uuid = safe_unique_id(browse_records, database_id) - edi_token = hashlib.sha256('%s-%s-%s' % (time.time(),db_uuid,time.time())).hexdigest() - - + def new_edi_token(self, record): + """ + Return a new, random unique token to identify an edi.document + :param record: It's a object of browse_record of any model + """ + db_uuid = safe_unique_id(record._name, record.id) + edi_token = hashlib.sha256('%s-%s-%s' % (time.time(), db_uuid, time.time())).hexdigest() return edi_token - def serialize(self,edi_list_of_dicts): - - """Serialize the list of dictionaries using json dumps method""" - - serialized_list = json.dumps(edi_list_of_dicts) + def serialize(self, edi_documents): + """Serialize the list of dictionaries using json dumps method + perform a JSON serialization of a list of dicts prepared by generate_edi() and return a UTF-8 encoded string that could be passed to deserialize() + :param edi_dicts: it's list of edi_dict + """ + serialized_list = json.dumps(edi_dicts) return serialized_list - def generate_edi(self,browse_record_list): + def generate_edi(self, cr, uid, records, context=None): + """ + Generate the list of dictionaries using edi_export method of edi class + :param records: it's a object of browse_record_list of any model + """ - """ Generate the list of dictionaries using edi_export method of edi class """ - edi_list = [] - for record in browse_record_list: - edi_struct = record.edi_export(record) - edi_list.append(edi_struct) + for record in records: + record_model_obj = self.pool.get(record._name) + edi_list += record_model_obj.edi_export(cr, uid, [record], context=context) return self.serialize(edi_list) - def get_document(self,cr,uid,edi_token,context=None): + def get_document(self, cr, uid, edi_token, context=None): + """ + Get the edi document from database using given edi token + returns the string serialization that is in the database (column: document) for the given edi_token or raise. + """ - """Get the edi document from database using given edi token """ - - records = self.search(cr,uid,[('name','=',edi_token)],context=context) + records = self.name_search(cr, uid, edi_token, context=context) if records: - doc_string = self.read(cr,uid,records,['document'],context=context) + edi = self.browse(cr, uid, records[0], context=context) + return edi.document else: raise osv.except_osv(_('Error !'), _('Desired EDI Document does not Exist')) - return doc_string + - def load_edi(self,cr,uid,ids,edi_list_of_dicts,context=None): - - """loads the values from list of dictionaries to the corresponding OpenERP records - using the edi_import method of edi class""" - - edi_list = [] - for record in edi_list_of_dicts: - edi_record = record.edi_import(record) - - - return True - - def deserialize(self,cr,uid,edi_document_string,context=None): - """ Deserialized the edi document string""" - - deserialize_string = json.loads(edi_document_string) - return self.load_edi(cr,uid,deserialize_string,context=context) - - def export_doc(self, cr, uid, browse_records, database_id, context=None): - """the method handles the flow of the edi document generation and store it in - the database and return the edi_token of the particular document""" - - edi_document = { - 'name': self.new_edi_token(browse_records,database_id), - 'document': self.generate_edi(browse_records) - } - - unique_id = self.create(cr,uid,edi_document,context=context) - return edi_document['name'] - - def import_doc(self, cr, uid, ids, edi_document=None,edi_url=None,context=None): - - """the method handles the flow of importing particular edi document and - updates the database values on the basis of the edi document using - edi_loads method """ - - if edi_document is not None: - self.deserialize(cr,uid,edi_document,context=context) - elif edi_url is not None: - edi_document = urllib2.urlopen(edi_url).read() - self.deserialize(cr,uid,edi_document,context=context) + def load_edi(self, cr, uid, edi_documents, context=None): + """ + loads the values from list of dictionaries to the corresponding OpenERP records + using the edi_import method of edi class + For each edi record (dict) in the list, call the corresponding osv.edi_import() method, based on the __model attribute (see section 2.4 of for spec of + osv.edi_import) + :param edi_dicts: list of edi_dict + """ + + for edi_document in edi_documents: + model = edi_document.get('__model') + assert model, _('model should be provided in EDI Dict') + model_obj = self.pool.get(model) + model_obj.edi_import(cr, uid, edi_document, context=context) return True + def deserialize(self, edi_document_string): + """ Deserialized the edi document string + perform JSON deserialization from an edi document string, and returns a list of dicts + """ + edi_document = json.loads(edi_document_string) + return edi_document + + def export(self, cr, uid, records, context=None): + """ + The method handles the flow of the edi document generation and store it in + the database and return the edi_token of the particular document + Steps: + * call generate_edi() to get a serialization and new_edi_token() to get a unique ID + * serialize the list returned by generate_edi() using serialize(), and save it in database with unique ID. + * return the unique ID + + : param records: list of browse_record of any model + """ + exported_ids = [] + for record in records: + document = self.generate_edi(cr, uid, [record], context) + token = self.new_edi_token(record) + self.create(cr, uid, { + 'name': token, + 'document': document + }, context=context) + + exported_ids.append(token) + return exported_ids + + def import(self, cr, uid, edi_document=None, edi_url=None, context=None): + """ + The method handles the flow of importing particular edi document and + updates the database values on the basis of the edi document using + edi_loads method + + * N: a serialized edi.document or the URL to download a serialized document + * If a URL is provided, download it first to get the document + * Calls deserialize() to get the resulting list of dicts from the document + * Call load_edi() with the list of dicts, to create or update the corresponding OpenERP records based on the edi.document. + """ + + if edi_url and not edi_document: + edi_document = urllib2.urlopen(edi_url).read() + assert edi_document, _('EDI Document should be provided') + edi_documents = self.deserialize(edi_document) + return self.load_edi(cr, uid, edi_documents, context=context) + ir_edi_document() @@ -148,102 +191,104 @@ class edi(object): specific behavior, based on the primitives provided by this superclass.""" - def edi_metadata(self, cr, uid,browse_rec_list, context=None): + def edi_metadata(self, cr, uid, records, context=None): """Return a list representing the boilerplate EDI structure for exporting the record in the given browse_rec_list, including - the metadata fields""" - # generic implementation! - data_object = self.pool.get('ir.model.data') + the metadata fields - if self.edi_document['__model'] is not None: - model_ids = data_object.search(cr,uid,[('model','=',self.edi_document['__model'])],context=context) - for data in data_object.browse(cr,uid,model_ids): - self.edi_document['__module'] = data.module - if data.name is not None: - self.edi_document['__id'] = context['active_id']+':'+data.name - else: - self.edi_document['__id'] = context['active_id']+':'+safe_unique_id(data.model,) - self.edi_document['__last_updata'] = context['__last_update'].values()[0] + The metadata fields MUST always include: + - __model': the OpenERP model name + - __module': the OpenERP module containing the model + - __id': the unique (cross-DB) identifier for this record + - __last_update': last update date of record, ISO date string in UTC + - __version': a list of components for the version + - __attachments': a list (possibly empty) of dicts describing the files attached to this record. + """ + # generic implementation! + attachment_object = self.pool.get('ir.attachment') + edi_dict_list = [] + for record in records: + attachment_ids = attachment_object.search(cr, uid, [('res_model','=', record._name), ('res_id', '=', record.id)]) + attachment_dict_list = [] + for attachment in attachment_object.browse(cr, uid, attachment_ids, context=context): + attachment_dict_list.append({ + 'name' : attachment.name, + 'content': base64.encodestring(attachment.datas), + 'file_name': attachment.datas_fname, + }) + version = [] + for ver in release.version.split('.'): + version.append(int(ver)) + edi_dict = { + '__model' : record._name, + '__module' : record._module, + '__id': record.id, + '__last_update': record.write_date, #TODO: convert into UTC + '__version': version, + '__attachments': attachment_dict_list + } + edi_dict_list.append(edi_dict) - return True + return edi_dict_list - def edi_m2o(self, cr, uid,browse_rec): + def edi_m2o(self, cr, uid, record, context=None): """Return a list representing a M2O EDI value for - the given browse_record.""" + the given browse_record. + M2O are passed as pair (ID, Name) + Exmaple: ['db-uuid:xml-id', 'Partner Name'] + """ # generic implementation! - - model = browse_rec._name - res = [] data_object = self.pool.get('ir.model.data') - uu_id = data_model.search(cr,uid,[('model','=',model)]) - for xml in data_model.browse(cr,uid,uu_id): - if xml: - xml_ID = xml.name - else: - xml_ID = None - for data in browse_rec: - if data.name is not None: - if xml_ID is not None: - res.append(safe_unique_id(model,data.id)+':'+xml_ID) - else: - res.append(safe_unique_id(model,data.id)+':'+safe_unique_id(model,data.id)) - res.append(data.name) - return res + db_uuid = safe_unique_id(record._name, record.id) + xml_ids = data_object.search(cr, uid, [('model','=',record._name),('res_id','=',record.id)]) + if xml_ids: + xml_id = data_object.browse(cr, uid, xml_ids[0], context=context) + xml_id = xml_id.name + db_uuid = '%s:%s' %(db_uuid, xml_id) + name = record.name + return [db_uuid, name] + - def edi_o2m(self, cr, uid,line = None,browse_rec_list, context=None): + def edi_o2m(self, cr, uid, records, context=None): """Return a list representing a O2M EDI value for - the browse_records from the given browse_record_list.""" - # generic implementation! - res = [] - init_data = {} - model = browse_rec_list._name - data_object = self.pool.get('ir.model.data') - uu_id = data_model.search(cr,uid,[('model','=',model)]) - - for xml in data_model.browse(cr,uid,uu_id): - if xml: - xml_ID = xml.name - else: - xml_ID = None - - data_model = self.pool.get(model) - for record in browse_rec_list: - if xml_ID is not None: - init_data['__id'] = safe_unique_id(model,record.id)+':'+xml_ID - else: - init_data['__id'] = safe_unique_id(model,record.id)+':'+model+'-'+safe_unique_id(model,record.id) - init_data['__last_update'] =record.write_date - init_data.update(self.determine_type(cr,uid,ids,line,record,context=context)) - - res.append(init_data) - return res + the browse_records from the given browse_record_list. - def edi_m2m(self, cr, uid, line = None, browse_rec_list, context=None): - """Return a list representing a M2M EDI value for - the browse_records from the given browse_record_list.""" + Example: + [ # O2M fields would be a list of dicts, with their + { '__id': 'db-uuid:xml-id', # own __id. + '__last_update': 'iso date', # The update date, just in case... + 'name': 'some name', + ... + }], + """ # generic implementation! - model = browse_rec_list[0]._name - res = [] - init_data = [] - data_object = self.pool.get('ir.model.data') - uu_id = data_model.search(cr,uid,[('model','=',model)]) - for xml in data_model.browse(cr,uid,uu_id): - if xml: - xml_ID = xml.name - else: - xml_ID = None - data_model = self.pool.get(model) - for record in browse_rec_list: - if xml_ID is not None: - init_data.append(safe_unique_id(model,record.id)+':'+xml_ID) - else: - init_data.append(safe_unique_id(model,record.id)+':'+model+'-'+safe_unique_id(model,record.id)) - if record.name is not None: - init_data.append('name:'+record.name) - res.append(init_data) - - # blob: - return res + dict_list = [] + for record in records: + model_obj = self.pool.get(record._name) + dict_list += model_obj.edi_export(cr, uid, [record.id], context=context) + return dict_list + + + def edi_m2m(self, cr, uid, records, context=None): + """Return a list representing a M2M EDI value for + the browse_records from the given browse_record_list. + + Example: + 'related_tasks': [ # M2M fields would exported as a list of pairs, + ['db-uuid:xml-id1', # similar to a list of M2O values. + 'Task 01: bla bla'], + ['db-uuid:xml-id2', + 'Task 02: bla bla'] + ] + """ + # generic implementation! + dict_list = [] + for record in records: + dict_list.append(self.edi_o2m(cr, uid, record, context=None)) + + return dict_list + + def edi_export(self, cr, uid, ids, edi_struct=None, context=None): """Returns a list of dicts representing an edi.document containing the @@ -265,135 +310,172 @@ class edi(object): be included in the exported data. """ # generic implementation! - - self.edi_document = { - - '__model':'', - '__module':'', - '__id':'', - '__last_update':'', - '__version':'', - '__attachments':'', - - } - self.edi_document['__model'] = self._name + if context is None: + context = {} + _fields = self.fields_get(cr, uid, context=context) - fields_object = self.pool.get('ir.model.fields') - model_object = self.pool.get(self.edi_document['__model']) - edi_metadata(cr, uid, edi_struct, context=context) - record = model_object.read(cr,uid,ids,[],context=context) - for field in edi_struct.keys(): - f_ids = fields_object.search(cr,uid,[('name','=',field),('model','=',self.edi_document['__model'])],context=context) - - for fname in fields_object.browse(cr,uid,f_ids): - if fname.ttype == 'many2one': - - self.edi_document[field] = self.edi_m2o(cr, uid, browse_rec = record[0][field]) - elif fname.ttype == 'one2many': - - self.edi_ducument[field] = self.edi_o2m(cr,uid,line = edi_struct[field],browse_rec_list = record[0][field]) - elif fname.ttype == 'many2many': - self.edi_document[field] = self.edi_m2m(cr,uid,line = edi_struct[field],browse_rec_list = record[0][field]) + fields_to_export = edi_struct and edi_struct.keys() or _fields.key() + + edi_dict_list = [] + for row in self.browse(cr, uid, ids, fields_to_export, context): + edi_dict = {} + edi_dict.update(self.edi_metadata(cr, uid, [row.id], context=context)[0]) + for field in fields_to_export: + cols = self._fields[field] + if cols['type'] == 'many2one': + m2o_record = row.getattrs(field) + value = self.m2o(cr, uid, m2o_record, context=context) + elif cols['type'] == 'many2many': + m2m_records = row.getattrs(field) + value = self.m2m(cr, uid, m2m_records, context=context) + elif cols['type'] == 'one2many': + o2m_records = row.getattrs(field) + value = self.o2m(cr, uid, o2m_records, context=context) else: - self.edi_document[field] = record[0][field] + value = row.getattrs(field) + edi_dict[field] = value + edi_dict_list.append(edi_dict) + return edi_dict_list - - return self.edi_document - def determine_type(self,cr,uid,ids,record_line,browse_record=None,context=None): - model = browse_record._name - fields_object = self.pool.get('ir.model.fields') - obj = self.pool.get(model) - field_data = obj.read(cr,uid,browse_record.id,record_line.keys())[0] - for field in field_data.keys(): - field_ids = field_object.search(cr,uid,[field]) - for field_type in field_object.browse(cr,uid,field_ids): - if field_type.ttype == 'many2one': - record_line.update(field,self.edi_m2o(cr,uid,browse_rec = field_data[field])) - elif field_type.ttype == 'one2many': - pass - elif field_type.ttype == 'many2many': - pass - else: - record_line.update(field,field_data[field]) - return record_line def edi_import(self, cr, uid, edi_document, context=None): """Imports a list of dicts representing an edi.document, using the generic algorithm. + + All relationship fields are exported in a special way, and provide their own + unique identifier, so that we can avoid duplication of records when importing. + Note: for all ir.model.data entries, the "module" value to use for read/write + should always be "edi_import", and the "name" value should be the full + db_id provided in the EDI. + + 1: Many2One + M2O fields are always exported as a pair [db_id, name], where db_id + is in the form "db_uuid:xml_id", both being values that come directly + from the original database. + The import should behave like this: + a. Look in ir.model.data for a record that matches the db_id. + If found, replace the m2o value with the correct database ID and stop. + If not found, continue to next step. + b. Perform name_search(name) to look for a record that matches the + given m2o name. If only one record is found, create the missing + ir.model.data record to link it to the db_id, and the replace the m2o + value with the correct database ID, then stop. If zero result or + multiple results are found, go to next step. + c. Create the new record using the only field value that is known: the + name, and create the ir.model.data entry to map to it. + This should work for many models, and if not, the module should + provide a custom EDI import logic to care for it. + + 2: One2Many + O2M fields are always exported as a list of dicts, where each dict corresponds + to a full EDI record. The import should not update existing records + if they already exist, it should only link them to the parent object. + a. Look for a record that matches the db_id provided in the __id field. If + found, keep the corresponding database id, and connect it to the parent + using a write value like (4,db_id). + b. If not found via db_id, create a new entry using the same method that + imports a full EDI record (recursive call!), grab the resulting db id, + and use it to connect to the parent via a write value like (4, db_id). + + 3: Many2Many + M2M fields are always exported as a list of pairs similar to M2O. + For each pair in the M2M: + a. Perform the same steps as for a Many2One (see 1.2.1.1) + b. After finding the database ID of the final record in the database, + connect it to the parent record via a write value like (4, db_id). """ # generic implementation! - new = True - resource = {} - model = edi_document['__model'] - model_object = self.pool.get(model) - data_model = self.pool.get('ir.model.data') - field_model = self.pool.get('ir.model.fields') + fields = edi_document.keys() + fields_to_import = [] + data_line = [] + model_data = self.pool.get('ir.model.data') + _fields = self.fields_get(cr, uid, context=context) + current_module = 'edi_import' + values = {} for field in edi_document.keys(): - if field == '__id': - db_id , xml_ID = edi_document[field].split(':') - self.data_ids = data_model.search(cr,uid,[('name','=',xml_ID)]) - - - f_ids = fields_object.search(cr,uid,[('name','=',field),('model','=',model)],context=context) - for fname in fields_object.browse(cr,uid,f_ids): - - if fname.ttype == 'many2one': - field_ids,field_xml = edi_document[field][0].split(':') - field_ids = int(field_ids) - model_ids = data_model.search(cr,uid,[('name','=',field_xml)]) - if not len(model_ids): - obj = self.pool.get(fname.relation) - model1_ids = obj.search(cr,uid,[('name','=',edi_document[field][1])]) - if len(model1_ids) == 1: - resourse[field] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':model1_ids})[0] - - elif not len(model1_ids) or len(model1_ids) > 1: - edi_document[field][1] = obj.create(cr,uid,{'name':edi_document[field][1]}) - resource[field] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':edi_document[field][0]})[0] + if not field.startwith('__'): + field_to_import.append(field) + edi_field_value = edi_document[field] + if _fields[field]['type'] in ('many2one', 'many2many'): + + if _fields[field]['type'] == 'many2one': + edi_parent_documents = [edi_field_value] else: - resource[field] = model_ids[0] - - elif fname.ttype == 'one2many': - for records in edi_document[field]: - res = {} - field_ids,field_xml = records['__id'].split(':') - field_ids = int(field_ids) - model3_ids = data_model.search(cr,uid,[('name','=',field_xml)]) - for fields in records.keys(): - if fields != '__id' or fields != '__last_update': - res[fields] = records[fields] - if not len(model3_ids): - resource[field][records] = (4,self.edi_import(cr,uid,res,context = context)[0]) + edi_parent_documents = edi_field_value + + parent_lines = [] + + for edi_parent_document in edi_parent_documents: + #Look in ir.model.data for a record that matches the db_id. + #If found, replace the m2o value with the correct database ID and stop. + #If not found, continue to next step + + db_uuid, xml_id = tuple(edi_parent_document[0].split(':')) + data_ids = model_data.name_search(cr, uid, xml_id) + if data_ids: + d = model_data.browse(cr, uid, data_ids[0], context=context) + parent_lines.append(d.res_id) + else: - resource[field][records] = (4,model3_ids[0])obj = self.pool.get(model) - record_ids = obj.create(cr,uid,resource,context=context) - return record_ids + #Perform name_search(name) to look for a record that matches the + #given m2o name. If only one record is found, create the missing + #ir.model.data record to link it to the db_id, and the replace the m2o + #value with the correct database ID, then stop. If zero result or + #multiple results are found, go to next step. + #Create the new record using the only field value that is known: the + #name, and create the ir.model.data entry to map to it. + + relation_object = self.get.pool(_fields[field]['relation']) + relation_ids = relation_object.name_search(cr, uid, edi_parent_document[1]) + if relation_ids and len(relation_ids) == 1: + relation_id = relation_ids[0] + else: + relation_id = relation_object.create(cr, uid, {'name': edi_parent_document[1]}, context=context) + model_data.create(cr, uid, { + 'name': xml_id, + 'model': relation_object._name, + #'module':relation_object._module, + 'res_id':relation_id, + },context=context) + + parent_lines.append(relation_id) + + if _fields[field]['type'] == 'many2one': + values[field] = parent_lines[0] + else: + many2many_ids = [] + for m2m_id in parent_lines: + many2many_ids.append((4,m2m_id)) + values[field] = many2many_ids - elif fname.ttype == 'many2many': - for records in edi_document[field]: - field_ids,field_xml = records[0].split(':') - field_ids = int(field_ids) - model4_ids = data_model.search(cr,uid,[('name','=',field_xml)]) - if not len(model4_ids): - obj = self.pool.get(fname.relation) - name,value = records[1].split(':') - model5_ids = obj.search(cr,uid,[(name,'=',value)]) - if len(model5_ids) == 1: - resource[field][records] = data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':model5_ids})[0] - elif not len(model5_ids) or len(model5_ids) > 1: - edi_document[field][records][1] = obj.create(cr,uid,{'name':edi_document[field][1]}) - resource[field] = (4,data_model.create(cr,uid,{'module':'edi_import','name':field_xml,'res_id':edi_document[field][records][1]})[0]) + + elif _fields[field]['type'] == 'one2many': + #Look for a record that matches the db_id provided in the __id field. If + #found, keep the corresponding database id, and connect it to the parent + #using a write value like (4,db_id). + #If not found via db_id, create a new entry using the same method that + #imports a full EDI record (recursive call!), grab the resulting db id, + #and use it to connect to the parent via a write value like (4, db_id). + + relations = [] + relation_object = self.get.pool(_fields[field]['relation']) + for edi_relation_document in edi_field_value: + db_uuid, xml_id = tuple(edi_relation_document.get('__id').split(':')) + data_ids = model_data.name_search(cr, uid, xml_id) + if data_ids: + d = model_data.browse(cr, uid, data_ids[0], context=context) + relations.append(d.res_id) else: - resource[field] = (4,model4_ids[0]) + r = relation_object.edi_import(cr, uid, edi_relation_document, context=context) + relations.append(r) + one2many_ids = [] + for o2m_id in relations: + one2many_ids.append((4,o2m_id)) + values[field] = one2many_ids else: - resource[field] = edi_document[field] - if len(self.data_ids): - obj = self.pool.get(model) - record_ids = obj.write(cr,uid,resource,context=context) - return record_ids - else: - obj = self.pool.get(model) - record_ids = obj.create(cr,uid,resource,context=context) - return record_ids + values[field] = edi_field_value + + return model_data._update(cr, uid, self._name, current_module, values, context=context) +