From 8752ac03b4acc461bd9156c2b2107635591404e9 Mon Sep 17 00:00:00 2001 From: "tfr@openerp.com" <> Date: Thu, 12 May 2011 10:49:08 +0200 Subject: [PATCH] [ADD] import framework bzr revid: tfr@openerp.com-20110512084908-j3o9lsrja52ybflf --- addons/import_base/__init__.py | 23 ++ addons/import_base/__openerp__.py | 39 +++ addons/import_base/import_framework.py | 420 +++++++++++++++++++++++++ addons/import_base/mapper.py | 167 ++++++++++ 4 files changed, 649 insertions(+) create mode 100644 addons/import_base/__init__.py create mode 100644 addons/import_base/__openerp__.py create mode 100644 addons/import_base/import_framework.py create mode 100644 addons/import_base/mapper.py diff --git a/addons/import_base/__init__.py b/addons/import_base/__init__.py new file mode 100644 index 00000000000..0a5d7fb20b7 --- /dev/null +++ b/addons/import_base/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Openerp sa (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +import import_framework +import mapper \ No newline at end of file diff --git a/addons/import_base/__openerp__.py b/addons/import_base/__openerp__.py new file mode 100644 index 00000000000..962541f427a --- /dev/null +++ b/addons/import_base/__openerp__.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +{ + 'name': 'Base framework for module that need to import complex data', + 'version': '0.9', + 'category': 'Generic Modules', + 'description': """ + This module provide a class import_framework to help importing + complex data from other software + """, + 'author': 'OpenERP SA', + 'website': 'http://www.openerp.com', + 'depends': ['base'], + 'init_xml': [], + 'update_xml': [], + 'demo_xml': [], + 'test': [], #TODO provide test + 'installable': True, + 'active': False, +} diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py new file mode 100644 index 00000000000..febfbb68d44 --- /dev/null +++ b/addons/import_base/import_framework.py @@ -0,0 +1,420 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import pprint +import mapper +pp = pprint.PrettyPrinter(indent=4) + + + + +class import_framework(): + """ + This class should be extends, + get_data and get_mapping have to extends + get_state_map and initialize can be extended + for advanced purpose get_default_hook can also be extended + @see dummy import for a minimal exemple + """ + + """ + for import_object, this domain will avoid to find an already existing object + """ + DO_NOT_FIND_DOMAIN = [('id', '=', 0)] + + def __init__(self, obj, cr, uid, instance_name, module_name, context): + self.obj = obj + self.cr = cr + self.uid = uid + self.instance_name = instance_name + self.module_name = module_name + self.context = context or {} + self.initialize() + + + """ + Abstract Method to be implemented in + the real instance + """ + def initialize(self): + """ + init before import + usually for the login + """ + pass + + def get_data(self, table): + """ + @return: a list of dictionaries + each dictionnaries contains the list of pair external_field_name : value + """ + return [{}] + + def get_mapping(self): + """ + @return: { TABLE_NAME : { + 'model' : 'openerp.model.name', + #if true import the table if not just resolve dependencies, use for meta package, by default => True + #Not required + 'import' : True or False, + #Not required + 'dependencies' : [TABLE_1, TABLE_2], + #Not required + 'hook' : self.function_name, #get the val dict of the object, return the same val dict or False + 'map' : { @see mapper + 'openerp_field_name' : 'external_field_name', or val('external_field_name') + 'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right + 'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id + 'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state + 'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one + 'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting + 'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable + 'field' : callable + 'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field + 'field' : const(value) #always set this field to value + + any custom mapper that you will define + } + }, + + } + """ + return {} + + def default_hook(self, val): + """ + this hook will be apply on each table that don't have hook + here we define the identity hook + """ + return val + + def _import_table(self, table): + data = self.get_data(table) + map = self.get_mapping()[table]['map'] + hook = self.get_mapping()[table].get('hook', self.default_hook) + model = self.get_mapping()[table]['model'] + + final_data = [] + for val in data: + res = hook(val) + if res: + final_data.append(res) + self._save_data(model, map, final_data, table) + + def _save_data(self, model, mapping, datas, table): + """ + @param model: the model of the object to import + @param table : the external table where the data come from + @param mapping : definition of the mapping + @see: get_mapping + @param datas : list of dictionnaries + datas = [data_1, data_2, ..] + data_i is a map external field_name => value + and each data_i have a external id => in data_id['id'] + """ + if not datas: + return + mapping['id'] = 'id_new' + res = [] + for data in datas: + + self_dependencies = [] + for k in mapping.keys(): + if '_parent' in k: + self_dependencies.append(k[:-7]) + field_name = mapping.pop(k) + data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table) + + data['id_new'] = self._generate_xml_id(data['id'], table) + fields, values = self._fields_mapp(data, mapping, table) + res.append(values) + + model_obj = self.obj.pool.get(model) + if not model_obj: + raise ValueError("%s is not a valid model name" % model) + + model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context) + for field in self_dependencies: + self._import_self_dependencies(model_obj, field, datas) + + def _import_self_dependencies(self, obj, parent_field, datas): + """ + @param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this + field the parent of the object + @param datas: a list of dictionnaries + Dictionnaries need to contains + id_new : the xml_id of the object + field_new : the xml_id of the parent + """ + fields = ['id', parent_field] + for data in datas: + if data.get(parent_field + '_parent'): + values = [data['id_new'], data[parent_field + '_parent']] + obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context) + + def _preprocess_mapping(self, mapping): + """ + Preprocess the mapping : + after the preprocces, everything is + callable in the val of the dictionary + + use to allow syntaxical sugar like 'field': 'external_field' + instead of 'field' : value('external_field') + """ + map = dict(mapping) + for key, value in map.items(): + if isinstance(value, basestring): + map[key] = mapper.value(value) + #set parent for instance of dbmapper + elif isinstance(value, mapper.dbmapper): + value.set_parent(self) + return map + + + def _fields_mapp(self,dict_sugar, openerp_dict, table): + """ + call all the mapper and transform data + to be compatible with import_data + """ + fields=[] + data_lst = [] + mapping = self._preprocess_mapping(openerp_dict) + for key,val in mapping.items(): + if key not in fields and dict_sugar: + fields.append(key) + value = val(dict(dict_sugar)) + data_lst.append(value) + + return fields, data_lst + + def _generate_xml_id(self, name, table): + """ + @param name: name of the object, has to be unique in for a given table + @param table : table where the record we want generate come from + @return: a unique xml id for record, the xml_id will be the same given the same table and same name + To be used to avoid duplication of data that don't have ids + """ + sugar_instance = "sugarcrm" #TODO need to be changed we information is known in the wizard + name = name.replace('.', '_').replace(',', '_') + return sugar_instance + "_" + table + "_" + name + + + """ + Public interface of the framework + those function can be use in the callable function defined in the mapping + """ + def xml_id_exist(self, table, external_id): + """ + Check if the external id exist in the openerp database + in order to check if the id exist the table where it come from + should be provide + """ + if not external_id: + return False + + xml_id = self._generate_xml_id(external_id, table) + id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)]) + return id and xml_id or False + + def name_exist(self, table, name, model): + """ + Check if the object with the name exist in the openerp database + in order to check if the id exist the table where it come from + should be provide and the model of the object + """ + fields = ['name'] + data = [name] + return self.import_object(fields, data, model, table, name, [('name', '=', name)]) + + def get_mapped_id(self, table, external_id, context=None): + """ + @return return the databse id linked with the external_id + """ + if not external_id: + return False + + xml_id = self._generate_xml_id(external_id, table) + return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1] + + def import_object_mapping(self, mapping, data, model, table, name, domain_search=False): + """ + same as import_objects but instead of two list fields and data, + this method take a dictionnaries : external_field : value + and the mapping similar to the one define in 'map' key + @see import_object, get_mapping + """ + fields, datas = self._fields_mapp(data, mapping, table) + return self.import_object(fields, datas, model, table, name, domain_search) + + def import_object(self, fields, data, model, table, name, domain_search=False): + """ + This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp + use import_data that will take care to create/update or do nothing with the data + this method return the xml_id + + To be use, when you want to create an object or link if already exist + use DO_NOT_LINK_DOMAIN to create always a new object + @param fields: list of fields needed to create the object without id + @param data: the list of the data, in the same order as the field + ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds'] + @param model: the openerp's model of the create/update object + @param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique + @param unique_name: the name of the object that we want to create/update get the id + @param domain_search : the domain that should find the unique existing record + + @return: the xml_id of the ressources + """ + domain_search = not domain_search and [('name', 'ilike', name)] or domain_search + obj = self.obj.pool.get(model) + xml_id = self._generate_xml_id(name, table) + + xml_ref = self.mapped_id_if_exist(model, domain_search, table, name) + fields.append('id') + data.append(xml_id) + obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context) + return xml_ref or xml_id + + + def mapped_id_if_exist(self, model, domain, table, name): + """ + To be use when we want link with and existing object, if the object don't exist + just ignore. + @param domain : search domain to find existing record, should return a unique record + @param xml_id: xml_id give to the mapping + @param name: external_id or name of the object to create if there is no id + @param table: the name of the table of the object to map + @return : the xml_id if the record exist in the db, False otherwise + """ + obj = self.obj.pool.get(model) + ids = obj.search(self.cr, self.uid, domain, context=self.context) + if ids: + xml_id = self._generate_xml_id(name, table) + ir_model_data_obj = obj.pool.get('ir.model.data') + id = ir_model_data_obj._update(self.cr, self.uid, model, + self.module_name, {}, mode='update', xml_id=xml_id, + noupdate=True, res_id=ids[0], context=self.context) + return xml_id + return False + + + def import_all(self, table_list): + """Import all data into openerp, + this is the Entry point to launch the process of import + @param table_list: the list of external table to import + ['Leads', 'Opportunity'] + + """ + imported = set() #to invoid importing 2 times the sames modules + for table in table_list: + to_import = self.get_mapping()[table].get('import', True) + if not table in imported: + self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported) + if to_import: + self._import_table(table) + imported.add(table) + + def _resolve_dependencies(self, dep, imported): + """ + import dependencies recursively + and avoid to import twice the same table + """ + for dependency in dep: + if not dependency in imported: + to_import = self.get_mapping()[dependency].get('import', True) + self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported) + if to_import: + self._import_table(dependency) + imported.add(dependency) + + + + +class dummy_import(import_framework): + """ + this is a really simple exemple to show how to use the framework + """ + + TABLE_LEAD = 'Leads' + TABLE_STAGE = 'Stage' + TABLE_USER = 'User' + + def get_lead_status(self, sugar_val): + fields = ['name', 'type'] + name = 'lead_' + sugar_val.get('status', '') + data = [sugar_val.get('status', ''), 'lead'] + return self.import_object(fields, data, 'crm.case.stage', self.TABLE_STAGE, name, [('type', '=', 'lead'), ('name', 'ilike', sugar_val.get('status', ''))]) + + + + + lead_state = { + 'New' : 'draft', + 'Assigned':'open', + 'In Progress': 'open', + 'Recycled': 'cancel', + 'Dead': 'done', + } + + def get_mapping(self): + return { + self.TABLE_LEAD : { + 'model' : 'crm.lead', + 'dependencies' : [self.TABLE_USER], + 'map' : { + 'name' : 'name', + 'stage_id/id' : self.get_lead_status, + 'state' : mapper.map_val('status', self.lead_state), + 'user_id/id' : mapper.ref(self.TABLE_USER, 'assigned_user_id'), + } + + }, + self.TABLE_USER : { + 'model' : 'res.users', + 'map' : { + 'name' : 'name', + 'login' : 'name' + } + } + } + + + #DUMMY way to get data, just demonstration + def get_data(self, table): + if table == self.TABLE_LEAD: + return self.get_lead() + if table == self.TABLE_USER: + return self.get_user() + + + def get_lead(self): + return [ + {'id' : 'lead1', + 'name' : 'name1', + 'status' : 'Assigned', + 'assigned_user_id' : 'user1'}, + {'id' : 'lead2', + 'name' : 'name2' + }, + ] + + def get_user(self): + return [{ + 'id' : 'user1', + 'name' : 'name1', + }] diff --git a/addons/import_base/mapper.py b/addons/import_base/mapper.py new file mode 100644 index 00000000000..582a45ee4fb --- /dev/null +++ b/addons/import_base/mapper.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# OpenERP, Open Source Management Solution +# Copyright (C) 2004-2010 Tiny SPRL (). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + + +class mapper(object): + """ + super class for all mapper class + They are call before import data + to transform the mapping into real value that we + will import + + the call function receive a dictionary with external data + 'external_field' : value + """ + def __call__(self, external_values): + raise NotImplementedError() + +class dbmapper(mapper): + """ + Super class for mapper that need to access to + data base or any function of the import_framework + + self.parent contains a reference to the instance of + the import framework + """ + def set_parent(self, parent): + self.parent = parent + + +class concat(mapper): + """ + Use : contact('field_name1', 'field_name2', delimiter='_') + concat value of fields using the delimiter, delimiter is optional + and by default is a space + + """ + def __init__(self, *arg, **delimiter): + self.arg = arg + self.delimiter = delimiter and delimiter.get('delimiter', ' ') or ' ' + + def __call__(self, external_values): + return self.delimiter.join(map(lambda x : external_values.get(x,''), self.arg)) + +class ppconcat(mapper): + """ + Use : contact('field_name1', 'field_name2', delimiter='_') + concat external field name and value of fields using the delimiter, + delimiter is optional and by default is a two line feeds + + """ + def __init__(self, *arg, **delimiter): + self.arg = arg + self.delimiter = delimiter and delimiter.get('delimiter', ' ') or '\n\n' + + def __call__(self, external_values): + return self.delimiter.join(map(lambda x : x + ": " + external_values.get(x,''), self.arg)) + +class const(mapper): + """ + Use : const(arg) + return always arg + """ + def __init__(self, val): + self.val = val + + def __call__(self, external_values): + return self.val + +class value(mapper): + """ + Use : value(external_field_name) + Return the value of the external field name + this is equivalent to the a single string + + usefull for call if you want your call get the value + and don't care about the name of the field + call(self.method, value('field1')) + """ + def __init__(self, val, default=''): + self.val = val + self.default = default + + def __call__(self, external_values): + return external_values.get(self.val, self.default) + +class map_val(mapper): + """ + Use : map_val(external_field, val_mapping) + where val_mapping is a dictionary + with external_val : openerp_val + + usefull for selection field like state + to map value + """ + def __init__(self, val, map, default='draft'): + self.val = value(val) + self.map = map + self.default = default + + def __call__(self, external_values): + return self.map.get(self.val(external_values), self.default) + +class ref(dbmapper): + """ + Use : ref(table_name, external_id) + return the xml_id of the ressource + + to associate an already imported object with the current object + """ + def __init__(self, table, field_name): + self.table = table + self.field_name = field_name + + def __call__(self, external_values): + return self.parent.xml_id_exist(self.table, external_values.get(self.field_name)) + +class refbyname(dbmapper): + """ + Use : refbyname(table_name, external_name, res.model) + same as ref but use the name of the ressource to find it + """ + def __init__(self, table, field_name, model): + self.table = table + self.field_name = field_name + self.model = model + + def __call__(self, external_values): + v = external_values.get(self.field_name, '') + return self.parent.name_exist(self.table, v , self.model) + +class call(mapper): + """ + Use : call(function, arg1, arg2) + to call the function with external val follow by the arg specified + """ + def __init__(self, fun, *arg): + self.fun = fun + self.arg = arg + + def __call__(self, external_values): + args = [] + for arg in self.arg: + if isinstance(arg, mapper): + args.append(arg(external_values)) + else: + args.append(arg) + return self.fun(external_values, *args) + + \ No newline at end of file