From 8752ac03b4acc461bd9156c2b2107635591404e9 Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Thu, 12 May 2011 10:49:08 +0200
Subject: [PATCH 1/6] [ADD] import framework

bzr revid: tfr@openerp.com-20110512084908-j3o9lsrja52ybflf
---
 addons/import_base/__init__.py         |  23 ++
 addons/import_base/__openerp__.py      |  39 +++
 addons/import_base/import_framework.py | 420 +++++++++++++++++++++++++
 addons/import_base/mapper.py           | 167 ++++++++++
 4 files changed, 649 insertions(+)
 create mode 100644 addons/import_base/__init__.py
 create mode 100644 addons/import_base/__openerp__.py
 create mode 100644 addons/import_base/import_framework.py
 create mode 100644 addons/import_base/mapper.py

diff --git a/addons/import_base/__init__.py b/addons/import_base/__init__.py
new file mode 100644
index 00000000000..0a5d7fb20b7
--- /dev/null
+++ b/addons/import_base/__init__.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2004-2010 Openerp sa (<http://openerp.com>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+import import_framework
+import mapper
\ No newline at end of file
diff --git a/addons/import_base/__openerp__.py b/addons/import_base/__openerp__.py
new file mode 100644
index 00000000000..962541f427a
--- /dev/null
+++ b/addons/import_base/__openerp__.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+{
+    'name': 'Base framework for module that need to import complex data',
+    'version': '0.9',
+    'category': 'Generic Modules',
+    'description': """
+        This module provide a class import_framework to help importing 
+        complex data from other software
+    """,
+    'author': 'OpenERP SA',
+    'website': 'http://www.openerp.com',
+    'depends': ['base'],
+    'init_xml': [],
+    'update_xml': [],
+    'demo_xml': [],
+    'test': [], #TODO provide test
+    'installable': True,
+    'active': False,
+}
diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
new file mode 100644
index 00000000000..febfbb68d44
--- /dev/null
+++ b/addons/import_base/import_framework.py
@@ -0,0 +1,420 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import pprint
+import mapper
+pp = pprint.PrettyPrinter(indent=4)
+
+
+
+
+class import_framework():
+    """
+        This class should be extends,
+        get_data and get_mapping have to extends
+        get_state_map and initialize can be extended
+        for advanced purpose get_default_hook can also be extended
+        @see dummy import for a minimal exemple
+    """
+    
+    """
+        for import_object, this domain will avoid to find an already existing object
+    """
+    DO_NOT_FIND_DOMAIN = [('id', '=', 0)]
+  
+    def __init__(self, obj, cr, uid, instance_name, module_name, context):
+        self.obj = obj
+        self.cr = cr
+        self.uid = uid
+        self.instance_name = instance_name
+        self.module_name = module_name
+        self.context = context or {}
+        self.initialize()
+        
+      
+    """
+        Abstract Method to be implemented in 
+        the real instance
+    """  
+    def initialize(self):
+        """
+            init before import
+            usually for the login
+        """
+        pass
+    
+    def get_data(self, table):
+        """
+            @return: a list of dictionaries
+                each dictionnaries contains the list of pair  external_field_name : value 
+        """
+        return [{}]
+    
+    def get_mapping(self):
+        """
+            @return: { TABLE_NAME : {
+                'model' : 'openerp.model.name',
+                #if true import the table if not just resolve dependencies, use for meta package, by default => True  
+                #Not required
+                'import' : True or False, 
+                #Not required
+                'dependencies' : [TABLE_1, TABLE_2],
+                #Not required
+                'hook' : self.function_name, #get the val dict of the object, return the same val dict or False
+                'map' : { @see mapper
+                    'openerp_field_name' : 'external_field_name', or val('external_field_name')
+                    'openerp_field_id/id' : ref(TABLE_1, 'external_id_field'), #make the mapping between the external id and the xml on the right
+                    'openerp_field2_id/id_parent' : ref(TABLE_1,'external_id_field') #indicate a self dependencies on openerp_field2_id
+                    'state' : map_val('state_equivalent_field', mapping), # use get_state_map to make the mapping between the value of the field and the value of the state
+                    'text_field' : concat('field_1', 'field_2', .., delimiter=':'), #concat the value of the list of field in one
+                    'description_field' : ppconcat('field_1', 'field_2', .., delimiter='\n\t'), #same as above but with a prettier formatting
+                    'field' : call(callable, arg1, arg2, ..), #call the function with all the value, the function should send the value : self.callable
+                    'field' : callable
+                    'field' : call(method, val('external_field') interface of method is self, val where val is the value of the field
+                    'field' : const(value) #always set this field to value
+                    + any custom mapper that you will define
+                } 
+            },
+            
+            } 
+        """
+        return {}
+    
+    def default_hook(self, val):
+        """
+            this hook will be apply on each table that don't have hook
+            here we define the identity hook
+        """
+        return val
+        
+    def _import_table(self, table):
+        data = self.get_data(table)
+        map = self.get_mapping()[table]['map']
+        hook = self.get_mapping()[table].get('hook', self.default_hook)
+        model = self.get_mapping()[table]['model']
+        
+        final_data = []
+        for val in data:
+            res = hook(val)
+            if res:
+                final_data.append(res)
+        self._save_data(model, map, final_data, table)
+        
+    def _save_data(self, model, mapping, datas, table):
+        """
+            @param model: the model of the object to import
+            @param table : the external table where the data come from 
+            @param mapping : definition of the mapping
+                             @see: get_mapping
+            @param datas : list of dictionnaries 
+                datas = [data_1, data_2, ..]
+                data_i is a map external field_name => value
+                and each data_i have a external id => in data_id['id']
+        """
+        if not datas:
+            return
+        mapping['id'] = 'id_new'
+        res = []
+        for data in datas:
+            
+            self_dependencies = []
+            for k in mapping.keys():
+                if '_parent' in k:
+                    self_dependencies.append(k[:-7])
+                    field_name = mapping.pop(k)
+                    data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table)
+                    
+            data['id_new'] = self._generate_xml_id(data['id'], table)
+            fields, values = self._fields_mapp(data, mapping, table)
+            res.append(values)
+        
+        model_obj = self.obj.pool.get(model)
+        if not model_obj:
+            raise ValueError("%s is not a valid model name" % model)
+        
+        model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context)
+        for field in self_dependencies:
+            self._import_self_dependencies(model_obj, field, datas)
+            
+    def _import_self_dependencies(self, obj, parent_field, datas):
+        """
+            @param parent_field: the name of the field that generate a self_dependencies, we call the object referenced in this
+                field the parent of the object
+            @param datas: a list of dictionnaries
+                Dictionnaries need to contains 
+                    id_new : the xml_id of the object
+                    field_new : the xml_id of the parent
+        """
+        fields = ['id', parent_field]
+        for data in datas:
+            if data.get(parent_field + '_parent'):
+                values = [data['id_new'], data[parent_field + '_parent']]
+                obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context) 
+    
+    def _preprocess_mapping(self, mapping):
+        """
+            Preprocess the mapping :
+            after the preprocces, everything is 
+            callable in the val of the dictionary
+            
+            use to allow syntaxical sugar like 'field': 'external_field'
+            instead of 'field' : value('external_field')
+        """
+        map = dict(mapping)
+        for key, value in map.items():
+            if isinstance(value, basestring):
+                map[key] = mapper.value(value)
+            #set parent for instance of dbmapper
+            elif isinstance(value, mapper.dbmapper): 
+                value.set_parent(self)
+        return map   
+            
+            
+    def _fields_mapp(self,dict_sugar, openerp_dict, table):
+        """
+            call all the mapper and transform data 
+            to be compatible with import_data
+        """
+        fields=[]
+        data_lst = []
+        mapping = self._preprocess_mapping(openerp_dict)
+        for key,val in mapping.items():
+            if key not in fields and dict_sugar:
+                fields.append(key)
+                value = val(dict(dict_sugar))
+                data_lst.append(value)
+                
+        return fields, data_lst
+    
+    def _generate_xml_id(self, name, table):
+        """
+            @param name: name of the object, has to be unique in for a given table
+            @param table : table where the record we want generate come from
+            @return: a unique xml id for record, the xml_id will be the same given the same table and same name
+                     To be used to avoid duplication of data that don't have ids
+        """
+        sugar_instance = "sugarcrm" #TODO need to be changed we information is known in the wizard
+        name = name.replace('.', '_').replace(',', '_')
+        return sugar_instance + "_" + table + "_" + name
+    
+    
+    """
+        Public interface of the framework
+        those function can be use in the callable function defined in the mapping
+    """
+    def xml_id_exist(self, table, external_id):
+        """
+            Check if the external id exist in the openerp database
+            in order to check if the id exist the table where it come from
+            should be provide
+        """
+        if not external_id:
+            return False
+        
+        xml_id = self._generate_xml_id(external_id, table)
+        id = self.obj.pool.get('ir.model.data').search(self.cr, self.uid, [('name', '=', xml_id), ('module', '=', self.module_name)])
+        return id and xml_id or False
+    
+    def name_exist(self, table, name, model):
+        """
+            Check if the object with the name exist in the openerp database
+            in order to check if the id exist the table where it come from
+            should be provide and the model of the object
+        """
+        fields = ['name']
+        data = [name]
+        return self.import_object(fields, data, model, table, name, [('name', '=', name)])
+    
+    def get_mapped_id(self, table, external_id, context=None):
+        """
+            @return return the databse id linked with the external_id
+        """
+        if not external_id:
+            return False
+    
+        xml_id = self._generate_xml_id(external_id, table)
+        return self.obj.pool.get('ir.model.data').get_object_reference(self.cr, self.uid, self.module_name, xml_id)[1]
+    
+    def import_object_mapping(self, mapping, data, model, table, name, domain_search=False):
+        """
+            same as import_objects but instead of two list fields and data, 
+            this method take a dictionnaries : external_field : value 
+                            and the mapping similar to the one define in 'map' key
+            @see import_object, get_mapping
+        """
+        fields, datas = self._fields_mapp(data, mapping, table)
+        return self.import_object(fields, datas, model, table, name, domain_search)
+
+    def import_object(self, fields, data, model, table, name, domain_search=False):
+        """
+            This method will import an object in the openerp, usefull for field that is only a char in sugar and is an object in openerp
+            use import_data that will take care to create/update or do nothing with the data
+            this method return the xml_id
+            
+            To be use, when you want to create an object or link if already exist
+            use DO_NOT_LINK_DOMAIN to create always a new object
+            @param fields: list of fields needed to create the object without id
+            @param data: the list of the data, in the same order as the field
+                ex : fields = ['firstname', 'lastname'] ; data = ['John', 'Mc donalds']
+            @param model: the openerp's model of the create/update object
+            @param table: the table where data come from in sugarcrm, no need to fit the real name of openerp name, just need to be unique
+            @param unique_name: the name of the object that we want to create/update get the id
+            @param domain_search : the domain that should find the unique existing record
+            
+            @return: the xml_id of the ressources
+        """
+        domain_search = not domain_search and [('name', 'ilike', name)] or domain_search
+        obj = self.obj.pool.get(model)
+        xml_id = self._generate_xml_id(name, table)
+        
+        xml_ref = self.mapped_id_if_exist(model, domain_search, table, name)
+        fields.append('id')
+        data.append(xml_id)
+        obj.import_data(self.cr, self.uid, fields, [data], mode='update', current_module=self.module_name, noupdate=True, context=self.context)
+        return xml_ref or xml_id
+    
+
+    def mapped_id_if_exist(self, model, domain, table, name):
+        """
+            To be use when we want link with and existing object, if the object don't exist
+            just ignore.
+            @param domain : search domain to find existing record, should return a unique record
+            @param xml_id: xml_id give to the mapping
+            @param name: external_id or name of the object to create if there is no id
+            @param table: the name of the table of the object to map
+            @return : the xml_id if the record exist in the db, False otherwise
+        """
+        obj = self.obj.pool.get(model)
+        ids = obj.search(self.cr, self.uid, domain, context=self.context)
+        if ids:
+            xml_id = self._generate_xml_id(name, table)
+            ir_model_data_obj = obj.pool.get('ir.model.data')
+            id = ir_model_data_obj._update(self.cr, self.uid, model,
+                             self.module_name, {}, mode='update', xml_id=xml_id,
+                             noupdate=True, res_id=ids[0], context=self.context)
+            return xml_id
+        return False
+    
+    
+    def import_all(self, table_list):
+        """Import all data into openerp, 
+           this is the Entry point to launch the process of import
+           @param table_list: the list of external table to import
+               ['Leads', 'Opportunity']
+        
+        """
+        imported = set() #to invoid importing 2 times the sames modules
+        for table in table_list:
+            to_import = self.get_mapping()[table].get('import', True)
+            if not table in imported:
+                self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
+                if to_import:
+                    self._import_table(table)
+                imported.add(table)
+
+    def _resolve_dependencies(self, dep, imported):
+        """ 
+            import dependencies recursively
+            and avoid to import twice the same table
+        """
+        for dependency in dep:
+            if not dependency in imported:
+                to_import = self.get_mapping()[dependency].get('import', True)
+                self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported)
+                if to_import:
+                    self._import_table(dependency)
+                imported.add(dependency)
+    
+    
+    
+    
+class dummy_import(import_framework):
+    """
+        this is a really simple exemple to show how to use the framework
+    """
+    
+    TABLE_LEAD = 'Leads'
+    TABLE_STAGE = 'Stage'
+    TABLE_USER = 'User'
+    
+    def get_lead_status(self, sugar_val):
+        fields = ['name', 'type']
+        name = 'lead_' + sugar_val.get('status', '')
+        data = [sugar_val.get('status', ''), 'lead']
+        return self.import_object(fields, data, 'crm.case.stage', self.TABLE_STAGE, name, [('type', '=', 'lead'), ('name', 'ilike', sugar_val.get('status', ''))])
+
+    
+    
+  
+    lead_state =  {
+            'New' : 'draft',
+            'Assigned':'open',
+            'In Progress': 'open',
+            'Recycled': 'cancel',
+            'Dead': 'done',
+        }
+    
+    def get_mapping(self):
+        return {
+                self.TABLE_LEAD : {
+                    'model' : 'crm.lead',
+                    'dependencies' : [self.TABLE_USER],
+                    'map' : {
+                            'name' : 'name',
+                            'stage_id/id' : self.get_lead_status, 
+                            'state' : mapper.map_val('status', self.lead_state),
+                            'user_id/id' : mapper.ref(self.TABLE_USER, 'assigned_user_id'),
+                    }         
+                              
+                },
+                self.TABLE_USER : {
+                    'model' : 'res.users',
+                    'map' : {
+                             'name' : 'name',
+                             'login' : 'name'
+                    }              
+                }
+        }
+        
+       
+    #DUMMY way to get data, just demonstration 
+    def get_data(self, table):
+        if table == self.TABLE_LEAD:
+            return self.get_lead()
+        if table == self.TABLE_USER:
+            return self.get_user()
+        
+    
+    def get_lead(self):
+        return [
+                {'id' : 'lead1',
+                 'name' : 'name1',
+                 'status' : 'Assigned',
+                 'assigned_user_id' : 'user1'},
+                {'id' : 'lead2',
+                 'name' : 'name2'
+                 },
+                ]
+        
+    def get_user(self):
+        return [{
+            'id' : 'user1',
+            'name' : 'name1',
+                }]
diff --git a/addons/import_base/mapper.py b/addons/import_base/mapper.py
new file mode 100644
index 00000000000..582a45ee4fb
--- /dev/null
+++ b/addons/import_base/mapper.py
@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
+
+class mapper(object):
+    """
+        super class for all mapper class
+        They are call before import data
+        to transform the mapping into real value that we
+        will import
+        
+        the call function receive a dictionary with external data
+            'external_field' : value
+    """
+    def __call__(self, external_values):
+        raise NotImplementedError()
+
+class dbmapper(mapper):
+    """
+        Super class for mapper that need to access to 
+        data base or any function of the import_framework
+        
+        self.parent contains a reference to the instance of
+        the import framework
+    """
+    def set_parent(self, parent):
+        self.parent = parent
+        
+
+class concat(mapper):
+    """
+        Use : contact('field_name1', 'field_name2', delimiter='_')
+        concat value of fields using the delimiter, delimiter is optional
+        and by default is a space
+        
+    """
+    def __init__(self, *arg, **delimiter):
+        self.arg = arg
+        self.delimiter = delimiter and delimiter.get('delimiter', ' ') or ' '
+        
+    def __call__(self, external_values):
+        return self.delimiter.join(map(lambda x : external_values.get(x,''), self.arg))
+    
+class ppconcat(mapper):
+    """
+        Use : contact('field_name1', 'field_name2', delimiter='_')
+        concat external field name and value of fields using the delimiter, 
+        delimiter is optional and by default is a two line feeds
+        
+    """
+    def __init__(self, *arg, **delimiter):
+        self.arg = arg
+        self.delimiter = delimiter and delimiter.get('delimiter', ' ') or '\n\n'
+        
+    def __call__(self, external_values):
+        return self.delimiter.join(map(lambda x : x + ": " + external_values.get(x,''), self.arg))
+    
+class const(mapper):
+    """
+        Use : const(arg)
+        return always arg
+    """
+    def __init__(self, val):
+        self.val = val
+        
+    def __call__(self, external_values):
+        return self.val 
+    
+class value(mapper):
+    """
+        Use : value(external_field_name)
+        Return the value of the external field name
+        this is equivalent to the a single string
+        
+        usefull for call if you want your call get the value
+        and don't care about the name of the field
+        call(self.method, value('field1'))
+    """
+    def __init__(self, val, default=''):
+        self.val = val
+        self.default = default
+        
+    def __call__(self, external_values):
+        return external_values.get(self.val, self.default) 
+    
+class map_val(mapper):
+    """
+        Use : map_val(external_field, val_mapping)
+        where val_mapping is a dictionary 
+        with external_val : openerp_val
+        
+        usefull for selection field like state
+        to map value 
+    """
+    def __init__(self, val, map, default='draft'):
+        self.val = value(val)
+        self.map = map
+        self.default = default
+        
+    def __call__(self, external_values):
+        return self.map.get(self.val(external_values), self.default)
+    
+class ref(dbmapper):
+    """
+        Use : ref(table_name, external_id)
+        return the xml_id of the ressource
+        
+        to associate an already imported object with the current object
+    """
+    def __init__(self, table, field_name):
+        self.table = table
+        self.field_name = field_name
+        
+    def __call__(self, external_values):
+        return self.parent.xml_id_exist(self.table, external_values.get(self.field_name))
+  
+class refbyname(dbmapper):  
+    """
+        Use : refbyname(table_name, external_name, res.model)
+        same as ref but use the name of the ressource to find it
+    """
+    def __init__(self, table, field_name, model):
+        self.table = table
+        self.field_name = field_name
+        self.model = model
+        
+    def __call__(self, external_values):
+        v = external_values.get(self.field_name, '')
+        return self.parent.name_exist(self.table, v , self.model)
+        
+class call(mapper):
+    """
+        Use : call(function, arg1, arg2)
+        to call the function with external val follow by the arg specified 
+    """
+    def __init__(self, fun, *arg):
+        self.fun = fun
+        self.arg = arg
+    
+    def __call__(self, external_values):
+        args = []
+        for arg in self.arg:
+            if isinstance(arg, mapper):
+                args.append(arg(external_values))
+            else:
+                args.append(arg)
+        return self.fun(external_values, *args)
+    
+    
\ No newline at end of file

From 1d470e1c8cc4ada54baf35285c952b149f6983bb Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Mon, 16 May 2011 11:22:59 +0200
Subject: [PATCH 2/6] [IMP] run in a different thread, don't freezee the
 client, email notification + begin of error handling

bzr revid: tfr@openerp.com-20110516092259-m7j07g2k9mbypakk
---
 addons/import_base/import_framework.py | 77 +++++++++++++++++++++-----
 1 file changed, 62 insertions(+), 15 deletions(-)

diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
index febfbb68d44..b2b0c9a7494 100644
--- a/addons/import_base/import_framework.py
+++ b/addons/import_base/import_framework.py
@@ -20,12 +20,20 @@
 ##############################################################################
 import pprint
 import mapper
+import pooler
+import tools
+
+
+from threading import Thread
+import datetime
+from reportlab.lib import logger
+import logging
 pp = pprint.PrettyPrinter(indent=4)
 
 
 
 
-class import_framework():
+class import_framework(Thread):
     """
         This class should be extends,
         get_data and get_mapping have to extends
@@ -39,13 +47,15 @@ class import_framework():
     """
     DO_NOT_FIND_DOMAIN = [('id', '=', 0)]
   
-    def __init__(self, obj, cr, uid, instance_name, module_name, context):
+    def __init__(self, obj, cr, uid, instance_name, module_name, email_to_notify, context):
+        Thread.__init__(self)
         self.obj = obj
         self.cr = cr
         self.uid = uid
         self.instance_name = instance_name
         self.module_name = module_name
         self.context = context or {}
+        self.email = email_to_notify
         self.initialize()
         
       
@@ -115,7 +125,7 @@ class import_framework():
             res = hook(val)
             if res:
                 final_data.append(res)
-        self._save_data(model, map, final_data, table)
+        return self._save_data(model, map, final_data, table)
         
     def _save_data(self, model, mapping, datas, table):
         """
@@ -149,9 +159,10 @@ class import_framework():
         if not model_obj:
             raise ValueError("%s is not a valid model name" % model)
         
-        model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context)
+        (p, r, warning, s) = model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context)
         for field in self_dependencies:
             self._import_self_dependencies(model_obj, field, datas)
+        return (len(res), warning)
             
     def _import_self_dependencies(self, obj, parent_field, datas):
         """
@@ -313,22 +324,38 @@ class import_framework():
         return False
     
     
-    def import_all(self, table_list):
+    def set_table_list(self, table_list):
+        self.table_list = table_list
+    
+    def run(self):
         """Import all data into openerp, 
            this is the Entry point to launch the process of import
            @param table_list: the list of external table to import
                ['Leads', 'Opportunity']
         
         """
-        imported = set() #to invoid importing 2 times the sames modules
-        for table in table_list:
-            to_import = self.get_mapping()[table].get('import', True)
-            if not table in imported:
-                self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
-                if to_import:
-                    self._import_table(table)
-                imported.add(table)
-
+        
+        self.data_started = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        self.cr = pooler.get_db(self.cr.dbname).cursor()
+        try:
+            result = []
+            imported = set() #to invoid importing 2 times the sames modules
+            for table in self.table_list:
+                to_import = self.get_mapping()[table].get('import', True)
+                if not table in imported:
+                    self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
+                    if to_import:
+                        (position, warning) = self._import_table(table)
+                        result.append((table, position, warning))
+                    imported.add(table)
+            self.cr.commit()
+            
+        finally:
+            self.cr.close()
+        self.date_ended = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        self._send_notification_email(result)
+        
+        
     def _resolve_dependencies(self, dep, imported):
         """ 
             import dependencies recursively
@@ -341,8 +368,28 @@ class import_framework():
                 if to_import:
                     self._import_table(dependency)
                 imported.add(dependency)
+                
+    def _send_notification_email(self, result):
+        subject = "Openerp has finish to import your data at %s" % self.date_ended
+        tools.email_send(
+                'import_sugarcrm@module.openerp',
+                self.email,
+                subject,
+                self._get_email_body(result),
+            )
+        logger = logging.getLogger('import_sugarcam')
+        logger.info("Import finished, notification email sended")
     
-    
+    def _get_email_body(self, result):
+        header = "The import of sugarcrm data \n instance name : %s \n started at %s is finished at %s" % (self.instance_name, self.data_started, self.date_ended)
+        body = ""
+        for (table, nb, warning) in result:
+            if not warning:
+                warning = "with no warning"
+            else:
+                warning = "with warning : %s" % warning
+            body += "%s records were imported from table %s, %s \n" % (nb, table, warning)
+        return header + "\n\n" + body
     
     
 class dummy_import(import_framework):

From 63f989735f04a32e64fe6b3a6c2d478e523f7a89 Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Mon, 16 May 2011 11:32:16 +0200
Subject: [PATCH 3/6] [FIX] email to notify not required

bzr revid: tfr@openerp.com-20110516093216-iu61m6xlx1vi4i47
---
 addons/import_base/import_framework.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
index b2b0c9a7494..0e4f4c00b7f 100644
--- a/addons/import_base/import_framework.py
+++ b/addons/import_base/import_framework.py
@@ -47,7 +47,7 @@ class import_framework(Thread):
     """
     DO_NOT_FIND_DOMAIN = [('id', '=', 0)]
   
-    def __init__(self, obj, cr, uid, instance_name, module_name, email_to_notify, context):
+    def __init__(self, obj, cr, uid, instance_name, module_name, email_to_notify=False, context=None):
         Thread.__init__(self)
         self.obj = obj
         self.cr = cr
@@ -370,6 +370,8 @@ class import_framework(Thread):
                 imported.add(dependency)
                 
     def _send_notification_email(self, result):
+		if not self.email:
+			return
         subject = "Openerp has finish to import your data at %s" % self.date_ended
         tools.email_send(
                 'import_sugarcrm@module.openerp',

From 5d251c741faf139301d3c626ca1753d225fbd15f Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Wed, 25 May 2011 15:24:55 +0200
Subject: [PATCH 4/6] [IMP] email notfication + [FIX] instance name not hard
 coded anymore

bzr revid: tfr@openerp.com-20110525132455-kcspd1huwj36jubs
---
 addons/import_base/import_framework.py | 141 +++++++++----------------
 1 file changed, 51 insertions(+), 90 deletions(-)

diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
index 0e4f4c00b7f..385e86ee847 100644
--- a/addons/import_base/import_framework.py
+++ b/addons/import_base/import_framework.py
@@ -26,7 +26,6 @@ import tools
 
 from threading import Thread
 import datetime
-from reportlab.lib import logger
 import logging
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -56,8 +55,10 @@ class import_framework(Thread):
         self.module_name = module_name
         self.context = context or {}
         self.email = email_to_notify
+        self.table_list = []
         self.initialize()
         
+        
       
     """
         Abstract Method to be implemented in 
@@ -221,7 +222,7 @@ class import_framework(Thread):
             @return: a unique xml id for record, the xml_id will be the same given the same table and same name
                      To be used to avoid duplication of data that don't have ids
         """
-        sugar_instance = "sugarcrm" #TODO need to be changed we information is known in the wizard
+        sugar_instance = self.instance_name
         name = name.replace('.', '_').replace(',', '_')
         return sugar_instance + "_" + table + "_" + name
     
@@ -325,13 +326,18 @@ class import_framework(Thread):
     
     
     def set_table_list(self, table_list):
+        """
+            Set the list of table to import, this method should be call before run
+            @param table_list: the list of external table to import
+               ['Leads', 'Opportunity']
+        """
         self.table_list = table_list
     
     def run(self):
-        """Import all data into openerp, 
-           this is the Entry point to launch the process of import
-           @param table_list: the list of external table to import
-               ['Leads', 'Opportunity']
+        """
+            Import all data into openerp, 
+            this is the Entry point to launch the process of import
+            
         
         """
         
@@ -343,7 +349,8 @@ class import_framework(Thread):
             for table in self.table_list:
                 to_import = self.get_mapping()[table].get('import', True)
                 if not table in imported:
-                    self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
+                    res = self._resolve_dependencies(self.get_mapping()[table].get('dependencies', []), imported)
+                    result.extend(res)
                     if to_import:
                         (position, warning) = self._import_table(table)
                         result.append((table, position, warning))
@@ -361,109 +368,63 @@ class import_framework(Thread):
             import dependencies recursively
             and avoid to import twice the same table
         """
+        result = []
         for dependency in dep:
             if not dependency in imported:
                 to_import = self.get_mapping()[dependency].get('import', True)
                 self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported)
                 if to_import:
-                    self._import_table(dependency)
+                    (position, warning) = self._import_table(dependency)
+                    result.append((dependency, position, warning))
                 imported.add(dependency)
+        return result
                 
     def _send_notification_email(self, result):
-		if not self.email:
-			return
-        subject = "Openerp has finish to import your data at %s" % self.date_ended
+        if not self.email:
+            return 		 
         tools.email_send(
                 'import_sugarcrm@module.openerp',
                 self.email,
-                subject,
-                self._get_email_body(result),
+                self.get_email_subject(result),
+                self.get_email_body(result),
             )
         logger = logging.getLogger('import_sugarcam')
         logger.info("Import finished, notification email sended")
     
-    def _get_email_body(self, result):
-        header = "The import of sugarcrm data \n instance name : %s \n started at %s is finished at %s" % (self.instance_name, self.data_started, self.date_ended)
-        body = ""
+    def get_email_subject(self, result):
+        """
+            This method define the subject of the email send by openerp at the end of import
+            @param result: a list of tuple 
+                (table_name, number_of_record_created/updated, warning) for each table
+            @return the subject of the mail
+        
+        """
+        return "Import of your data finished at %s" % self.date_ended
+    
+    def get_email_body(self, result):
+        """
+            This method define the body of the email send by openerp at the end of import. The body is separated in two part
+            the header (@see get_body_header), and the generate body with the list of table and number of record imported.
+            If you want to keep this it's better to overwrite get_body_header
+            @param result: a list of tuple 
+                (table_name, number_of_record_created/updated, warning) for each table
+            @return the subject of the mail
+        
+        """
+        
+        body = "started at %s and finished at %s \n" % (self.data_started, self.date_ended)
         for (table, nb, warning) in result:
             if not warning:
                 warning = "with no warning"
             else:
                 warning = "with warning : %s" % warning
             body += "%s records were imported from table %s, %s \n" % (nb, table, warning)
-        return header + "\n\n" + body
+        return self.get_body_header(result) + "\n\n" + body
     
+    def get_body_header(self, result):
+        """
+            @return the first sentences written in the mail's body
+        """
+        return "The import of data \n instance name : %s \n" % self.instance_name
     
-class dummy_import(import_framework):
-    """
-        this is a really simple exemple to show how to use the framework
-    """
-    
-    TABLE_LEAD = 'Leads'
-    TABLE_STAGE = 'Stage'
-    TABLE_USER = 'User'
-    
-    def get_lead_status(self, sugar_val):
-        fields = ['name', 'type']
-        name = 'lead_' + sugar_val.get('status', '')
-        data = [sugar_val.get('status', ''), 'lead']
-        return self.import_object(fields, data, 'crm.case.stage', self.TABLE_STAGE, name, [('type', '=', 'lead'), ('name', 'ilike', sugar_val.get('status', ''))])
-
-    
-    
-  
-    lead_state =  {
-            'New' : 'draft',
-            'Assigned':'open',
-            'In Progress': 'open',
-            'Recycled': 'cancel',
-            'Dead': 'done',
-        }
-    
-    def get_mapping(self):
-        return {
-                self.TABLE_LEAD : {
-                    'model' : 'crm.lead',
-                    'dependencies' : [self.TABLE_USER],
-                    'map' : {
-                            'name' : 'name',
-                            'stage_id/id' : self.get_lead_status, 
-                            'state' : mapper.map_val('status', self.lead_state),
-                            'user_id/id' : mapper.ref(self.TABLE_USER, 'assigned_user_id'),
-                    }         
-                              
-                },
-                self.TABLE_USER : {
-                    'model' : 'res.users',
-                    'map' : {
-                             'name' : 'name',
-                             'login' : 'name'
-                    }              
-                }
-        }
-        
-       
-    #DUMMY way to get data, just demonstration 
-    def get_data(self, table):
-        if table == self.TABLE_LEAD:
-            return self.get_lead()
-        if table == self.TABLE_USER:
-            return self.get_user()
-        
-    
-    def get_lead(self):
-        return [
-                {'id' : 'lead1',
-                 'name' : 'name1',
-                 'status' : 'Assigned',
-                 'assigned_user_id' : 'user1'},
-                {'id' : 'lead2',
-                 'name' : 'name2'
-                 },
-                ]
-        
-    def get_user(self):
-        return [{
-            'id' : 'user1',
-            'name' : 'name1',
-                }]
+#For example of use see import_sugarcrm    
\ No newline at end of file

From ddf020103e5f7403d0b3f468fbfd414f0a746c92 Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Wed, 25 May 2011 17:57:44 +0200
Subject: [PATCH 5/6] [FIX] self dependencies

bzr revid: tfr@openerp.com-20110525155744-zw7odokw76ngvz38
---
 addons/import_base/import_framework.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
index 385e86ee847..6380af7736a 100644
--- a/addons/import_base/import_framework.py
+++ b/addons/import_base/import_framework.py
@@ -126,7 +126,7 @@ class import_framework(Thread):
             res = hook(val)
             if res:
                 final_data.append(res)
-        return self._save_data(model, map, final_data, table)
+        return self._save_data(model, dict(map), final_data, table)
         
     def _save_data(self, model, mapping, datas, table):
         """
@@ -143,14 +143,16 @@ class import_framework(Thread):
             return
         mapping['id'] = 'id_new'
         res = []
+        
+        
+        self_dependencies = []
+        for k in mapping.keys():
+            if '_parent' in k:
+                self_dependencies.append((k[:-7], mapping.pop(k)))
+        
         for data in datas:
-            
-            self_dependencies = []
-            for k in mapping.keys():
-                if '_parent' in k:
-                    self_dependencies.append(k[:-7])
-                    field_name = mapping.pop(k)
-                    data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table)
+            for k, field_name in self_dependencies:
+                data[k] = data.get(field_name) and self._generate_xml_id(data.get(field_name), table)
                     
             data['id_new'] = self._generate_xml_id(data['id'], table)
             fields, values = self._fields_mapp(data, mapping, table)
@@ -161,7 +163,7 @@ class import_framework(Thread):
             raise ValueError("%s is not a valid model name" % model)
         
         (p, r, warning, s) = model_obj.import_data(self.cr, self.uid, fields, res, mode='update', current_module=self.module_name, noupdate=True, context=self.context)
-        for field in self_dependencies:
+        for (field, field_name) in self_dependencies:
             self._import_self_dependencies(model_obj, field, datas)
         return (len(res), warning)
             
@@ -176,8 +178,8 @@ class import_framework(Thread):
         """
         fields = ['id', parent_field]
         for data in datas:
-            if data.get(parent_field + '_parent'):
-                values = [data['id_new'], data[parent_field + '_parent']]
+            if data.get(parent_field):
+                values = [data['id_new'], data[parent_field]]
                 obj.import_data(self.cr, self.uid, fields, [values], mode='update', current_module=self.module_name, noupdate=True, context=self.context) 
     
     def _preprocess_mapping(self, mapping):

From a843faf44cc8fada36fb483de831569fb961d33f Mon Sep 17 00:00:00 2001
From: "tfr@openerp.com" <>
Date: Tue, 31 May 2011 16:25:45 +0200
Subject: [PATCH 6/6] [FIX] import table with no data inside made crash resolve
 dependencies

bzr revid: tfr@openerp.com-20110531142545-l130b9b6xffdrcew
---
 addons/import_base/import_framework.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/addons/import_base/import_framework.py b/addons/import_base/import_framework.py
index 6380af7736a..d041951ca11 100644
--- a/addons/import_base/import_framework.py
+++ b/addons/import_base/import_framework.py
@@ -140,7 +140,7 @@ class import_framework(Thread):
                 and each data_i have a external id => in data_id['id']
         """
         if not datas:
-            return
+            return (0, 'No data in this table')
         mapping['id'] = 'id_new'
         res = []
         
@@ -374,9 +374,11 @@ class import_framework(Thread):
         for dependency in dep:
             if not dependency in imported:
                 to_import = self.get_mapping()[dependency].get('import', True)
-                self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported)
+                res = self._resolve_dependencies(self.get_mapping()[dependency].get('dependencies', []), imported)
+                result.extend(res)
                 if to_import:
-                    (position, warning) = self._import_table(dependency)
+                    r = self._import_table(dependency)
+                    (position, warning) = r
                     result.append((dependency, position, warning))
                 imported.add(dependency)
         return result