diff --git a/addons/anonymization/__init__.py b/addons/anonymization/__init__.py index 6ac286f9033..11529f98733 100644 --- a/addons/anonymization/__init__.py +++ b/addons/anonymization/__init__.py @@ -1,4 +1,5 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import anonymization +import models +import wizard diff --git a/addons/anonymization/__openerp__.py b/addons/anonymization/__openerp__.py index 3f82cc44b45..1ddd445e9d9 100644 --- a/addons/anonymization/__openerp__.py +++ b/addons/anonymization/__openerp__.py @@ -1,4 +1,4 @@ -# -*- encoding: utf-8 -*- +# -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. @@ -19,11 +19,12 @@ team. Once you get back your migrated database, you restore it and reverse the anonymization process to recover your previous data. """, 'depends': ['base'], - 'demo': ['anonymization_demo.xml'], + 'demo': ['data/anonymization_demo.xml'], 'data': [ - 'ir.model.fields.anonymization.csv', + 'data/ir.model.fields.anonymization.csv', 'security/ir.model.access.csv', - 'anonymization_view.xml', + 'views/anonymization_views.xml', + 'wizard/anonymize_wizard_views.xml' ], 'installable': True, 'auto_install': False, diff --git a/addons/anonymization/anonymization.py b/addons/anonymization/anonymization.py deleted file mode 100644 index 5179850e7de..00000000000 --- a/addons/anonymization/anonymization.py +++ /dev/null @@ -1,625 +0,0 @@ -# -*- encoding: utf-8 -*- -# Part of Odoo. See LICENSE file for full copyright and licensing details. - -from lxml import etree -import os -import base64 -try: - import cPickle as pickle -except ImportError: - import pickle -import random -import datetime -from openerp.release import version_info -from openerp.osv import fields, osv -from openerp.tools.translate import _ -from openerp.tools.safe_eval import safe_eval as eval - -from itertools import groupby -from operator import itemgetter -from openerp.exceptions import UserError - - -FIELD_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('not_existing', 'Not Existing'), ('new', 'New')] -ANONYMIZATION_STATES = FIELD_STATES + [('unstable', 'Unstable')] -WIZARD_ANONYMIZATION_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('unstable', 'Unstable')] -ANONYMIZATION_HISTORY_STATE = [('started', 'Started'), ('done', 'Done'), ('in_exception', 'Exception occured')] -ANONYMIZATION_DIRECTION = [('clear -> anonymized', 'clear -> anonymized'), ('anonymized -> clear', 'anonymized -> clear')] - - -def group(lst, cols): - if isinstance(cols, basestring): - cols = [cols] - return dict((k, [v for v in itr]) for k, itr in groupby(sorted(lst, key=itemgetter(*cols)), itemgetter(*cols))) - - -class ir_model_fields_anonymization(osv.osv): - _name = 'ir.model.fields.anonymization' - _rec_name = 'field_id' - - _columns = { - 'model_name': fields.char('Object Name', required=True), - 'model_id': fields.many2one('ir.model', 'Object', ondelete='set null'), - 'field_name': fields.char('Field Name', required=True), - 'field_id': fields.many2one('ir.model.fields', 'Field', ondelete='set null'), - 'state': fields.selection(selection=FIELD_STATES, String='Status', required=True, readonly=True), - } - - _sql_constraints = [ - ('model_id_field_id_uniq', 'unique (model_name, field_name)', _("You cannot have two fields with the same name on the same object!")), - ] - - def _get_global_state(self, cr, uid, context=None): - ids = self.search(cr, uid, [('state', '<>', 'not_existing')], context=context) - fields = self.browse(cr, uid, ids, context=context) - if not len(fields) or len(fields) == len([f for f in fields if f.state == 'clear']): - state = 'clear' # all fields are clear - elif len(fields) == len([f for f in fields if f.state == 'anonymized']): - state = 'anonymized' # all fields are anonymized - else: - state = 'unstable' # fields are mixed: this should be fixed - - return state - - def _check_write(self, cr, uid, context=None): - """check that the field is created from the menu and not from an database update - otherwise the database update can crash:""" - if context is None: - context = {} - - if context.get('manual'): - global_state = self._get_global_state(cr, uid, context=context) - if global_state == 'anonymized': - raise UserError(_("The database is currently anonymized, you cannot create, modify or delete fields.")) - elif global_state == 'unstable': - msg = _("The database anonymization is currently in an unstable state. Some fields are anonymized," - " while some fields are not anonymized. You should try to solve this problem before trying to create, write or delete fields.") - raise UserError(msg) - - return True - - def _get_model_and_field_ids(self, cr, uid, vals, context=None): - model_and_field_ids = (False, False) - - if 'field_name' in vals and vals['field_name'] and 'model_name' in vals and vals['model_name']: - ir_model_fields_obj = self.pool.get('ir.model.fields') - ir_model_obj = self.pool.get('ir.model') - - model_ids = ir_model_obj.search(cr, uid, [('model', '=', vals['model_name'])], context=context) - if model_ids: - field_ids = ir_model_fields_obj.search(cr, uid, [('name', '=', vals['field_name']), ('model_id', '=', model_ids[0])], context=context) - if field_ids: - field_id = field_ids[0] - model_and_field_ids = (model_ids[0], field_id) - - return model_and_field_ids - - def create(self, cr, uid, vals, context=None): - # check field state: all should be clear before we can add a new field to anonymize: - self._check_write(cr, uid, context=context) - - global_state = self._get_global_state(cr, uid, context=context) - - if 'field_name' in vals and vals['field_name'] and 'model_name' in vals and vals['model_name']: - vals['model_id'], vals['field_id'] = self._get_model_and_field_ids(cr, uid, vals, context=context) - - # check not existing fields: - if not vals.get('field_id'): - vals['state'] = 'not_existing' - else: - vals['state'] = global_state - - res = super(ir_model_fields_anonymization, self).create(cr, uid, vals, context=context) - - return res - - def write(self, cr, uid, ids, vals, context=None): - # check field state: all should be clear before we can modify a field: - if not (len(vals.keys()) == 1 and vals.get('state') == 'clear'): - self._check_write(cr, uid, context=context) - - if 'field_name' in vals and vals['field_name'] and 'model_name' in vals and vals['model_name']: - vals['model_id'], vals['field_id'] = self._get_model_and_field_ids(cr, uid, vals, context=context) - - # check not existing fields: - if 'field_id' in vals: - if not vals.get('field_id'): - vals['state'] = 'not_existing' - else: - global_state = self._get_global_state(cr, uid, context) - if global_state != 'unstable': - vals['state'] = global_state - - res = super(ir_model_fields_anonymization, self).write(cr, uid, ids, vals, context=context) - - return res - - def unlink(self, cr, uid, ids, context=None): - # check field state: all should be clear before we can unlink a field: - self._check_write(cr, uid, context=context) - - res = super(ir_model_fields_anonymization, self).unlink(cr, uid, ids, context=context) - return res - - def onchange_model_id(self, cr, uid, ids, model_id, context=None): - res = {'value': { - 'field_name': False, - 'field_id': False, - 'model_name': False, - }} - - if model_id: - ir_model_obj = self.pool.get('ir.model') - model_ids = ir_model_obj.search(cr, uid, [('id', '=', model_id)]) - model_id = model_ids and model_ids[0] or None - model_name = model_id and ir_model_obj.browse(cr, uid, model_id).model or False - res['value']['model_name'] = model_name - - return res - - def onchange_model_name(self, cr, uid, ids, model_name, context=None): - res = {'value': { - 'field_name': False, - 'field_id': False, - 'model_id': False, - }} - - if model_name: - ir_model_obj = self.pool.get('ir.model') - model_ids = ir_model_obj.search(cr, uid, [('model', '=', model_name)]) - model_id = model_ids and model_ids[0] or False - res['value']['model_id'] = model_id - - return res - - def onchange_field_name(self, cr, uid, ids, field_name, model_name): - res = {'value': { - 'field_id': False, - }} - - if field_name and model_name: - ir_model_fields_obj = self.pool.get('ir.model.fields') - field_ids = ir_model_fields_obj.search(cr, uid, [('name', '=', field_name), ('model', '=', model_name)]) - field_id = field_ids and field_ids[0] or False - res['value']['field_id'] = field_id - - return res - - def onchange_field_id(self, cr, uid, ids, field_id, model_name): - res = {'value': { - 'field_name': False, - }} - - if field_id: - ir_model_fields_obj = self.pool.get('ir.model.fields') - field = ir_model_fields_obj.browse(cr, uid, field_id) - res['value']['field_name'] = field.name - - return res - - _defaults = { - 'state': lambda *a: 'clear', - } - - -class ir_model_fields_anonymization_history(osv.osv): - _name = 'ir.model.fields.anonymization.history' - _order = "date desc" - - _columns = { - 'date': fields.datetime('Date', required=True, readonly=True), - 'field_ids': fields.many2many('ir.model.fields.anonymization', 'anonymized_field_to_history_rel', 'field_id', 'history_id', 'Fields', readonly=True), - 'state': fields.selection(selection=ANONYMIZATION_HISTORY_STATE, string='Status', required=True, readonly=True), - 'direction': fields.selection(selection=ANONYMIZATION_DIRECTION, string='Direction', size=20, required=True, readonly=True), - 'msg': fields.text('Message', readonly=True), - 'filepath': fields.char(string='File path', readonly=True), - } - - -class ir_model_fields_anonymize_wizard(osv.osv_memory): - _name = 'ir.model.fields.anonymize.wizard' - - def _get_state(self, cr, uid, ids, name, arg, context=None): - res = {} - - state = self._get_state_value(cr, uid, context=None) - for id in ids: - res[id] = state - - return res - - def _get_summary(self, cr, uid, ids, name, arg, context=None): - res = {} - summary = self._get_summary_value(cr, uid, context) - for id in ids: - res[id] = summary - - return res - - _columns = { - 'name': fields.char(string='File Name'), - 'summary': fields.function(_get_summary, type='text', string='Summary'), - 'file_export': fields.binary(string='Export'), - 'file_import': fields.binary(string='Import', help="This is the file created by the anonymization process. It should have the '.pickle' extention."), - 'state': fields.function(_get_state, string='Status', type='selection', selection=WIZARD_ANONYMIZATION_STATES, readonly=False), - 'msg': fields.text(string='Message'), - } - - def _get_state_value(self, cr, uid, context=None): - state = self.pool.get('ir.model.fields.anonymization')._get_global_state(cr, uid, context=context) - return state - - def _get_summary_value(self, cr, uid, context=None): - summary = u'' - anon_field_obj = self.pool.get('ir.model.fields.anonymization') - ir_model_fields_obj = self.pool.get('ir.model.fields') - - anon_field_ids = anon_field_obj.search(cr, uid, [('state', '<>', 'not_existing')], context=context) - anon_fields = anon_field_obj.browse(cr, uid, anon_field_ids, context=context) - - field_ids = [anon_field.field_id.id for anon_field in anon_fields if anon_field.field_id] - fields = ir_model_fields_obj.browse(cr, uid, field_ids, context=context) - - fields_by_id = dict([(f.id, f) for f in fields]) - - for anon_field in anon_fields: - field = fields_by_id.get(anon_field.field_id.id) - - values = { - 'model_name': field.model_id.name, - 'model_code': field.model_id.model, - 'field_code': field.name, - 'field_name': field.field_description, - 'state': anon_field.state, - } - summary += u" * %(model_name)s (%(model_code)s) -> %(field_name)s (%(field_code)s): state: (%(state)s)\n" % values - - return summary - - def default_get(self, cr, uid, fields_list, context=None): - res = {} - res['name'] = '.pickle' - res['summary'] = self._get_summary_value(cr, uid, context) - res['state'] = self._get_state_value(cr, uid, context) - res['msg'] = _("""Before executing the anonymization process, you should make a backup of your database.""") - - return res - - def fields_view_get(self, cr, uid, view_id=None, view_type='form', context=None, toolbar=False, submenu=False): - state = self.pool.get('ir.model.fields.anonymization')._get_global_state(cr, uid, context=context) - - if context is None: - context = {} - - step = context.get('step', 'new_window') - - res = super(ir_model_fields_anonymize_wizard, self).fields_view_get(cr, uid, view_id=view_id, view_type=view_type, context=context, toolbar=toolbar, submenu=submenu) - - eview = etree.fromstring(res['arch']) - placeholder = eview.xpath("group[@name='placeholder1']") - if len(placeholder): - placeholder = placeholder[0] - if step == 'new_window' and state == 'clear': - # clicked in the menu and the fields are not anonymized: warn the admin that backuping the db is very important - placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) - placeholder.addnext(etree.Element('newline')) - placeholder.addnext(etree.Element('label', {'string': 'Warning'})) - eview.remove(placeholder) - elif step == 'new_window' and state == 'anonymized': - # clicked in the menu and the fields are already anonymized - placeholder.addnext(etree.Element('newline')) - placeholder.addnext(etree.Element('field', {'name': 'file_import', 'required': "1"})) - placeholder.addnext(etree.Element('label', {'string': 'Anonymization file'})) - eview.remove(placeholder) - elif step == 'just_anonymized': - # we just ran the anonymization process, we need the file export field - placeholder.addnext(etree.Element('newline')) - placeholder.addnext(etree.Element('field', {'name': 'file_export'})) - # we need to remove the button: - buttons = eview.xpath("button") - for button in buttons: - eview.remove(button) - # and add a message: - placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) - placeholder.addnext(etree.Element('newline')) - placeholder.addnext(etree.Element('label', {'string': 'Result'})) - # remove the placeholer: - eview.remove(placeholder) - elif step == 'just_desanonymized': - # we just reversed the anonymization process, we don't need any field - # we need to remove the button - buttons = eview.xpath("button") - for button in buttons: - eview.remove(button) - # and add a message - # and add a message: - placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) - placeholder.addnext(etree.Element('newline')) - placeholder.addnext(etree.Element('label', {'string': 'Result'})) - # remove the placeholer: - eview.remove(placeholder) - else: - msg = _("The database anonymization is currently in an unstable state. Some fields are anonymized," - " while some fields are not anonymized. You should try to solve this problem before trying to do anything else.") - raise UserError(msg) - - res['arch'] = etree.tostring(eview) - - return res - - def _raise_after_history_update(self, cr, uid, history_id, error_type, error_msg): - self.pool.get('ir.model.fields.anonymization.history').write(cr, uid, history_id, { - 'state': 'in_exception', - 'msg': error_msg, - }) - raise UserError('%s: %s' % (error_type, error_msg)) - - def anonymize_database(self, cr, uid, ids, context=None): - """Sets the 'anonymized' state to defined fields""" - - # create a new history record: - anonymization_history_model = self.pool.get('ir.model.fields.anonymization.history') - - vals = { - 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'state': 'started', - 'direction': 'clear -> anonymized', - } - history_id = anonymization_history_model.create(cr, uid, vals) - - # check that all the defined fields are in the 'clear' state - state = self.pool.get('ir.model.fields.anonymization')._get_global_state(cr, uid, context=context) - if state == 'anonymized': - self._raise_after_history_update(cr, uid, history_id, _('Error !'), _("The database is currently anonymized, you cannot anonymize it again.")) - elif state == 'unstable': - msg = _("The database anonymization is currently in an unstable state. Some fields are anonymized," - " while some fields are not anonymized. You should try to solve this problem before trying to do anything.") - self._raise_after_history_update(cr, uid, history_id, 'Error !', msg) - - # do the anonymization: - dirpath = os.environ.get('HOME') or os.getcwd() - rel_filepath = 'field_anonymization_%s_%s.pickle' % (cr.dbname, history_id) - abs_filepath = os.path.abspath(os.path.join(dirpath, rel_filepath)) - - ir_model_fields_anonymization_model = self.pool.get('ir.model.fields.anonymization') - field_ids = ir_model_fields_anonymization_model.search(cr, uid, [('state', '<>', 'not_existing')], context=context) - fields = ir_model_fields_anonymization_model.browse(cr, uid, field_ids, context=context) - - if not fields: - msg = "No fields are going to be anonymized." - self._raise_after_history_update(cr, uid, history_id, 'Error !', msg) - - data = [] - - for field in fields: - model_name = field.model_id.model - field_name = field.field_id.name - field_type = field.field_id.ttype - table_name = self.pool[model_name]._table - - # get the current value - sql = "select id, %s from %s" % (field_name, table_name) - cr.execute(sql) - records = cr.dictfetchall() - for record in records: - data.append({"model_id": model_name, "field_id": field_name, "id": record['id'], "value": record[field_name]}) - - # anonymize the value: - anonymized_value = None - - sid = str(record['id']) - if field_type == 'char': - anonymized_value = 'xxx'+sid - elif field_type == 'selection': - anonymized_value = 'xxx'+sid - elif field_type == 'text': - anonymized_value = 'xxx'+sid - elif field_type == 'html': - anonymized_value = 'xxx'+sid - elif field_type == 'boolean': - anonymized_value = random.choice([True, False]) - elif field_type == 'date': - anonymized_value = '2011-11-11' - elif field_type == 'datetime': - anonymized_value = '2011-11-11 11:11:11' - elif field_type == 'float': - anonymized_value = 0.0 - elif field_type == 'integer': - anonymized_value = 0 - elif field_type in ['binary', 'many2many', 'many2one', 'one2many', 'reference']: # cannot anonymize these kind of fields - msg = _("Cannot anonymize fields of these types: binary, many2many, many2one, one2many, reference.") - self._raise_after_history_update(cr, uid, history_id, 'Error !', msg) - - if anonymized_value is None: - self._raise_after_history_update(cr, uid, history_id, _('Error !'), _("Anonymized value can not be empty.")) - - sql = "update %(table)s set %(field)s = %%(anonymized_value)s where id = %%(id)s" % { - 'table': table_name, - 'field': field_name, - } - cr.execute(sql, { - 'anonymized_value': anonymized_value, - 'id': record['id'] - }) - - # save pickle: - fn = open(abs_filepath, 'w') - pickle.dump(data, fn, pickle.HIGHEST_PROTOCOL) - - # update the anonymization fields: - values = { - 'state': 'anonymized', - } - ir_model_fields_anonymization_model.write(cr, uid, field_ids, values, context=context) - - # add a result message in the wizard: - msgs = ["Anonymization successful.", - "", - "Donot forget to save the resulting file to a safe place because you will not be able to revert the anonymization without this file.", - "", - "This file is also stored in the %s directory. The absolute file path is: %s.", - ] - msg = '\n'.join(msgs) % (dirpath, abs_filepath) - - fn = open(abs_filepath, 'r') - - self.write(cr, uid, ids, { - 'msg': msg, - 'file_export': base64.encodestring(fn.read()), - }) - fn.close() - - # update the history record: - anonymization_history_model.write(cr, uid, history_id, { - 'field_ids': [[6, 0, field_ids]], - 'msg': msg, - 'filepath': abs_filepath, - 'state': 'done', - }) - - # handle the view: - view_id = self.pool['ir.model.data'].xmlid_to_res_id( - cr, uid, 'anonymization.view_ir_model_fields_anonymize_wizard_form' - ) - - return { - 'res_id': ids[0], - 'view_id': [view_id], - 'view_type': 'form', - "view_mode": 'form', - 'res_model': 'ir.model.fields.anonymize.wizard', - 'type': 'ir.actions.act_window', - 'context': {'step': 'just_anonymized'}, - 'target':'new', - } - - def reverse_anonymize_database(self, cr, uid, ids, context=None): - """Set the 'clear' state to defined fields""" - ir_model_fields_anonymization_model = self.pool.get('ir.model.fields.anonymization') - anonymization_history_model = self.pool.get('ir.model.fields.anonymization.history') - - # create a new history record: - vals = { - 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), - 'state': 'started', - 'direction': 'anonymized -> clear', - } - history_id = anonymization_history_model.create(cr, uid, vals) - - # check that all the defined fields are in the 'anonymized' state - state = ir_model_fields_anonymization_model._get_global_state(cr, uid, context=context) - if state == 'clear': - raise UserError(_("The database is not currently anonymized, you cannot reverse the anonymization.")) - elif state == 'unstable': - msg = _("The database anonymization is currently in an unstable state. Some fields are anonymized," - " while some fields are not anonymized. You should try to solve this problem before trying to do anything.") - raise UserError(msg) - - wizards = self.browse(cr, uid, ids, context=context) - for wizard in wizards: - if not wizard.file_import: - msg = _("It is not possible to reverse the anonymization process without supplying the anonymization export file.") - self._raise_after_history_update(cr, uid, history_id, 'Error !', msg) - - # reverse the anonymization: - # load the pickle file content into a data structure: - data = pickle.loads(base64.decodestring(wizard.file_import)) - - migration_fix_obj = self.pool.get('ir.model.fields.anonymization.migration.fix') - fix_ids = migration_fix_obj.search(cr, uid, [('target_version', '=', '.'.join(map(str, version_info[:2])))]) - fixes = migration_fix_obj.read(cr, uid, fix_ids, ['model_name', 'field_name', 'query', 'query_type', 'sequence']) - fixes = group(fixes, ('model_name', 'field_name')) - - for line in data: - queries = [] - table_name = self.pool[line['model_id']]._table if line['model_id'] in self.pool else None - - # check if custom sql exists: - key = (line['model_id'], line['field_id']) - custom_updates = fixes.get(key) - if custom_updates: - custom_updates.sort(key=itemgetter('sequence')) - queries = [(record['query'], record['query_type']) for record in custom_updates if record['query_type']] - elif table_name: - queries = [("update %(table)s set %(field)s = %%(value)s where id = %%(id)s" % { - 'table': table_name, - 'field': line['field_id'], - }, 'sql')] - - for query in queries: - if query[1] == 'sql': - sql = query[0] - cr.execute(sql, { - 'value': line['value'], - 'id': line['id'] - }) - elif query[1] == 'python': - raw_code = query[0] - code = raw_code % line - eval(code) - else: - raise Exception("Unknown query type '%s'. Valid types are: sql, python." % (query['query_type'], )) - - # update the anonymization fields: - ir_model_fields_anonymization_model = self.pool.get('ir.model.fields.anonymization') - field_ids = ir_model_fields_anonymization_model.search(cr, uid, [('state', '<>', 'not_existing')], context=context) - values = { - 'state': 'clear', - } - ir_model_fields_anonymization_model.write(cr, uid, field_ids, values, context=context) - - # add a result message in the wizard: - msg = '\n'.join(["Successfully reversed the anonymization.", - "", - ]) - - self.write(cr, uid, ids, {'msg': msg}) - - # update the history record: - anonymization_history_model.write(cr, uid, history_id, { - 'field_ids': [[6, 0, field_ids]], - 'msg': msg, - 'filepath': False, - 'state': 'done', - }) - - # handle the view: - view_id = self.pool['ir.model.data'].xmlid_to_res_id( - cr, uid, 'anonymization.view_ir_model_fields_anonymize_wizard_form' - ) - - - return { - 'res_id': ids[0], - 'view_id': [view_id], - 'view_type': 'form', - "view_mode": 'form', - 'res_model': 'ir.model.fields.anonymize.wizard', - 'type': 'ir.actions.act_window', - 'context': {'step': 'just_desanonymized'}, - 'target':'new', - } - - def _id_get(self, cr, uid, model, id_str, mod): - if '.' in id_str: - mod, id_str = id_str.split('.') - try: - idn = self.pool.get('ir.model.data')._get_id(cr, uid, mod, id_str) - res = int(self.pool.get('ir.model.data').read(cr, uid, [idn], ['res_id'])[0]['res_id']) - except: - res = None - return res - - -class ir_model_fields_anonymization_migration_fix(osv.osv): - _name = 'ir.model.fields.anonymization.migration.fix' - _order = "sequence" - - _columns = { - 'target_version': fields.char('Target Version'), - 'model_name': fields.char('Model'), - 'field_name': fields.char('Field'), - 'query': fields.text('Query'), - 'query_type': fields.selection(string='Query', selection=[('sql', 'sql'), ('python', 'python')]), - 'sequence': fields.integer('Sequence'), - } diff --git a/addons/anonymization/anonymization_demo.xml b/addons/anonymization/data/anonymization_demo.xml similarity index 93% rename from addons/anonymization/anonymization_demo.xml rename to addons/anonymization/data/anonymization_demo.xml index 1183cc975ea..91a107bae26 100644 --- a/addons/anonymization/anonymization_demo.xml +++ b/addons/anonymization/data/anonymization_demo.xml @@ -1,5 +1,5 @@ - - + + @@ -18,5 +18,5 @@ - + diff --git a/addons/anonymization/ir.model.fields.anonymization.csv b/addons/anonymization/data/ir.model.fields.anonymization.csv similarity index 100% rename from addons/anonymization/ir.model.fields.anonymization.csv rename to addons/anonymization/data/ir.model.fields.anonymization.csv diff --git a/addons/anonymization/models/__init__.py b/addons/anonymization/models/__init__.py new file mode 100644 index 00000000000..5d2a1a794f0 --- /dev/null +++ b/addons/anonymization/models/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import anonymization diff --git a/addons/anonymization/models/anonymization.py b/addons/anonymization/models/anonymization.py new file mode 100644 index 00000000000..06ffcf92216 --- /dev/null +++ b/addons/anonymization/models/anonymization.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from itertools import groupby +from operator import itemgetter + +from odoo import api, fields, models, _ +from odoo.exceptions import UserError + +FIELD_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('not_existing', 'Not Existing'), ('new', 'New')] +ANONYMIZATION_HISTORY_STATE = [('started', 'Started'), ('done', 'Done'), ('in_exception', 'Exception occured')] +ANONYMIZATION_DIRECTION = [('clear -> anonymized', 'clear -> anonymized'), ('anonymized -> clear', 'anonymized -> clear')] + + +def group(lst, cols): + if isinstance(cols, basestring): + cols = [cols] + return dict((k, [v for v in itr]) for k, itr in groupby(sorted(lst, key=itemgetter(*cols)), itemgetter(*cols))) + + +class IrModelFieldsAnonymization(models.Model): + _name = 'ir.model.fields.anonymization' + _rec_name = 'field_id' + + model_name = fields.Char('Object Name', required=True) + model_id = fields.Many2one('ir.model', string='Object', ondelete='set null') + field_name = fields.Char(required=True) + field_id = fields.Many2one('ir.model.fields', string='Field', ondelete='set null') + state = fields.Selection(selection=FIELD_STATES, string='Status', required=True, readonly=True, default='clear') + + _sql_constraints = [ + ('model_id_field_id_uniq', 'unique (model_name, field_name)', _("You cannot have two fields with the same name on the same object!")), + ] + + @api.model + def _get_global_state(self): + field_ids = self.search([('state', '!=', 'not_existing')]) + if not field_ids or len(field_ids) == len(field_ids.filtered(lambda field: field.state == "clear")): + state = 'clear' # all fields are clear + elif len(field_ids) == len(field_ids.filtered(lambda field: field.state == "anonymized")): + state = 'anonymized' # all fields are anonymized + else: + state = 'unstable' # fields are mixed: this should be fixed + return state + + @api.model + def _check_write(self): + """check that the field is created from the menu and not from an database update + otherwise the database update can crash:""" + if self.env.context.get('manual'): + global_state = self._get_global_state() + if global_state == 'anonymized': + raise UserError(_("The database is currently anonymized, you cannot create, modify or delete fields.")) + elif global_state == 'unstable': + raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized," + " while some fields are not anonymized. You should try to solve this problem before trying to create, write or delete fields.")) + return True + + @api.model + def _get_model_and_field_ids(self, vals): + if vals.get('field_name') and vals.get('model_name'): + model_id = self.env['ir.model'].search([('model', '=', vals['model_name'])], limit=1).id + if model_id: + field_id = self.env['ir.model.fields'].search([('name', '=', vals['field_name']), ('model_id', '=', model_id)], limit=1).id + if field_id: + return (model_id, field_id) + return (False, False) + + @api.model + def create(self, vals): + # check field state: all should be clear before we can add a new field to anonymize: + self._check_write() + if vals.get('field_name') and vals.get('model_name'): + vals['model_id'], vals['field_id'] = self._get_model_and_field_ids(vals) + # check not existing fields: + vals['state'] = self._get_global_state() if vals.get('field_id') else 'not_existing' + return super(IrModelFieldsAnonymization, self).create(vals) + + @api.multi + def write(self, vals): + # check field state: all should be clear before we can modify a field: + if not len(vals.keys()) == 1 and vals.get('state') == 'clear': + self._check_write() + if vals.get('field_name') and vals.get('model_name'): + vals['model_id'], vals['field_id'] = self._get_model_and_field_ids(vals) + # check not existing fields: + if 'field_id' in vals: + if not vals['field_id']: + vals['state'] = 'not_existing' + else: + global_state = self._get_global_state() + if global_state != 'unstable': + vals['state'] = global_state + return super(IrModelFieldsAnonymization, self).write(vals) + + @api.multi + def unlink(self): + # check field state: all should be clear before we can unlink a field: + self._check_write() + return super(IrModelFieldsAnonymization, self).unlink() + + @api.onchange('model_id') + def _onchange_model_id(self): + self.field_name = False + self.field_id = False + self.model_name = self.model_id.model + + @api.onchange('model_name') + def _onchange_model_name(self): + self.field_name = False + self.field_id = False + if self.model_name: + self.model_id = self.env['ir.model'].search([('model', '=', self.model_name)], limit=1) + else: + self.model_id = False + + @api.onchange('field_name') + def _onchange_field_name(self): + if self.field_name and self.model_name: + self.field_id = self.env['ir.model.fields'].search([ + ('name', '=', self.field_name), ('model', '=', self.model_name) + ], limit=1) + else: + self.field_id = False + + @api.onchange('field_id') + def _onchange_field_id(self): + self.field_name = self.field_id.name + + +class IrModelFieldsAnonymizationHistory(models.Model): + _name = 'ir.model.fields.anonymization.history' + _order = "date desc" + + date = fields.Datetime(required=True, readonly=True) + field_ids = fields.Many2many( + 'ir.model.fields.anonymization', 'anonymized_field_to_history_rel', + 'field_id', 'history_id', string='Fields', readonly=True + ) + state = fields.Selection(selection=ANONYMIZATION_HISTORY_STATE, string='Status', required=True, readonly=True) + direction = fields.Selection(selection=ANONYMIZATION_DIRECTION, required=True, readonly=True) + msg = fields.Text('Message', readonly=True) + filepath = fields.Char('File path', readonly=True) + + +class IrModelFieldsAnonymizationMigrationFix(models.Model): + _name = 'ir.model.fields.anonymization.migration.fix' + _order = "sequence" + + target_version = fields.Char('Target Version') + model_name = fields.Char('Model') + field_name = fields.Char('Field') + query = fields.Text() + query_type = fields.Selection(selection=[('sql', 'sql'), ('python', 'python')], string='Query') + sequence = fields.Integer() diff --git a/addons/anonymization/anonymization_view.xml b/addons/anonymization/views/anonymization_views.xml similarity index 58% rename from addons/anonymization/anonymization_view.xml rename to addons/anonymization/views/anonymization_views.xml index 6b3ca271976..df81688365a 100644 --- a/addons/anonymization/anonymization_view.xml +++ b/addons/anonymization/views/anonymization_views.xml @@ -1,22 +1,20 @@ - - - + + - + ir.model.fields.anonymization.form ir.model.fields.anonymization
- - + + - + @@ -24,7 +22,7 @@ - + ir.model.fields.anonymization.tree ir.model.fields.anonymization @@ -38,7 +36,7 @@ - + Anonymized Fields ir.model.fields.anonymization form @@ -46,55 +44,16 @@ tree,form - - - - - ir.model.fields.anonymize.wizard.form - ir.model.fields.anonymize.wizard - - -
-
- - - - - - - - - - - - - - Anonymize Database - ir.model.fields.anonymize.wizard - new - form - form - - - + Anonymization History ir.model.fields.anonymization.history form tree,form - + ir.model.fields.anonymization.history.form ir.model.fields.anonymization.history @@ -122,7 +81,7 @@ - + ir.model.fields.anonymization.history.tree ir.model.fields.anonymization.history @@ -154,11 +113,4 @@ parent="menu_administration_anonymization" sequence="20" /> - - -
 - + diff --git a/addons/anonymization/wizard/__init__.py b/addons/anonymization/wizard/__init__.py new file mode 100644 index 00000000000..d22126ebe0d --- /dev/null +++ b/addons/anonymization/wizard/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import anonymize_wizard diff --git a/addons/anonymization/wizard/anonymize_wizard.py b/addons/anonymization/wizard/anonymize_wizard.py new file mode 100644 index 00000000000..8879ce5dc65 --- /dev/null +++ b/addons/anonymization/wizard/anonymize_wizard.py @@ -0,0 +1,338 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import base64 +import os +import random +try: + import cPickle as pickle +except ImportError: + import pickle +from lxml import etree +from operator import itemgetter + +from odoo import api, fields, models, _ +from odoo.exceptions import UserError +from odoo.release import version_info +from odoo.tools.safe_eval import safe_eval as eval +from odoo.addons.anonymization.models.anonymization import group + +WIZARD_ANONYMIZATION_STATES = [('clear', 'Clear'), ('anonymized', 'Anonymized'), ('unstable', 'Unstable')] + + +class IrModelFieldsAnonymizeWizard(models.TransientModel): + _name = 'ir.model.fields.anonymize.wizard' + + name = fields.Char('File Name') + summary = fields.Text(compute='_compute_summary') + file_export = fields.Binary('Export') + file_import = fields.Binary('Import', + help="This is the file created by the anonymization process. It should have the '.pickle' extention.") + state = fields.Selection(compute='_compute_state', string='Status', selection=WIZARD_ANONYMIZATION_STATES) + msg = fields.Text('Message') + + @api.multi + def _compute_summary(self): + for anonymize_wizard in self: + anonymize_wizard.summary = anonymize_wizard._get_summary_value() + + @api.multi + def _compute_state(self): + for anonymize_wizard in self: + anonymize_wizard.state = anonymize_wizard._get_state_value() + + @api.model + def _get_state_value(self): + return self.env['ir.model.fields.anonymization']._get_global_state() + + @api.model + def _get_summary_value(self): + summary = u'' + for anon_field in self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')]): + field = anon_field.field_id + values = { + 'model_name': field.model_id.name, + 'model_code': field.model_id.model, + 'field_code': field.name, + 'field_name': field.field_description, + 'state': anon_field.state, + } + summary += u" * %(model_name)s (%(model_code)s) -> %(field_name)s (%(field_code)s): state: (%(state)s)\n" % values + return summary + + @api.model + def default_get(self, fields_list): + res = {} + res['name'] = '.pickle' + res['summary'] = self._get_summary_value() + res['state'] = self._get_state_value() + res['msg'] = _("Before executing the anonymization process, you should make a backup of your database.") + return res + + @api.model + def fields_view_get(self, view_id=None, view_type='form', toolbar=False, submenu=False): + state = self.env['ir.model.fields.anonymization']._get_global_state() + step = self.env.context.get('step', 'new_window') + res = super(IrModelFieldsAnonymizeWizard, self).fields_view_get(view_id=view_id, view_type=view_type, toolbar=toolbar, submenu=submenu) + eview = etree.fromstring(res['arch']) + placeholder = eview.xpath("group[@name='placeholder1']") + if len(placeholder): + placeholder = placeholder[0] + if step == 'new_window' and state == 'clear': + # clicked in the menu and the fields are not anonymized: warn the admin that backuping the db is very important + placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) + placeholder.addnext(etree.Element('newline')) + placeholder.addnext(etree.Element('label', {'string': 'Warning'})) + eview.remove(placeholder) + elif step == 'new_window' and state == 'anonymized': + # clicked in the menu and the fields are already anonymized + placeholder.addnext(etree.Element('newline')) + placeholder.addnext(etree.Element('field', {'name': 'file_import', 'required': "1"})) + placeholder.addnext(etree.Element('label', {'string': 'Anonymization file'})) + eview.remove(placeholder) + elif step == 'just_anonymized': + # we just ran the anonymization process, we need the file export field + placeholder.addnext(etree.Element('newline')) + placeholder.addnext(etree.Element('field', {'name': 'file_export'})) + # we need to remove the button: + buttons = eview.xpath("button") + for button in buttons: + eview.remove(button) + # and add a message: + placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) + placeholder.addnext(etree.Element('newline')) + placeholder.addnext(etree.Element('label', {'string': 'Result'})) + # remove the placeholer: + eview.remove(placeholder) + elif step == 'just_desanonymized': + # we just reversed the anonymization process, we don't need any field + # we need to remove the button + buttons = eview.xpath("button") + for button in buttons: + eview.remove(button) + # and add a message + placeholder.addnext(etree.Element('field', {'name': 'msg', 'colspan': '4', 'nolabel': '1'})) + placeholder.addnext(etree.Element('newline')) + placeholder.addnext(etree.Element('label', {'string': 'Result'})) + # remove the placeholer: + eview.remove(placeholder) + else: + raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized," + " while some fields are not anonymized. You should try to solve this problem before trying to do anything else.")) + res['arch'] = etree.tostring(eview) + return res + + def _raise_after_history_update(self, history, error_type, error_msg): + history.write({ + 'state': 'in_exception', + 'msg': error_msg, + }) + raise UserError('%s: %s' % (error_type, error_msg)) + + @api.multi + def anonymize_database(self): + """Sets the 'anonymized' state to defined fields""" + self.ensure_one() + + # create a new history record: + history = self.env['ir.model.fields.anonymization.history'].create({ + 'date': fields.Datetime.now(), + 'state': 'started', + 'direction': 'clear -> anonymized' + }) + + # check that all the defined fields are in the 'clear' state + state = self.env['ir.model.fields.anonymization']._get_global_state() + if state == 'anonymized': + self._raise_after_history_update(history, _('Error !'), _("The database is currently anonymized, you cannot anonymize it again.")) + elif state == 'unstable': + self._raise_after_history_update(history, _('Error !'), _("The database anonymization is currently in an unstable state. Some fields are anonymized," + " while some fields are not anonymized. You should try to solve this problem before trying to do anything.")) + + # do the anonymization: + dirpath = os.environ.get('HOME') or os.getcwd() + rel_filepath = 'field_anonymization_%s_%s.pickle' % (self.env.cr.dbname, history.id) + abs_filepath = os.path.abspath(os.path.join(dirpath, rel_filepath)) + + ano_fields = self.env['ir.model.fields.anonymization'].search([('state', '!=', 'not_existing')]) + if not ano_fields: + self._raise_after_history_update(history, _('Error !'), _("No fields are going to be anonymized.")) + + data = [] + + for field in ano_fields: + model_name = field.model_id.model + field_name = field.field_id.name + field_type = field.field_id.ttype + table_name = self.env[model_name]._table + + # get the current value + self.env.cr.execute("select id, %s from %s" % (field_name, table_name)) + for record in self.env.cr.dictfetchall(): + data.append({"model_id": model_name, "field_id": field_name, "id": record['id'], "value": record[field_name]}) + + # anonymize the value: + anonymized_value = None + + sid = str(record['id']) + if field_type == 'char': + anonymized_value = 'xxx' + sid + elif field_type == 'selection': + anonymized_value = 'xxx' + sid + elif field_type == 'text': + anonymized_value = 'xxx' + sid + elif field_type == 'html': + anonymized_value = 'xxx' + sid + elif field_type == 'boolean': + anonymized_value = random.choice([True, False]) + elif field_type == 'date': + anonymized_value = '2011-11-11' + elif field_type == 'datetime': + anonymized_value = '2011-11-11 11:11:11' + elif field_type == 'float': + anonymized_value = 0.0 + elif field_type == 'integer': + anonymized_value = 0 + elif field_type in ['binary', 'many2many', 'many2one', 'one2many', 'reference']: # cannot anonymize these kind of fields + self._raise_after_history_update(history, _('Error !'), _("Cannot anonymize fields of these types: binary, many2many, many2one, one2many, reference.")) + + if anonymized_value is None: + self._raise_after_history_update(history, _('Error !'), _("Anonymized value can not be empty.")) + + sql = "update %(table)s set %(field)s = %%(anonymized_value)s where id = %%(id)s" % { + 'table': table_name, + 'field': field_name, + } + self.env.cr.execute(sql, { + 'anonymized_value': anonymized_value, + 'id': record['id'] + }) + + # save pickle: + fn = open(abs_filepath, 'w') + pickle.dump(data, fn, pickle.HIGHEST_PROTOCOL) + + # update the anonymization fields: + ano_fields.write({'state': 'anonymized'}) + + # add a result message in the wizard: + msgs = ["Anonymization successful.", + "", + "Donot forget to save the resulting file to a safe place because you will not be able to revert the anonymization without this file.", + "", + "This file is also stored in the %s directory. The absolute file path is: %s." + ] + msg = '\n'.join(msgs) % (dirpath, abs_filepath) + + fn = open(abs_filepath, 'r') + + self.write({ + 'msg': msg, + 'file_export': base64.encodestring(fn.read()), + }) + fn.close() + + # update the history record: + history.write({ + 'field_ids': [[6, 0, ano_fields.ids]], + 'msg': msg, + 'filepath': abs_filepath, + 'state': 'done', + }) + + return { + 'res_id': self.id, + 'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids, + 'view_type': 'form', + "view_mode": 'form', + 'res_model': 'ir.model.fields.anonymize.wizard', + 'type': 'ir.actions.act_window', + 'context': {'step': 'just_anonymized'}, + 'target': 'new' + } + + @api.multi + def reverse_anonymize_database(self): + """Set the 'clear' state to defined fields""" + self.ensure_one() + IrModelFieldsAnonymization = self.env['ir.model.fields.anonymization'] + + # create a new history record: + history = self.env['ir.model.fields.anonymization.history'].create({ + 'date': fields.Datetime.now(), + 'state': 'started', + 'direction': 'anonymized -> clear' + }) + + # check that all the defined fields are in the 'anonymized' state + state = IrModelFieldsAnonymization._get_global_state() + if state == 'clear': + raise UserError(_("The database is not currently anonymized, you cannot reverse the anonymization.")) + elif state == 'unstable': + raise UserError(_("The database anonymization is currently in an unstable state. Some fields are anonymized," + " while some fields are not anonymized. You should try to solve this problem before trying to do anything.")) + + if not self.file_import: + raise UserError('%s: %s' % (_('Error !'), _("It is not possible to reverse the anonymization process without supplying the anonymization export file."))) + + # reverse the anonymization: + # load the pickle file content into a data structure: + data = pickle.loads(base64.decodestring(self.file_import)) + + fixes = self.env['ir.model.fields.anonymization.migration.fix'].search_read([ + ('target_version', '=', '.'.join(map(str, version_info[:2]))) + ], ['model_name', 'field_name', 'query', 'query_type', 'sequence']) + fixes = group(fixes, ('model_name', 'field_name')) + + for line in data: + queries = [] + table_name = self.env[line['model_id']]._table if line['model_id'] in self.env else None + + # check if custom sql exists: + key = (line['model_id'], line['field_id']) + custom_updates = fixes.get(key) + if custom_updates: + custom_updates.sort(key=itemgetter('sequence')) + queries = [(record['query'], record['query_type']) for record in custom_updates if record['query_type']] + elif table_name: + queries = [("update %(table)s set %(field)s = %%(value)s where id = %%(id)s" % { + 'table': table_name, + 'field': line['field_id'], + }, 'sql')] + + for query in queries: + if query[1] == 'sql': + self.env.cr.execute(query[0], { + 'value': line['value'], + 'id': line['id'] + }) + elif query[1] == 'python': + eval(query[0] % line) + else: + raise Exception("Unknown query type '%s'. Valid types are: sql, python." % (query['query_type'], )) + + # update the anonymization fields: + ano_fields = IrModelFieldsAnonymization.search([('state', '!=', 'not_existing')]) + ano_fields.write({'state': 'clear'}) + + # add a result message in the wizard: + self.msg = '\n'.join(["Successfully reversed the anonymization.", ""]) + + # update the history record: + history.write({ + 'field_ids': [[6, 0, ano_fields.ids]], + 'msg': self.msg, + 'filepath': False, + 'state': 'done' + }) + + return { + 'res_id': self.id, + 'view_id': self.env.ref('anonymization.view_ir_model_fields_anonymize_wizard_form').ids, + 'view_type': 'form', + "view_mode": 'form', + 'res_model': 'ir.model.fields.anonymize.wizard', + 'type': 'ir.actions.act_window', + 'context': {'step': 'just_desanonymized'}, + 'target': 'new' + } diff --git a/addons/anonymization/wizard/anonymize_wizard_views.xml b/addons/anonymization/wizard/anonymize_wizard_views.xml new file mode 100644 index 00000000000..abe577e855b --- /dev/null +++ b/addons/anonymization/wizard/anonymize_wizard_views.xml @@ -0,0 +1,44 @@ + + + + + ir.model.fields.anonymize.wizard.form + ir.model.fields.anonymize.wizard + +
+
+
+ + + + + + + +
+ + + + Anonymize Database + ir.model.fields.anonymize.wizard + new + form + form + + + + + diff --git a/addons/google_drive/views/google_drive_views.xml b/addons/google_drive/views/google_drive_views.xml index 5dde683f301..bc0ff8f23e5 100644 --- a/addons/google_drive/views/google_drive_views.xml +++ b/addons/google_drive/views/google_drive_views.xml @@ -22,7 +22,7 @@ - +