From ab452d00f89127c6cb144d79f899758fa889baac Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 23 Jun 2015 11:20:08 +0200 Subject: [PATCH 01/15] [IMP] models: remove deprecated methods `read_string` and `write_string` --- openerp/models.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/openerp/models.py b/openerp/models.py index c601d309512..50e605c16b6 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -1714,46 +1714,6 @@ class BaseModel(object): res = self.name_get(cr, access_rights_uid, ids, context) return res - def read_string(self, cr, uid, id, langs, fields=None, context=None): - res = {} - res2 = {} - self.pool.get('ir.translation').check_access_rights(cr, uid, 'read') - if not fields: - fields = self._columns.keys() + self._inherit_fields.keys() - #FIXME: collect all calls to _get_source into one SQL call. - for lang in langs: - res[lang] = {'code': lang} - for f in fields: - if f in self._columns: - res_trans = self.pool.get('ir.translation')._get_source(cr, uid, self._name+','+f, 'field', lang) - if res_trans: - res[lang][f] = res_trans - else: - res[lang][f] = self._columns[f].string - for table in self._inherits: - cols = intersect(self._inherit_fields.keys(), fields) - res2 = self.pool[table].read_string(cr, uid, id, langs, cols, context) - for lang in res2: - if lang in res: - res[lang]['code'] = lang - for f in res2[lang]: - res[lang][f] = res2[lang][f] - return res - - def write_string(self, cr, uid, id, langs, vals, context=None): - self.pool.get('ir.translation').check_access_rights(cr, uid, 'write') - #FIXME: try to only call the translation in one SQL - for lang in langs: - for field in vals: - if field in self._columns: - src = self._columns[field].string - self.pool.get('ir.translation')._set_ids(cr, uid, self._name+','+field, 'field', lang, [0], vals[field], src) - for table in self._inherits: - cols = intersect(self._inherit_fields.keys(), vals) - if cols: - self.pool[table].write_string(cr, uid, id, langs, vals, context) - return True - def _add_missing_default_values(self, cr, uid, values, context=None): # avoid overriding inherited values when parent is set avoid_tables = [] From 1fc4a3424f6f741bcdca96a636c40a235265c6d4 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Mon, 29 Jun 2015 11:00:25 +0200 Subject: [PATCH 02/15] [IMP] fields: add method `traverse_related` to ease the handling of related fields --- openerp/fields.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/openerp/fields.py b/openerp/fields.py index 56d177d9c9e..bcda4ac39d1 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -514,28 +514,28 @@ class Field(object): if self.inherited and field.required: self.required = True + def traverse_related(self, record): + """ Traverse the fields of the related field `self` except for the last + one, and return it as a pair `(last_record, last_field)`. """ + for name in self.related[:-1]: + record = record[name][:1] + return record, self.related_field + def _compute_related(self, records): """ Compute the related field `self` on `records`. """ # when related_sudo, bypass access rights checks when reading values others = records.sudo() if self.related_sudo else records for record, other in zip(records, others): - if not record.id: - # draft record, do not switch to another environment - other = record - # traverse the intermediate fields; follow the first record at each step - for name in self.related[:-1]: - other = other[name][:1] - record[self.name] = other[self.related[-1]] + # do not switch to another environment if record is a draft one + other, field = self.traverse_related(other if record.id else record) + record[self.name] = other[field.name] def _inverse_related(self, records): """ Inverse the related field `self` on `records`. """ for record in records: - other = record - # traverse the intermediate fields, and keep at most one record - for name in self.related[:-1]: - other = other[name][:1] + other, field = self.traverse_related(record) if other: - other[self.related[-1]] = record[self.name] + other[field.name] = record[self.name] def _search_related(self, records, operator, value): """ Determine the domain to search on field `self`. """ @@ -1702,11 +1702,8 @@ class _RelationalMulti(_Relational): def _compute_related(self, records): """ Compute the related field `self` on `records`. """ for record in records: - value = record - # traverse the intermediate fields, and keep at most one record - for name in self.related[:-1]: - value = value[name][:1] - record[self.name] = value[self.related[-1]] + other, field = self.traverse_related(record) + record[self.name] = other[field.name] class One2many(_RelationalMulti): From 848c220180708f313a180c377592e81a13e96a76 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 23 Jun 2015 11:26:56 +0200 Subject: [PATCH 03/15] [IMP] ir_model: use `ir.model.fields` to store translations of fields string and help * add the necessary fields in `ir.model.fields` * when loading translations, convert types 'field', 'help' into 'model' * add specific methods to retrieve translations of field string and help --- openerp/addons/base/ir/ir_model.py | 3 +- openerp/addons/base/ir/ir_translation.py | 62 ++++++++++++++++++------ openerp/fields.py | 13 +++-- openerp/models.py | 11 +++-- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index d035e30b09d..85d247bc7b9 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -220,7 +220,8 @@ class ir_model_fields(osv.osv): help="For one2many fields, the field on the target model that implement the opposite many2one relationship"), 'model_id': fields.many2one('ir.model', 'Model', required=True, select=True, ondelete='cascade', help="The model this field belongs to"), - 'field_description': fields.char('Field Label', required=True), + 'field_description': fields.char('Field Label', required=True, translate=True), + 'help': fields.text('Field Help', translate=True), 'ttype': fields.selection(_get_fields_type, 'Field Type', required=True), 'selection': fields.char('Selection Options', help="List of options for a selection field, " "specified as a Python expression defining a list of (key, label) pairs. " diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index 3eefef97902..fb0fbab55c6 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -3,7 +3,7 @@ import logging -from openerp import tools +from openerp import api, tools import openerp.modules from openerp.osv import fields, osv from openerp.tools.translate import _ @@ -12,17 +12,12 @@ from openerp.exceptions import UserError _logger = logging.getLogger(__name__) TRANSLATION_TYPE = [ - ('field', 'Field'), + ('field', 'Field'), # deprecated ('model', 'Object'), - ('rml', 'RML (deprecated - use Report)'), # Pending deprecation - to be replaced by report! ('report', 'Report/Template'), ('selection', 'Selection'), ('view', 'View'), - ('wizard_button', 'Wizard Button'), - ('wizard_field', 'Wizard Field'), - ('wizard_view', 'Wizard View'), - ('xsl', 'XSL'), - ('help', 'Help'), + ('help', 'Help'), # deprecated ('code', 'Code'), ('constraint', 'Constraint'), ('sql_constraint', 'SQL Constraint') @@ -70,6 +65,21 @@ class ir_translation_import_cursor(object): elif params['res_id'] is None: params['res_id'] = 0 + # backward compatibility: convert 'field', 'help' into 'model' + if params['type'] == 'field': + model, field = params['name'].split(',') + params['type'] = 'model' + params['name'] = 'ir.model.fields,field_description' + params['imd_model'] = 'ir.model.fields' + params['imd_name'] = 'field_%s_%s' % (model.replace('.', '_'), field) + + elif params['type'] == 'help': + model, field = params['name'].split(',') + params['type'] = 'model' + params['name'] = 'ir.model.fields,help' + params['imd_model'] = 'ir.model.fields' + params['imd_name'] = 'field_%s_%s' % (model.replace('.', '_'), field) + self._cr.execute("""INSERT INTO %s (name, lang, res_id, src, type, imd_model, module, imd_name, value, state, comments) VALUES (%%(name)s, %%(lang)s, %%(res_id)s, %%(src)s, %%(type)s, %%(imd_model)s, %%(module)s, %%(imd_name)s, %%(value)s, %%(state)s, %%(comments)s)""" % self._table_name, @@ -267,8 +277,7 @@ class ir_translation(osv.osv): return translations def _set_ids(self, cr, uid, name, tt, lang, ids, value, src=None): - self._get_ids.clear_cache(self) - self.__get_source.clear_cache(self) + self.clear_caches() cr.execute('delete from ir_translation ' 'where lang=%s ' @@ -354,12 +363,35 @@ class ir_translation(osv.osv): res_id = tuple(res_id) return self.__get_source(cr, uid, name, types, lang, source, res_id) + @api.model + @tools.ormcache_context('model_name', keys=('lang',)) + def get_field_string(self, model_name): + """ Return the translation of fields strings in the context's language. + Note that the result contains the available translations only. + + :param model_name: the name of a model + :return: the model's fields' strings as a dictionary `{field_name: field_string}` + """ + fields = self.env['ir.model.fields'].search([('model', '=', model_name)]) + return {field.name: field.field_description for field in fields} + + @api.model + @tools.ormcache_context('model_name', keys=('lang',)) + def get_field_help(self, model_name): + """ Return the translation of fields help in the context's language. + Note that the result contains the available translations only. + + :param model_name: the name of a model + :return: the model's fields' help as a dictionary `{field_name: field_help}` + """ + fields = self.env['ir.model.fields'].search([('model', '=', model_name)]) + return {field.name: field.help for field in fields} + def create(self, cr, uid, vals, context=None): if context is None: context = {} ids = super(ir_translation, self).create(cr, uid, vals, context=context) - self.__get_source.clear_cache(self) - self._get_ids.clear_cache(self) + self.clear_caches() self.pool['ir.ui.view'].clear_cache() return ids @@ -373,8 +405,7 @@ class ir_translation(osv.osv): if vals.get('value'): vals.update({'state':'translated'}) result = super(ir_translation, self).write(cursor, user, ids, vals, context=context) - self.__get_source.clear_cache(self) - self._get_ids.clear_cache(self) + self.clear_caches() self.pool['ir.ui.view'].clear_cache() return result @@ -384,8 +415,7 @@ class ir_translation(osv.osv): if isinstance(ids, (int, long)): ids = [ids] - self.__get_source.clear_cache(self) - self._get_ids.clear_cache(self) + self.clear_caches() result = super(ir_translation, self).unlink(cursor, user, ids, context=context) return result diff --git a/openerp/fields.py b/openerp/fields.py index bcda4ac39d1..6c0253b5ea7 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -651,17 +651,16 @@ class Field(object): def _description_string(self, env): if self.string and env.lang: - field = self.base_field - name = "%s,%s" % (field.model_name, field.name) - trans = env['ir.translation']._get_source(name, 'field', env.lang) - return trans or self.string + model_name = self.base_field.model_name + field_string = env['ir.translation'].get_field_string(model_name) + return field_string.get(self.name) or self.string return self.string def _description_help(self, env): if self.help and env.lang: - name = "%s,%s" % (self.model_name, self.name) - trans = env['ir.translation']._get_source(name, 'help', env.lang) - return trans or self.help + model_name = self.base_field.model_name + field_help = env['ir.translation'].get_field_help(model_name) + return field_help.get(self.name) or self.help return self.help ############################################################################ diff --git a/openerp/models.py b/openerp/models.py index 50e605c16b6..54859c3adf4 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -380,6 +380,7 @@ class BaseModel(object): 'model': self._name, 'name': k, 'field_description': f.string, + 'help': f.help or False, 'ttype': f.type, 'relation': f.comodel_name or '', 'select_level': tools.ustr(int(f.index)), @@ -1166,11 +1167,11 @@ class BaseModel(object): Converter = self.pool['ir.fields.converter'] Translation = self.pool['ir.translation'] fields = dict(self._fields) - field_names = dict( - (f, (Translation._get_source(cr, uid, self._name + ',' + f, 'field', - context.get('lang')) - or field.string)) - for f, field in fields.iteritems()) + field_names = {name: field.string for name, field in fields.iteritems()} + if context.get('lang'): + field_names.update( + Translation.get_field_string(cr, uid, self._name, context=context) + ) convert = Converter.for_model(cr, uid, self, context=context) From d5a1d7b6ae9711864e03c69271d4b42898d3dcf0 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 2 Dec 2014 10:58:54 +0100 Subject: [PATCH 04/15] [IMP] models: improve reflection of fields into `ir.model.fields` * add missing columns in base.sql * add missing fields 'help', 'readonly', 'required', 'selectable' in reflection * add field 'help' in `ir.model.fields` views * do not patch registry with translated field string and help upon `ir.model.fields` update --- openerp/addons/base/base.sql | 12 ++++--- openerp/addons/base/ir/ir_model.py | 4 +++ openerp/addons/base/ir/ir_model_view.xml | 14 ++++++--- openerp/models.py | 40 +++++++++--------------- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/openerp/addons/base/base.sql b/openerp/addons/base/base.sql index 7084148885b..c0fd67ea56c 100644 --- a/openerp/addons/base/base.sql +++ b/openerp/addons/base/base.sql @@ -28,12 +28,16 @@ CREATE TABLE ir_model_fields ( model varchar NOT NULL, model_id integer references ir_model on delete cascade, name varchar NOT NULL, - relation varchar, - select_level varchar, - field_description varchar, - ttype varchar, state varchar default 'base', + field_description varchar, + help varchar, + ttype varchar, + relation varchar, relation_field varchar, + select_level varchar, + readonly boolean default False, + required boolean default False, + selectable boolean default False, translate boolean default False, serialization_field_id integer references ir_model_fields on delete cascade, primary key(id) diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index 85d247bc7b9..c3ba59f4f71 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -395,6 +395,7 @@ class ir_model_fields(osv.osv): # static table of properties model_props = [ # (our-name, fields.prop, set_fn) ('field_description', 'string', tools.ustr), + ('help', 'help', lambda s: s and tools.ustr(s) or None), ('required', 'required', bool), ('readonly', 'readonly', bool), ('domain', 'domain', eval), @@ -404,6 +405,9 @@ class ir_model_fields(osv.osv): ('select_level', 'index', lambda x: bool(int(x))), ('selection', 'selection', eval), ] + if context.get('lang') not in (None, 'en_US'): + # do not patch registry with translations of field string and help + model_props = model_props[2:] if vals and ids: checked_selection = False # need only check it once, so defer diff --git a/openerp/addons/base/ir/ir_model_view.xml b/openerp/addons/base/ir/ir_model_view.xml index 1b6c46083f4..a62b8518eda 100644 --- a/openerp/addons/base/ir/ir_model_view.xml +++ b/openerp/addons/base/ir/ir_model_view.xml @@ -53,10 +53,15 @@
- - - - + + + + + + + + + @@ -158,6 +163,7 @@ + diff --git a/openerp/models.py b/openerp/models.py index 54859c3adf4..a147615bc54 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -384,11 +384,11 @@ class BaseModel(object): 'ttype': f.type, 'relation': f.comodel_name or '', 'select_level': tools.ustr(int(f.index)), - 'readonly': (f.readonly and 1) or 0, - 'required': (f.required and 1) or 0, - 'selectable': (f.search or f.store and 1) or 0, - 'translate': (f.translate if hasattr(f,'translate') else False and 1) or 0, - 'relation_field': f.inverse_name if hasattr(f, 'inverse_name') else '', + 'readonly': bool(f.readonly), + 'required': bool(f.required), + 'selectable': bool(f.search or f.store), + 'translate': bool(getattr(f, 'translate', False)), + 'relation_field': getattr(f, 'inverse_name', ''), 'serialization_field_id': None, } if getattr(f, 'serialization_field', None): @@ -410,16 +410,11 @@ class BaseModel(object): cr.execute('select nextval(%s)', ('ir_model_fields_id_seq',)) id = cr.fetchone()[0] vals['id'] = id - cr.execute("""INSERT INTO ir_model_fields ( - id, model_id, model, name, field_description, ttype, - relation,state,select_level,relation_field, translate, serialization_field_id - ) VALUES ( - %s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s - )""", ( - id, vals['model_id'], vals['model'], vals['name'], vals['field_description'], vals['ttype'], - vals['relation'], 'base', - vals['select_level'], vals['relation_field'], bool(vals['translate']), vals['serialization_field_id'] - )) + query = "INSERT INTO ir_model_fields (%s) VALUES (%s)" % ( + ",".join(vals), + ",".join("%%(%s)s" % name for name in vals), + ) + cr.execute(query, vals) if 'module' in context: name1 = 'field_' + self._table + '_' + k cr.execute("select name from ir_model_data where name=%s", (name1,)) @@ -431,16 +426,11 @@ class BaseModel(object): else: for key, val in vals.items(): if cols[k][key] != vals[key]: - cr.execute('update ir_model_fields set field_description=%s where model=%s and name=%s', (vals['field_description'], vals['model'], vals['name'])) - cr.execute("""UPDATE ir_model_fields SET - model_id=%s, field_description=%s, ttype=%s, relation=%s, - select_level=%s, readonly=%s ,required=%s, selectable=%s, relation_field=%s, translate=%s, serialization_field_id=%s - WHERE - model=%s AND name=%s""", ( - vals['model_id'], vals['field_description'], vals['ttype'], - vals['relation'], - vals['select_level'], bool(vals['readonly']), bool(vals['required']), bool(vals['selectable']), vals['relation_field'], bool(vals['translate']), vals['serialization_field_id'], vals['model'], vals['name'] - )) + names = set(vals) - set(['model', 'name']) + query = "UPDATE ir_model_fields SET %s WHERE model=%%(model)s and name=%%(name)s" % ( + ",".join("%s=%%(%s)s" % (name, name) for name in names), + ) + cr.execute(query, vals) break self.invalidate_cache(cr, SUPERUSER_ID) From 8ddfb1888b9f5ef20f00f2a4c9d5fe3a16c319cf Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 24 Jun 2015 10:24:33 +0200 Subject: [PATCH 05/15] [IMP] models, fields: handle the case where field.translate is a callable Using `field.translate(callback, value)` returns the translation of the given field value by using the function `callback` to translate simple terms in `value`. In practice, the function `callback` uses a translation mapping retrieved from `ir.translation`. --- addons/base_gengo/ir_translation.py | 19 ++++++- openerp/addons/base/ir/ir_translation.py | 31 ++++++++++++ .../addons/base/ir/ir_translation_view.xml | 3 +- openerp/fields.py | 46 +++++++++++++++-- openerp/models.py | 32 +++++++++--- openerp/osv/expression.py | 2 +- openerp/tools/translate.py | 49 +++++-------------- 7 files changed, 133 insertions(+), 49 deletions(-) diff --git a/addons/base_gengo/ir_translation.py b/addons/base_gengo/ir_translation.py index 9885c24df65..f1297f96e97 100644 --- a/addons/base_gengo/ir_translation.py +++ b/addons/base_gengo/ir_translation.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from openerp.osv import fields, osv +from openerp.osv import fields, osv, api from openerp.tools.translate import _ from openerp.exceptions import UserError @@ -84,3 +84,20 @@ class ir_translation(osv.Model): """ params += ('machine', 'standard', 'ultra', 'pro',) return (query, params) + + @api.model + def _get_terms_query(self, field, records): + query, params = super(ir_translation, self)._get_terms_query(field, records) + # order translations from worst to best + query += """ + ORDER BY + CASE + WHEN gengo_translation=%s then 10 + WHEN gengo_translation=%s then 20 + WHEN gengo_translation=%s then 30 + WHEN gengo_translation=%s then 40 + ELSE 0 + END ASC + """ + params += ('machine', 'standard', 'ultra', 'pro') + return query, params diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index fb0fbab55c6..46502a2f358 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -363,6 +363,37 @@ class ir_translation(osv.osv): res_id = tuple(res_id) return self.__get_source(cr, uid, name, types, lang, source, res_id) + @api.model + def _get_terms_query(self, field, records): + """ Utility function that makes the query for field terms. """ + query = """ SELECT id, res_id, src, value FROM ir_translation + WHERE lang=%s AND type=%s AND name=%s AND res_id IN %s """ + name = "%s,%s" % (field.model_name, field.name) + params = (records.env.lang, 'model', name, tuple(records.ids)) + return query, params + + @api.model + def _get_terms_mapping(self, field, records): + """ Return a function mapping a (trans_id, source, value) to a value. + This method is called before querying the database for translations. + """ + return lambda tid, src, value: value + + @api.model + def _get_terms_translations(self, field, records): + """ Return the terms and translations of a given `field` on `records`. + + :return: {record_id: {source: value}} + """ + result = {rid: {} for rid in records.ids} + if records: + map_trans = self._get_terms_mapping(field, records) + query, params = self._get_terms_query(field, records) + self._cr.execute(query, params) + for trans_id, res_id, src, value in self._cr.fetchall(): + result[res_id][src] = map_trans(trans_id, src, value) + return result + @api.model @tools.ormcache_context('model_name', keys=('lang',)) def get_field_string(self, model_name): diff --git a/openerp/addons/base/ir/ir_translation_view.xml b/openerp/addons/base/ir/ir_translation_view.xml index 3fd80f90da9..6bf3d783040 100644 --- a/openerp/addons/base/ir/ir_translation_view.xml +++ b/openerp/addons/base/ir/ir_translation_view.xml @@ -8,7 +8,7 @@ + domain="['|','|',('state','=','to_translate'),('value', '=', False),('value','=','')]"/> @@ -61,6 +61,7 @@ + diff --git a/openerp/fields.py b/openerp/fields.py index 6c0253b5ea7..89cb9325f88 100644 --- a/openerp/fields.py +++ b/openerp/fields.py @@ -1110,15 +1110,50 @@ class _String(Field): _column_translate = property(attrgetter('translate')) _related_translate = property(attrgetter('translate')) - _description_translate = property(attrgetter('translate')) + + def _description_translate(self, env): + return bool(self.translate) + + def get_trans_terms(self, value): + """ Return the sequence of terms to translate found in `value`. """ + if not callable(self.translate): + return [value] if value else [] + terms = [] + self.translate(terms.append, value) + return terms + + def get_trans_func(self, records): + """ Return a translation function `translate` for `self` on the given + records; the function call `translate(record_id, value)` translates the + field value to the language given by the environment of `records`. + """ + if callable(self.translate): + rec_src_trans = records.env['ir.translation']._get_terms_translations(self, records) + + def translate(record_id, value): + src_trans = rec_src_trans[record_id] + return self.translate(src_trans.get, value) + + else: + rec_trans = records.env['ir.translation']._get_ids( + '%s,%s' % (self.model_name, self.name), 'model', records.env.lang, records.ids) + + def translate(record_id, value): + return rec_trans.get(record_id) or value + + return translate class Char(_String): """ Basic string field, can be length-limited, usually displayed as a - single-line string in clients + single-line string in clients. :param int size: the maximum size of values stored for that field - :param bool translate: whether the values of this field can be translated + + :param translate: enable the translation of the field's values; use + `translate=True` to translate field values as a whole; `translate` may + also be a callable such that `translate(callback, value)` translates + `value` by using `callback(term)` to retrieve the translation of terms. """ type = 'char' _slots = { @@ -1143,7 +1178,10 @@ class Text(_String): """ Very similar to :class:`~.Char` but used for longer contents, does not have a size and usually displayed as a multiline text box. - :param translate: whether the value of this field can be translated + :param translate: enable the translation of the field's values; use + `translate=True` to translate field values as a whole; `translate` may + also be a callable such that `translate(callback, value)` translates + `value` by using `callback(term)` to retrieve the translation of terms. """ type = 'text' diff --git a/openerp/models.py b/openerp/models.py index a147615bc54..4a6f380feba 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -2135,7 +2135,8 @@ class BaseModel(object): ) model, alias = parent_model, parent_alias # handle the case where the field is translated - if model._columns[field].translate: + translate = model._columns[field].translate + if translate and not callable(translate): return model._generate_translated_field(alias, field, query) else: return '"%s"."%s"' % (alias, field) @@ -3222,6 +3223,7 @@ class BaseModel(object): field for field in fields if field.base_field.column._classic_write + if not (field.inherited and callable(field.base_field.column.translate)) ] # the query may involve several tables: we need fully-qualified names @@ -3254,8 +3256,18 @@ class BaseModel(object): result.extend(cr.dictfetchall()) ids = [vals['id'] for vals in result] + fetched = self.browse(ids) if ids: + # translate the fields if necessary + if context.get('lang'): + for field in fields_pre: + if not field.inherited and callable(field.column.translate): + f = field.name + translate = field.get_trans_func(fetched) + for vals in result: + vals[f] = translate(vals['id'], vals[f]) + # apply the symbol_get functions of the fields we just read for field in fields_pre: symbol_get = field.base_field.column._symbol_get @@ -3312,7 +3324,6 @@ class BaseModel(object): record._cache.update(record._convert_to_cache(vals, validate=False)) # store failed values in cache for the records that could not be read - fetched = self.browse(ids) missing = self - fetched if missing: extras = fetched - self @@ -3795,7 +3806,7 @@ class BaseModel(object): upd_todo = [] updend = [] direct = [] - totranslate = context.get('lang', False) and (context['lang'] != 'en_US') + totranslate = context.get('lang') and context['lang'] != 'en_US' for field in vals: ffield = self._fields.get(field) if ffield and ffield.deprecated: @@ -3805,7 +3816,7 @@ class BaseModel(object): if hasattr(column, 'selection') and vals[field]: self._check_selection_field_value(cr, user, field, vals[field], context=context) if column._classic_write and not hasattr(column, '_fnct_inv'): - if (not totranslate) or not column.translate: + if not (totranslate and column.translate): updates.append((field, '%s', column._symbol_set[1](vals[field]))) direct.append(field) else: @@ -3833,7 +3844,16 @@ class BaseModel(object): if totranslate: # TODO: optimize for f in direct: - if self._columns[f].translate: + if callable(self._columns[f].translate): + # We do not handle the update of a field where translate + # is a callable. One should update the value without a + # language, or update translated terms separately. + raise ValidationError(_( + "The translation of the field '%s' cannot be " + "directly updated. You can either update its value " + "in English, or update translated terms separately." + ) % self._fields[f].string) + elif self._columns[f].translate: src_trans = self.pool[self._name].read(cr, user, ids, [f])[0][f] if not src_trans: src_trans = vals[f] @@ -4555,7 +4575,7 @@ class BaseModel(object): elif order_field in self._columns: order_column = self._columns[order_field] if order_column._classic_read: - if order_column.translate: + if order_column.translate and not callable(order_column.translate): inner_clause = self._generate_translated_field(self._table, order_field, query) else: inner_clause = '"%s"."%s"' % (self._table, order_field) diff --git a/openerp/osv/expression.py b/openerp/osv/expression.py index 171ff9e4047..bbc373e7f0e 100644 --- a/openerp/osv/expression.py +++ b/openerp/osv/expression.py @@ -1026,7 +1026,7 @@ class expression(object): right += ' 00:00:00' push(create_substitution_leaf(leaf, (left, operator, right), model)) - elif column.translate and right: + elif column.translate and not callable(column.translate) and right: need_wildcard = operator in ('like', 'ilike', 'not like', 'not ilike') sql_operator = {'=like': 'like', '=ilike': 'ilike'}.get(operator, operator) if need_wildcard: diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index 2f8d779215a..9028a235acd 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -723,33 +723,14 @@ def trans_generate(lang, modules, cr): except AttributeError, exc: _logger.error("name error in %s: %s", xml_name, str(exc)) continue - objmodel = registry.get(obj.model) - if (objmodel is None or field_name not in objmodel._columns - or not objmodel._translate): + field_model = registry.get(obj.model) + if (field_model is None or not field_model._translate or + field_name not in field_model._fields): continue - field_def = objmodel._columns[field_name] - - name = "%s,%s" % (encode(obj.model), field_name) - push_translation(module, 'field', name, 0, encode(field_def.string)) - - if field_def.help: - push_translation(module, 'help', name, 0, encode(field_def.help)) - - if field_def.translate: - ids = objmodel.search(cr, uid, []) - obj_values = objmodel.read(cr, uid, ids, [field_name]) - for obj_value in obj_values: - res_id = obj_value['id'] - if obj.name in ('ir.model', 'ir.ui.menu'): - res_id = 0 - model_data_ids = model_data_obj.search(cr, uid, [ - ('model', '=', model), - ('res_id', '=', res_id), - ]) - if not model_data_ids: - push_translation(module, 'model', name, 0, encode(obj_value[field_name])) + field_def = field_model._fields[field_name] if hasattr(field_def, 'selection') and isinstance(field_def.selection, (list, tuple)): + name = "%s,%s" % (encode(obj.model), field_name) for dummy, val in field_def.selection: push_translation(module, 'selection', name, 0, encode(val)) @@ -761,9 +742,7 @@ def trans_generate(lang, modules, cr): parse_func = trans_parse_rml report_type = "report" elif obj.report_xsl: - fname = obj.report_xsl - parse_func = trans_parse_xsl - report_type = "xsl" + continue if fname and obj.report_type in ('pdf', 'xsl'): try: report_file = misc.file_open(fname) @@ -776,17 +755,15 @@ def trans_generate(lang, modules, cr): except (IOError, etree.XMLSyntaxError): _logger.exception("couldn't export translation for report %s %s %s", name, report_type, fname) - for field_name, field_def in obj._columns.items(): - if model == 'ir.model' and field_name == 'name' and obj.name == obj.model: - # ignore model name if it is the technical one, nothing to translate - continue - if field_def.translate: + for field_name, field_def in obj._fields.iteritems(): + if getattr(field_def, 'translate', None): name = model + "," + field_name try: - term = obj[field_name] or '' - except: - term = '' - push_translation(module, 'model', name, xml_name, encode(term)) + value = obj[field_name] or '' + except Exception: + continue + for term in set(field_def.get_trans_terms(value)): + push_translation(module, 'model', name, xml_name, encode(term)) # End of data for ir.model.data query results From 2ba2cbca5f539219dc0e97f2ee71bcb6a0fe1952 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 24 Jun 2015 10:27:31 +0200 Subject: [PATCH 06/15] [IMP] translations: synchronize terms when English value of field is modified Also adapt the insertion of missing translations when clicking on the translation icon. --- openerp/addons/base/ir/ir_translation.py | 201 +++++++++++++++++------ openerp/models.py | 46 +++--- 2 files changed, 172 insertions(+), 75 deletions(-) diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index 46502a2f358..ea18178d57a 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. +from difflib import get_close_matches import logging from openerp import api, tools import openerp.modules from openerp.osv import fields, osv from openerp.tools.translate import _ -from openerp.exceptions import UserError +from openerp.exceptions import AccessError, UserError _logger = logging.getLogger(__name__) @@ -172,16 +173,18 @@ class ir_translation(osv.osv): context = {} res = dict.fromkeys(ids, False) for record in self.browse(cr, uid, ids, context=context): - if record.type != 'model': - res[record.id] = record.src - else: - model_name, field = record.name.split(',') + res[record.id] = record.src + if record.type == 'model': + model_name, field_name = record.name.split(',') model = self.pool.get(model_name) - if model is not None: + if model is None: + continue + field = model._fields[field_name] + if not callable(field.translate): # Pass context without lang, need to read real stored field, not translation context_no_lang = dict(context, lang=None) - result = model.read(cr, uid, [record.res_id], [field], context=context_no_lang) - res[record.id] = result[0][field] if result else False + result = model.read(cr, uid, [record.res_id], [field_name], context=context_no_lang) + res[record.id] = result[0][field_name] if result else False return res def _set_src(self, cr, uid, id, name, value, args, context=None): @@ -191,15 +194,18 @@ class ir_translation(osv.osv): if context is None: context = {} record = self.browse(cr, uid, id, context=context) - if record.type == 'model': - model_name, field = record.name.split(',') + if record.type == 'model': + model_name, field_name = record.name.split(',') model = self.pool.get(model_name) - #We need to take the context without the language information, because we want to write on the - #value store in db and not on the one associate with current language. - #Also not removing lang from context trigger an error when lang is different - context_wo_lang = context.copy() - context_wo_lang.pop('lang', None) - model.write(cr, uid, [record.res_id], {field: value}, context=context_wo_lang) + field = model._fields[field_name] + if not callable(field.translate): + # Make a context without language information, because we want + # to write on the value stored in db and not on the one + # associated with the current language. Also not removing lang + # from context trigger an error when lang is different. + context_wo_lang = context.copy() + context_wo_lang.pop('lang', None) + model.write(cr, uid, [record.res_id], {field_name: value}, context=context_wo_lang) return self.write(cr, uid, id, {'src': value}, context=context) _columns = { @@ -394,6 +400,49 @@ class ir_translation(osv.osv): result[res_id][src] = map_trans(trans_id, src, value) return result + @api.model + def _sync_terms_translations(self, field, records): + """ Synchronize the translations to the terms to translate, after the + English value of a field is modified. The algorithm tries to match + existing translations to the terms to translate, provided the distance + between modified strings is not too large. It allows to not retranslate + data where a typo has been fixed in the English value. + """ + if not callable(getattr(field, 'translate', None)): + return + + trans = self.env['ir.translation'] + outdated = trans + discarded = trans + + for record in records: + # get field value and terms to translate + value = record[field.name] + terms = set(field.get_trans_terms(value)) + record_trans = trans.search([ + ('type', '=', 'model'), + ('name', '=', "%s,%s" % (field.model_name, field.name)), + ('res_id', '=', record.id), + ]) + + if not terms: + # discard all translations for that field + discarded += record_trans + continue + + # remap existing translations on terms when possible + for trans in record_trans: + if trans.src not in terms: + matches = get_close_matches(trans.src, terms, 1, 0.9) + if matches: + trans.write({'src': matches[0], 'state': trans.state}) + else: + outdated += trans + + # process outdated and discarded translations + outdated.write({'state': 'to_translate'}) + discarded.unlink() + @api.model @tools.ormcache_context('model_name', keys=('lang',)) def get_field_string(self, model_name): @@ -431,10 +480,10 @@ class ir_translation(osv.osv): context = {} if isinstance(ids, (int, long)): ids = [ids] - if vals.get('src') or ('value' in vals and not(vals.get('value'))): - vals.update({'state':'to_translate'}) if vals.get('value'): - vals.update({'state':'translated'}) + vals.setdefault('state', 'translated') + elif vals.get('src') or not vals.get('value', True): + vals.setdefault('state', 'to_translate') result = super(ir_translation, self).write(cursor, user, ids, vals, context=context) self.clear_caches() self.pool['ir.ui.view'].clear_cache() @@ -450,38 +499,81 @@ class ir_translation(osv.osv): result = super(ir_translation, self).unlink(cursor, user, ids, context=context) return result - def translate_fields(self, cr, uid, model, id, field=None, context=None): - trans_model = self.pool[model] - domain = ['&', ('res_id', '=', id), ('name', '=like', model + ',%')] - langs_ids = self.pool.get('res.lang').search(cr, uid, [('code', '!=', 'en_US')], context=context) - if not langs_ids: - raise UserError(_("Translation features are unavailable until you install an extra Odoo translation.")) - langs = [lg.code for lg in self.pool.get('res.lang').browse(cr, uid, langs_ids, context=context)] + @api.model + def insert_missing(self, field, records): + """ Insert missing translations for `field` on `records`. """ + records = records.with_context(lang=None) + if callable(field.translate): + # insert missing translations for each term in src + query = """ INSERT INTO ir_translation (lang, type, name, res_id, src, value) + SELECT l.code, 'model', %(name)s, %(res_id)s, %(src)s, %(src)s + FROM res_lang l + WHERE l.code != 'en_US' AND NOT EXISTS ( + SELECT 1 FROM ir_translation + WHERE lang=l.code AND type='model' AND name=%(name)s AND res_id=%(res_id)s AND src=%(src)s + ); + """ + for record in records: + src = record[field.name] or None + for term in set(field.get_trans_terms(src)): + self._cr.execute(query, { + 'name': "%s,%s" % (field.model_name, field.name), + 'res_id': record.id, + 'src': term, + }) + else: + # insert missing translations for src + query = """ INSERT INTO ir_translation (lang, type, name, res_id, src, value) + SELECT l.code, 'model', %(name)s, %(res_id)s, %(src)s, %(src)s + FROM res_lang l + WHERE l.code != 'en_US' AND NOT EXISTS ( + SELECT 1 FROM ir_translation + WHERE lang=l.code AND type='model' AND name=%(name)s AND res_id=%(res_id)s + ); + UPDATE ir_translation SET src=%(src)s + WHERE type='model' AND name=%(name)s AND res_id=%(res_id)s; + """ + for record in records: + self._cr.execute(query, { + 'name': "%s,%s" % (field.model_name, field.name), + 'res_id': record.id, + 'src': record[field.name] or None, + }) + self.clear_caches() + + @api.model + def translate_fields(self, model, id, field=None): + """ Open a view for translating the field(s) of the record (model, id). """ main_lang = 'en_US' - translatable_fields = [] - for k, f in trans_model._fields.items(): - if getattr(f, 'translate', False): - if f.inherited: - parent_id = trans_model.read(cr, uid, [id], [f.related[0]], context=context)[0][f.related[0]][0] - translatable_fields.append({'name': k, 'id': parent_id, 'model': f.base_field.model_name}) - domain.insert(0, '|') - domain.extend(['&', ('res_id', '=', parent_id), ('name', '=', "%s,%s" % (f.base_field.model_name, k))]) - else: - translatable_fields.append({'name': k, 'id': id, 'model': model }) - if len(langs): - fields = [f.get('name') for f in translatable_fields] - record = trans_model.read(cr, uid, [id], fields, context={ 'lang': main_lang })[0] - for lg in langs: - for f in translatable_fields: - # Check if record exists, else create it (at once) - sql = """INSERT INTO ir_translation (lang, src, name, type, res_id, value) - SELECT %s, %s, %s, 'model', %s, %s WHERE NOT EXISTS - (SELECT 1 FROM ir_translation WHERE lang=%s AND name=%s AND res_id=%s AND type='model'); - UPDATE ir_translation SET src = %s WHERE lang=%s AND name=%s AND res_id=%s AND type='model'; - """ - src = record[f['name']] or None - name = "%s,%s" % (f['model'], f['name']) - cr.execute(sql, (lg, src , name, f['id'], src, lg, name, f['id'], src, lg, name, id)) + if not self.env['res.lang'].search_count([('code', '!=', main_lang)]): + raise UserError(_("Translation features are unavailable until you install an extra translation.")) + + # determine domain for selecting translations + record = self.env[model].with_context(lang=main_lang).browse(id) + domain = ['&', ('res_id', '=', id), ('name', '=like', model + ',%')] + + def make_domain(fld, rec): + name = "%s,%s" % (fld.model_name, fld.name) + return ['&', ('res_id', '=', rec.id), ('name', '=', name)] + + # insert missing translations, and extend domain for related fields + for name, fld in record._fields.items(): + if not getattr(fld, 'translate', False): + continue + + rec = record + if fld.related: + try: + # traverse related fields up to their data source + while fld.related: + rec, fld = fld.traverse_related(rec) + if rec: + domain = ['|'] + domain + make_domain(fld, rec) + except AccessError: + continue + + assert fld.translate and rec._name == fld.model_name + self.insert_missing(fld, rec) action = { 'name': 'Translate', @@ -492,10 +584,11 @@ class ir_translation(osv.osv): 'domain': domain, } if field: - f = trans_model._fields[field] - action['context'] = { - 'search_default_name': "%s,%s" % (f.base_field.model_name, field) - } + fld = record._fields[field] + if not fld.related: + action['context'] = { + 'search_default_name': "%s,%s" % (fld.model_name, fld.name), + } return action def _get_import_cursor(self, cr, uid, context=None): diff --git a/openerp/models.py b/openerp/models.py index 4a6f380feba..6955ff46207 100644 --- a/openerp/models.py +++ b/openerp/models.py @@ -3816,7 +3816,12 @@ class BaseModel(object): if hasattr(column, 'selection') and vals[field]: self._check_selection_field_value(cr, user, field, vals[field], context=context) if column._classic_write and not hasattr(column, '_fnct_inv'): - if not (totranslate and column.translate): + if totranslate and column.translate: + # vals[field] is a translation: do not update the table + if callable(column.translate): + flabel = ffield.get_description(recs.env)['string'] + raise UserError(_("You cannot update the translated value of field '%s'") % flabel) + else: updates.append((field, '%s', column._symbol_set[1](vals[field]))) direct.append(field) else: @@ -3841,26 +3846,25 @@ class BaseModel(object): if cr.rowcount != len(sub_ids): raise MissingError(_('One of the records you are trying to modify has already been deleted (Document type: %s).') % self._description) - if totranslate: - # TODO: optimize - for f in direct: - if callable(self._columns[f].translate): - # We do not handle the update of a field where translate - # is a callable. One should update the value without a - # language, or update translated terms separately. - raise ValidationError(_( - "The translation of the field '%s' cannot be " - "directly updated. You can either update its value " - "in English, or update translated terms separately." - ) % self._fields[f].string) - elif self._columns[f].translate: - src_trans = self.pool[self._name].read(cr, user, ids, [f])[0][f] - if not src_trans: - src_trans = vals[f] - # Inserting value to DB - context_wo_lang = dict(context, lang=None) - self.write(cr, user, ids, {f: vals[f]}, context=context_wo_lang) - self.pool.get('ir.translation')._set_ids(cr, user, self._name+','+f, 'model', context['lang'], ids, vals[f], src_trans) + # TODO: optimize + for f in direct: + column = self._columns[f] + if callable(column.translate): + # The English value of a field has been modified, + # synchronize translated terms when possible. + assert not totranslate + self.pool['ir.translation']._sync_terms_translations( + cr, user, self._fields[f], recs, context=context) + + elif column.translate and totranslate: + # The translated value of a field has been modified. + src_trans = self.pool[self._name].read(cr, user, ids, [f])[0][f] + if not src_trans: + # Insert value to DB + src_trans = vals[f] + context_wo_lang = dict(context, lang=None) + self.write(cr, user, ids, {f: vals[f]}, context=context_wo_lang) + self.pool['ir.translation']._set_ids(cr, user, self._name+','+f, 'model', context['lang'], ids, vals[f], src_trans) # invalidate and mark new-style fields to recompute; do this before # setting other fields, because it can require the value of computed From 0af707594e5e36aac8b65e1c4e2597b963cef417 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 24 Jun 2015 11:06:20 +0200 Subject: [PATCH 07/15] [IMP] translate: introduce function to translate XML fields --- openerp/tools/translate.py | 51 ++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index 9028a235acd..3361ed85de2 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -128,6 +128,52 @@ class UNIX_LINE_TERMINATOR(csv.excel): csv.register_dialect("UNIX", UNIX_LINE_TERMINATOR) + +# +# Helper functions for translating fields +# +def encode(s): + if isinstance(s, unicode): + return s.encode('utf8') + return s + +TRANSLATED_ATTRS = { + 'string', 'help', 'sum', 'avg', 'confirm', 'placeholder', 'alt', 'title', +} + + +def xml_translate(callback, value): + """ Translate an XML value (string), using `callback` for translating text + appearing in `value`. + """ + if not value: + return value + + def process_text(text): + term = text.strip() + trans = term and callback(term) + return text.replace(term, trans) if trans else text + + def process(node): + if not isinstance(node, SKIPPED_ELEMENT_TYPES): + if node.text: + node.text = process_text(node.text) + if node.tail: + node.tail = process_text(node.tail) + if node.tag == 'attribute' and node.get('name') in TRANSLATED_ATTRS: + if node.text: + node.text = process_text(node.text) + for attr in TRANSLATED_ATTRS: + if node.get(attr): + node.set(attr, process_text(node.get(attr))) + for child in node: + process(child) + + root = etree.fromstring(encode(value)) + process(root) + return etree.tostring(root, encoding='utf8') + + # # Warning: better use self.pool.get('ir.translation')._get_source if you can # @@ -668,11 +714,6 @@ def trans_generate(lang, modules, cr): tnx = (module, source, name, id, type, tuple(comments or ())) _to_translate.add(tnx) - def encode(s): - if isinstance(s, unicode): - return s.encode('utf8') - return s - def push(mod, type, name, res_id, term): term = (term or '').strip() if len(term) > 2 or term in ENGLISH_SMALL_WORDS: From 2eb257ff944abc4d170275102b6be15300f2b3a6 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Mon, 20 Apr 2015 13:36:30 +0200 Subject: [PATCH 08/15] [IMP] translate: in XML translations, include phrasing elements in translations HTML phrasing elements (like ``, ``, etc.) should be included in translations. This avoids splitting phrases when translating HTML content. Note that elements with qweb-related attributes (t-*) are not translated inline. Add tests on xml_translate(). --- openerp/addons/base/tests/test_translate.py | 54 ++++++++- openerp/tools/translate.py | 126 ++++++++++++++++---- 2 files changed, 157 insertions(+), 23 deletions(-) diff --git a/openerp/addons/base/tests/test_translate.py b/openerp/addons/base/tests/test_translate.py index 4607cd64b9e..f9ce5189e20 100644 --- a/openerp/addons/base/tests/test_translate.py +++ b/openerp/addons/base/tests/test_translate.py @@ -2,7 +2,7 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import unittest -from openerp.tools.translate import quote, unquote +from openerp.tools.translate import quote, unquote, xml_translate class TranslationToolsTestCase(unittest.TestCase): @@ -27,3 +27,55 @@ class TranslationToolsTestCase(unittest.TestCase): self.assertRaises(AssertionError, quote, """test \nall kinds\n\no\r \\\\nope\n\n" """) + + def test_translate_xml_base(self): + """ Test xml_translate() without formatting elements. """ + terms = [] + source = """ +

Blah blah blah

+ Put some more text here + + """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['Form stuff', 'Blah blah blah', 'Put some more text here']) + + def test_translate_xml_inline1(self): + """ Test xml_translate() with formatting elements. """ + terms = [] + source = """
+

Blah blah blah

+ Put some more text here + + """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['Form stuff', 'Blah blah blah', 'Put some more text here']) + + def test_translate_xml_inline2(self): + """ Test xml_translate() with formatting elements embedding other elements. """ + terms = [] + source = """
+

Blah blah blah

+ Put some more text here + + """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['Form stuff', 'Blah blah blah', 'Put some more text here']) + + def test_translate_xml_t(self): + """ Test xml_translate() with t-* attributes. """ + terms = [] + source = """ + stuff before + + stuff after + """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['stuff before', 'stuff after']) diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index 3361ed85de2..1d6dbedaf93 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -18,6 +18,7 @@ from collections import defaultdict from datetime import datetime from lxml import etree from os.path import join +from xml.sax.saxutils import escape import config import misc @@ -137,10 +138,110 @@ def encode(s): return s.encode('utf8') return s +TRANSLATED_ELEMENTS = { + 'a', 'abbr', 'audio', 'b', 'bdi', 'bdo', 'br', 'canvas', 'cite', 'code', + 'data', 'datalist', 'del', 'dfn', 'em', 'embed', 'font', 'i', 'iframe', + 'ins', 'kbd', 'keygen', 'map', 'mark', 'math', 'meter', 'object', 'output', + 'progress', 'q', 'ruby', 's', 'samp', 'select', 'small', 'span', 'strong', + 'sub', 'sup', 'svg', 'template', 'textarea', 'time', 'u', 'var', 'video', + 'wbr', 'text', +} + TRANSLATED_ATTRS = { 'string', 'help', 'sum', 'avg', 'confirm', 'placeholder', 'alt', 'title', } +def serialize(tag, attrib, content): + """ Return a serialized element with the given `tag`, attributes + `attrib`, and already-serialized `content`. + """ + elem = etree.tostring(etree.Element(tag, attrib)) + assert elem.endswith("/>") + return "%s>%s" % (elem[:-2], content, tag) if content else elem + +class XMLTranslator(object): + """ A sequence of serialized xml items, with some of them to translate + (todo) and others already translated (done). The purpose of this object + is to simplify the handling of phrasing elements (like ) that must be + translated together with their surrounding text. + + For instance, the content of the "div" element below will be translated + as a whole (without surrounding spaces): + +
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit, + sed do eiusmod tempor incididunt ut labore et dolore + magna aliqua. Ut enim ad minim veniam, + quis nostrud exercitation ullamco laboris nisi ut + aliquip ex ea commodo consequat. +
+ + """ + def __init__(self, callback): + self.callback = callback # callback function to translate terms + self._done = [] # translated strings + self._todo = [] # todo strings that come after _done + + def todo(self, text): + self._todo.append(text) + + def all_todo(self): + return not self._done + + def get_todo(self): + return "".join(self._todo) + + def flush(self): + if self._todo: + todo = "".join(self._todo) + self._done.append(self.process_text(todo)) + del self._todo[:] + + def done(self, text): + self.flush() + self._done.append(text) + + def get_done(self): + """ Complete the translations and return the result. """ + self.flush() + return "".join(self._done) + + def process_text(self, text): + """ Translate text.strip(), but keep the surrounding spaces from text. """ + term = text.strip() + trans = term and self.callback(term) + return text.replace(term, trans) if trans else text + + def process(self, node): + """ Process the given xml `node`: collect `todo` and `done` items. """ + if isinstance(node, SKIPPED_ELEMENT_TYPES) or node.tag in SKIPPED_ELEMENTS: + # serialize the node as done, but keep its tail as todo + tail, node.tail = node.tail, None + self.done(etree.tostring(node)) + self.todo(escape(tail or "")) + return + + # process children nodes locally in child_trans + child_trans = XMLTranslator(self.callback) + child_trans.todo(escape(node.text or "")) + for child in node: + child_trans.process(child) + + if (child_trans.all_todo() and + node.tag in TRANSLATED_ELEMENTS and + not any(attr.startswith("t-") for attr in node.attrib)): + # serialize the node element as todo + self.todo(serialize(node.tag, node.attrib, child_trans.get_todo())) + else: + # complete translations and serialize result as done + for attr in TRANSLATED_ATTRS: + if node.get(attr): + node.set(attr, self.process_text(node.get(attr))) + self.done(serialize(node.tag, node.attrib, child_trans.get_done())) + + # add node tail as todo + self.todo(escape(node.tail or "")) + def xml_translate(callback, value): """ Translate an XML value (string), using `callback` for translating text @@ -149,29 +250,10 @@ def xml_translate(callback, value): if not value: return value - def process_text(text): - term = text.strip() - trans = term and callback(term) - return text.replace(term, trans) if trans else text - - def process(node): - if not isinstance(node, SKIPPED_ELEMENT_TYPES): - if node.text: - node.text = process_text(node.text) - if node.tail: - node.tail = process_text(node.tail) - if node.tag == 'attribute' and node.get('name') in TRANSLATED_ATTRS: - if node.text: - node.text = process_text(node.text) - for attr in TRANSLATED_ATTRS: - if node.get(attr): - node.set(attr, process_text(node.get(attr))) - for child in node: - process(child) - + trans = XMLTranslator(callback) root = etree.fromstring(encode(value)) - process(root) - return etree.tostring(root, encoding='utf8') + trans.process(root) + return trans.get_done() # From 33c8aab97195bf0b28580c9b0abce999b3cb5089 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Tue, 12 May 2015 16:36:21 +0200 Subject: [PATCH 09/15] [IMP] translate: do not translate XML elements without actual text --- openerp/addons/base/tests/test_translate.py | 17 +++++++++++++++++ openerp/tools/translate.py | 12 +++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/openerp/addons/base/tests/test_translate.py b/openerp/addons/base/tests/test_translate.py index f9ce5189e20..c3691563aef 100644 --- a/openerp/addons/base/tests/test_translate.py +++ b/openerp/addons/base/tests/test_translate.py @@ -67,6 +67,23 @@ class TranslationToolsTestCase(unittest.TestCase): self.assertItemsEqual(terms, ['Form stuff', 'Blah blah blah', 'Put some more text here']) + def test_translate_xml_inline3(self): + """ Test xml_translate() with formatting elements without actual text. """ + terms = [] + source = """
+
+ +

Blah blah blah

+ + + +
+ """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['Form stuff', 'Blah blah blah']) + def test_translate_xml_t(self): """ Test xml_translate() with t-* attributes. """ terms = [] diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index 1d6dbedaf93..b4e54b06537 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -181,9 +181,12 @@ class XMLTranslator(object): self.callback = callback # callback function to translate terms self._done = [] # translated strings self._todo = [] # todo strings that come after _done + self.needs_trans = False # whether todo needs translation - def todo(self, text): + def todo(self, text, needs_trans=True): self._todo.append(text) + if needs_trans and text.strip(): + self.needs_trans = True def all_todo(self): return not self._done @@ -194,8 +197,10 @@ class XMLTranslator(object): def flush(self): if self._todo: todo = "".join(self._todo) - self._done.append(self.process_text(todo)) + done = self.process_text(todo) if self.needs_trans else todo + self._done.append(done) del self._todo[:] + self.needs_trans = False def done(self, text): self.flush() @@ -231,7 +236,8 @@ class XMLTranslator(object): node.tag in TRANSLATED_ELEMENTS and not any(attr.startswith("t-") for attr in node.attrib)): # serialize the node element as todo - self.todo(serialize(node.tag, node.attrib, child_trans.get_todo())) + self.todo(serialize(node.tag, node.attrib, child_trans.get_todo()), + child_trans.needs_trans) else: # complete translations and serialize result as done for attr in TRANSLATED_ATTRS: From 4e572e351eb26d83b4f573213929433ebeb0fed5 Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Thu, 25 Jun 2015 10:07:18 +0200 Subject: [PATCH 10/15] [IMP] translate: add cases where the content of XML elements is not translated - do not translate the content of elements with translate="off" - translate `` only for some values of X --- openerp/addons/base/tests/test_translate.py | 25 +++++++++++++++++++++ openerp/tools/translate.py | 9 ++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/tests/test_translate.py b/openerp/addons/base/tests/test_translate.py index c3691563aef..ce3e7fd51ae 100644 --- a/openerp/addons/base/tests/test_translate.py +++ b/openerp/addons/base/tests/test_translate.py @@ -96,3 +96,28 @@ class TranslationToolsTestCase(unittest.TestCase): self.assertEquals(result, source) self.assertItemsEqual(terms, ['stuff before', 'stuff after']) + + def test_translate_xml_off(self): + """ Test xml_translate() with attribute translate="off". """ + terms = [] + source = """
+ stuff before +
Do not translate this
+ stuff after +
""" + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['stuff before', 'stuff after']) + + def test_translate_xml_attribute(self): + """ Test xml_translate() with elements. """ + terms = [] + source = """ + Translate this + Do not translate this + """ + result = xml_translate(terms.append, source) + self.assertEquals(result, source) + self.assertItemsEqual(terms, + ['Translate this']) diff --git a/openerp/tools/translate.py b/openerp/tools/translate.py index b4e54b06537..2c2bdaffab2 100644 --- a/openerp/tools/translate.py +++ b/openerp/tools/translate.py @@ -219,8 +219,13 @@ class XMLTranslator(object): def process(self, node): """ Process the given xml `node`: collect `todo` and `done` items. """ - if isinstance(node, SKIPPED_ELEMENT_TYPES) or node.tag in SKIPPED_ELEMENTS: - # serialize the node as done, but keep its tail as todo + if ( + isinstance(node, SKIPPED_ELEMENT_TYPES) or + node.tag in SKIPPED_ELEMENTS or + node.get("translation", "").strip() == "off" or + node.tag == "attribute" and node.get("name") not in TRANSLATED_ATTRS + ): + # do not translate the contents of the node tail, node.tail = node.tail, None self.done(etree.tostring(node)) self.todo(escape(tail or "")) From f2be4715589fb068e043f217734925eb3d045a7a Mon Sep 17 00:00:00 2001 From: Raphael Collet Date: Wed, 24 Jun 2015 11:15:20 +0200 Subject: [PATCH 11/15] [IMP] ir_ui_view: adapt translation of views by using callable field.translate Also add constraint to prevent using translated attributes in view inheritance. You may no longer select elements based on attributes 'string', 'help', etc: ... ... --- addons/website/controllers/main.py | 12 +-- addons/website_version/models/ir_ui_view.py | 7 +- openerp/addons/base/ir/ir_translation.py | 16 +++- openerp/addons/base/ir/ir_ui_view.py | 86 ++++++++++----------- openerp/addons/base/ir/ir_ui_view_view.xml | 6 +- openerp/addons/base/tests/test_views.py | 11 ++- openerp/tools/translate.py | 58 +------------- 7 files changed, 75 insertions(+), 121 deletions(-) diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py index f300487e7b8..9f01f71b2a0 100644 --- a/addons/website/controllers/main.py +++ b/addons/website/controllers/main.py @@ -253,7 +253,8 @@ class Website(openerp.addons.web.controllers.main.Home): old_trans = irt.search_read( request.cr, request.uid, [ - ('type', '=', 'view'), + ('type', '=', 'model'), + ('name', '=', 'ir.ui.view,arch_db'), ('res_id', '=', view_id), ('lang', '=', lang), ('src', '=', initial_content), @@ -261,15 +262,16 @@ class Website(openerp.addons.web.controllers.main.Home): if old_trans: tid = old_trans[0]['id'] if tid: - vals = {'value': new_content} + vals = {'value': new_content, 'state': 'translated'} irt.write(request.cr, request.uid, [tid], vals) else: new_trans = { - 'name': 'website', + 'type': 'model', + 'name': 'ir.ui.view,arch_db', + 'state': 'translated', 'res_id': view_id, 'lang': lang, - 'type': 'view', - 'source': initial_content, + 'src': initial_content, 'value': new_content, } if t.get('gengo_translation'): diff --git a/addons/website_version/models/ir_ui_view.py b/addons/website_version/models/ir_ui_view.py index 94fa6cab45e..4a3ded70f43 100644 --- a/addons/website_version/models/ir_ui_view.py +++ b/addons/website_version/models/ir_ui_view.py @@ -170,7 +170,12 @@ class view(models.Model): for v in view_list: views = self.customize_template_get(v.id, full=True) views_ids += [v.get('id') for v in views if v.get('active')] - domain = [('type', '=', 'view'), ('res_id', 'in', views_ids), ('lang', '=', lang)] + domain = [ + ('type', '=', 'model'), + ('name', '=', 'ir.ui.view,arch_db'), + ('res_id', 'in', views_ids), + ('lang', '=', lang), + ] element_list = irt.search_read(self.env.cr, self.env.uid, domain, field, context=self.env.context) for element in element_list: diff --git a/openerp/addons/base/ir/ir_translation.py b/openerp/addons/base/ir/ir_translation.py index ea18178d57a..1b0c78c9991 100644 --- a/openerp/addons/base/ir/ir_translation.py +++ b/openerp/addons/base/ir/ir_translation.py @@ -17,7 +17,7 @@ TRANSLATION_TYPE = [ ('model', 'Object'), ('report', 'Report/Template'), ('selection', 'Selection'), - ('view', 'View'), + ('view', 'View'), # deprecated ('help', 'Help'), # deprecated ('code', 'Code'), ('constraint', 'Constraint'), @@ -62,11 +62,14 @@ class ir_translation_import_cursor(object): # ugly hack for QWeb views - pending refactoring of translations in master if params['imd_model'] == 'website': params['imd_model'] = "ir.ui.view" - # non-QWeb views do not need a matching res_id -> force to 0 to avoid dropping them - elif params['res_id'] is None: + # non-QWeb views do not need a matching res_id in case they do not + # have an xml id -> force to 0 to avoid dropping them + elif params['res_id'] is None and not params['imd_name']: + # maybe we should insert this translation for all views of the + # given model? params['res_id'] = 0 - # backward compatibility: convert 'field', 'help' into 'model' + # backward compatibility: convert 'field', 'help', 'view' into 'model' if params['type'] == 'field': model, field = params['name'].split(',') params['type'] = 'model' @@ -81,6 +84,11 @@ class ir_translation_import_cursor(object): params['imd_model'] = 'ir.model.fields' params['imd_name'] = 'field_%s_%s' % (model.replace('.', '_'), field) + elif params['type'] == 'view': + params['type'] = 'model' + params['name'] = 'ir.ui.view,arch_db' + params['imd_model'] = "ir.ui.view" + self._cr.execute("""INSERT INTO %s (name, lang, res_id, src, type, imd_model, module, imd_name, value, state, comments) VALUES (%%(name)s, %%(lang)s, %%(res_id)s, %%(src)s, %%(type)s, %%(imd_model)s, %%(module)s, %%(imd_name)s, %%(value)s, %%(state)s, %%(comments)s)""" % self._table_name, diff --git a/openerp/addons/base/ir/ir_ui_view.py b/openerp/addons/base/ir/ir_ui_view.py index 25101e8d6ff..58f3b12a412 100644 --- a/openerp/addons/base/ir/ir_ui_view.py +++ b/openerp/addons/base/ir/ir_ui_view.py @@ -3,11 +3,11 @@ import collections import copy import datetime -import dateutil from dateutil.relativedelta import relativedelta import fnmatch import logging import os +import re import time from operator import itemgetter @@ -18,6 +18,7 @@ from lxml import etree import openerp from openerp import tools, api +from openerp.exceptions import ValidationError from openerp.http import request from openerp.modules.module import get_resource_path, get_resource_from_path from openerp.osv import fields, osv, orm @@ -27,7 +28,7 @@ from openerp.tools.parse_version import parse_version from openerp.tools.safe_eval import safe_eval as eval from openerp.tools.view_validation import valid_view from openerp.tools import misc -from openerp.tools.translate import _ +from openerp.tools.translate import TRANSLATED_ATTRS, encode, xml_translate, _ _logger = logging.getLogger(__name__) @@ -118,6 +119,22 @@ def get_view_arch_from_file(filename, xmlid): xpath_utils = etree.FunctionNamespace(None) xpath_utils['hasclass'] = _hasclass +TRANSLATED_ATTRS_RE = re.compile(r"@(%s)\b" % "|".join(TRANSLATED_ATTRS)) + +def valid_inheritance(arch): + """ Check whether view inheritance is based on translated attribute. """ + for node in arch.xpath('//*[@position]'): + # inheritance may not use a translated attribute as selector + if node.tag == 'xpath': + match = TRANSLATED_ATTRS_RE.search(node.get('expr', '')) + if match: + raise ValidationError("View inheritance may not use attribute %r as a selector." % match.group(1)) + else: + for attr in TRANSLATED_ATTRS: + if node.get(attr): + raise ValidationError("View inheritance may not use attribute %r as a selector." % attr) + return True + class view(osv.osv): _name = 'ir.ui.view' @@ -135,7 +152,6 @@ class view(osv.osv): def _arch_get(self, cr, uid, ids, name, arg, context=None): result = {} - for view in self.browse(cr, uid, ids, context=context): arch_fs = None if config['dev_mode'] and view.arch_fs and view.xml_id: @@ -164,6 +180,16 @@ class view(osv.osv): return True + @api.multi + def _arch_base_get(self, name, arg): + """ Return the field 'arch' without translation. """ + return self.with_context(lang=None)._arch_get(name, arg) + + @api.multi + def _arch_base_set(self, name, value, arg): + """ Assign the field 'arch' without translation. """ + return self.with_context(lang=None)._arch_set(name, value, arg) + _columns = { 'name': fields.char('View Name', required=True), 'model': fields.char('Object', select=True), @@ -182,7 +208,8 @@ class view(osv.osv): ('search','Search'), ('qweb', 'QWeb')], string='View Type'), 'arch': fields.function(_arch_get, fnct_inv=_arch_set, string='View Architecture', type="text", nodrop=True), - 'arch_db': fields.text('Arch Blob', oldname='arch'), + 'arch_base': fields.function(_arch_base_get, fnct_inv=_arch_base_set, string='View Architecture', type="text"), + 'arch_db': fields.text('Arch Blob', translate=xml_translate, oldname='arch'), 'arch_fs': fields.char('Arch Filename'), 'inherit_id': fields.many2one('ir.ui.view', 'Inherited View', ondelete='restrict', select=True), 'inherit_children_ids': fields.one2many('ir.ui.view','inherit_id', 'Inherit Views'), @@ -249,6 +276,9 @@ class view(osv.osv): # Sanity checks: the view should not break anything upon rendering! # Any exception raised below will cause a transaction rollback. for view in self.browse(cr, uid, ids, context): + view_arch = etree.fromstring(encode(view.arch)) + if not valid_inheritance(view_arch): + return False view_def = self.read_combined(cr, uid, view.id, None, context=context) view_arch_utf8 = view_def['arch'] if view.type != 'qweb': @@ -278,7 +308,7 @@ class view(osv.osv): " extend an other view"), ] _constraints = [ - (_check_xml, 'Invalid view definition', ['arch']), + (_check_xml, 'Invalid view definition', ['arch', 'arch_base']), ] def _auto_init(self, cr, context=None): @@ -462,7 +492,6 @@ class view(osv.osv): self.inherit_branding(node, view_id, root_id) else: node.set('data-oe-id', str(view_id)) - node.set('data-oe-source-id', str(root_id)) node.set('data-oe-xpath', xpath) node.set('data-oe-model', 'ir.ui.view') node.set('data-oe-field', 'arch') @@ -735,46 +764,6 @@ class view(osv.osv): # Get the attrs before they are (possibly) deleted by check_group below orm.transfer_node_to_modifiers(node, modifiers, context, in_tree_view) - # TODO remove attrs counterpart in modifiers when invisible is true ? - - # translate view - if 'lang' in context: - Translations = self.pool['ir.translation'] - if node.text and node.text.strip(): - term = node.text.strip() - trans = Translations._get_source(cr, user, model, 'view', context['lang'], term) - if trans: - node.text = node.text.replace(term, trans) - if node.tail and node.tail.strip(): - term = node.tail.strip() - trans = Translations._get_source(cr, user, model, 'view', context['lang'], term) - if trans: - node.tail = node.tail.replace(term, trans) - - if node.get('string') and node.get('string').strip() and not result: - term = node.get('string').strip() - trans = Translations._get_source(cr, user, model, 'view', context['lang'], term) - if trans == term: - if 'base_model_name' in context: - # If translation is same as source, perhaps we'd have more luck with the alternative model name - # (in case we are in a mixed situation, such as an inherited view where parent_view.model != model - trans = Translations._get_source(cr, user, context['base_model_name'], 'view', context['lang'], term) - else: - inherit_model = self.browse(cr, user, view_id, context=context).inherit_id.model or model - if inherit_model != model: - # parent view has a different model, if the terms belongs to the parent view, the translation - # should be checked on the parent model as well - trans = Translations._get_source(cr, user, inherit_model, 'view', context['lang'], term) - if trans: - node.set('string', trans) - - for attr_name in ('confirm', 'sum', 'avg', 'help', 'placeholder'): - attr_value = node.get(attr_name) - if attr_value and attr_value.strip(): - trans = Translations._get_source(cr, user, model, 'view', context['lang'], attr_value.strip()) - if trans: - node.set(attr_name, trans) - for f in node: if children or (node.tag == 'field' and f.tag in ('filter','separator')): fields.update(self.postprocess(cr, user, model, f, view_id, in_tree_view, model_fields, context)) @@ -1104,6 +1093,11 @@ class view(osv.osv): # Misc #------------------------------------------------------ + @api.multi + def open_translations(self): + """ Open a view for editing the translations of field 'arch_db'. """ + return self.env['ir.translation'].translate_fields('ir.ui.view', self.id, 'arch_db') + def graph_get(self, cr, uid, id, model, node_obj, conn_obj, src_node, des_node, label, scale, context=None): nodes=[] nodes_name=[] diff --git a/openerp/addons/base/ir/ir_ui_view_view.xml b/openerp/addons/base/ir/ir_ui_view_view.xml index 171d04d5651..02c0f69e83e 100644 --- a/openerp/addons/base/ir/ir_ui_view_view.xml +++ b/openerp/addons/base/ir/ir_ui_view_view.xml @@ -6,7 +6,7 @@ ir.ui.view
- + @@ -25,7 +25,9 @@ - +