From d57d8534bb5b416ee64f4ba4298414c29dee8d36 Mon Sep 17 00:00:00 2001 From: "Chong Wang (cwg)" Date: Thu, 3 Nov 2022 17:35:39 +0000 Subject: [PATCH] [FIX] core: fix search with accent For a trigram indexed translated field field_x, domain leaf ('field_x', '=', value) and ('field_x', 'ilike', pattern) will miss records if the pattern contains accent characters This commit set ensure_ascii=False for json.dumps to avoid escaping special characters opw-3048182 closes odoo/odoo#105240 X-original-commit: 26b5c8c2baeaee9fcd5384aafa5f9b39543d6903 Signed-off-by: Raphael Collet Signed-off-by: Wang Chong (cwg) --- odoo/addons/base/tests/test_translate.py | 7 +++ .../tests/test_indexed_translation.py | 60 +++++++++++++------ odoo/tools/sql.py | 11 ++-- 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/odoo/addons/base/tests/test_translate.py b/odoo/addons/base/tests/test_translate.py index efb36b3a2fd..4ceeb8f7efd 100644 --- a/odoo/addons/base/tests/test_translate.py +++ b/odoo/addons/base/tests/test_translate.py @@ -15,6 +15,9 @@ from odoo.tests.common import TransactionCase, BaseCase, new_test_user, tagged _stats_logger = logging.getLogger('odoo.tests.stats') +# a string with various unicode characters +SPECIAL_CHARACTERS = " ¥®°²Æçéðπ⁉€∇⓵▲☑♂♥✓➔『にㄅ㊀中한︸🌈🌍👌😀" + class TranslationToolsTestCase(BaseCase): def assertItemsEqual(self, a, b, msg=None): @@ -858,6 +861,7 @@ class TestLanguageInstallPerformance(TransactionCase): class TestTranslationTrigramIndexPatterns(BaseCase): def test_value_conversion(self): + sc = SPECIAL_CHARACTERS cases = [ # pylint: disable=bad-whitespace ( 'abc', '%abc%', 'simple text is not escaped correctly'), @@ -867,11 +871,13 @@ class TestTranslationTrigramIndexPatterns(BaseCase): ( 'a_bc', r'%a\_bc%', '_ is not escaped correctly'), ( 'a%bc', r'%a\%bc%', '% is not escaped correctly'), ( 'a_', '%', 'values with less than 3 characters should be dropped'), + ( sc, f'%{sc}%', 'special characters should not be escaped'), ] for value, expected, message in cases: self.assertEqual(sql.value_to_translated_trigram_pattern(value), expected, message) def test_pattern_conversion(self): + sc = SPECIAL_CHARACTERS cases = [ # pylint: disable=bad-whitespace ( 'abc', '%abc%', 'simple pattern is not escaped correctly'), @@ -885,6 +891,7 @@ class TestTranslationTrigramIndexPatterns(BaseCase): (r'a\bc', '%abc%', r'redundant \ for pattern should be removed'), ( 'abc_de', '%abc%', 'sub patterns less than 3 characters should be dropped'), ( 'ab', '%', 'patterns without trigram should be simplified'), + ( sc, f'%{sc}%', 'special characters should not be escaped'), ] for original_pattern, escaped_pattern, message in cases: self.assertEqual(sql.pattern_to_translated_trigram_pattern(original_pattern), escaped_pattern, message) diff --git a/odoo/addons/test_new_api/tests/test_indexed_translation.py b/odoo/addons/test_new_api/tests/test_indexed_translation.py index 90356750803..c5b51bdc104 100644 --- a/odoo/addons/test_new_api/tests/test_indexed_translation.py +++ b/odoo/addons/test_new_api/tests/test_indexed_translation.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import odoo.tests +from odoo.addons.base.tests.test_translate import SPECIAL_CHARACTERS @odoo.tests.tagged('post_install', '-at_install') @@ -10,31 +11,56 @@ class TestIndexedTranslation(odoo.tests.TransactionCase): def setUpClass(cls): super().setUpClass() cls.env['res.lang']._activate_lang('fr_FR') - cls.record_en = cls.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({}) - cls.record_fr = cls.record_en.with_context(lang='fr_FR') - cls.record_en.name = '
hello
\n
' - cls.record_fr.name = '
%bonjour\\
\a
' def test_search_ilike(self): - self.assertEqual(self.record_en.name, '
hello
\n
') - self.assertEqual(self.record_fr.name, '
%bonjour\\
\a
') + record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({}) + record_fr = record_en.with_context(lang='fr_FR') + record_en.name = '
hello
\n
' + record_fr.name = '
%bonjour\\
\a
' + + self.assertEqual(record_en.name, '
hello
\n
') + self.assertEqual(record_fr.name, '
%bonjour\\
\a
') # matching double quotes - self.assertEqual(self.record_en.search([('name', 'ilike', 'class="my_class')]), self.record_en) + self.assertEqual(record_en.search([('name', 'ilike', 'class="my_class')]), record_en) # escaped and unescaped PG wildcards - self.assertEqual(self.record_en.search([('name', 'ilike', 'class%class')]), self.record_en) - self.assertEqual(self.record_en.search([('name', 'ilike', r'class="m_\_class')]), self.record_en) - self.assertEqual(self.record_en.search([('name', 'ilike', 'bonjour')]), self.record_en.browse()) - self.assertEqual(self.record_en.search([('name', 'ilike', '
\n
')]), self.record_en) - self.assertEqual(self.record_fr.search([('name', 'ilike', '
\a
')]), self.record_fr) - self.assertEqual(self.record_fr.search([('name', 'ilike', r'\%bonjour\\')]), self.record_fr) + self.assertEqual(record_en.search([('name', 'ilike', 'class%class')]), record_en) + self.assertEqual(record_en.search([('name', 'ilike', r'class="m_\_class')]), record_en) + self.assertEqual(record_en.search([('name', 'ilike', 'bonjour')]), record_en.browse()) + self.assertEqual(record_en.search([('name', 'ilike', '
\n
')]), record_en) + self.assertEqual(record_fr.search([('name', 'ilike', '
\a
')]), record_fr) + self.assertEqual(record_fr.search([('name', 'ilike', r'\%bonjour\\')]), record_fr) self.assertEqual( - self.record_en.search([('name', '=', '
hello
\n
')]), - self.record_en, + record_en.search([('name', '=', '
hello
\n
')]), + record_en, ) self.assertEqual( - self.record_fr.search([('name', '=', '
%bonjour\\
\a
')]), - self.record_fr, + record_fr.search([('name', '=', '
%bonjour\\
\a
')]), + record_fr, ) + + def test_search_special_characters(self): + name_en = f'{SPECIAL_CHARACTERS}_en' + name_fr = f'{SPECIAL_CHARACTERS}_fr' + record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({}) + record_fr = record_en.with_context(lang='fr_FR') + record_en.name = name_en + record_fr.name = name_fr + + self.assertEqual(record_en.search([('name', 'ilike', name_en)]), record_en) + self.assertEqual(record_en.search([('name', '=', name_en)]), record_en) + self.assertEqual(record_en.search([('name', 'in', [name_en])]), record_en) + + self.assertEqual(record_fr.search([('name', 'ilike', name_fr)]), record_en) + self.assertEqual(record_fr.search([('name', '=', name_fr)]), record_en) + self.assertEqual(record_fr.search([('name', 'in', [name_fr])]), record_en) + + self.assertFalse(record_fr.search([('name', 'ilike', name_en)])) + self.assertFalse(record_fr.search([('name', '=', name_en)])) + self.assertFalse(record_fr.search([('name', 'in', [name_en])])) + + self.assertFalse(record_en.search([('name', 'ilike', name_fr)])) + self.assertFalse(record_en.search([('name', '=', name_fr)])) + self.assertFalse(record_en.search([('name', 'in', [name_fr])])) diff --git a/odoo/tools/sql.py b/odoo/tools/sql.py index 2d20fede732..dc2313fc3fe 100644 --- a/odoo/tools/sql.py +++ b/odoo/tools/sql.py @@ -380,8 +380,10 @@ def value_to_translated_trigram_pattern(value): # matching less than 3 characters will not take advantage of the index return '%' - # apply JSON escaping to value - json_escaped = json.dumps(value)[1:-1] + # apply JSON escaping to value; the argument ensure_ascii=False prevents + # json.dumps from escaping unicode to ascii, which is consistent with the + # index function jsonb_path_query_array("column_name", '$.*')::text + json_escaped = json.dumps(value, ensure_ascii=False)[1:-1] # apply PG wildcard escaping to JSON-escaped text wildcard_escaped = re.sub(r'(_|%|\\)', r'\\\1', json_escaped) @@ -412,8 +414,9 @@ def pattern_to_translated_trigram_pattern(pattern): # unescape PG wildcards from each sub pattern (\% becomes %) sub_texts = [re.sub(r'\\(.|$)', r'\1', t, flags=re.DOTALL) for t in sub_patterns] - # apply JSON escaping to sub texts having at least 3 characters (" becomes \") - json_escaped = [json.dumps(t)[1:-1] for t in sub_texts if len(t) >= 3] + # apply JSON escaping to sub texts having at least 3 characters (" becomes \"); + # the argument ensure_ascii=False prevents from escaping unicode to ascii + json_escaped = [json.dumps(t, ensure_ascii=False)[1:-1] for t in sub_texts if len(t) >= 3] # apply PG wildcard escaping to JSON-escaped texts (% becomes \%) wildcard_escaped = [re.sub(r'(_|%|\\)', r'\\\1', t) for t in json_escaped]