diff --git a/odoo/addons/base/tests/test_translate.py b/odoo/addons/base/tests/test_translate.py
index efb36b3a2fd..4ceeb8f7efd 100644
--- a/odoo/addons/base/tests/test_translate.py
+++ b/odoo/addons/base/tests/test_translate.py
@@ -15,6 +15,9 @@ from odoo.tests.common import TransactionCase, BaseCase, new_test_user, tagged
_stats_logger = logging.getLogger('odoo.tests.stats')
+# a string with various unicode characters
+SPECIAL_CHARACTERS = " ¥®°²Æçéðπ⁉€∇⓵▲☑♂♥✓➔『にㄅ㊀中한︸🌈🌍👌😀"
+
class TranslationToolsTestCase(BaseCase):
def assertItemsEqual(self, a, b, msg=None):
@@ -858,6 +861,7 @@ class TestLanguageInstallPerformance(TransactionCase):
class TestTranslationTrigramIndexPatterns(BaseCase):
def test_value_conversion(self):
+ sc = SPECIAL_CHARACTERS
cases = [
# pylint: disable=bad-whitespace
( 'abc', '%abc%', 'simple text is not escaped correctly'),
@@ -867,11 +871,13 @@ class TestTranslationTrigramIndexPatterns(BaseCase):
( 'a_bc', r'%a\_bc%', '_ is not escaped correctly'),
( 'a%bc', r'%a\%bc%', '% is not escaped correctly'),
( 'a_', '%', 'values with less than 3 characters should be dropped'),
+ ( sc, f'%{sc}%', 'special characters should not be escaped'),
]
for value, expected, message in cases:
self.assertEqual(sql.value_to_translated_trigram_pattern(value), expected, message)
def test_pattern_conversion(self):
+ sc = SPECIAL_CHARACTERS
cases = [
# pylint: disable=bad-whitespace
( 'abc', '%abc%', 'simple pattern is not escaped correctly'),
@@ -885,6 +891,7 @@ class TestTranslationTrigramIndexPatterns(BaseCase):
(r'a\bc', '%abc%', r'redundant \ for pattern should be removed'),
( 'abc_de', '%abc%', 'sub patterns less than 3 characters should be dropped'),
( 'ab', '%', 'patterns without trigram should be simplified'),
+ ( sc, f'%{sc}%', 'special characters should not be escaped'),
]
for original_pattern, escaped_pattern, message in cases:
self.assertEqual(sql.pattern_to_translated_trigram_pattern(original_pattern), escaped_pattern, message)
diff --git a/odoo/addons/test_new_api/tests/test_indexed_translation.py b/odoo/addons/test_new_api/tests/test_indexed_translation.py
index 90356750803..c5b51bdc104 100644
--- a/odoo/addons/test_new_api/tests/test_indexed_translation.py
+++ b/odoo/addons/test_new_api/tests/test_indexed_translation.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import odoo.tests
+from odoo.addons.base.tests.test_translate import SPECIAL_CHARACTERS
@odoo.tests.tagged('post_install', '-at_install')
@@ -10,31 +11,56 @@ class TestIndexedTranslation(odoo.tests.TransactionCase):
def setUpClass(cls):
super().setUpClass()
cls.env['res.lang']._activate_lang('fr_FR')
- cls.record_en = cls.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
- cls.record_fr = cls.record_en.with_context(lang='fr_FR')
- cls.record_en.name = 'hello
\n'
- cls.record_fr.name = '%bonjour\\
\a'
def test_search_ilike(self):
- self.assertEqual(self.record_en.name, 'hello
\n')
- self.assertEqual(self.record_fr.name, '%bonjour\\
\a')
+ record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
+ record_fr = record_en.with_context(lang='fr_FR')
+ record_en.name = 'hello
\n'
+ record_fr.name = '%bonjour\\
\a'
+
+ self.assertEqual(record_en.name, 'hello
\n')
+ self.assertEqual(record_fr.name, '%bonjour\\
\a')
# matching double quotes
- self.assertEqual(self.record_en.search([('name', 'ilike', 'class="my_class')]), self.record_en)
+ self.assertEqual(record_en.search([('name', 'ilike', 'class="my_class')]), record_en)
# escaped and unescaped PG wildcards
- self.assertEqual(self.record_en.search([('name', 'ilike', 'class%class')]), self.record_en)
- self.assertEqual(self.record_en.search([('name', 'ilike', r'class="m_\_class')]), self.record_en)
- self.assertEqual(self.record_en.search([('name', 'ilike', 'bonjour')]), self.record_en.browse())
- self.assertEqual(self.record_en.search([('name', 'ilike', '\n')]), self.record_en)
- self.assertEqual(self.record_fr.search([('name', 'ilike', '\a')]), self.record_fr)
- self.assertEqual(self.record_fr.search([('name', 'ilike', r'\%bonjour\\')]), self.record_fr)
+ self.assertEqual(record_en.search([('name', 'ilike', 'class%class')]), record_en)
+ self.assertEqual(record_en.search([('name', 'ilike', r'class="m_\_class')]), record_en)
+ self.assertEqual(record_en.search([('name', 'ilike', 'bonjour')]), record_en.browse())
+ self.assertEqual(record_en.search([('name', 'ilike', '\n')]), record_en)
+ self.assertEqual(record_fr.search([('name', 'ilike', '\a')]), record_fr)
+ self.assertEqual(record_fr.search([('name', 'ilike', r'\%bonjour\\')]), record_fr)
self.assertEqual(
- self.record_en.search([('name', '=', 'hello
\n')]),
- self.record_en,
+ record_en.search([('name', '=', 'hello
\n')]),
+ record_en,
)
self.assertEqual(
- self.record_fr.search([('name', '=', '%bonjour\\
\a')]),
- self.record_fr,
+ record_fr.search([('name', '=', '%bonjour\\
\a')]),
+ record_fr,
)
+
+ def test_search_special_characters(self):
+ name_en = f'{SPECIAL_CHARACTERS}_en'
+ name_fr = f'{SPECIAL_CHARACTERS}_fr'
+ record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
+ record_fr = record_en.with_context(lang='fr_FR')
+ record_en.name = name_en
+ record_fr.name = name_fr
+
+ self.assertEqual(record_en.search([('name', 'ilike', name_en)]), record_en)
+ self.assertEqual(record_en.search([('name', '=', name_en)]), record_en)
+ self.assertEqual(record_en.search([('name', 'in', [name_en])]), record_en)
+
+ self.assertEqual(record_fr.search([('name', 'ilike', name_fr)]), record_en)
+ self.assertEqual(record_fr.search([('name', '=', name_fr)]), record_en)
+ self.assertEqual(record_fr.search([('name', 'in', [name_fr])]), record_en)
+
+ self.assertFalse(record_fr.search([('name', 'ilike', name_en)]))
+ self.assertFalse(record_fr.search([('name', '=', name_en)]))
+ self.assertFalse(record_fr.search([('name', 'in', [name_en])]))
+
+ self.assertFalse(record_en.search([('name', 'ilike', name_fr)]))
+ self.assertFalse(record_en.search([('name', '=', name_fr)]))
+ self.assertFalse(record_en.search([('name', 'in', [name_fr])]))
diff --git a/odoo/tools/sql.py b/odoo/tools/sql.py
index 2d20fede732..dc2313fc3fe 100644
--- a/odoo/tools/sql.py
+++ b/odoo/tools/sql.py
@@ -380,8 +380,10 @@ def value_to_translated_trigram_pattern(value):
# matching less than 3 characters will not take advantage of the index
return '%'
- # apply JSON escaping to value
- json_escaped = json.dumps(value)[1:-1]
+ # apply JSON escaping to value; the argument ensure_ascii=False prevents
+ # json.dumps from escaping unicode to ascii, which is consistent with the
+ # index function jsonb_path_query_array("column_name", '$.*')::text
+ json_escaped = json.dumps(value, ensure_ascii=False)[1:-1]
# apply PG wildcard escaping to JSON-escaped text
wildcard_escaped = re.sub(r'(_|%|\\)', r'\\\1', json_escaped)
@@ -412,8 +414,9 @@ def pattern_to_translated_trigram_pattern(pattern):
# unescape PG wildcards from each sub pattern (\% becomes %)
sub_texts = [re.sub(r'\\(.|$)', r'\1', t, flags=re.DOTALL) for t in sub_patterns]
- # apply JSON escaping to sub texts having at least 3 characters (" becomes \")
- json_escaped = [json.dumps(t)[1:-1] for t in sub_texts if len(t) >= 3]
+ # apply JSON escaping to sub texts having at least 3 characters (" becomes \");
+ # the argument ensure_ascii=False prevents from escaping unicode to ascii
+ json_escaped = [json.dumps(t, ensure_ascii=False)[1:-1] for t in sub_texts if len(t) >= 3]
# apply PG wildcard escaping to JSON-escaped texts (% becomes \%)
wildcard_escaped = [re.sub(r'(_|%|\\)', r'\\\1', t) for t in json_escaped]