[FIX] core: fix search with accent
For a trigram indexed translated field field_x,
domain leaf ('field_x', '=', value) and ('field_x', 'ilike', pattern)
will miss records if the pattern contains accent characters
This commit set ensure_ascii=False for json.dumps to avoid escaping special
characters
opw-3048182
closes odoo/odoo#105240
X-original-commit: 26b5c8c2baeaee9fcd5384aafa5f9b39543d6903
Signed-off-by: Raphael Collet <rco@odoo.com>
Signed-off-by: Wang Chong (cwg) <cwg@odoo.com>
This commit is contained in:
Chong Wang (cwg)
committed by
Raphael Collet
Raphael Collet
parent
8d5b11205d
commit
d57d8534bb
@@ -15,6 +15,9 @@ from odoo.tests.common import TransactionCase, BaseCase, new_test_user, tagged
|
||||
|
||||
_stats_logger = logging.getLogger('odoo.tests.stats')
|
||||
|
||||
# a string with various unicode characters
|
||||
SPECIAL_CHARACTERS = " ¥®°²Æçéðπ⁉€∇⓵▲☑♂♥✓➔『にㄅ㊀中한︸🌈🌍👌😀"
|
||||
|
||||
|
||||
class TranslationToolsTestCase(BaseCase):
|
||||
def assertItemsEqual(self, a, b, msg=None):
|
||||
@@ -858,6 +861,7 @@ class TestLanguageInstallPerformance(TransactionCase):
|
||||
|
||||
class TestTranslationTrigramIndexPatterns(BaseCase):
|
||||
def test_value_conversion(self):
|
||||
sc = SPECIAL_CHARACTERS
|
||||
cases = [
|
||||
# pylint: disable=bad-whitespace
|
||||
( 'abc', '%abc%', 'simple text is not escaped correctly'),
|
||||
@@ -867,11 +871,13 @@ class TestTranslationTrigramIndexPatterns(BaseCase):
|
||||
( 'a_bc', r'%a\_bc%', '_ is not escaped correctly'),
|
||||
( 'a%bc', r'%a\%bc%', '% is not escaped correctly'),
|
||||
( 'a_', '%', 'values with less than 3 characters should be dropped'),
|
||||
( sc, f'%{sc}%', 'special characters should not be escaped'),
|
||||
]
|
||||
for value, expected, message in cases:
|
||||
self.assertEqual(sql.value_to_translated_trigram_pattern(value), expected, message)
|
||||
|
||||
def test_pattern_conversion(self):
|
||||
sc = SPECIAL_CHARACTERS
|
||||
cases = [
|
||||
# pylint: disable=bad-whitespace
|
||||
( 'abc', '%abc%', 'simple pattern is not escaped correctly'),
|
||||
@@ -885,6 +891,7 @@ class TestTranslationTrigramIndexPatterns(BaseCase):
|
||||
(r'a\bc', '%abc%', r'redundant \ for pattern should be removed'),
|
||||
( 'abc_de', '%abc%', 'sub patterns less than 3 characters should be dropped'),
|
||||
( 'ab', '%', 'patterns without trigram should be simplified'),
|
||||
( sc, f'%{sc}%', 'special characters should not be escaped'),
|
||||
]
|
||||
for original_pattern, escaped_pattern, message in cases:
|
||||
self.assertEqual(sql.pattern_to_translated_trigram_pattern(original_pattern), escaped_pattern, message)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import odoo.tests
|
||||
from odoo.addons.base.tests.test_translate import SPECIAL_CHARACTERS
|
||||
|
||||
|
||||
@odoo.tests.tagged('post_install', '-at_install')
|
||||
@@ -10,31 +11,56 @@ class TestIndexedTranslation(odoo.tests.TransactionCase):
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.env['res.lang']._activate_lang('fr_FR')
|
||||
cls.record_en = cls.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
|
||||
cls.record_fr = cls.record_en.with_context(lang='fr_FR')
|
||||
cls.record_en.name = '<div class="my_class">hello</div>\n<div/>'
|
||||
cls.record_fr.name = '<div class="my_class">%bonjour\\</div>\a<div/>'
|
||||
|
||||
def test_search_ilike(self):
|
||||
self.assertEqual(self.record_en.name, '<div class="my_class">hello</div>\n<div/>')
|
||||
self.assertEqual(self.record_fr.name, '<div class="my_class">%bonjour\\</div>\a<div/>')
|
||||
record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
|
||||
record_fr = record_en.with_context(lang='fr_FR')
|
||||
record_en.name = '<div class="my_class">hello</div>\n<div/>'
|
||||
record_fr.name = '<div class="my_class">%bonjour\\</div>\a<div/>'
|
||||
|
||||
self.assertEqual(record_en.name, '<div class="my_class">hello</div>\n<div/>')
|
||||
self.assertEqual(record_fr.name, '<div class="my_class">%bonjour\\</div>\a<div/>')
|
||||
|
||||
# matching double quotes
|
||||
self.assertEqual(self.record_en.search([('name', 'ilike', 'class="my_class')]), self.record_en)
|
||||
self.assertEqual(record_en.search([('name', 'ilike', 'class="my_class')]), record_en)
|
||||
|
||||
# escaped and unescaped PG wildcards
|
||||
self.assertEqual(self.record_en.search([('name', 'ilike', 'class%class')]), self.record_en)
|
||||
self.assertEqual(self.record_en.search([('name', 'ilike', r'class="m_\_class')]), self.record_en)
|
||||
self.assertEqual(self.record_en.search([('name', 'ilike', 'bonjour')]), self.record_en.browse())
|
||||
self.assertEqual(self.record_en.search([('name', 'ilike', '</div>\n<div/>')]), self.record_en)
|
||||
self.assertEqual(self.record_fr.search([('name', 'ilike', '</div>\a<div/>')]), self.record_fr)
|
||||
self.assertEqual(self.record_fr.search([('name', 'ilike', r'\%bonjour\\')]), self.record_fr)
|
||||
self.assertEqual(record_en.search([('name', 'ilike', 'class%class')]), record_en)
|
||||
self.assertEqual(record_en.search([('name', 'ilike', r'class="m_\_class')]), record_en)
|
||||
self.assertEqual(record_en.search([('name', 'ilike', 'bonjour')]), record_en.browse())
|
||||
self.assertEqual(record_en.search([('name', 'ilike', '</div>\n<div/>')]), record_en)
|
||||
self.assertEqual(record_fr.search([('name', 'ilike', '</div>\a<div/>')]), record_fr)
|
||||
self.assertEqual(record_fr.search([('name', 'ilike', r'\%bonjour\\')]), record_fr)
|
||||
|
||||
self.assertEqual(
|
||||
self.record_en.search([('name', '=', '<div class="my_class">hello</div>\n<div/>')]),
|
||||
self.record_en,
|
||||
record_en.search([('name', '=', '<div class="my_class">hello</div>\n<div/>')]),
|
||||
record_en,
|
||||
)
|
||||
self.assertEqual(
|
||||
self.record_fr.search([('name', '=', '<div class="my_class">%bonjour\\</div>\a<div/>')]),
|
||||
self.record_fr,
|
||||
record_fr.search([('name', '=', '<div class="my_class">%bonjour\\</div>\a<div/>')]),
|
||||
record_fr,
|
||||
)
|
||||
|
||||
def test_search_special_characters(self):
|
||||
name_en = f'{SPECIAL_CHARACTERS}_en'
|
||||
name_fr = f'{SPECIAL_CHARACTERS}_fr'
|
||||
record_en = self.env['test_new_api.indexed_translation'].with_context(lang='en_US').create({})
|
||||
record_fr = record_en.with_context(lang='fr_FR')
|
||||
record_en.name = name_en
|
||||
record_fr.name = name_fr
|
||||
|
||||
self.assertEqual(record_en.search([('name', 'ilike', name_en)]), record_en)
|
||||
self.assertEqual(record_en.search([('name', '=', name_en)]), record_en)
|
||||
self.assertEqual(record_en.search([('name', 'in', [name_en])]), record_en)
|
||||
|
||||
self.assertEqual(record_fr.search([('name', 'ilike', name_fr)]), record_en)
|
||||
self.assertEqual(record_fr.search([('name', '=', name_fr)]), record_en)
|
||||
self.assertEqual(record_fr.search([('name', 'in', [name_fr])]), record_en)
|
||||
|
||||
self.assertFalse(record_fr.search([('name', 'ilike', name_en)]))
|
||||
self.assertFalse(record_fr.search([('name', '=', name_en)]))
|
||||
self.assertFalse(record_fr.search([('name', 'in', [name_en])]))
|
||||
|
||||
self.assertFalse(record_en.search([('name', 'ilike', name_fr)]))
|
||||
self.assertFalse(record_en.search([('name', '=', name_fr)]))
|
||||
self.assertFalse(record_en.search([('name', 'in', [name_fr])]))
|
||||
|
||||
+7
-4
@@ -380,8 +380,10 @@ def value_to_translated_trigram_pattern(value):
|
||||
# matching less than 3 characters will not take advantage of the index
|
||||
return '%'
|
||||
|
||||
# apply JSON escaping to value
|
||||
json_escaped = json.dumps(value)[1:-1]
|
||||
# apply JSON escaping to value; the argument ensure_ascii=False prevents
|
||||
# json.dumps from escaping unicode to ascii, which is consistent with the
|
||||
# index function jsonb_path_query_array("column_name", '$.*')::text
|
||||
json_escaped = json.dumps(value, ensure_ascii=False)[1:-1]
|
||||
|
||||
# apply PG wildcard escaping to JSON-escaped text
|
||||
wildcard_escaped = re.sub(r'(_|%|\\)', r'\\\1', json_escaped)
|
||||
@@ -412,8 +414,9 @@ def pattern_to_translated_trigram_pattern(pattern):
|
||||
# unescape PG wildcards from each sub pattern (\% becomes %)
|
||||
sub_texts = [re.sub(r'\\(.|$)', r'\1', t, flags=re.DOTALL) for t in sub_patterns]
|
||||
|
||||
# apply JSON escaping to sub texts having at least 3 characters (" becomes \")
|
||||
json_escaped = [json.dumps(t)[1:-1] for t in sub_texts if len(t) >= 3]
|
||||
# apply JSON escaping to sub texts having at least 3 characters (" becomes \");
|
||||
# the argument ensure_ascii=False prevents from escaping unicode to ascii
|
||||
json_escaped = [json.dumps(t, ensure_ascii=False)[1:-1] for t in sub_texts if len(t) >= 3]
|
||||
|
||||
# apply PG wildcard escaping to JSON-escaped texts (% becomes \%)
|
||||
wildcard_escaped = [re.sub(r'(_|%|\\)', r'\\\1', t) for t in json_escaped]
|
||||
|
||||
Reference in New Issue
Block a user