Benoit Socias
07290bfe8d
Since [1] when fuzzy search was introduced, fuzzy search was also used when the search term contained numbers. This is a problem for companies where the shop search is used on references: the fuzzy search does not match against the whole database and therefore the exact match might not be found. After this commit the exact search is used whenever the search term is made of 80% of numbers or more - because it is very likely to be a reference or an identifier. [1]: https://github.com/odoo/odoo/commit/7559626c54e34b41e1549e28276a650accec6986 opw-2715647 closes odoo/odoo#84943 X-original-commit: 7b44907c322a7b78ef4a5cb46627df4f72b21240 Signed-off-by: Romain Derie (rde) <rde@odoo.com>
248 lines
12 KiB
Python
248 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
|
|
import logging
|
|
from lxml import etree
|
|
import re
|
|
from markupsafe import Markup
|
|
|
|
from odoo.addons.website.controllers.main import Website
|
|
from odoo.addons.website.tools import distance, MockRequest
|
|
import odoo.tests
|
|
from odoo.tests.common import TransactionCase
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
@odoo.tests.tagged('-at_install', 'post_install')
|
|
class TestFuzzy(TransactionCase):
|
|
def test_01_fuzzy_names(self):
|
|
# Models from other modules commented out on commit: they make the test much longer
|
|
# Last results observed across modules:
|
|
# - 698 words in target dictionary
|
|
# - 21 wrong guesses over 9379 tested typos (0.22%)
|
|
# - Duration: ~5.7 seconds
|
|
fields_per_model = {
|
|
'website.page': ['name', 'arch'],
|
|
# 'product.public.category': ['name', 'website_description'],
|
|
# 'product.template': ['name', 'description_sale'],
|
|
# 'blog.blog': ['name', 'subtitle'],
|
|
# 'blog.post': ['name', 'subtitle'],
|
|
# 'slide.channel': ['name', 'description_short'],
|
|
# 'slide.slide': ['name', 'description'],
|
|
# 'event.event': ['name', 'subtitle'],
|
|
}
|
|
match_pattern = '\\w{4,}'
|
|
words = set()
|
|
for model_name, fields in fields_per_model.items():
|
|
if model_name not in self.env:
|
|
continue
|
|
model = self.env[model_name]
|
|
if 'description' not in fields and 'description' in model:
|
|
fields.append('description') # larger target dataset
|
|
records = model.sudo().search_read([], fields, limit=100)
|
|
for record in records:
|
|
for field, value in record.items():
|
|
if isinstance(value, str):
|
|
if field == 'arch':
|
|
view_arch = etree.fromstring(value.encode('utf-8'))
|
|
value = ' '.join(view_arch.itertext())
|
|
for word in re.findall(match_pattern, value):
|
|
words.add(word.lower())
|
|
_logger.info("%s words in target dictionary", len(words))
|
|
|
|
website = self.env.ref('website.default_website')
|
|
|
|
typos = {}
|
|
|
|
def add_typo(expected, typo):
|
|
if typo not in words:
|
|
typos.setdefault(typo, set()).add(expected)
|
|
|
|
for search in words:
|
|
for index in range(2, len(search)):
|
|
# swap letters
|
|
if search[index] != search[index - 1]:
|
|
add_typo(search, search[:index - 1] + search[index] + search[index - 1] + search[index + 1:])
|
|
# miss letter
|
|
if len(search) > 4:
|
|
add_typo(search, search[:index - 1] + search[index:])
|
|
# wrong letter
|
|
add_typo(search, search[:index - 1] + '!' + search[index:])
|
|
|
|
words = list(words)
|
|
words.sort() # guarantee results stability
|
|
mismatch_count = 0
|
|
for search, expected in typos.items():
|
|
fuzzy_guess = website._search_find_fuzzy_term({}, search, word_list=words)
|
|
if not fuzzy_guess or (fuzzy_guess not in expected and fuzzy_guess not in [exp[:-1] for exp in expected]):
|
|
mismatch_count += 1
|
|
_logger.info("'%s' fuzzy matched to '%s' instead of %s", search, fuzzy_guess, expected)
|
|
|
|
ratio = 100.0 * mismatch_count / len(typos)
|
|
_logger.info("%s wrong guesses over %s tested typos (%.2f%%)", mismatch_count, len(typos), ratio)
|
|
typos.clear()
|
|
words.clear()
|
|
self.assertTrue(ratio < 1, "Too many wrong fuzzy guesses")
|
|
|
|
def test_02_distance(self):
|
|
self.assertEqual(distance("gravity", "granity", 3), 1)
|
|
self.assertEqual(distance("gravity", "graity", 3), 1)
|
|
self.assertEqual(distance("gravity", "grait", 3), 2)
|
|
self.assertEqual(distance("gravity", "griaty", 3), 3)
|
|
self.assertEqual(distance("gravity", "giraty", 3), 3)
|
|
self.assertEqual(distance("gravity", "giraty", 2), -1)
|
|
self.assertEqual(distance("gravity", "girafe", 3), -1)
|
|
self.assertEqual(distance("warranty", "warantl", 3), 2)
|
|
|
|
# non-optimized cases still have to return correct results
|
|
self.assertEqual(distance("warranty", "warranty", 3), 0)
|
|
self.assertEqual(distance("", "warranty", 3), -1)
|
|
self.assertEqual(distance("", "warranty", 10), 8)
|
|
self.assertEqual(distance("warranty", "", 10), 8)
|
|
self.assertEqual(distance("", "", 10), 0)
|
|
|
|
|
|
@odoo.tests.tagged('-at_install', 'post_install')
|
|
class TestAutoComplete(TransactionCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
super().setUpClass()
|
|
cls.website = cls.env['website'].browse(1)
|
|
cls.WebsiteController = Website()
|
|
cls.options = {
|
|
'displayDescription': True,
|
|
}
|
|
cls.expectedParts = {
|
|
'name': True,
|
|
'description': True,
|
|
'website_url': True,
|
|
}
|
|
texts = [
|
|
"This page only matches few",
|
|
"This page matches both few and many",
|
|
"This page only matches many",
|
|
"Many results contain this page",
|
|
"How many times does the word many appear",
|
|
"Will there be many results",
|
|
"Welcome to our many friends",
|
|
"This should only be approximately matched",
|
|
]
|
|
for text in texts:
|
|
cls._create_page(text, text, f"/{text.lower().replace(' ', '-')}")
|
|
|
|
@classmethod
|
|
def _create_page(cls, name, content, url):
|
|
cls.env['website.page'].create({
|
|
'name': name,
|
|
'type': 'qweb',
|
|
'arch': f'<div>{content}</div>',
|
|
'url': url,
|
|
'is_published': True,
|
|
})
|
|
|
|
def _autocomplete(self, term):
|
|
""" Calls the autocomplete for a given term and performs general checks """
|
|
with MockRequest(self.env, website=self.website):
|
|
suggestions = self.WebsiteController.autocomplete(
|
|
search_type="pages", term=term, max_nb_chars=50, options=self.options,
|
|
)
|
|
if suggestions['results_count']:
|
|
self.assertDictEqual(self.expectedParts, suggestions['parts'],
|
|
f"Parts should contain {self.expectedParts.keys()}")
|
|
for result in suggestions['results']:
|
|
self.assertEqual("fa-file-o", result['_fa'], "Expect an fa icon")
|
|
for field in suggestions['parts'].keys():
|
|
value = result[field]
|
|
if value:
|
|
self.assertTrue(
|
|
isinstance(value, Markup),
|
|
f"All fields should be wrapped in Markup: found {type(value)}: '{value}' in {field}"
|
|
)
|
|
return suggestions
|
|
|
|
def _check_highlight(self, term, value):
|
|
""" Verifies if a term is highlighted in a value """
|
|
self.assertTrue(f'<span class="text-primary">{term}</span>' in value.lower(),
|
|
"Term must be highlighted")
|
|
|
|
def test_01_few_results(self):
|
|
""" Tests an autocomplete with exact match and less than the maximum number of results """
|
|
suggestions = self._autocomplete("few")
|
|
self.assertEqual(2, suggestions['results_count'], "Text data contains two pages with 'few'")
|
|
self.assertEqual(2, len(suggestions['results']), "All results must be present")
|
|
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
|
|
for result in suggestions['results']:
|
|
self._check_highlight("few", result['name'])
|
|
self._check_highlight("few", result['description'])
|
|
|
|
def test_02_many_results(self):
|
|
""" Tests an autocomplete with exact match and more than the maximum number of results """
|
|
suggestions = self._autocomplete("many")
|
|
self.assertEqual(6, suggestions['results_count'], "Test data contains six pages with 'many'")
|
|
self.assertEqual(5, len(suggestions['results']), "Results must be limited to 5")
|
|
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
|
|
for result in suggestions['results']:
|
|
self._check_highlight("many", result['name'])
|
|
self._check_highlight("many", result['description'])
|
|
|
|
def test_03_no_result(self):
|
|
""" Tests an autocomplete without matching results """
|
|
suggestions = self._autocomplete("nothing")
|
|
self.assertEqual(0, suggestions['results_count'], "Text data contains no page with 'nothing'")
|
|
self.assertEqual(0, len(suggestions['results']), "No result must be present")
|
|
|
|
def test_04_fuzzy_results(self):
|
|
""" Tests an autocomplete with fuzzy matching results """
|
|
suggestions = self._autocomplete("appoximtly")
|
|
self.assertEqual("approximately", suggestions['fuzzy_search'], "")
|
|
self.assertEqual(1, suggestions['results_count'], "Text data contains one page with 'approximately'")
|
|
self.assertEqual(1, len(suggestions['results']), "Single result must be present")
|
|
for result in suggestions['results']:
|
|
self._check_highlight("approximately", result['name'])
|
|
self._check_highlight("approximately", result['description'])
|
|
|
|
def test_05_long_url(self):
|
|
""" Ensures that long URL do not get truncated """
|
|
url = "/this-url-is-so-long-it-would-be-truncated-without-the-fix"
|
|
self._create_page("Too long", "Way too long URL", url)
|
|
suggestions = self._autocomplete("long url")
|
|
self.assertEqual(1, suggestions['results_count'], "Text data contains one page with 'long url'")
|
|
self.assertEqual(1, len(suggestions['results']), "Single result must be present")
|
|
self.assertEqual(url, suggestions['results'][0]['website_url'], 'URL must not be truncated')
|
|
|
|
def test_06_case_insensitive_results(self):
|
|
""" Tests an autocomplete with exact match and more than the maximum
|
|
number of results.
|
|
"""
|
|
suggestions = self._autocomplete("Many")
|
|
self.assertEqual(6, suggestions['results_count'], "Test data contains six pages with 'Many'")
|
|
self.assertEqual(5, len(suggestions['results']), "Results must be limited to 5")
|
|
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
|
|
for result in suggestions['results']:
|
|
self._check_highlight("many", result['name'])
|
|
self._check_highlight("many", result['description'])
|
|
|
|
def test_07_no_fuzzy_for_mostly_number(self):
|
|
""" Ensures exact match is used when search contains mostly numbers. """
|
|
self._create_page('Product P7935432254U7 page', 'Product P7935432254U7', '/numberpage')
|
|
suggestions = self._autocomplete("54321")
|
|
self.assertEqual(0, suggestions['results_count'], "Test data contains no exact match")
|
|
suggestions = self._autocomplete("54322")
|
|
self.assertEqual(1, suggestions['results_count'], "Test data contains one exact match")
|
|
suggestions = self._autocomplete("P79355")
|
|
self.assertEqual(0, suggestions['results_count'], "Test data contains no exact match")
|
|
suggestions = self._autocomplete("P79354")
|
|
self.assertEqual(1, suggestions['results_count'], "Test data contains one exact match")
|
|
self.assertFalse(suggestions['fuzzy_search'], "Expects an exact match")
|
|
suggestions = self._autocomplete("produkt")
|
|
self.assertEqual(1, suggestions['results_count'], "Test data contains one fuzzy match")
|
|
self.assertTrue(suggestions['fuzzy_search'], "Expects a fuzzy match")
|
|
|
|
def test_08_fuzzy_classic_numbers(self):
|
|
""" Ensures fuzzy match is used when search contains a few numbers. """
|
|
self._create_page('iPhone 6', 'iPhone6', '/iphone6')
|
|
suggestions = self._autocomplete("iphone7")
|
|
self.assertEqual(1, suggestions['results_count'], "Test data contains one fuzzy match")
|
|
self.assertTrue(suggestions['fuzzy_search'], "Expects an fuzzy match")
|