pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] core: add unaccent to trigram indexes

Browse Source 
We added trigram index for char fields since https://github.com/odoo/odoo/pull/83015.
But if `unaccent` is installed in the database (and isn't force to
`False` on field), these new trigram indexes are pointless and cost a
lot for nothing (almost nothing, it still can be used for equality
operator but in this case a btree will be far more efficient).

The simple way to fix it is to add `unaccent(<column>)` in the index
trigram definition, but unfortunately `unaccent` is not immutable and
may therefore not be indexed.  In order to make `unaccent` indexable, we
must declare it as immutable (see
https://stackoverflow.com/questions/11005036/does-postgresql-support-accent-insensitive-collations/11007216#11007216
for more information and how to do that).

With this patch, trigram indexes are created with `unaccent(<column>)`
if the function `unaccent` is available in the database, and for the
fields that are not declared with `unaccent=False`.  Moreover, we issue
a warning when `unaccent` is available but is not immutable, in which
case most trigram indexes will be useless.

odoo/upgrade#3736
task-2551518

closes odoo/odoo#95943

Signed-off-by: Rémy Voet <ryv@odoo.com>
This commit is contained in:
Rémy Voet (ryv)
2022-09-05 18:34:53 +02:00
parent 7f0f7bcdce
commit 4087bcdc5e
5 changed files with 62 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/account/models/account_move.py
+3 -3
View File
@@ -77,7 +77,7 @@ class AccountMove(models.Model):
compute='_compute_name', readonly=False, store=True,
copy=False,
tracking=True,
index='btree',
index='btree', # We need the btree index for unicity constraint (`_check_unique_sequence_number`)
)
ref = fields.Char(string='Reference', copy=False, tracking=True)
date = fields.Date(
@@ -543,8 +543,8 @@ class AccountMove(models.Model):
def _auto_init(self):
super()._auto_init()
if sql.install_pg_trgm(self._cr):
# We need the btree index for unicity constraint (on field) AND this one for human searches
if self.pool.has_trigram:
# This index is for human searches
sql.create_index(self._cr, 'account_move_name_trigram_index', self._table, ['"name" gin_trgm_ops'], 'gin')
self.env.cr.execute("""
odoo/modules/db.py
+25 -4
View File
@@ -1,8 +1,10 @@
# -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import odoo.modules
import logging
from enum import IntEnum
import odoo.modules
_logger = logging.getLogger(__name__)
@@ -137,15 +139,34 @@ def create_categories(cr, categories):
categories = categories[1:]
return p_id
class FunctionStatus(IntEnum):
MISSING = 0 # function is not present (falsy)
PRESENT = 1 # function is present but not indexable (not immutable)
INDEXABLE = 2 # function is present and indexable (immutable)
def has_unaccent(cr):
""" Test if the database has an unaccent function.
""" Test whether the database has function 'unaccent' and return its status.
The unaccent is supposed to be provided by the PostgreSQL unaccent contrib
module but any similar function will be picked by OpenERP.
:rtype: FunctionStatus
"""
cr.execute("SELECT proname FROM pg_proc WHERE proname='unaccent'")
return len(cr.fetchall()) > 0
cr.execute("""
SELECT p.provolatile
FROM pg_proc p
LEFT JOIN pg_catalog.pg_namespace ns ON p.pronamespace = ns.oid
WHERE p.proname = 'unaccent'
AND p.pronargs = 1
AND ns.nspname = 'public'
""")
result = cr.fetchone()
if not result:
return FunctionStatus.MISSING
# The `provolatile` of unaccent allows to know whether the unaccent function
# can be used to create index (it should be 'i' - means immutable), see
# https://www.postgresql.org/docs/current/catalog-pg-proc.html.
return FunctionStatus.INDEXABLE if result[0] == 'i' else FunctionStatus.PRESENT
def has_trigram(cr):
""" Test if the database has the a word_similarity function.
odoo/modules/registry.py
+19 -7
View File
@@ -13,10 +13,13 @@ import logging
import os
import threading
import time
import warnings
import psycopg2
import odoo
from odoo.modules.db import FunctionStatus
from odoo.osv.expression import get_unaccent_wrapper
from .. import SUPERUSER_ID
from odoo.sql_db import TestCursor
from odoo.tools import (config, existing_tables, ignore,
@@ -453,7 +456,7 @@ class Registry(Mapping):
def check_indexes(self, cr, model_names):
""" Create or drop column indexes for the given models. """
expected = [
(f"{Model._table}_{field.name}_index", Model._table, field.name, field.index)
(f"{Model._table}_{field.name}_index", Model._table, field.name, field.index, getattr(field, 'unaccent', False))
for model_name in model_names
for Model in [self.models[model_name]]
if Model._auto and not Model._abstract
@@ -467,23 +470,32 @@ class Registry(Mapping):
[tuple(row[0] for row in expected)])
existing = {row[0] for row in cr.fetchall()}
if not self.has_trigram and any(row[3] == 'trigram' for row in expected):
self.has_trigram = sql.install_pg_trgm(cr)
for indexname, tablename, columnname, index in expected:
for indexname, tablename, column_name, index, unaccent in expected:
assert index in ('btree', 'btree_not_null', 'trigram', True, False, None)
if index and indexname not in existing:
column_expression = f'"{column_name}"'
method = 'btree'
operator = ''
where = ''
if index == 'btree_not_null':
where = f'"{columnname}" IS NOT NULL'
where = f'{column_expression} IS NOT NULL'
elif index == 'trigram' and self.has_trigram:
method = 'gin'
operator = 'gin_trgm_ops'
# add `unaccent` to the trigram index only because the
# trigram indexes are mainly used for (i/=)like search and
# unaccent is added only in these cases when searching
if unaccent and self.has_unaccent:
if self.has_unaccent == FunctionStatus.INDEXABLE:
column_expression = get_unaccent_wrapper(cr)(column_expression)
else:
warnings.warn(
"PostgreSQL function 'unaccent' is present but not immutable, "
"therefore trigram indexes may not be effective.",
)
try:
with cr.savepoint(flush=False):
expression = f'"{columnname}" {operator}'
expression = f'{column_expression} {operator}'
sql.create_index(cr, indexname, tablename, [expression], method, where)
except psycopg2.OperationalError:
_schema.error("Unable to add index for %s", self)
odoo/service/db.py
+15 -7
View File
@@ -116,14 +116,22 @@ def _create_empty_database(name):
sql.Identifier(name), collate, sql.Identifier(chosen_template)
))
if odoo.tools.config['unaccent']:
try:
db = odoo.sql_db.db_connect(name)
with closing(db.cursor()) as cr:
# TODO: add --extension=trigram,unaccent
try:
db = odoo.sql_db.db_connect(name)
with db.cursor() as cr:
cr.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
if odoo.tools.config['unaccent']:
cr.execute("CREATE EXTENSION IF NOT EXISTS unaccent")
cr.commit()
except psycopg2.Error:
pass
# From PostgreSQL's point of view, making 'unaccent' immutable is incorrect
# because it depends on external data - see
# https://www.postgresql.org/message-id/flat/201012021544.oB2FiTn1041521@wwwmaster.postgresql.org#201012021544.oB2FiTn1041521@wwwmaster.postgresql.org
# But in the case of Odoo, we consider that those data don't
# change in the lifetime of a database. If they do change, all
# indexes created with this function become corrupted!
cr.execute("ALTER FUNCTION unaccent(text) IMMUTABLE")
except psycopg2.Error as e:
_logger.warning("Unable to create PostgreSQL extensions : %s", e)
@check_db_management_enabled
def exp_create_database(db_name, demo, lang, user_password='admin', login='admin', country_code=None, phone=None):
odoo/tools/sql.py
-20
View File
@@ -234,26 +234,6 @@ def fix_foreign_key(cr, tablename1, columnname1, tablename2, columnname2, ondele
if not found:
return add_foreign_key(cr, tablename1, columnname1, tablename2, columnname2, ondelete)
def install_pg_trgm(cr):
cr.execute("SELECT installed_version FROM pg_available_extensions WHERE name='pg_trgm'")
version = cr.fetchone()
if version is None:
return False
if version[0]:
return True
cr.execute('SELECT usesuper FROM pg_user WHERE usename = CURRENT_USER')
if not cr.fetchone()[0]:
return False
try:
db = odoo.sql_db.db_connect(cr.dbname)
with closing(db.cursor()) as cr:
cr.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
cr.commit()
return True
except psycopg2.Error:
return False
def index_exists(cr, indexname):
""" Return whether the given index exists. """
cr.execute("SELECT 1 FROM pg_indexes WHERE indexname=%s", (indexname,))