From bb24757d0819eb4cc206692df7f806916bb3443d Mon Sep 17 00:00:00 2001 From: Pratik Raval Date: Fri, 8 Apr 2022 11:55:00 +0000 Subject: [PATCH] [IMP] crm: detect leads based on similar phone/mobile number Currently, the 'similar lead detection' mechanism only considers email, contact name, and partner name while finding duplicate leads in CRM. After this commit, it will also consider the mobile/phone number for the same. Note that the mobile numbers and phone numbers both will be matched with each other while finding duplicate leads. Task-2817884 closes odoo/odoo#88372 Signed-off-by: Thibault Delavallee (tde) --- addons/crm/models/crm_lead.py | 27 ++++----- addons/crm/tests/test_crm_lead_duplicates.py | 57 +++++++++++++++++++ addons/iap/tools/iap_tools.py | 35 +++++++++++- .../test_crm_full/tests/test_performance.py | 4 +- odoo/tools/mail.py | 11 +++- 5 files changed, 112 insertions(+), 22 deletions(-) diff --git a/addons/crm/models/crm_lead.py b/addons/crm/models/crm_lead.py index 8a71718cd83..9e42790a824 100644 --- a/addons/crm/models/crm_lead.py +++ b/addons/crm/models/crm_lead.py @@ -522,6 +522,7 @@ class Lead(models.Model): def _compute_potential_lead_duplicates(self): MIN_EMAIL_LENGTH = 7 MIN_NAME_LENGTH = 6 + MIN_PHONE_LENGTH = 8 SEARCH_RESULT_LIMIT = 21 def return_if_relevant(model_name, domain): @@ -542,20 +543,6 @@ class Lead(models.Model): res = model.search(domain, limit=SEARCH_RESULT_LIMIT) return res if len(res) < SEARCH_RESULT_LIMIT else model - def get_email_to_search(email): - """ Returns the full email address if the domain of the email address - is common (i.e: in the mail domain blacklist). Otherwise, returns - the domain of the email address. A minimal length is required to avoid - returning false positives records. """ - if not email or len(email) < MIN_EMAIL_LENGTH: - return False - parts = email.rsplit('@', maxsplit=1) - if len(parts) > 1: - email_domain = parts[1] - if email_domain not in iap_tools._MAIL_DOMAIN_BLACKLIST: - return '@' + email_domain - return email - for lead in self: lead_id = lead._origin.id if isinstance(lead.id, models.NewId) else lead.id common_lead_domain = [ @@ -563,11 +550,11 @@ class Lead(models.Model): ] duplicate_lead_ids = self.env['crm.lead'] - email_search = get_email_to_search(lead.email_from) + email_search = iap_tools.mail_prepare_for_domain_search(lead.email_from, min_email_length=MIN_EMAIL_LENGTH) if email_search: duplicate_lead_ids |= return_if_relevant('crm.lead', common_lead_domain + [ - ('email_from', 'ilike', email_search) + '|', ('email_normalized', 'ilike', email_search), ('email_from', 'ilike', email_search) ]) if lead.partner_name and len(lead.partner_name) >= MIN_NAME_LENGTH: duplicate_lead_ids |= return_if_relevant('crm.lead', common_lead_domain + [ @@ -581,6 +568,14 @@ class Lead(models.Model): duplicate_lead_ids |= lead.with_context(active_test=False).search(common_lead_domain + [ ("partner_id", "child_of", lead.partner_id.commercial_partner_id.id) ]) + if lead.phone and len(lead.phone) >= MIN_PHONE_LENGTH: + duplicate_lead_ids |= return_if_relevant('crm.lead', common_lead_domain + [ + ('phone_mobile_search', 'ilike', lead.phone) + ]) + if lead.mobile and len(lead.mobile) >= MIN_PHONE_LENGTH: + duplicate_lead_ids |= return_if_relevant('crm.lead', common_lead_domain + [ + ('phone_mobile_search', 'ilike', lead.mobile) + ]) lead.duplicate_lead_ids = duplicate_lead_ids + lead lead.duplicate_lead_count = len(duplicate_lead_ids) diff --git a/addons/crm/tests/test_crm_lead_duplicates.py b/addons/crm/tests/test_crm_lead_duplicates.py index c420f2b0a30..35c7927715c 100644 --- a/addons/crm/tests/test_crm_lead_duplicates.py +++ b/addons/crm/tests/test_crm_lead_duplicates.py @@ -135,6 +135,63 @@ class TestLeadConvert(TestCrmCommon): self.assertEqual(lead_13 + lead_14, lead_13.duplicate_lead_ids) self.assertEqual(lead_13 + lead_14, lead_14.duplicate_lead_ids) + @users('user_sales_manager') + def test_potential_duplicates_with_phone(self): + customer = self.env['res.partner'].create({ + 'email': 'customer1@duplicate.example.com', + 'mobile': '+32485001122', + 'name': 'Customer1', + 'phone': '(803)-456-6126', + }) + base_lead = self.env['crm.lead'].create({ + 'name': 'Base Lead', + 'partner_id': customer.id, + 'type': 'lead', + }) + + self.assertEqual(base_lead.contact_name, customer.name) + self.assertEqual(base_lead.mobile, customer.mobile) + self.assertFalse(base_lead.partner_name) + self.assertEqual(base_lead.phone, customer.phone) + + dup1_1 = self.env['crm.lead'].create({ + 'name': 'Base Lead Dup1', + 'type': 'lead', + 'phone': '456-6126', # shorter version of base_lead + 'partner_name': 'Partner Name 1', + }) + dup1_2 = self.env['crm.lead'].create({ + 'name': 'Base Lead Dup2', + 'mobile': '8034566126', + 'partner_name': 'Partner Name 2', + 'type': 'lead', + }) + dup1_3 = self.env['crm.lead'].create({ + 'name': 'Base Lead Dup3', + 'partner_name': 'Partner Name 3', + 'phone': '(803)-456-6126', + 'type': 'lead', + }) + dup1_4 = self.env['crm.lead'].create({ + 'mobile': '0032485001122', + # 'mobile': '0485001122', # note: does not work + 'name': 'Base Lead Dup4', + 'partner_name': 'Partner Name 4', + 'phone': False, + 'type': 'lead', + }) + + expected = base_lead + dup1_2 + dup1_3 + dup1_4 # dup1_1 is shorter than lead -> not a dupe + self.assertEqual( + base_lead.duplicate_lead_ids, expected, + 'CRM: missing %s, extra %s' % ((expected - base_lead.duplicate_lead_ids).mapped('name'), (base_lead.duplicate_lead_ids - expected).mapped('name')) + ) + expected = base_lead + dup1_1 + dup1_2 + dup1_3 # dup1_4 has mobile of customer, but no link with dup1_1 + self.assertEqual( + dup1_1.duplicate_lead_ids, expected, + 'CRM: missing %s, extra %s' % ((expected - dup1_1.duplicate_lead_ids).mapped('name'), (dup1_1.duplicate_lead_ids - expected).mapped('name')) + ) + @users('user_sales_manager') def test_potential_duplicates_with_invalid_email(self): lead_1 = self.env['crm.lead'].create({ diff --git a/addons/iap/tools/iap_tools.py b/addons/iap/tools/iap_tools.py index a9e30792ab1..8ef951da082 100644 --- a/addons/iap/tools/iap_tools.py +++ b/addons/iap/tools/iap_tools.py @@ -10,7 +10,7 @@ from unittest.mock import patch from odoo import exceptions, _ from odoo.tests.common import BaseCase -from odoo.tools import pycompat +from odoo.tools import email_normalize, pycompat _logger = logging.getLogger(__name__) @@ -73,6 +73,39 @@ _STATES_FILTER_COUNTRIES_WHITELIST = set([ 'AR', 'AU', 'BR', 'CA', 'IN', 'MY', 'MX', 'NZ', 'AE', 'US' ]) + +#---------------------------------------------------------- +# Tools +#---------------------------------------------------------- + +def mail_prepare_for_domain_search(email, min_email_length=0): + """ Return an email address to use for a domain-based search. For generic + email providers like gmail (see ``_MAIL_DOMAIN_BLACKLIST``) we consider + each email as being independant (and return the whole email). Otherwise + we return only the right-part of the email (aka "mydomain.com" if email is + "Raoul Lachignole" ). + + :param integer min_email_length: skip if email has not the sufficient minimal + length, indicating a probably fake / wrong value (skip if 0); + """ + if not email: + return False + email_tocheck = email_normalize(email, strict=False) + if not email_tocheck: + email_tocheck = email.casefold() + + if email_tocheck and min_email_length and len(email_tocheck) < min_email_length: + return False + + parts = email_tocheck.rsplit('@', maxsplit=1) + if len(parts) == 1: + return email_tocheck + email_domain = parts[1] + if email_domain not in _MAIL_DOMAIN_BLACKLIST: + return '@' + email_domain + return email_tocheck + + #---------------------------------------------------------- # Helpers for both clients and proxy #---------------------------------------------------------- diff --git a/addons/test_crm_full/tests/test_performance.py b/addons/test_crm_full/tests/test_performance.py index b82b9530b37..64ee80c2490 100644 --- a/addons/test_crm_full/tests/test_performance.py +++ b/addons/test_crm_full/tests/test_performance.py @@ -70,7 +70,7 @@ class TestCrmPerformance(CrmPerformanceCase): country_be = self.env.ref('base.be') lang_be = self.env['res.lang']._lang_get('fr_BE') - with freeze_time(self.reference_now), self.assertQueryCount(user_sales_leads=178): # tcf 164 / com 166 + with freeze_time(self.reference_now), self.assertQueryCount(user_sales_leads=186): # tcf 174 / com 175 self.env.cr._now = self.reference_now # force create_date to check schedulers with Form(self.env['crm.lead']) as lead_form: lead_form.country_id = country_be @@ -89,7 +89,7 @@ class TestCrmPerformance(CrmPerformanceCase): @warmup def test_lead_create_form_partner(self): """ Test a single lead creation using Form with a partner """ - with freeze_time(self.reference_now), self.assertQueryCount(user_sales_leads=186): # tcf 173 / com 175 + with freeze_time(self.reference_now), self.assertQueryCount(user_sales_leads=199): # tcf 186 / com 188 self.env.cr._now = self.reference_now # force create_date to check schedulers with self.debug_mode(): # {'invisible': ['|', ('type', '=', 'opportunity'), ('is_partner_visible', '=', False)]} diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py index 75b931f0ac7..3f169cbd66b 100644 --- a/odoo/tools/mail.py +++ b/odoo/tools/mail.py @@ -519,7 +519,7 @@ def email_split_and_format(text): return [] return [formataddr((name, email)) for (name, email) in email_split_tuples(text)] -def email_normalize(text): +def email_normalize(text, strict=True): """ Sanitize and standardize email address entries. A normalized email is considered as : - having a left part + @ + a right part (the domain can be without '.something') @@ -528,13 +528,18 @@ def email_normalize(text): Ex: - Possible Input Email : 'Name ' - Normalized Output Email : 'name@domain.com' + + :param bool strict: text should contain exactly one email (default behavior + and unique behavior before Odoo16); + + :return: False if no email found (or if more than 1 email found when being + in strict mode); normalized email otherwise; """ emails = email_split(text) - if not emails or len(emails) != 1: + if not emails or (strict and len(emails) != 1): return False return emails[0].lower() - def email_domain_extract(email): """ Extract the company domain to be used by IAP services notably. Domain is extracted from email information e.g: