diff --git a/addons/crm/data/crm_lead_prediction_data.xml b/addons/crm/data/crm_lead_prediction_data.xml
index f390b6e6138..1a1cbc3675f 100644
--- a/addons/crm/data/crm_lead_prediction_data.xml
+++ b/addons/crm/data/crm_lead_prediction_data.xml
@@ -20,6 +20,18 @@
+
+ crm.pls_fields
+
+
+
+ crm.pls_start_date
+
+
+
+ crm.pls_rebuild_threshold
+ 100
+
diff --git a/addons/crm/models/crm_lead.py b/addons/crm/models/crm_lead.py
index abe9e285c82..c2a2bb40759 100644
--- a/addons/crm/models/crm_lead.py
+++ b/addons/crm/models/crm_lead.py
@@ -105,17 +105,15 @@ class Lead(models.Model):
date_conversion = fields.Datetime('Conversion Date', readonly=True)
# Probability - Only used for type opportunity
- probability = fields.Float('Probability', group_operator="avg", default=10.0)
+ probability = fields.Float('Probability', group_operator="avg")
automated_probability = fields.Float('Automated Probability', readonly=True)
is_automated_probability = fields.Boolean('Is automated probability?', compute="_compute_is_automated_probability")
phone_state = fields.Selection([
('correct', 'Correct'),
- ('incorrect', 'Incorrect'),
- ('empty', 'Empty')], string='Phone Quality', default="empty", compute="_compute_phone_state", store=True)
+ ('incorrect', 'Incorrect')], string='Phone Quality', compute="_compute_phone_state", store=True)
email_state = fields.Selection([
('correct', 'Correct'),
- ('incorrect', 'Incorrect'),
- ('empty', 'Empty')], string='Email Quality', default="empty", compute="_compute_email_state", store=True)
+ ('incorrect', 'Incorrect')], string='Email Quality', compute="_compute_email_state", store=True)
# Only used for type opportunity
planned_revenue = fields.Monetary('Expected Revenue', currency_field='company_currency', tracking=True)
@@ -222,7 +220,7 @@ class Lead(models.Model):
@api.depends('phone', 'country_id.code')
def _compute_phone_state(self):
for lead in self:
- phone_status = 'empty'
+ phone_status = False
if lead.phone:
country_code = lead.country_id.code if lead.country_id and lead.country_id.code else None
try:
@@ -235,11 +233,11 @@ class Lead(models.Model):
@api.depends('email_from')
def _compute_email_state(self):
for lead in self:
- email_state = 'empty'
+ email_state = False
if lead.email_from:
email_state = 'incorrect'
for email in email_split(lead.email_from):
- if tools.email_validate(email):
+ if tools.email_normalize(email):
email_state = 'correct'
break
lead.email_state = email_state
@@ -483,9 +481,24 @@ class Lead(models.Model):
# Actions Methods
# ----------------------------------------
+ def _rebuild_pls_frequency_table_threshold(self):
+ """ Called by action_set_lost and action_set_won.
+ Will run the cron to update the frequency table only if the number of lead is above
+ a specified value (from config_parameter) for onboarding purpose.
+ Once the threshold is reached, the config_param is set to 0 to avoid re-run the cron
+ and, mainly, to avoid making useless search_count in the future."""
+ pls_threshold = int(self.env['ir.config_parameter'].sudo().get_param('crm.pls_rebuild_threshold'))
+ if pls_threshold:
+ lead_count = self.env['crm.lead'].sudo().search_count([])
+ if lead_count < pls_threshold:
+ self.sudo()._cron_update_automated_probabilities()
+ else:
+ self.env['ir.config_parameter'].sudo().set_param('crm.pls_rebuild_threshold', 0)
+
def action_set_lost(self, **additional_values):
""" Lost semantic: probability = 0 or active = False """
result = self.write({'active': False, 'probability': 0, **additional_values})
+ self._rebuild_pls_frequency_table_threshold()
return result
def action_set_won(self):
@@ -493,6 +506,7 @@ class Lead(models.Model):
for lead in self:
stage_id = lead._stage_find(domain=[('is_won', '=', True)])
lead.write({'stage_id': stage_id.id, 'probability': 100})
+ self._rebuild_pls_frequency_table_threshold()
return True
def action_set_automated_probability(self):
@@ -1320,6 +1334,7 @@ class Lead(models.Model):
# get stages
first_stage_id = self.env['crm.stage'].search([], order='sequence', limit=1)
+ won_stage_ids = self.env['crm.stage'].search([('is_won', '=', True)]).ids
# Get all leads values, no matter the team_id
leads_values_dict = self._pls_get_lead_pls_values(batch_mode=batch_mode)
@@ -1328,8 +1343,11 @@ class Lead(models.Model):
# Get unique couples to search in frequency table
leads_values = set()
- for values in leads_values_dict.values():
+ won_leads = set()
+ for lead_id, values in leads_values_dict.items():
for couple in values['values']:
+ if couple[0] == 'stage_id' and couple[1] in won_stage_ids:
+ won_leads.add(lead_id)
leads_values.add(couple)
# get all variable related records from frequency table, no matter the team_id
@@ -1374,6 +1392,16 @@ class Lead(models.Model):
save_team_id = None
p_won, p_lost = 1, 1
for lead_id, lead_values in leads_values_dict.items():
+ # if stage_id is null, return 0 and bypass computation
+ lead_fields = [value[0] for value in lead_values.get('values', [])]
+ if not 'stage_id' in lead_fields:
+ lead_probabilities[lead_id] = 0
+ continue
+ # if lead stage is won, return 100
+ elif lead_id in won_leads:
+ lead_probabilities[lead_id] = 100
+ continue
+
lead_team_id = lead_values['team_id'] if lead_values['team_id'] else 0 # team_id = None -> Convert to 0
lead_team_id = lead_team_id if lead_team_id in result else -1 # team_id not in frequency Table -> convert to -1
if lead_team_id != save_team_id:
@@ -1428,15 +1456,24 @@ class Lead(models.Model):
# Recompute all the leads. Won : probability = 100 | Lost : probability = 0 or inactive
# Here, inactive won't be returned anyway
+ # Get also all the lead without probability --> These are the new leads. Activate auto probability on them.
pls_start_date = self._pls_get_safe_start_date()
if pls_start_date:
- pending_lead_domain = [('probability', '<', 100), ('probability', '>', 0), ('create_date', '>', pls_start_date)]
+ pending_lead_domain = ['&', '&', ('stage_id', '!=', False), ('create_date', '>', pls_start_date),
+ '|', ('probability', '=', False), '&', ('probability', '<', 100), ('probability', '>', 0)]
leads_to_update = self.env['crm.lead'].search(pending_lead_domain)
lead_probabilities = leads_to_update._pls_get_naive_bayes_probabilities(batch_mode=True)
# Update in execute batch to avoid server roundtrips + page_size to 10000 to avoid memory errors
- sql = "update crm_lead set automated_probability = %s where id = %s"
- batch_params = [(lead_probabilities[lead.id], lead.id) for lead in leads_to_update if lead.id in lead_probabilities]
+ # Update both probability and automated_probability if they were equal, else, update only automated_probability
+ sql = """UPDATE crm_lead
+ SET automated_probability = %s,
+ probability = CASE WHEN (ROUND(probability::numeric, 2) = ROUND(automated_probability::numeric, 2) or probability is null)
+ THEN (%s)
+ ELSE (probability)
+ END
+ WHERE id = %s"""
+ batch_params = [(lead_probabilities[lead.id], lead_probabilities[lead.id], lead.id) for lead in leads_to_update if lead.id in lead_probabilities]
extras.execute_batch(self._cr, sql, batch_params, page_size=10000)
_logger.info("Predictive Lead Scoring : all automated probability updated (count: %d)" % (len(leads_to_update)))
@@ -1464,7 +1501,7 @@ class Lead(models.Model):
that are defined on the model. """
pls_fields_config = self.env['ir.config_parameter'].sudo().get_param('crm.pls_fields')
pls_fields = pls_fields_config.split(',') if pls_fields_config else []
- pls_safe_fields = [field for field in pls_fields if field in self._fields]
+ pls_safe_fields = [field for field in pls_fields if field in self._fields.keys()]
return pls_safe_fields
# Rebuild Frequency Table Tools
@@ -1525,7 +1562,7 @@ class Lead(models.Model):
lost = result['count'] if result['probability'] == 0 else 0
for field in fields:
value = result[field]
- if value:
+ if value or field in ('email_state', 'phone_state'):
if field == 'stage_id':
if won: # increment all stages if won
stages_to_increment = [stage['id'] for stage in stage_ids]
@@ -1600,7 +1637,7 @@ class Lead(models.Model):
self.flush(['probability'])
query = """SELECT id, %s
FROM crm_lead l
- WHERE probability > 0 AND probability < 100 AND active = True AND id in %%s order by team_id asc"""
+ WHERE ((probability > 0 AND probability < 100) OR probability is null) AND active = True AND id in %%s order by team_id asc"""
query = sql.SQL(query % str_fields).format(*args)
self._cr.execute(query, [tuple(self.ids)])
@@ -1610,7 +1647,7 @@ class Lead(models.Model):
FROM crm_lead l
LEFT JOIN crm_lead_tag_rel rel ON l.id = rel.lead_id
LEFT JOIN crm_lead_tag t ON rel.tag_id = t.id
- WHERE l.probability > 0 AND l.probability < 100 AND l.active = True AND l.id in %s order by l.team_id asc"""
+ WHERE ((l.probability > 0 AND l.probability < 100) OR l.probability is null) AND l.active = True AND l.id in %s order by l.team_id asc"""
self._cr.execute(query, [tuple(self.ids)])
tag_results = self._cr.dictfetchall()
@@ -1621,7 +1658,7 @@ class Lead(models.Model):
if field == 'team_id': # ignore team_id as stored separately in leads_values_dict[lead_id][team_id]
continue
value = lead[field]
- if value:
+ if value or field in ('email_state', 'phone_state'):
lead_values.append((field, value))
leads_values_dict[lead['id']] = {'values': lead_values, 'team_id': lead['team_id']}
@@ -1636,7 +1673,7 @@ class Lead(models.Model):
if field == 'team_id': # ignore team_id as stored separately in leads_values_dict[lead_id][team_id]
continue
value = lead[field].id if isinstance(lead[field], models.BaseModel) else lead[field]
- if value:
+ if value or field in ('email_state', 'phone_state'):
lead_values.append((field, value))
for tag in lead.tag_ids:
lead_values.append(('tag_id', tag.id))
@@ -1652,7 +1689,7 @@ class Lead(models.Model):
:param first_stage_id: stage with smallest sequence
:return: won count, lost count and total count for all records in frequencies
"""
- if str(first_stage_id.id) not in team_results['stage_id']:
+ if str(first_stage_id.id) not in team_results.get('stage_id', []):
return 0, 0, 0
stage_result = team_results['stage_id'][str(first_stage_id.id)]
return stage_result['won'], stage_result['lost'], stage_result['won'] + stage_result['lost']
diff --git a/addons/crm/models/res_config_settings.py b/addons/crm/models/res_config_settings.py
index deb4fc322a9..07b91adef98 100644
--- a/addons/crm/models/res_config_settings.py
+++ b/addons/crm/models/res_config_settings.py
@@ -16,7 +16,7 @@ class ResConfigSettings(models.TransientModel):
module_crm_iap_lead_enrich = fields.Boolean("Enrich your leads automatically with company data based on their email address.")
lead_mining_in_pipeline = fields.Boolean("Create a lead mining request directly from the opportunity pipeline.", config_parameter='crm.lead_mining_in_pipeline')
predictive_lead_scoring_start_date = fields.Date(string='Lead Scoring Starting Date', compute="_compute_pls_start_date", inverse="_inverse_pls_start_date_str")
- predictive_lead_scoring_start_date_str = fields.Char(string='Lead Scoring Starting Date in String', default=date.today().strftime('%Y-%m-%d'), config_parameter='crm.pls_start_date')
+ predictive_lead_scoring_start_date_str = fields.Char(string='Lead Scoring Starting Date in String', config_parameter='crm.pls_start_date')
predictive_lead_scoring_fields = fields.Many2many('crm.lead.scoring.frequency.field', string='Lead Scoring Frequency Fields', compute="_compute_pls_fields", inverse="_inverse_pls_fields_str")
predictive_lead_scoring_fields_str = fields.Char(string='Lead Scoring Frequency Fields in String', config_parameter='crm.pls_fields')
@@ -39,7 +39,8 @@ class ResConfigSettings(models.TransientModel):
for setting in self:
if setting.predictive_lead_scoring_fields_str:
names = setting.predictive_lead_scoring_fields_str.split(',')
- setting.predictive_lead_scoring_fields = self.env['crm.lead.scoring.frequency.field'].search([('name', 'in', names)])
+ fields = self.env['ir.model.fields'].search([('name', 'in', names), ('model', '=', 'crm.lead')])
+ setting.predictive_lead_scoring_fields = self.env['crm.lead.scoring.frequency.field'].search([('field_id', 'in', fields.ids)])
else:
setting.predictive_lead_scoring_fields = None
@@ -48,7 +49,7 @@ class ResConfigSettings(models.TransientModel):
we store the fields with a comma separated string into a Char config field """
for setting in self:
if setting.predictive_lead_scoring_fields:
- setting.predictive_lead_scoring_fields_str = ','.join(setting.predictive_lead_scoring_fields.mapped('name'))
+ setting.predictive_lead_scoring_fields_str = ','.join(setting.predictive_lead_scoring_fields.mapped('field_id.name'))
else:
setting.predictive_lead_scoring_fields_str = ''
diff --git a/addons/crm/views/res_config_settings_views.xml b/addons/crm/views/res_config_settings_views.xml
index baa8dc3d925..4431b46aae6 100644
--- a/addons/crm/views/res_config_settings_views.xml
+++ b/addons/crm/views/res_config_settings_views.xml
@@ -66,7 +66,7 @@
Consider leads created as of the
-
+
diff --git a/addons/crm_iap_lead_enrich/models/crm_lead.py b/addons/crm_iap_lead_enrich/models/crm_lead.py
index 48df18f3c77..dd6e4b9ee1d 100644
--- a/addons/crm_iap_lead_enrich/models/crm_lead.py
+++ b/addons/crm_iap_lead_enrich/models/crm_lead.py
@@ -19,10 +19,11 @@ class Lead(models.Model):
@api.model
def _iap_enrich_leads_cron(self):
timeDelta = fields.datetime.now() - datetime.timedelta(hours=1)
+ # Get all leads not lost nor won (lost: active = False)
leads = self.search([
('iap_enrich_done', '=', False),
('reveal_id', '=', False),
- ('probability', 'not in', (0, 100)),
+ ('probability', '<', 100),
('create_date', '>', timeDelta)
])
leads._iap_enrich(from_cron=True)
@@ -30,7 +31,8 @@ class Lead(models.Model):
def _iap_enrich(self, from_cron=False):
lead_emails = {}
for lead in self:
- if lead.probability in (0, 100) or lead.iap_enrich_done:
+ # If lead is lost, active == False, but is anyway removed from the search in the cron.
+ if lead.probability == 100 or lead.iap_enrich_done:
continue
normalized_email = tools.email_normalize(lead.partner_address_email) or tools.email_normalize(lead.email_from)
if normalized_email:
diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py
index 50ccd6b0375..f9a5cfa67cf 100644
--- a/odoo/tools/mail.py
+++ b/odoo/tools/mail.py
@@ -20,31 +20,6 @@ from odoo.tools import misc
_logger = logging.getLogger(__name__)
-# Optional Flanker dependency for email validation.
-_flanker_warning = False
-try:
- from flanker.addresslib import address
- if hasattr(address, 'six'):
- # Python 3 supported
- def checkmail(mail):
- return bool(address.validate_address(mail))
- else:
- def checkmail(mail):
- global _flanker_warning
- if not _flanker_warning:
- _logger.info("Flanker version 0.9 or greater required for Python 3 compatibility")
- _flanker_warning = True
- return True
-
-except ImportError:
- _logger.info('The flanker Python module is not installed, so email validation with flanker is unavailable')
- def checkmail(mail):
- global _flanker_warning
- if not _flanker_warning:
- _logger.info('The flanker Python module is not installed, so email validation with flanker is unavailable')
- _flanker_warning = True
- return True
-
#----------------------------------------------------------
# HTML Sanitizer
#----------------------------------------------------------
@@ -531,16 +506,6 @@ def email_normalize(text):
return False
return emails[0].lower()
-def email_validate(email):
- """
- Check is the email is valid using email normalization + optional Flanker module.
- If Flanker is installed, it will check if email is valid (checking DNS and other stuff).
- If Flanker is not installed, this method only normalizes the email
- :param text: string containing the email to validate
- :return: normalized email if mail is valid or False
- """
- return email_normalize(email) if checkmail(email) else False
-
def email_escape_char(email_address):
""" Escape problematic characters in the given email address string"""
return email_address.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')