pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[MERGE] crm : fix and ease PLS settings and flow

Browse Source 
This commit applies multiples things:
- Fix PLS settings that were not loaded correctly (was not using technical field names)
- Add data for PLS settings to check all optional fields and set the start date.
- Set date pls field required
- use null value instead of empty for phone and email state fields to lessen the DB storage
- Remove Flanker that crashes anyway while validating a simple email address.
- Handle stage_id = False
- Remove default value for lead probability -> To let Bayes makes the magic.
- Set default pls date to today - 8 days to include all demo data.
- Set won should give 100% predicted probability

See sub commits for more details.

Task ID:2044539

closes odoo/odoo#37413

Signed-off-by: Thibault Delavallee (tde) <tde@openerp.com>
This commit is contained in:
Odoo's Mergebot
2019-09-25 19:33:09 +02:00
committed by GitHub
parent 403bf9e76f 9f6540c41e
commit 458355405f
6 changed files with 77 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/crm/data/crm_lead_prediction_data.xml
+12
View File
@@ -20,6 +20,18 @@
<record id="frequency_field_lang_id" model="crm.lead.scoring.frequency.field">
<field name="field_id" eval="ref('crm.field_crm_lead__lang_id')"/>
</record>
<record id="crm_pls_fields_param" model="ir.config_parameter">
<field name="key">crm.pls_fields</field>
<field name="value" eval="'state_id,country_id,phone_state,email_state,source_id'"/>
</record>
<record id="crm_pls_start_date_param" model="ir.config_parameter">
<field name="key">crm.pls_start_date</field>
<field name="value" eval="(datetime.now() - timedelta(days=8)).strftime('%Y-%m-%d')"/>
</record>
<record id="crm_pls_rebuild_threshold_param" model="ir.config_parameter">
<field name="key">crm.pls_rebuild_threshold</field>
<field name="value">100</field>
</record>
</data>
<record id="website_crm_score_cron" model="ir.cron">
addons/crm/models/crm_lead.py
+56 -19
View File
@@ -105,17 +105,15 @@ class Lead(models.Model):
date_conversion = fields.Datetime('Conversion Date', readonly=True)
# Probability - Only used for type opportunity
probability = fields.Float('Probability', group_operator="avg", default=10.0)
probability = fields.Float('Probability', group_operator="avg")
automated_probability = fields.Float('Automated Probability', readonly=True)
is_automated_probability = fields.Boolean('Is automated probability?', compute="_compute_is_automated_probability")
phone_state = fields.Selection([
('correct', 'Correct'),
('incorrect', 'Incorrect'),
('empty', 'Empty')], string='Phone Quality', default="empty", compute="_compute_phone_state", store=True)
('incorrect', 'Incorrect')], string='Phone Quality', compute="_compute_phone_state", store=True)
email_state = fields.Selection([
('correct', 'Correct'),
('incorrect', 'Incorrect'),
('empty', 'Empty')], string='Email Quality', default="empty", compute="_compute_email_state", store=True)
('incorrect', 'Incorrect')], string='Email Quality', compute="_compute_email_state", store=True)
# Only used for type opportunity
planned_revenue = fields.Monetary('Expected Revenue', currency_field='company_currency', tracking=True)
@@ -222,7 +220,7 @@ class Lead(models.Model):
@api.depends('phone', 'country_id.code')
def _compute_phone_state(self):
for lead in self:
phone_status = 'empty'
phone_status = False
if lead.phone:
country_code = lead.country_id.code if lead.country_id and lead.country_id.code else None
try:
@@ -235,11 +233,11 @@ class Lead(models.Model):
@api.depends('email_from')
def _compute_email_state(self):
for lead in self:
email_state = 'empty'
email_state = False
if lead.email_from:
email_state = 'incorrect'
for email in email_split(lead.email_from):
if tools.email_validate(email):
if tools.email_normalize(email):
email_state = 'correct'
break
lead.email_state = email_state
@@ -483,9 +481,24 @@ class Lead(models.Model):
# Actions Methods
# ----------------------------------------
def _rebuild_pls_frequency_table_threshold(self):
""" Called by action_set_lost and action_set_won.
Will run the cron to update the frequency table only if the number of lead is above
a specified value (from config_parameter) for onboarding purpose.
Once the threshold is reached, the config_param is set to 0 to avoid re-run the cron
and, mainly, to avoid making useless search_count in the future."""
pls_threshold = int(self.env['ir.config_parameter'].sudo().get_param('crm.pls_rebuild_threshold'))
if pls_threshold:
lead_count = self.env['crm.lead'].sudo().search_count([])
if lead_count < pls_threshold:
self.sudo()._cron_update_automated_probabilities()
else:
self.env['ir.config_parameter'].sudo().set_param('crm.pls_rebuild_threshold', 0)
def action_set_lost(self, **additional_values):
""" Lost semantic: probability = 0 or active = False """
result = self.write({'active': False, 'probability': 0, **additional_values})
self._rebuild_pls_frequency_table_threshold()
return result
def action_set_won(self):
@@ -493,6 +506,7 @@ class Lead(models.Model):
for lead in self:
stage_id = lead._stage_find(domain=[('is_won', '=', True)])
lead.write({'stage_id': stage_id.id, 'probability': 100})
self._rebuild_pls_frequency_table_threshold()
return True
def action_set_automated_probability(self):
@@ -1320,6 +1334,7 @@ class Lead(models.Model):
# get stages
first_stage_id = self.env['crm.stage'].search([], order='sequence', limit=1)
won_stage_ids = self.env['crm.stage'].search([('is_won', '=', True)]).ids
# Get all leads values, no matter the team_id
leads_values_dict = self._pls_get_lead_pls_values(batch_mode=batch_mode)
@@ -1328,8 +1343,11 @@ class Lead(models.Model):
# Get unique couples to search in frequency table
leads_values = set()
for values in leads_values_dict.values():
won_leads = set()
for lead_id, values in leads_values_dict.items():
for couple in values['values']:
if couple[0] == 'stage_id' and couple[1] in won_stage_ids:
won_leads.add(lead_id)
leads_values.add(couple)
# get all variable related records from frequency table, no matter the team_id
@@ -1374,6 +1392,16 @@ class Lead(models.Model):
save_team_id = None
p_won, p_lost = 1, 1
for lead_id, lead_values in leads_values_dict.items():
# if stage_id is null, return 0 and bypass computation
lead_fields = [value[0] for value in lead_values.get('values', [])]
if not 'stage_id' in lead_fields:
lead_probabilities[lead_id] = 0
continue
# if lead stage is won, return 100
elif lead_id in won_leads:
lead_probabilities[lead_id] = 100
continue
lead_team_id = lead_values['team_id'] if lead_values['team_id'] else 0 # team_id = None -> Convert to 0
lead_team_id = lead_team_id if lead_team_id in result else -1 # team_id not in frequency Table -> convert to -1
if lead_team_id != save_team_id:
@@ -1428,15 +1456,24 @@ class Lead(models.Model):
# Recompute all the leads. Won : probability = 100 | Lost : probability = 0 or inactive
# Here, inactive won't be returned anyway
# Get also all the lead without probability --> These are the new leads. Activate auto probability on them.
pls_start_date = self._pls_get_safe_start_date()
if pls_start_date:
pending_lead_domain = [('probability', '<', 100), ('probability', '>', 0), ('create_date', '>', pls_start_date)]
pending_lead_domain = ['&', '&', ('stage_id', '!=', False), ('create_date', '>', pls_start_date),
'|', ('probability', '=', False), '&', ('probability', '<', 100), ('probability', '>', 0)]
leads_to_update = self.env['crm.lead'].search(pending_lead_domain)
lead_probabilities = leads_to_update._pls_get_naive_bayes_probabilities(batch_mode=True)
# Update in execute batch to avoid server roundtrips + page_size to 10000 to avoid memory errors
sql = "update crm_lead set automated_probability = %s where id = %s"
batch_params = [(lead_probabilities[lead.id], lead.id) for lead in leads_to_update if lead.id in lead_probabilities]
# Update both probability and automated_probability if they were equal, else, update only automated_probability
sql = """UPDATE crm_lead
SET automated_probability = %s,
probability = CASE WHEN (ROUND(probability::numeric, 2) = ROUND(automated_probability::numeric, 2) or probability is null)
THEN (%s)
ELSE (probability)
END
WHERE id = %s"""
batch_params = [(lead_probabilities[lead.id], lead_probabilities[lead.id], lead.id) for lead in leads_to_update if lead.id in lead_probabilities]
extras.execute_batch(self._cr, sql, batch_params, page_size=10000)
_logger.info("Predictive Lead Scoring : all automated probability updated (count: %d)" % (len(leads_to_update)))
@@ -1464,7 +1501,7 @@ class Lead(models.Model):
that are defined on the model. """
pls_fields_config = self.env['ir.config_parameter'].sudo().get_param('crm.pls_fields')
pls_fields = pls_fields_config.split(',') if pls_fields_config else []
pls_safe_fields = [field for field in pls_fields if field in self._fields]
pls_safe_fields = [field for field in pls_fields if field in self._fields.keys()]
return pls_safe_fields
# Rebuild Frequency Table Tools
@@ -1525,7 +1562,7 @@ class Lead(models.Model):
lost = result['count'] if result['probability'] == 0 else 0
for field in fields:
value = result[field]
if value:
if value or field in ('email_state', 'phone_state'):
if field == 'stage_id':
if won: # increment all stages if won
stages_to_increment = [stage['id'] for stage in stage_ids]
@@ -1600,7 +1637,7 @@ class Lead(models.Model):
self.flush(['probability'])
query = """SELECT id, %s
FROM crm_lead l
WHERE probability > 0 AND probability < 100 AND active = True AND id in %%s order by team_id asc"""
WHERE ((probability > 0 AND probability < 100) OR probability is null) AND active = True AND id in %%s order by team_id asc"""
query = sql.SQL(query % str_fields).format(*args)
self._cr.execute(query, [tuple(self.ids)])
@@ -1610,7 +1647,7 @@ class Lead(models.Model):
FROM crm_lead l
LEFT JOIN crm_lead_tag_rel rel ON l.id = rel.lead_id
LEFT JOIN crm_lead_tag t ON rel.tag_id = t.id
WHERE l.probability > 0 AND l.probability < 100 AND l.active = True AND l.id in %s order by l.team_id asc"""
WHERE ((l.probability > 0 AND l.probability < 100) OR l.probability is null) AND l.active = True AND l.id in %s order by l.team_id asc"""
self._cr.execute(query, [tuple(self.ids)])
tag_results = self._cr.dictfetchall()
@@ -1621,7 +1658,7 @@ class Lead(models.Model):
if field == 'team_id': # ignore team_id as stored separately in leads_values_dict[lead_id][team_id]
continue
value = lead[field]
if value:
if value or field in ('email_state', 'phone_state'):
lead_values.append((field, value))
leads_values_dict[lead['id']] = {'values': lead_values, 'team_id': lead['team_id']}
@@ -1636,7 +1673,7 @@ class Lead(models.Model):
if field == 'team_id': # ignore team_id as stored separately in leads_values_dict[lead_id][team_id]
continue
value = lead[field].id if isinstance(lead[field], models.BaseModel) else lead[field]
if value:
if value or field in ('email_state', 'phone_state'):
lead_values.append((field, value))
for tag in lead.tag_ids:
lead_values.append(('tag_id', tag.id))
@@ -1652,7 +1689,7 @@ class Lead(models.Model):
:param first_stage_id: stage with smallest sequence
:return: won count, lost count and total count for all records in frequencies
"""
if str(first_stage_id.id) not in team_results['stage_id']:
if str(first_stage_id.id) not in team_results.get('stage_id', []):
return 0, 0, 0
stage_result = team_results['stage_id'][str(first_stage_id.id)]
return stage_result['won'], stage_result['lost'], stage_result['won'] + stage_result['lost']
addons/crm/models/res_config_settings.py
+4 -3
View File
@@ -16,7 +16,7 @@ class ResConfigSettings(models.TransientModel):
module_crm_iap_lead_enrich = fields.Boolean("Enrich your leads automatically with company data based on their email address.")
lead_mining_in_pipeline = fields.Boolean("Create a lead mining request directly from the opportunity pipeline.", config_parameter='crm.lead_mining_in_pipeline')
predictive_lead_scoring_start_date = fields.Date(string='Lead Scoring Starting Date', compute="_compute_pls_start_date", inverse="_inverse_pls_start_date_str")
predictive_lead_scoring_start_date_str = fields.Char(string='Lead Scoring Starting Date in String', default=date.today().strftime('%Y-%m-%d'), config_parameter='crm.pls_start_date')
predictive_lead_scoring_start_date_str = fields.Char(string='Lead Scoring Starting Date in String', config_parameter='crm.pls_start_date')
predictive_lead_scoring_fields = fields.Many2many('crm.lead.scoring.frequency.field', string='Lead Scoring Frequency Fields', compute="_compute_pls_fields", inverse="_inverse_pls_fields_str")
predictive_lead_scoring_fields_str = fields.Char(string='Lead Scoring Frequency Fields in String', config_parameter='crm.pls_fields')
@@ -39,7 +39,8 @@ class ResConfigSettings(models.TransientModel):
for setting in self:
if setting.predictive_lead_scoring_fields_str:
names = setting.predictive_lead_scoring_fields_str.split(',')
setting.predictive_lead_scoring_fields = self.env['crm.lead.scoring.frequency.field'].search([('name', 'in', names)])
fields = self.env['ir.model.fields'].search([('name', 'in', names), ('model', '=', 'crm.lead')])
setting.predictive_lead_scoring_fields = self.env['crm.lead.scoring.frequency.field'].search([('field_id', 'in', fields.ids)])
else:
setting.predictive_lead_scoring_fields = None
@@ -48,7 +49,7 @@ class ResConfigSettings(models.TransientModel):
we store the fields with a comma separated string into a Char config field """
for setting in self:
if setting.predictive_lead_scoring_fields:
setting.predictive_lead_scoring_fields_str = ','.join(setting.predictive_lead_scoring_fields.mapped('name'))
setting.predictive_lead_scoring_fields_str = ','.join(setting.predictive_lead_scoring_fields.mapped('field_id.name'))
else:
setting.predictive_lead_scoring_fields_str = ''
addons/crm/views/res_config_settings_views.xml
+1 -1
View File
@@ -66,7 +66,7 @@
<div class="mt16">
Consider leads created as of the
<field name="predictive_lead_scoring_start_date_str" invisible="1"/>
<field name="predictive_lead_scoring_start_date"/>
<field name="predictive_lead_scoring_start_date" required="1"/>
</div>
</div>
</div>
addons/crm_iap_lead_enrich/models/crm_lead.py
+4 -2
View File
@@ -19,10 +19,11 @@ class Lead(models.Model):
@api.model
def _iap_enrich_leads_cron(self):
timeDelta = fields.datetime.now() - datetime.timedelta(hours=1)
# Get all leads not lost nor won (lost: active = False)
leads = self.search([
('iap_enrich_done', '=', False),
('reveal_id', '=', False),
('probability', 'not in', (0, 100)),
('probability', '<', 100),
('create_date', '>', timeDelta)
])
leads._iap_enrich(from_cron=True)
@@ -30,7 +31,8 @@ class Lead(models.Model):
def _iap_enrich(self, from_cron=False):
lead_emails = {}
for lead in self:
if lead.probability in (0, 100) or lead.iap_enrich_done:
# If lead is lost, active == False, but is anyway removed from the search in the cron.
if lead.probability == 100 or lead.iap_enrich_done:
continue
normalized_email = tools.email_normalize(lead.partner_address_email) or tools.email_normalize(lead.email_from)
if normalized_email:
odoo/tools/mail.py
-35
View File
@@ -20,31 +20,6 @@ from odoo.tools import misc
_logger = logging.getLogger(__name__)
# Optional Flanker dependency for email validation.
_flanker_warning = False
try:
from flanker.addresslib import address
if hasattr(address, 'six'):
# Python 3 supported
def checkmail(mail):
return bool(address.validate_address(mail))
else:
def checkmail(mail):
global _flanker_warning
if not _flanker_warning:
_logger.info("Flanker version 0.9 or greater required for Python 3 compatibility")
_flanker_warning = True
return True
except ImportError:
_logger.info('The flanker Python module is not installed, so email validation with flanker is unavailable')
def checkmail(mail):
global _flanker_warning
if not _flanker_warning:
_logger.info('The flanker Python module is not installed, so email validation with flanker is unavailable')
_flanker_warning = True
return True
#----------------------------------------------------------
# HTML Sanitizer
#----------------------------------------------------------
@@ -531,16 +506,6 @@ def email_normalize(text):
return False
return emails[0].lower()
def email_validate(email):
"""
Check is the email is valid using email normalization + optional Flanker module.
If Flanker is installed, it will check if email is valid (checking DNS and other stuff).
If Flanker is not installed, this method only normalizes the email
:param text: string containing the email to validate
:return: normalized email if mail is valid or False
"""
return email_normalize(email) if checkmail(email) else False
def email_escape_char(email_address):
""" Escape problematic characters in the given email address string"""
return email_address.replace('\\', '\\\\').replace('%', '\\%').replace('_', '\\_')