pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMP] gamification,gamification_sale_crm: compute goals using batch mode

Browse Source 
Trying to improve the resources used by the challenge update cron
operation.

Before this commit some count-based goals and all sum-based goals were
computed with one select count or sum per user.

After this commit each count or sum based goal is computed in a single select
for all users. The goal form now also allows to enable batch mode for
sum computations. Other changes include:
- adding an index on the challenge_id of goals after verifying the
positive impact of such an index on high volumes on a staging server
- introducing additional intermediary commits to allow massive crons to
catch up over several attempts.

task-internal

closes odoo/odoo#63265

X-original-commit: 0f9411d10753aa96103ce77702dbdc0516e9969a
Signed-off-by: Jérémy Kersten (jke) <jke@openerp.com>
This commit is contained in:
Benoit Socias
2020-12-11 17:52:11 +00:00
committed by Jeremy Kersten
parent 7a5e08bdd3
commit 4402f873be
5 changed files with 81 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/gamification/data/goal_base.xml
+5 -2
View File
@@ -8,9 +8,12 @@
<field name="computation_mode">count</field>
<field name="display_mode">boolean</field>
<field name="model_id" ref="base.model_res_users"/>
<field name="domain">[('id','=',user.id),('partner_id.tz', '!=', False)]</field>
<field name="domain">[('partner_id.tz', '!=', False)]</field>
<field name="action_id" ref="base.action_res_users_my"/>
<field name="res_id_field">user.id</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="base.field_res_users__id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_base_company_data">
@@ -30,7 +33,7 @@
<field name="computation_mode">count</field>
<field name="display_mode">boolean</field>
<field name="model_id" ref="base.model_res_company"/>
<field name="domain">[('user_ids', 'in', user.id),('logo', '!=', False)]</field>
<field name="domain">[('user_ids', 'in', [user.id]),('logo', '!=', False)]</field>
<field name="action_id" ref="base.action_res_company_form"/>
<field name="res_id_field">user.company_id.id</field>
</record>
addons/gamification/models/challenge.py
+8 -3
View File
@@ -207,6 +207,10 @@ class Challenge(models.Model):
- Create the missing goals (eg: modified the challenge to add lines)
- Update every running challenge
"""
# in cron mode, will do intermediate commits
# cannot be replaced by a parameter because it is intended to impact side-effects of
# write operations
self = self.with_context(commit_gamification=commit)
# start scheduled challenges
planned_challenges = self.search([
('state', '=', 'draft'),
@@ -225,9 +229,7 @@ class Challenge(models.Model):
records = self.browse(ids) if ids else self.search([('state', '=', 'inprogress')])
# in cron mode, will do intermediate commits
# FIXME: replace by parameter
return records.with_context(commit_gamification=commit)._update_all()
return records._update_all()
def _update_all(self):
"""Update the challenges and related goals
@@ -391,6 +393,9 @@ class Challenge(models.Model):
to_update.update_goal()
if self.env.context.get('commit_gamification'):
self.env.cr.commit()
return True
##### JS utilities #####
addons/gamification/models/goal.py
+25 -11
View File
@@ -155,7 +155,7 @@ class Goal(models.Model):
user_id = fields.Many2one('res.users', string="User", required=True, auto_join=True, ondelete="cascade")
line_id = fields.Many2one('gamification.challenge.line', string="Challenge Line", ondelete="cascade")
challenge_id = fields.Many2one(
related='line_id.challenge_id', store=True, readonly=True,
related='line_id.challenge_id', store=True, readonly=True, index=True,
help="Challenge that generated the goal, assign challenge to users "
"to generate goals with a value in this field.")
start_date = fields.Date("Start Date", default=fields.Date.today)
@@ -291,11 +291,11 @@ class Goal(models.Model):
"of code for definition %s, expected a number",
result, definition.name)
else: # count or sum
elif definition.computation_mode in ('count', 'sum'): # count or sum
Obj = self.env[definition.model_id.model]
field_date_name = definition.field_date_id.name
if definition.computation_mode == 'count' and definition.batch_mode:
if definition.batch_mode:
# batch mode, trying to do as much as possible in one request
general_domain = ast.literal_eval(definition.domain)
field_name = definition.batch_distinctive_field.name
@@ -314,12 +314,22 @@ class Goal(models.Model):
if end_date:
subquery_domain.append((field_date_name, '<=', end_date))
if field_name == 'id':
# grouping on id does not work and is similar to search anyway
users = Obj.search(subquery_domain)
user_values = [{'id': user.id, 'id_count': 1} for user in users]
else:
user_values = Obj.read_group(subquery_domain, fields=[field_name], groupby=[field_name])
if definition.computation_mode == 'count':
value_field_name = field_name + '_count'
if field_name == 'id':
# grouping on id does not work and is similar to search anyway
users = Obj.search(subquery_domain)
user_values = [{'id': user.id, value_field_name: 1} for user in users]
else:
user_values = Obj.read_group(subquery_domain, fields=[field_name], groupby=[field_name])
else: # sum
value_field_name = definition.field_id.name
if field_name == 'id':
user_values = Obj.search_read(subquery_domain, fields=['id', value_field_name])
else:
user_values = Obj.read_group(subquery_domain, fields=[field_name, "%s:sum" % value_field_name], groupby=[field_name])
# user_values has format of read_group: [{'partner_id': 42, 'partner_id_count': 3},...]
for goal in [g for g in goals if g.id in query_goals]:
for user_value in user_values:
@@ -327,7 +337,7 @@ class Goal(models.Model):
if isinstance(queried_value, tuple) and len(queried_value) == 2 and isinstance(queried_value[0], int):
queried_value = queried_value[0]
if queried_value == query_goals[goal.id]:
new_value = user_value.get(field_name+'_count', goal.current)
new_value = user_value.get(value_field_name, goal.current)
goals_to_write.update(goal._get_write_values(new_value))
else:
@@ -343,7 +353,6 @@ class Goal(models.Model):
if definition.computation_mode == 'sum':
field_name = definition.field_id.name
# TODO for master: group on user field in batch mode
res = Obj.read_group(domain, [field_name], [])
new_value = res and res[0][field_name] or 0.0
@@ -352,6 +361,11 @@ class Goal(models.Model):
goals_to_write.update(goal._get_write_values(new_value))
else:
_logger.error(
"Invalid computation mode '%s' in definition %s",
definition.computation_mode, definition.name)
for goal, values in goals_to_write.items():
if not values:
continue
addons/gamification/views/goal.xml
+1 -1
View File
@@ -252,7 +252,7 @@
<field name="compute_code" attrs="{'invisible':[('computation_mode','!=','python')], 'required':[('computation_mode','=','python')]}"/>
<field name="condition" widget="radio"/>
</group>
<group string="Optimisation" name="optimisation" attrs="{'invisible': [('computation_mode', '!=', 'count')]}">
<group string="Optimisation" name="optimisation" attrs="{'invisible': [('computation_mode', 'not in', ('sum', 'count'))]}">
<field name="batch_mode" />
<div colspan="2">In batch mode, the domain is evaluated globally. If enabled, do not use keyword 'user' in above filter domain.</div>
<field name="batch_distinctive_field" attrs="{'invisible': [('batch_mode', '=', False)], 'required': [('batch_mode', '=', True)]}"
addons/gamification_sale_crm/data/gamification_sale_crm_data.xml
+42 -12
View File
@@ -10,7 +10,10 @@
<field name="model_id" ref="account.model_account_invoice_report"/>
<field name="field_id" ref="account.field_account_invoice_report__price_subtotal"/>
<field name="field_date_id" ref="account.field_account_invoice_report__invoice_date"/>
<field name="domain">[('state','!=','cancel'),('invoice_user_id','=',user.id),('move_type','=','out_invoice')]</field>
<field name="domain">[('state','!=','cancel'),('move_type','=','out_invoice')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="account.field_account_invoice_report__invoice_user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_nbr_new_leads">
@@ -21,7 +24,10 @@
<field name="model_id" ref="crm.model_crm_lead"/>
<field name="field_date_id" search="[('model','=','crm.lead'),('name','=','create_date')]" />
<!-- lead AND opportunity as don't want to be penalised for lead converted to opportunity -->
<field name="domain">[('user_id','=',user.id), '|', ('type', '=', 'lead'), ('type', '=', 'opportunity')]</field>
<field name="domain">['|', ('type', '=', 'lead'), ('type', '=', 'opportunity')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="crm.field_crm_lead__user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_lead_delay_open">
@@ -33,7 +39,10 @@
<field name="model_id" ref="crm.model_crm_lead"/>
<field name="field_id" ref="crm.field_crm_lead__day_close"/>
<field name="field_date_id" ref="crm.field_crm_lead__date_closed"/>
<field name="domain">[('user_id','=',user.id),('type', '=', 'lead')]</field>
<field name="domain">[('type', '=', 'lead')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="crm.field_crm_lead__user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_lead_delay_close">
@@ -45,7 +54,10 @@
<field name="model_id" ref="crm.model_crm_lead"/>
<field name="field_id" ref="crm.field_crm_lead__day_open"/>
<field name="field_date_id" ref="crm.field_crm_lead__date_open"/>
<field name="domain">[('user_id','=',user.id)]</field>
<field name="domain">[]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="crm.field_crm_lead__user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
@@ -56,7 +68,10 @@
<field name="suffix">opportunities</field>
<field name="model_id" ref="crm.model_crm_lead"/>
<field name="field_date_id" ref="crm.field_crm_lead__date_open"/>
<field name="domain">[('user_id','=',user.id),('type','=','opportunity')]</field>
<field name="domain">[('type','=','opportunity')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="crm.field_crm_lead__user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_nbr_sale_order_created">
@@ -66,7 +81,10 @@
<field name="suffix">orders</field>
<field name="model_id" ref="sale.model_sale_order"/>
<field name="field_date_id" ref="sale.field_sale_order__date_order"/>
<field name="domain">[('user_id','=',user.id),('state','not in',('draft', 'sent', 'cancel'))]</field>
<field name="domain">[('state','not in',('draft', 'sent', 'cancel'))]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="sale.field_sale_order__user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_nbr_paid_sale_order">
@@ -76,7 +94,10 @@
<field name="suffix">orders</field>
<field name="model_id" ref="account.model_account_invoice_report"/>
<field name="field_date_id" ref="account.field_account_invoice_report__invoice_date"/>
<field name="domain">[('payment_state','in',('paid', 'in_payment')),('invoice_user_id','=',user.id),('move_type','=','out_invoice')]</field>
<field name="domain">[('payment_state','in',('paid', 'in_payment')),('move_type','=','out_invoice')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="account.field_account_invoice_report__invoice_user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_tot_paid_sale_order">
<field name="name">Total Paid Sales Orders</field>
@@ -86,7 +107,10 @@
<field name="model_id" ref="account.model_account_invoice_report"/>
<field name="field_id" ref="account.field_account_invoice_report__price_subtotal"/>
<field name="field_date_id" ref="account.field_account_invoice_report__invoice_date"/>
<field name="domain">[('payment_state','in',('paid', 'in_payment')),('invoice_user_id','=',user.id),('move_type','=','out_invoice')]</field>
<field name="domain">[('payment_state','in',('paid', 'in_payment')),('move_type','=','out_invoice')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="account.field_account_invoice_report__invoice_user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
@@ -98,7 +122,10 @@
<field name="suffix">invoices</field>
<field name="model_id" ref="account.model_account_invoice_report"/>
<field name="field_date_id" ref="account.field_account_invoice_report__invoice_date"/>
<field name="domain">[('state','!=','cancel'),('invoice_user_id','=',user.id),('move_type','=','out_refund')]</field>
<field name="domain">[('state','!=','cancel'),('move_type','=','out_refund')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="account.field_account_invoice_report__invoice_user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
<record model="gamification.goal.definition" id="definition_crm_tot_customer_refunds">
<field name="name">Total Customer Credit Notes</field>
@@ -109,7 +136,10 @@
<field name="model_id" ref="account.model_account_invoice_report"/>
<field name="field_id" ref="account.field_account_invoice_report__price_subtotal"/>
<field name="field_date_id" ref="account.field_account_invoice_report__invoice_date"/>
<field name="domain">[('state','!=','cancel'),('invoice_user_id','=',user.id),('move_type','=','out_refund')]</field>
<field name="domain">[('state','!=','cancel'),('move_type','=','out_refund')]</field>
<field name="batch_mode">True</field>
<field name="batch_distinctive_field" ref="account.field_account_invoice_report__invoice_user_id"/>
<field name="batch_user_expression">user.id</field>
</record>
@@ -119,7 +149,7 @@
<field name="name">Monthly Sales Targets</field>
<field name="period">monthly</field>
<field name="visibility_mode">ranking</field>
<field name="user_domain" eval="str([('groups_id', 'in', ref('sales_team.group_sale_salesman'))])" />
<field name="user_domain" eval="str([('groups_id', 'in', [ref('sales_team.group_sale_salesman')])])" />
<field name="report_message_frequency">weekly</field>
</record>
@@ -127,7 +157,7 @@
<field name="name">Lead Acquisition</field>
<field name="period">monthly</field>
<field name="visibility_mode">ranking</field>
<field name="user_domain" eval="str([('groups_id', 'in', ref('sales_team.group_sale_salesman'))])" />
<field name="user_domain" eval="str([('groups_id', 'in', [ref('sales_team.group_sale_salesman')])])" />
<field name="report_message_frequency">weekly</field>
</record>