[ADD] crm_iap_lead_enrich: enrich leads based on email address

PURPOSE Allow the user to enrich a lead based on the domain of the email address of the client. SPECIFICATIONS Following fields will be populated if found using clearbit and if they are not yet populated : * description * partner_name * reveal_id * website * street, street2, zip, city, country_id, state_id * phone (using the first number found in clearbit) * mobile (using the second number found in clearbit, else the same as phone) A message will also be logged in the chatter. If no data is found using clearbit no credit is used and a message will be logged. If the user has no credits he will be redirected to buy credits for the service. It is possible to enrich using the 'enrich' action in listview or with a cron that will run every hour on leads not older than 1 hour. Add a "Lead Enrichment" setting in CRM installing lead enrichment. LINKS Task 193185 closes odoo/odoo#36419 Signed-off-by: Thibault Delavallee (tde) <tde@openerp.com>
2019-09-05 15:17:14 +00:00
parent e2c1226d91
commit 1a36d98e5f
16 changed files with 486 additions and 2 deletions
@@ -13,6 +13,7 @@ class ResConfigSettings(models.TransientModel):
    group_use_lead = fields.Boolean(string="Leads", implied_group='crm.group_use_lead')
    module_crm_iap_lead = fields.Boolean("Generate new leads based on their country, industries, size, etc.")
    module_crm_iap_lead_website = fields.Boolean("Create Leads/Opportunities from your website's traffic")
+    module_crm_iap_lead_enrich = fields.Boolean("Enrich your leads automatically with company data based on their email address.")
    lead_mining_in_pipeline = fields.Boolean("Create a lead mining request directly from the opportunity pipeline.", config_parameter='crm.lead_mining_in_pipeline')
    predictive_lead_scoring_start_date = fields.Date(string='Lead Scoring Starting Date', compute="_compute_pls_start_date", inverse="_inverse_pls_start_date_str")
    predictive_lead_scoring_start_date_str = fields.Char(string='Lead Scoring Starting Date in String', default=date.today().strftime('%Y-%m-%d'), config_parameter='crm.pls_start_date')
@@ -85,6 +85,20 @@
                                </div>
                            </div>
                        </div>
+                        <div class="col-lg-6 o_setting_box">
+                            <div class="o_setting_left_pane">
+                                <field name="module_crm_iap_lead_enrich"/>
+                            </div>
+
+                            <div class="o_setting_right_pane" id="crm_iap_lead_enrich">
+                                <label string="Lead Enrichment" for="module_crm_iap_lead_enrich"/>
+                                <div class="text-muted">
+                                    Enrich your leads automatically with company data based on their email address
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="row mt16 o_settings_container">
                        <div class="col-lg-6 o_setting_box">
                            <div class="o_setting_left_pane">
                                <field name="module_crm_iap_lead"/>
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from . import models
+from . import tests
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+{
+    'name': 'Lead Enrichment',
+    'summary': 'Enrich Leads/Opportunities using email address domain',
+    'category': 'Sales/CRM',
+    'depends': [
+        'iap',
+        'crm',
+    ],
+    'data': [
+        'data/ir_cron.xml',
+        'data/mail_data.xml',
+        'views/res_config_settings_views.xml',
+    ],
+}
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record id="ir_cron_lead_enrichment" model="ir.cron">
+        <field name="name">CRM: enrich leads (IAP)</field>
+        <field name="model_id" ref="crm.model_crm_lead"/>
+        <field name="user_id" ref="base.user_root"/>
+        <field name="state">code</field>
+        <field name="code">model._iap_enrich_leads_cron()</field>
+        <field name="interval_number">1</field>
+        <field name="interval_type">hours</field>
+        <field name="numbercall">-1</field>
+        <field name="doall" eval="False"/>
+    </record>
+
+</odoo>
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo><data noupdate="1">
+    <!-- MAIL TEMPLATE -->
+    <record id="mail_template_data_iap_lead_enrich_nocredit" model="mail.template">
+        <field name="name">CRM: IAP Lead Enrichment No Credit</field>
+        <field name="subject">IAP Lead Enrichment Notification</field>
+        <field name="model_id" ref="iap.model_iap_account"/>
+        <field name="email_from">iap@odoo.com</field>
+        <field name="email_to">${object.create_uid.email}</field>
+        <field name="body_html" type="html">
+<div style="margin: 0px; padding: 0px;">
+    <p>Dear ${object.create_uid.name or 'Madam/Sir'},</p><br/>
+    <p>Unfortunately, there are no more credits on your IAP Lead Enrichment account.<br/>
+    You can charge it back from the Settings of the CRM app or from your IAP portal.<br/></p><br/>
+    <p>Best regards,</p><br/>
+    <p>Odoo S.A.</p>
+</div>
+        </field>
+        <field name="lang">${object.create_uid.lang}</field>
+    </record>
+
+    <!-- VIEWS USED FOR MESSAGING -->
+    <template id="mail_message_lead_enrich_notfound">
+        <p>Lead Enrichment based on email address</p>
+        <div style="background-color:#ffffff;padding:15px;">
+            <span> No company data found based on the email address or email address is one of an email provider. No credit was consumed. </span>
+        </div>
+    </template>
+
+    <template id="mail_message_lead_enrich_no_email">
+        <p>Lead Enrichment based on email address</p>
+        <div style="background-color:#ffffff;padding:15px;">
+            <span>Enrichment could not be done as no email address was provided.</span>
+        </div>
+    </template>
+
+    <template id="mail_message_lead_enrich_no_credit">
+        <p>Lead enriched based on email address</p>
+        <div style="background-color:#ffffff;padding:15px;">
+            <span>Your balance for Lead Enrichment is insufficient. Please go to your <a t-attf-href="{{url}}" target="_blank">iap account</a> to buy credits.</span>
+        </div>
+    </template>
+
+    <template id="mail_message_lead_enrich_with_data">
+        <p>
+            <span>Lead enriched based on email address</span>
+        </p>
+        <div style="background-color:#ffffff;padding:15px;">
+            <table style="width:100%">
+                <tr valign="top">
+                    <td></td>
+                    <td rowspan="3">
+                        <img style="max-width: 80px;" t-attf-src="{{logo}}"/>
+                    </td>
+                </tr>
+                <tr>
+                    <td style="padding: 0 0 5px 0;">
+                        <span style="font-size:25px;vertical-align: middle;">
+                            <b><t t-esc="object.partner_name"/></b>
+                        </span>
+                        <span style="">
+                            <a t-if="twitter" style="margin: 0 0 0 8px;" target="_blank" t-attf-href="http://www.twitter.com/{{twitter}}">
+                                <img src="/web_editor/font_to_img/61569/rgb(0,132,180)/22"/>
+                            </a>
+                            <a t-if="facebook" style="margin: 0 0 0 8px;" target="_blank" t-attf-href="http://www.facebook.com/{{facebook}}">
+                                <img src="/web_editor/font_to_img/61570/rgb(59,89,152)/22"/>
+                            </a>
+                            <a t-if="linkedin" style="margin: 0 0 0 8px;" target="_blank" t-attf-href="https://www.linkedin.com/{{linkedin}}">
+                                <img src="/web_editor/font_to_img/61580/rgb(0,119,181)/22"/>
+                            </a>
+                            <a t-if="crunchbase" style="margin: 0 0 0 8px;" target="_blank" t-attf-href="https://www.crunchbase.com/{{crunchbase}}">
+                                <img width="18px" height="18px" src="/crm_iap_lead/static/img/crunchbase.ico"/>
+                            </a>
+                        </span>
+                    </td>
+                </tr>
+                <tr>
+                    <td style="padding: 0 0 5px 0;">
+                        <span><t t-esc="object.description"/></span>
+                    </td>
+                </tr>
+                <tr>
+                    <td t-if="timezone" style="padding: 0 0 5px 0;">
+                        <img src="/web_editor/font_to_img/61463/rgb(135,90,123)/16"/>
+                        <b>Timezone : </b>
+                        <span><t t-esc="timezone"/></span>
+                        <span>
+                            <a target="_blank" t-att-href="timezone_url">(Time Now)</a>
+                        </span>
+                    </td>
+                </tr>
+                <tr>
+                    <td t-if="phone_numbers" style="padding: 0 0 5px 0;">
+                        <img src="/web_editor/font_to_img/61589/rgb(135,90,123)/16"/>
+                        <b>Phone :</b>
+                        <t t-foreach="phone_numbers" t-as="phone_number">
+                            <span style="margin:0 5px;">
+                                <a target="_blank" t-attf-href="tel:{{phone_number}}"><t t-esc="phone_number"/></a>
+                            </span>
+                        </t>
+                    </td>
+                </tr>
+                <tr>
+                    <td t-if="tech" colspan="2">
+                        <b><img src="/web_editor/font_to_img/61874/rgb(135,90,123)/16"/> Technology Used :</b>
+                        <t t-foreach="tech" t-as="t">
+                            <label style="font-weight:normal;padding: 2px 10px;background-color: #eeeeee;margin: 0 0 4px 2px;border-radius: 13px;">
+                                <t t-esc="t"/>
+                            </label>
+                        </t>
+                    </td>
+                </tr>
+            </table>
+        </div>
+    </template>
+</data></odoo>
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from . import crm_lead
+from . import iap_enrich_api
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+import datetime
+import logging
+
+from odoo import api, fields, models, tools
+from odoo.addons.iap import InsufficientCreditError
+
+_logger = logging.getLogger(__name__)
+
+
+class Lead(models.Model):
+    _inherit = 'crm.lead'
+
+    reveal_id = fields.Char(string='Reveal ID', index=True)
+    iap_enrich_done = fields.Boolean(string='Enrichment done', help='Whether IAP service for lead enrichment based on email has been performed on this lead.')
+
+    @api.model
+    def _iap_enrich_leads_cron(self):
+        timeDelta = fields.datetime.now() - datetime.timedelta(hours=1)
+        leads = self.search([
+            ('iap_enrich_done', '=', False),
+            ('reveal_id', '=', False),
+            ('probability', 'not in', ('0', 100)),
+            ('create_date', '>', timeDelta)
+        ])
+        leads._iap_enrich(from_cron=True)
+
+    def _iap_enrich(self, from_cron=False):
+        lead_emails = {}
+        for lead in self:
+            if lead.probability in (0, 100) or lead.iap_enrich_done:
+                continue
+            normalized_email = tools.email_normalize(lead.partner_address_email) or tools.email_normalize(lead.email_from)
+            if normalized_email:
+                lead_emails[lead.id] = normalized_email.split('@')[1]
+            else:
+                lead.message_post_with_view(
+                    'crm_iap_lead_enrich.mail_message_lead_enrich_no_email',
+                    subtype_id=self.env.ref('mail.mt_note').id)
+
+        if lead_emails:
+            try:
+                iap_response = self.env['iap.enrich.api']._request_enrich(lead_emails)
+            except InsufficientCreditError:
+                _logger.info('Sent batch %s enrich requests: failed because of credit', len(lead_emails))
+                self._iap_enrich_notify_no_more_credit()
+                if not from_cron:
+                    data = {
+                        'url': self.env['iap.account'].get_credits_url('reveal'),
+                    }
+                    self[0].message_post_with_view(
+                        'crm_iap_lead_enrich.mail_message_lead_enrich_no_credit',
+                        values=data,
+                        subtype_id=self.env.ref('mail.mt_note').id)
+            except Exception as e:
+                _logger.info('Sent batch %s enrich requests: failed with exception %s', len(lead_emails), e)
+            else:
+                _logger.info('Sent batch %s enrich requests: success', len(lead_emails))
+                self._iap_enrich_from_response(iap_response)
+                self.env['ir.config_parameter'].sudo().set_param('crm_iap_lead_enrich.credit_notification', False)
+
+    @api.model
+    def _iap_enrich_from_response(self, iap_response):
+        """ Handle from the service and enrich the lead accordingly
+
+        :param iap_response: dict{lead_id: company data or False}
+        """
+        for lead in self.search([('id', 'in', list(iap_response.keys()))]):  # handle unlinked data by performing a search
+            iap_data = iap_response.get(lead.id)
+            if not iap_data:
+                lead.message_post_with_view('crm_iap_lead_enrich.mail_message_lead_enrich_notfound', subtype_id=self.env.ref('mail.mt_note').id)
+                continue
+
+            values = {'iap_enrich_done': True}
+            lead_fields = ['description', 'partner_name', 'reveal_id', 'street', 'city', 'zip']
+            iap_fields = ['description', 'name', 'clearbit_id', 'location', 'city', 'city', 'postal_code']
+            for lead_field, iap_field in zip(lead_fields, iap_fields):
+                if not lead[lead_field] and iap_data.get(iap_field):
+                    values[lead_field] = iap_data[iap_field]
+
+            if not lead.phone and iap_data.get('phone_numbers'):
+                values['phone'] = iap_data['phone_numbers'][0]
+            if not lead.mobile and iap_data.get('phone_numbers') and len(iap_data['phone_numbers']) > 1:
+                values['mobile'] = iap_data['phone_numbers'][1]
+            if not lead.country_id and iap_data.get('country_code'):
+                country = self.env['res.country'].search([('code', '=', iap_data['country_code'].upper())])
+                values['country_id'] = country.id
+            else:
+                country = lead.country_id
+            if not lead.state_id and country and iap_data.get('state_code'):
+                state = self.env['res.country.state'].search([
+                    ('code', '=', iap_data['state_code']),
+                    ('country_id', '=', country.id)
+                ])
+                values['state_id'] = state.id
+
+            lead.write(values)
+            lead.message_post_with_view(
+                'crm_iap_lead_enrich.mail_message_lead_enrich_with_data',
+                values=lead._iap_enrich_get_message_data(iap_data),
+                subtype_id=self.env.ref('mail.mt_note').id
+            )
+
+    def _iap_enrich_get_message_data(self, company_data):
+        log_data = {
+            'twitter': company_data.get('twitter'),
+            'logo': company_data.get('logo'),
+            'phone_numbers': company_data.get('phone_numbers'),
+            'facebook': company_data.get('facebook'),
+            'linkedin': company_data.get('linkedin'),
+            'crunchbase': company_data.get('crunchbase'),
+            'tech': [t.replace('_', ' ').title() for t in company_data.get('tech', [])],
+        }
+        timezone = company_data.get('timezone')
+        if timezone:
+            log_data.update({
+                'timezone': timezone.replace('_', ' ').title(),
+                'timezone_url': company_data.get('timezone_url'),
+            })
+        return log_data
+
+    @api.model
+    def _iap_enrich_notify_no_more_credit(self):
+        """ Notify when user has no credits anymore. In order to avoid to spam
+        people each hour, an ir.config_parameter is set. """
+        already_notified = self.env['ir.config_parameter'].sudo().get_param('crm_iap_lead_enrich.credit_notification')
+        if already_notified == 'True':
+            return
+
+        iap_account = self.env['iap.account'].search([('service_name', '=', 'reveal')], limit=1)
+        if not iap_account:
+            return
+
+        mail_template = self.env.ref('crm_iap_lead_enrich.mail_template_data_iap_lead_enrich_nocredit')
+        if not mail_template:
+            return
+
+        mail_template.sudo().send_mail(iap_account.id, force_send=False, notif_layout='mail.mail_notification_light')
+        self.env['ir.config_parameter'].sudo().set_param('lead_enrich.already_notified', True)
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo import models, api
+from odoo.addons import iap
+
+
+class IapEnrichAPI(models.AbstractModel):
+    _name = 'iap.enrich.api'
+    _description = 'IAP Lead Enrichment API'
+    _DEFAULT_ENDPOINT = 'https://iap-services.odoo.com'
+
+    @api.model
+    def _contact_iap(self, local_endpoint, params):
+        account = self.env['iap.account'].get('reveal')
+        dbuuid = self.env['ir.config_parameter'].sudo().get_param('database.uuid')
+        params['account_token'] = account.account_token
+        params['dbuuid'] = dbuuid
+        return iap.jsonrpc(self._DEFAULT_ENDPOINT + local_endpoint, params=params, timeout=300)
+
+    @api.model
+    def _request_enrich(self, lead_emails):
+        """ Contact endpoint to get enrichment data.
+
+        :param lead_emails: dict{lead_id: email}
+        :return: dict{lead_id: company data or False}
+        :raise: several errors, notably
+          * InsufficientCreditError: {
+            "credit": 4.0,
+            "service_name": "reveal",
+            "base_url": "https://iap.odoo.com/iap/1/credit",
+            "message": "You don't have enough credits on your account to use this service."
+            }
+        """
+        params = {
+            'domains': lead_emails,
+        }
+        return self._contact_iap('/iap/clearbit/1/lead_enrichment_email', params=params)
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from . import common
+from . import test_lead_enrich
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+
+from contextlib import contextmanager
+from functools import partial
+from unittest.mock import patch
+
+from odoo import exceptions
+from odoo.addons import iap
+from odoo.addons.crm_iap_lead_enrich.models.iap_enrich_api import IapEnrichAPI
+from odoo.tests import common, new_test_user
+
+crm_new_test_user = partial(new_test_user, context={'mail_create_nolog': True, 'mail_create_nosubscribe': True, 'mail_notrack': True, 'no_reset_password': True})
+
+
+class MockIAPEnrich(common.BaseCase):
+
+    @contextmanager
+    def mockIAPEnrichGateway(self, default_data=None, email_data=None, sim_error=None, failing_emails=None):
+
+        def _contact_iap(local_endpoint, params):
+            sim_result = {
+                'name': 'Simulator INC',
+                'location': 'Simulator Street',
+                'city': 'SimCity',
+                'postal_code': '9876',
+                'country_code': 'BE',
+                'clearbit_id': 'idontknow',
+                'phone_numbers': ['+3269001122', '+32456001122'],
+                'twitter': 'testtwitter',
+                'facebook': 'testfacebook',
+            }
+            if default_data:
+                sim_result.update(default_data)
+            # mock single sms sending
+            if local_endpoint == '/iap/clearbit/1/lead_enrichment_email':
+                result = {}
+                for lead_id, email in params['domains'].items():
+                    if sim_error and sim_error == 'credit':
+                        raise iap.InsufficientCreditError('InsufficientCreditError')
+                    elif sim_error and sim_error == 'jsonrpc_exception':
+                        raise exceptions.AccessError(
+                            'The url that this service requested returned an error. Please contact the author of the app. The url it tried to contact was ' + local_endpoint
+                        )
+                    result[lead_id] = dict(sim_result)
+                    if email_data and email_data.get(email):
+                        result[lead_id].update(email_data[email])
+                return result
+
+        try:
+            with patch.object(IapEnrichAPI, '_contact_iap', side_effect=_contact_iap) as contact_iap_mock:
+                yield
+        finally:
+            pass
+
+
+class CrmCase(common.SavepointCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(CrmCase, cls).setUpClass()
+
+        cls.sales_manager = crm_new_test_user(
+            cls.env, login='sales_manager', groups='base.group_user,sales_team.group_sale_manager',
+            name='Martine SalesManager', email='"Martine SalesManager" <martine@example.com>')
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+from odoo.addons.crm_iap_lead_enrich.tests import common as crm_iap_lead_enrich_common
+from odoo.tests.common import users
+
+
+class TestLeadEnrich(crm_iap_lead_enrich_common.CrmCase, crm_iap_lead_enrich_common.MockIAPEnrich):
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestLeadEnrich, cls).setUpClass()
+
+        cls.leads = cls.env['crm.lead']
+        for x in range(0, 4):
+            cls.leads += cls.env['crm.lead'].create({
+                'name': 'Test %s' % x,
+                'email_from': 'test_mail_%s@example.com' % x
+            })
+
+    @users('sales_manager')
+    def test_enrich_internals(self):
+        leads = self.env['crm.lead'].browse(self.leads.ids)
+        leads[0].write({'partner_name': 'Already set', 'email_from': 'test@test1'})
+        leads.flush()
+        with self.mockIAPEnrichGateway(email_data={'test1': {'country_code': 'AU', 'state_code': 'NSW'}}):
+            leads._iap_enrich()
+
+        leads.flush()
+        self.assertEqual(leads[0].partner_name, 'Already set')
+        self.assertEqual(leads[0].country_id, self.env.ref('base.au'))
+        self.assertEqual(leads[0].state_id, self.env.ref('base.state_au_2'))
+        for lead in leads[1:]:
+            self.assertEqual(lead.partner_name, 'Simulator INC')
+        for lead in leads:
+            self.assertEqual(lead.street, 'Simulator Street')
+
+    # @users('sales_manager')
+    # def test_enrich_error_credit(self):
+    #     leads = self.env['crm.lead'].browse(self.leads.ids)
+    #     with self.mockIAPEnrichGateway(sim_error='credit'):
+    #         leads._iap_enrich()
+
+    @users('sales_manager')
+    def test_enrich_error_jsonrpc_exception(self):
+        leads = self.env['crm.lead'].browse(self.leads.ids)
+        with self.mockIAPEnrichGateway(sim_error='jsonrpc_exception'):
+            leads._iap_enrich()
+
+        for lead in leads:
+            self.assertEqual(lead.street, False)
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record id="res_config_settings_view_form" model="ir.ui.view">
+        <field name="name">res.config.settings.view.form.inherit.crm.iap.lead.enrich</field>
+        <field name="model">res.config.settings</field>
+        <field name="inherit_id" ref="crm.res_config_settings_view_form"/>
+        <field name="arch" type="xml">
+            <div id="crm_iap_lead_enrich" position="inside">
+                <widget name="iap_buy_more_credits" service_name="reveal"/>
+            </div>
+        </field>
+    </record>
+</odoo>
@@ -46,7 +46,6 @@ def jsonrpc(url, method='call', params=None, timeout=15):
        'id': uuid.uuid4().hex,
    }

-
    _logger.info('iap jsonrpc %s', url)
    try:
        req = requests.post(url, json=payload, timeout=timeout)
@@ -20,7 +20,7 @@ CrashManager.include({
     */
    _getButtonMessage: function (isTrial){
        var isEnterprise = _.last(odoo.session_info.server_version_info) === 'e';
-        return isTrial && isEnterprise ? _t('Start a Trial at Odoo') : _t('Buy credits at Odoo');
+        return isTrial && isEnterprise ? _t('Start a Trial at Odoo') : _t('Buy credits');
    },
    /**
     * @override