From aaefa35e29f7e87e8e914cf1d9eb68af7d26464a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= Date: Wed, 11 Mar 2020 12:42:48 +0000 Subject: [PATCH] [MOV] link_tracker, various: more link shortening tools to mail.render.mixin PURPOSE Move links shortening tools on mail.render.mixin to have rendering and post procesing tools available on that mixin. SPECIFICATIONS Move html and text links shortening methods on mail.render.mixin class. That way rendering and post processing is done in that mixin instead of being specific to link.tracker model. Indeed those methods are tools and not really business related methods. Future commits will improve the way mail and sms composer use the render mixin. This commit is therefore both a cleaning and preparing commit. LINKS Task ID 1963529 Community PR odoo/odoo#32397 --- addons/link_tracker/__manifest__.py | 2 +- addons/link_tracker/models/__init__.py | 1 + addons/link_tracker/models/link_tracker.py | 58 ++--------------- .../link_tracker/models/mail_render_mixin.py | 62 +++++++++++++++++++ addons/link_tracker/models/utm.py | 2 +- addons/mass_mailing/models/mail_mail.py | 4 +- .../mass_mailing/models/mail_render_mixin.py | 2 +- addons/mass_mailing/models/mailing.py | 2 +- addons/mass_mailing_sms/models/sms_sms.py | 6 +- .../mass_mailing_sms/wizard/sms_composer.py | 2 +- addons/test_mail_full/tests/test_sms_sms.py | 8 +-- odoo/tools/mail.py | 16 ++++- 12 files changed, 94 insertions(+), 71 deletions(-) create mode 100644 addons/link_tracker/models/mail_render_mixin.py diff --git a/addons/link_tracker/__manifest__.py b/addons/link_tracker/__manifest__.py index 3f622632cc4..038fe807173 100644 --- a/addons/link_tracker/__manifest__.py +++ b/addons/link_tracker/__manifest__.py @@ -8,7 +8,7 @@ Shorten URLs and use them to track clicks and UTMs """, 'version': '1.0', - 'depends': ['utm'], + 'depends': ['utm', 'mail'], 'data': [ 'views/link_tracker_views.xml', 'views/utm_campaign_views.xml', diff --git a/addons/link_tracker/models/__init__.py b/addons/link_tracker/models/__init__.py index 847f935569e..6f884bd7c2e 100644 --- a/addons/link_tracker/models/__init__.py +++ b/addons/link_tracker/models/__init__.py @@ -2,4 +2,5 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. from . import link_tracker +from . import mail_render_mixin from . import utm diff --git a/addons/link_tracker/models/link_tracker.py b/addons/link_tracker/models/link_tracker.py index 3af4accb9ba..ec73046e1be 100644 --- a/addons/link_tracker/models/link_tracker.py +++ b/addons/link_tracker/models/link_tracker.py @@ -1,28 +1,14 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -import base64 -import datetime import random -import re import requests import string from lxml import html -from werkzeug import urls, utils +from werkzeug import urls -from odoo import models, fields, api, _ - - -URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])' -TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._\+~#=/-]+(?:\?\S+)?' - - -def VALIDATE_URL(url): - if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'): - return 'http://' + url - - return url +from odoo import tools, models, fields, api, _ class LinkTracker(models.Model): @@ -101,7 +87,7 @@ class LinkTracker(models.Model): if 'url' not in create_vals: raise ValueError('URL field required') else: - create_vals['url'] = VALIDATE_URL(vals['url']) + create_vals['url'] = tools.validate_url(vals['url']) search_domain = [] for fname, value in create_vals.items(): @@ -129,44 +115,10 @@ class LinkTracker(models.Model): @api.model def convert_links(self, html, vals, blacklist=None): - for match in re.findall(URL_REGEX, html): - - short_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/r/' - - href = match[0] - long_url = match[1] - - vals['url'] = utils.unescape(long_url) - - if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema): - link = self.create(vals) - shorten_url = self.browse(link.id)[0].short_url - - if shorten_url: - new_href = href.replace(long_url, shorten_url) - html = html.replace(href, new_href) - - return html + raise NotImplementedError('Moved on mail.render.mixin') def _convert_links_text(self, body, vals, blacklist=None): - shortened_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/r/' - unsubscribe_schema = self.env['ir.config_parameter'].sudo().get_param('web.base.url') + '/sms/' - for original_url in re.findall(TEXT_URL_REGEX, body): - # don't shorten already-shortened links or links towards unsubscribe page - if original_url.startswith(shortened_schema) or original_url.startswith(unsubscribe_schema): - continue - # support blacklist items in path, like /u/ - parsed = urls.url_parse(original_url, scheme='http') - if blacklist and any(item in parsed.path for item in blacklist): - continue - - vals['url'] = utils.unescape(original_url) - link = self.create(vals) - shortened_url = link.short_url - if shortened_url: - body = body.replace(original_url, shortened_url, 1) - - return body + raise NotImplementedError('Moved on mail.render.mixin') def action_view_statistics(self): action = self.env['ir.actions.act_window'].for_xml_id('link_tracker', 'link_tracker_click_action_statistics') diff --git a/addons/link_tracker/models/mail_render_mixin.py b/addons/link_tracker/models/mail_render_mixin.py new file mode 100644 index 00000000000..0bc30de960b --- /dev/null +++ b/addons/link_tracker/models/mail_render_mixin.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import re + +from werkzeug import urls, utils + +from odoo import api, models, tools + + +class MailRenderMixin(models.AbstractModel): + _inherit = "mail.render.mixin" + + # ------------------------------------------------------------ + # TOOLS + # ------------------------------------------------------------ + + @api.model + def _shorten_links(self, html, link_tracker_vals, blacklist=None, base_url=None): + """ + + :param link_tracker_vals: values given to the created link.tracker, containing + for example: campaign_id, medium_id, source_id, and any other relevant fields + like mass_mailing_id in mass_mailing; + + :return: update + """ + base_url = base_url or self.env['ir.config_parameter'].sudo().get_param('web.base.url') + short_schema = base_url + '/r/' + for match in re.findall(tools.URL_REGEX, html): + href = match[0] + long_url = match[1] + + if not blacklist or not [s for s in blacklist if s in long_url] and not long_url.startswith(short_schema): + create_vals = dict(link_tracker_vals, url=utils.unescape(long_url)) + link = self.env['link.tracker'].create(create_vals) + if link.short_url: + new_href = href.replace(long_url, link.short_url) + html = html.replace(href, new_href) + + return html + + @api.model + def _shorten_links_text(self, html, link_tracker_vals, blacklist=None, base_url=None): + base_url = base_url or self.env['ir.config_parameter'].sudo().get_param('web.base.url') + shortened_schema = base_url + '/r/' + unsubscribe_schema = base_url + '/sms/' + for original_url in re.findall(tools.TEXT_URL_REGEX, html): + # don't shorten already-shortened links or links towards unsubscribe page + if original_url.startswith(shortened_schema) or original_url.startswith(unsubscribe_schema): + continue + # support blacklist items in path, like /u/ + parsed = urls.url_parse(original_url, scheme='http') + if blacklist and any(item in parsed.path for item in blacklist): + continue + + create_vals = dict(link_tracker_vals, url= utils.unescape(original_url)) + link = self.env['link.tracker'].create(create_vals) + if link.short_url: + html = html.replace(original_url, link.short_url, 1) + + return html diff --git a/addons/link_tracker/models/utm.py b/addons/link_tracker/models/utm.py index ebb343472e7..75eff850150 100644 --- a/addons/link_tracker/models/utm.py +++ b/addons/link_tracker/models/utm.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Part of Odoo. See LICENSE file for full copyright and licensing details. -from odoo import fields, models, api, SUPERUSER_ID +from odoo import fields, models class UtmCampaign(models.Model): diff --git a/addons/mass_mailing/models/mail_mail.py b/addons/mass_mailing/models/mail_mail.py index 68a653a38db..d35ae72f4be 100644 --- a/addons/mass_mailing/models/mail_mail.py +++ b/addons/mass_mailing/models/mail_mail.py @@ -6,8 +6,6 @@ import werkzeug.urls from odoo import api, fields, models, tools -from odoo.addons.link_tracker.models.link_tracker import URL_REGEX - class MailMail(models.Model): """Add the mass mailing campaign data to mail""" @@ -54,7 +52,7 @@ class MailMail(models.Model): body = super(MailMail, self)._send_prepare_body() if self.mailing_id and body and self.mailing_trace_ids: - for match in re.findall(URL_REGEX, self.body_html): + for match in re.findall(tools.URL_REGEX, self.body_html): href = match[0] url = match[1] diff --git a/addons/mass_mailing/models/mail_render_mixin.py b/addons/mass_mailing/models/mail_render_mixin.py index d38667366ed..4447f3931c8 100644 --- a/addons/mass_mailing/models/mail_render_mixin.py +++ b/addons/mass_mailing/models/mail_render_mixin.py @@ -15,7 +15,7 @@ class MailRenderMixin(models.AbstractModel): # apply shortener after if self.env.context.get('post_convert_links'): for res_id, html in rendered.items(): - rendered[res_id] = self.env['link.tracker'].convert_links( + rendered[res_id] = self._shorten_links( html, self.env.context['post_convert_links'], blacklist=['/unsubscribe_from_list'] diff --git a/addons/mass_mailing/models/mailing.py b/addons/mass_mailing/models/mailing.py index 88c5a9c4574..1ce47e6cfa4 100644 --- a/addons/mass_mailing/models/mailing.py +++ b/addons/mass_mailing/models/mailing.py @@ -561,7 +561,7 @@ class MassMailing(models.Model): if mass_mailing.medium_id: vals['medium_id'] = mass_mailing.medium_id.id - res[mass_mailing.id] = self.env['link.tracker'].convert_links(html, vals, blacklist=['/unsubscribe_from_list']) + res[mass_mailing.id] = self._shorten_links(html, vals, blacklist=['/unsubscribe_from_list']) return res diff --git a/addons/mass_mailing_sms/models/sms_sms.py b/addons/mass_mailing_sms/models/sms_sms.py index d83359aa47a..c9c14e144cd 100644 --- a/addons/mass_mailing_sms/models/sms_sms.py +++ b/addons/mass_mailing_sms/models/sms_sms.py @@ -3,9 +3,7 @@ import re -from odoo import fields, models - -TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._+~#=/-]+' +from odoo import fields, models, tools class SmsSms(models.Model): @@ -25,7 +23,7 @@ class SmsSms(models.Model): continue body = sms.body - for url in re.findall(TEXT_URL_REGEX, body): + for url in re.findall(tools.TEXT_URL_REGEX, body): if url.startswith(shortened_schema): body = body.replace(url, url + '/s/%s' % sms.id) res[sms.id] = body diff --git a/addons/mass_mailing_sms/wizard/sms_composer.py b/addons/mass_mailing_sms/wizard/sms_composer.py index fefbeeb0137..b3e50234036 100644 --- a/addons/mass_mailing_sms/wizard/sms_composer.py +++ b/addons/mass_mailing_sms/wizard/sms_composer.py @@ -70,7 +70,7 @@ class SMSComposer(models.TransientModel): if self.mailing_id: tracker_values = self.mailing_id._get_link_tracker_values() for sms_id, body in all_bodies.items(): - body = self.env['link.tracker'].sudo()._convert_links_text(body, tracker_values) + body = self.env['mail.render.mixin'].sudo()._shorten_links_text(body, tracker_values) all_bodies[sms_id] = body return all_bodies diff --git a/addons/test_mail_full/tests/test_sms_sms.py b/addons/test_mail_full/tests/test_sms_sms.py index de7e37ac641..e404eaec572 100644 --- a/addons/test_mail_full/tests/test_sms_sms.py +++ b/addons/test_mail_full/tests/test_sms_sms.py @@ -64,7 +64,7 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock): def test_body_link_shorten(self): link = 'http://www.example.com' self.env['link.tracker'].search([('url', '=', link)]).unlink() - new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values) + new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values) self.assertNotIn(link, new_body) self.assertLinkTracker(link, {'utm_campaign': self.utm_c.name, 'utm_medium': self.utm_m.name}) link = self.env['link.tracker'].search([('url', '=', link)]) @@ -72,7 +72,7 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock): link = 'https://test.odoo.com/my/super_page?test[0]=42&toto=áâà#title3' self.env['link.tracker'].search([('url', '=', link)]).unlink() - new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values) + new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values) self.assertNotIn(link, new_body) self.assertLinkTracker(link, { 'utm_campaign': self.utm_c.name, @@ -86,14 +86,14 @@ class TestSMSPost(test_mail_full_common.TestSMSCommon, LinkTrackerMock): def test_body_link_shorten_wshort(self): link = 'https://test.odoo.com/r/RAOUL' self.env['link.tracker'].search([('url', '=', link)]).unlink() - new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values) + new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values) self.assertIn(link, new_body) self.assertFalse(self.env['link.tracker'].search([('url', '=', link)])) def test_body_link_shorten_wunsubscribe(self): link = 'https://test.odoo.com/sms/3/' self.env['link.tracker'].search([('url', '=', link)]).unlink() - new_body = self.env['link.tracker']._convert_links_text('Welcome to %s !' % link, self.tracker_values) + new_body = self.env['mail.render.mixin']._shorten_links_text('Welcome to %s !' % link, self.tracker_values) self.assertIn(link, new_body) self.assertFalse(self.env['link.tracker'].search([('url', '=', link)])) diff --git a/odoo/tools/mail.py b/odoo/tools/mail.py index 21756fe0640..d1bc961e8ff 100644 --- a/odoo/tools/mail.py +++ b/odoo/tools/mail.py @@ -13,6 +13,7 @@ import time from email.utils import getaddresses from lxml import etree +from werkzeug import urls import odoo from odoo.loglevels import ustr @@ -250,9 +251,19 @@ def html_sanitize(src, silent=True, sanitize_tags=True, sanitize_attributes=Fals return cleaned -#---------------------------------------------------------- +# ---------------------------------------------------------- # HTML/Text management -#---------------------------------------------------------- +# ---------------------------------------------------------- + +URL_REGEX = r'(\bhref=[\'"](?!mailto:|tel:|sms:)([^\'"]+)[\'"])' +TEXT_URL_REGEX = r'https?://[a-zA-Z0-9@:%._\+~#=/-]+(?:\?\S+)?' + + +def validate_url(url): + if urls.url_parse(url).scheme not in ('http', 'https', 'ftp', 'ftps'): + return 'http://' + url + + return url def is_html_empty(html_content): @@ -281,6 +292,7 @@ def html_keep_url(text): final += text[idx:] return final + def html2plaintext(html, body_id=None, encoding='utf-8'): """ From an HTML text, convert the HTML to plain text. If @param body_id is provided then this is the tag where the