From bd5a68a1a77055ae0b560379eb3dc021403bc0d0 Mon Sep 17 00:00:00 2001 From: Florian Charlier Date: Thu, 10 Aug 2023 11:43:21 +0200 Subject: [PATCH] [IMP] mail, mass_mailing, sms: process delivery reports Also impacts mass_mailing_sms, test_mail_sms, test_mass_mailing This PR adds support to receive sms delivery reports. Before this PR, an SMS was considered 'sent' when successfully handled by the third party. The user couldn't know if/when an SMS was actually sent for delivery or delivered to the recipient's device. This was similar to the behavior for emails as delivery reports are not commonly used (and not supported in Odoo). With this work, the SMS `pending` state is introduced in mail, mass_mailing, sms and mass_mailing_sms contexts although only fully used in the latter two modules (+tests of course). Because of the huge cost related to upgrading very large existing databases, the following compromises were made: 1. An email and sms notification/trace SENT means DELIVERED. Those that are sent but NOT DELIVERED are PENDING. The difference between email and sms traces reinforced with this PR is that an email sent will be counted as "sent" ~ "delivered" unless an error is returned for emails while for SMS it can only be reached if a delivery report is received. 2. The Link between an SMS uuid (shared with trusted parties) and the tracking records (notifications or traces) is done via an explicit relationship table (sms_tracker) instead of via a new field. This however allowed to nicely concentrate the state update logic. A `process` state is added to represent an intermediate step in the sending process, such as held at IAP for SMS. A few adjustments are also included to update for IAP api v3. Also, adapts and includes new tests. Task-2560666 Part-of: odoo/odoo#133392 --- addons/mail/models/mail_notification.py | 4 +- .../src/core/common/notification_model.js | 10 +- addons/mail/views/mail_notification_views.xml | 2 +- addons/mass_mailing/models/mail_mail.py | 11 +- addons/mass_mailing/models/mailing.py | 11 +- addons/mass_mailing/models/mailing_trace.py | 20 ++- .../report/mailing_trace_report.py | 8 +- .../report/mailing_trace_report_views.xml | 2 + .../views/mailing_mailing_views.xml | 12 +- .../views/mailing_trace_views.xml | 7 +- .../mass_mailing_sms/demo/mailing_trace.xml | 15 +- addons/mass_mailing_sms/models/__init__.py | 1 + .../models/mailing_mailing.py | 6 - .../mass_mailing_sms/models/mailing_trace.py | 12 +- addons/mass_mailing_sms/models/sms_sms.py | 17 -- addons/mass_mailing_sms/models/sms_tracker.py | 76 +++++++++ addons/mass_mailing_sms/tests/common.py | 17 +- .../views/mailing_mailing_views.xml | 18 +- .../views/mailing_trace_views.xml | 16 +- .../wizard/mailing_sms_test.py | 11 +- .../mass_mailing_sms/wizard/sms_composer.py | 9 +- addons/sms/__init__.py | 1 + addons/sms/__manifest__.py | 2 +- addons/sms/controllers/__init__.py | 3 + addons/sms/controllers/main.py | 48 ++++++ addons/sms/models/__init__.py | 1 + addons/sms/models/mail_notification.py | 11 +- addons/sms/models/mail_thread.py | 5 +- addons/sms/models/sms_api.py | 85 +++++----- addons/sms/models/sms_sms.py | 157 ++++++++---------- addons/sms/models/sms_tracker.py | 82 +++++++++ addons/sms/security/ir.model.access.csv | 2 + .../static/tests/thread/message_patch_test.js | 88 +++++----- addons/sms/tests/common.py | 68 +++++--- addons/sms/views/sms_sms_views.xml | 5 +- addons/sms/wizard/sms_composer.py | 17 +- addons/test_mail_sms/tests/__init__.py | 1 + .../test_mail_sms/tests/test_sms_composer.py | 8 +- .../tests/test_sms_controller.py | 98 +++++++++++ .../tests/test_sms_management.py | 16 +- .../tests/test_sms_performance.py | 10 +- addons/test_mail_sms/tests/test_sms_post.py | 12 +- addons/test_mail_sms/tests/test_sms_sms.py | 34 ++-- addons/test_mass_mailing/tests/__init__.py | 1 + .../tests/test_mailing_sms.py | 77 ++++++++- .../tests/test_mailing_statistics_sms.py | 14 +- .../tests/test_sms_controller.py | 63 +++++++ 47 files changed, 870 insertions(+), 324 deletions(-) create mode 100644 addons/mass_mailing_sms/models/sms_tracker.py create mode 100644 addons/sms/controllers/__init__.py create mode 100644 addons/sms/controllers/main.py create mode 100644 addons/sms/models/sms_tracker.py create mode 100644 addons/test_mail_sms/tests/test_sms_controller.py create mode 100644 addons/test_mass_mailing/tests/test_sms_controller.py diff --git a/addons/mail/models/mail_notification.py b/addons/mail/models/mail_notification.py index e7f79f65640..70523b5d418 100644 --- a/addons/mail/models/mail_notification.py +++ b/addons/mail/models/mail_notification.py @@ -27,7 +27,9 @@ class MailNotification(models.Model): ], string='Notification Type', default='inbox', index=True, required=True) notification_status = fields.Selection([ ('ready', 'Ready to Send'), - ('sent', 'Sent'), + ('process', 'Processing'), # being checked by intermediary like IAP for sms + ('pending', 'Sent'), # used with SMS; mail does not differentiate sent from delivered + ('sent', 'Delivered'), ('bounce', 'Bounced'), ('exception', 'Exception'), ('canceled', 'Canceled') diff --git a/addons/mail/static/src/core/common/notification_model.js b/addons/mail/static/src/core/common/notification_model.js index 265c42df285..687a5064f0e 100644 --- a/addons/mail/static/src/core/common/notification_model.js +++ b/addons/mail/static/src/core/common/notification_model.js @@ -68,6 +68,10 @@ export class Notification extends Record { get statusIcon() { switch (this.notification_status) { + case "process": + return "fa fa-hourglass-half"; + case "pending": + return "fa fa-paper-plane-o"; case "sent": return "fa fa-check"; case "bounce": @@ -84,8 +88,12 @@ export class Notification extends Record { get statusTitle() { switch (this.notification_status) { - case "sent": + case "process": + return _t("Processing"); + case "pending": return _t("Sent"); + case "sent": + return _t("Delivered"); case "bounce": return _t("Bounced"); case "exception": diff --git a/addons/mail/views/mail_notification_views.xml b/addons/mail/views/mail_notification_views.xml index 21e5290154c..31499fa7395 100644 --- a/addons/mail/views/mail_notification_views.xml +++ b/addons/mail/views/mail_notification_views.xml @@ -20,7 +20,7 @@
- +
diff --git a/addons/mass_mailing/models/mail_mail.py b/addons/mass_mailing/models/mail_mail.py index 7c650884337..738cf1c64e8 100644 --- a/addons/mass_mailing/models/mail_mail.py +++ b/addons/mass_mailing/models/mail_mail.py @@ -103,11 +103,8 @@ class MailMail(models.Model): return email_list def _postprocess_sent_message(self, success_pids, failure_reason=False, failure_type=None): - mail_sent = not failure_type # we consider that a recipient error is a failure with mass mailling and show them as failed - for mail in self: - if mail.mailing_id: - if mail_sent is True and mail.mailing_trace_ids: - mail.mailing_trace_ids.set_sent() - elif mail_sent is False and mail.mailing_trace_ids: - mail.mailing_trace_ids.set_failed(failure_type=failure_type) + if failure_type: # we consider that a recipient error is a failure with mass mailing and show them as failed + self.filtered('mailing_id').mailing_trace_ids.set_failed(failure_type=failure_type) + else: + self.filtered('mailing_id').mailing_trace_ids.set_sent() return super()._postprocess_sent_message(success_pids, failure_reason=failure_reason, failure_type=failure_type) diff --git a/addons/mass_mailing/models/mailing.py b/addons/mass_mailing/models/mailing.py index dbe302f8bd7..50e73aff0a9 100644 --- a/addons/mass_mailing/models/mailing.py +++ b/addons/mass_mailing/models/mailing.py @@ -212,6 +212,8 @@ class MassMailing(models.Model): expected = fields.Integer(compute="_compute_statistics") canceled = fields.Integer(compute="_compute_statistics") sent = fields.Integer(compute="_compute_statistics") + process = fields.Integer(compute="_compute_statistics") + pending = fields.Integer(compute="_compute_statistics") # used with SMS when not yet 'delivered' delivered = fields.Integer(compute="_compute_statistics") opened = fields.Integer(compute="_compute_statistics") clicked = fields.Integer(compute="_compute_statistics") @@ -297,8 +299,8 @@ class MassMailing(models.Model): def _compute_statistics(self): """ Compute statistics of the mass mailing """ for key in ( - 'scheduled', 'expected', 'canceled', 'sent', 'delivered', 'opened', - 'clicked', 'replied', 'bounced', 'failed', 'received_ratio', + 'scheduled', 'expected', 'canceled', 'sent', 'pending', 'delivered', 'opened', + 'process', 'clicked', 'replied', 'bounced', 'failed', 'received_ratio', 'opened_ratio', 'replied_ratio', 'bounced_ratio', ): self[key] = False @@ -314,6 +316,8 @@ class MassMailing(models.Model): COUNT(s.sent_datetime) AS sent, COUNT(s.trace_status) FILTER (WHERE s.trace_status = 'outgoing') AS scheduled, COUNT(s.trace_status) FILTER (WHERE s.trace_status = 'cancel') AS canceled, + COUNT(s.trace_status) FILTER (WHERE s.trace_status = 'process') AS process, + COUNT(s.trace_status) FILTER (WHERE s.trace_status = 'pending') AS pending, COUNT(s.trace_status) FILTER (WHERE s.trace_status in ('sent', 'open', 'reply')) AS delivered, COUNT(s.trace_status) FILTER (WHERE s.trace_status in ('open', 'reply')) AS opened, COUNT(s.links_click_datetime) AS clicked, @@ -671,6 +675,9 @@ class MassMailing(models.Model): def action_view_traces_failed(self): return self._action_view_traces_filtered('failed') + def action_view_traces_process(self): + return self._action_view_traces_filtered('process') + def action_view_traces_sent(self): return self._action_view_traces_filtered('sent') diff --git a/addons/mass_mailing/models/mailing_trace.py b/addons/mass_mailing/models/mailing_trace.py index e1a59010969..7025f6c7457 100644 --- a/addons/mass_mailing/models/mailing_trace.py +++ b/addons/mass_mailing/models/mailing_trace.py @@ -13,8 +13,8 @@ class MailingTrace(models.Model): Note:: State management / Error codes / Failure types summary * trace_status - 'outgoing', 'sent', 'opened', 'replied', - 'error', 'bouce', 'cancel' + 'outgoing', 'process', 'pending', 'sent', 'opened', 'replied', + 'error', 'bounce', 'cancel' * failure_type # generic 'unknown', @@ -24,8 +24,8 @@ class MailingTrace(models.Model): # mass mailing mass mode specific codes "mail_bl", "mail_optout", "mail_dup" # mass_mailing_sms - 'sms_number_missing', 'sms_number_format', 'sms_credit', - 'sms_server', 'sms_acc' + 'sms_number_missing', 'sms_number_format', 'sms_credit', 'sms_server', + 'sms_acc', 'sms_country_not_supported', 'sms_registration_needed', # mass_mailing_sms mass mode specific codes 'sms_blacklist', 'sms_duplicate', 'sms_optout', * cancel: @@ -39,8 +39,12 @@ class MailingTrace(models.Model): * invalid mail / invalid sms number -> error (RECIPIENT, sms_number_format) * exception: set in _postprocess_sent_message (_postprocess_iap_sent_sms) if mail (sms) not sent with failure type, reset if sent; - * sent: set in _postprocess_sent_message (_postprocess_iap_sent_sms) if - mail (sms) sent + * process: (used in sms): set in SmsTracker._update_sms_traces when held back + (at IAP) before actual sending to the sms_service. + * pending: (used in sms): default value for sent sms. + * sent: set in + * _postprocess_sent_message if mail + * SmsTracker._update_sms_traces if sms, when delivery report is received. * clicked: triggered by add_click * opened: triggered by add_click + blank gif (mail) + gateway reply (mail) * replied: triggered by gateway reply (mail) @@ -81,7 +85,9 @@ class MailingTrace(models.Model): reply_datetime = fields.Datetime('Replied On') trace_status = fields.Selection(selection=[ ('outgoing', 'Outgoing'), - ('sent', 'Sent'), + ('process', 'Processing'), + ('pending', 'Sent'), + ('sent', 'Delivered'), ('open', 'Opened'), ('reply', 'Replied'), ('bounce', 'Bounced'), diff --git a/addons/mass_mailing/report/mailing_trace_report.py b/addons/mass_mailing/report/mailing_trace_report.py index da9b7cf4999..64e2d7dab92 100644 --- a/addons/mass_mailing/report/mailing_trace_report.py +++ b/addons/mass_mailing/report/mailing_trace_report.py @@ -20,6 +20,8 @@ class MailingTraceReport(models.Model): email_from = fields.Char('From', readonly=True) # traces scheduled = fields.Integer(readonly=True) + processing = fields.Integer(readonly=True) + pending = fields.Integer(readonly=True) # Used with SMS before a delivery report is received sent = fields.Integer(readonly=True) delivered = fields.Integer(readonly=True) error = fields.Integer(readonly=True) @@ -59,8 +61,10 @@ class MailingTraceReport(models.Model): 'mailing.state', 'mailing.email_from', "COUNT(trace.id) as scheduled", - 'COUNT(trace.sent_datetime) as sent', - "(COUNT(trace.id) - COUNT(trace.trace_status) FILTER (WHERE trace.trace_status IN ('error', 'bounce', 'cancel'))) as delivered", + "COUNT(trace.sent_datetime) as sent", + "(COUNT(trace.id) - COUNT(trace.trace_status) FILTER (WHERE trace.trace_status IN ('outgoing', 'pending', 'process', 'error', 'bounce', 'cancel'))) as delivered", + "COUNT(trace.trace_status) FILTER (WHERE trace.trace_status = 'process') as processing", + "COUNT(trace.trace_status) FILTER (WHERE trace.trace_status = 'pending') as pending", "COUNT(trace.trace_status) FILTER (WHERE trace.trace_status = 'error') as error", "COUNT(trace.trace_status) FILTER (WHERE trace.trace_status = 'bounce') as bounced", "COUNT(trace.trace_status) FILTER (WHERE trace.trace_status = 'cancel') as canceled", diff --git a/addons/mass_mailing/report/mailing_trace_report_views.xml b/addons/mass_mailing/report/mailing_trace_report_views.xml index ecb16d3056d..e9ba3951492 100644 --- a/addons/mass_mailing/report/mailing_trace_report_views.xml +++ b/addons/mass_mailing/report/mailing_trace_report_views.xml @@ -12,6 +12,8 @@ + + diff --git a/addons/mass_mailing/views/mailing_mailing_views.xml b/addons/mass_mailing/views/mailing_mailing_views.xml index a6e954b0dbd..42201a103d4 100644 --- a/addons/mass_mailing/views/mailing_mailing_views.xml +++ b/addons/mass_mailing/views/mailing_mailing_views.xml @@ -72,7 +72,7 @@ +
+ +
-