diff --git a/addons/mail/models/mail_activity.py b/addons/mail/models/mail_activity.py
index 63772ae415e..74feb959b7c 100644
--- a/addons/mail/models/mail_activity.py
+++ b/addons/mail/models/mail_activity.py
@@ -363,7 +363,7 @@ class MailActivity(models.Model):
         for activity in activities_to_check:
             activity_to_documents.setdefault(activity['res_model'], list()).append(activity['res_id'])
 
-        allowed_ids = []
+        allowed_ids = set()
         for doc_model, doc_ids in activity_to_documents.items():
             # fall back on related document access right checks. Use the same as defined for mail.thread
             # if available; otherwise fall back on read
@@ -375,9 +375,10 @@ class MailActivity(models.Model):
             right = DocumentModel.check_access_rights(doc_operation, raise_exception=False)
             if right:
                 valid_docs = DocumentModel.browse(doc_ids)._filter_access_rules(doc_operation)
-                allowed_ids += [
+                valid_doc_ids = set(valid_docs.ids)
+                allowed_ids.update(
                     activity['id'] for activity in activities_to_check
-                    if activity['res_model'] == doc_model and activity['res_id'] in valid_docs.ids]
+                    if activity['res_model'] == doc_model and activity['res_id'] in valid_doc_ids)
 
         if count:
             return len(allowed_ids)