[FIX] base, mail, website_forum: correct populate of base models

Significantly improves the usability and the result of the populate for
base models and channel/member/message.

- make populate of base models re-entrant
- adapt size to values that make sense:
  - small shows something relevant, and can be quickly ran several times
  - medium is fast enough to be usuable but big enough to highlight
    performance issues
  - large is... still not to be attempted
- avoid conflicting conditions that either make no sense or can lead to
  crashes when populating the data or running the database
- better spread of data in channel/member/messages

closes odoo/odoo#139269

X-original-commit: 73a5e322f6e250569ee1942d678f8a4f87616bd7
Signed-off-by: Alexandre Kühn (aku) <aku@odoo.com>
Signed-off-by: Sébastien Theys (seb) <seb@odoo.com>

addons/mail/populate/discuss/discuss_channel.py

@@ -6,12 +6,19 @@ from odoo.tools import populate
 
 class Channel(models.Model):
     _inherit = "discuss.channel"
-    _populate_dependencies = ["res.partner"]
-    _populate_sizes = {"small": 10, "medium": 100, "large": 500}
+    _populate_sizes = {"small": 50, "medium": 500, "large": 5000}
 
     def _populate_factories(self):
+        groups = self.env.ref("base.group_portal") + self.env.ref("base.group_user") + self.env.ref("base.group_system")
+
+        def compute_group(values, counter, random):
+            if values["channel_type"] == "channel" and random.randrange(2):
+                return random.choice(groups.ids)
+            return False
+
         return [
             ("name", populate.constant("channel_{counter}")),
             ("channel_type", populate.randomize(["channel", "group"])),
             ("description", populate.constant("channel_{counter}_description")),
+            ("group_public_id", populate.compute(compute_group)),
         ]

addons/mail/populate/discuss/discuss_channel_member.py

@@ -6,21 +6,27 @@ from odoo.tools import populate
 
 class ChannelMember(models.Model):
     _inherit = "discuss.channel.member"
-    _populate_dependencies = ["res.partner", "discuss.channel"]
-    _populate_sizes = {"small": 10, "medium": 100, "large": 1000}
-
-    def _populate_factories(self):
-        partner_ids = self.env.registry.populated_models["res.partner"]
-        channel_ids = self.env.registry.populated_models["discuss.channel"]
-        return [
-            ("partner_id", populate.randomize(partner_ids)),
-            ("channel_id", populate.randomize(channel_ids)),
-        ]
+    _populate_dependencies = ["res.partner", "res.users", "discuss.channel"]
 
     def _populate(self, size):
-        channel_ids = self.env.registry.populated_models["discuss.channel"]
-        for channel_id in channel_ids:
-            self.env["discuss.channel.member"].create(
-                {"partner_id": self.env.ref("base.user_admin").partner_id.id, "channel_id": channel_id}
-            )
-        return super()._populate(size)
+        res = super()._populate(size)
+        random = populate.Random("discuss.channel.member")
+        channels = self.env["discuss.channel"].browse(self.env.registry.populated_models["discuss.channel"])
+        users = self.env["res.users"].browse(self.env.registry.populated_models["res.users"])
+        users = users.filtered(lambda user: user.active)
+        admin = self.env.ref("base.user_admin")
+        members = []
+        for channel in channels:
+            allowed_users = users
+            if channel.channel_type == "channel" and channel.group_public_id:
+                if channel.group_public_id in admin.groups_id:
+                    members.append({"partner_id": admin.partner_id.id, "channel_id": channel.id})
+                allowed_users = users.filtered(lambda user: channel.group_public_id in user.groups_id)
+            elif channel.channel_type in ["channel", "group"]:
+                members.append({"partner_id": admin.partner_id.id, "channel_id": channel.id})
+            if allowed_users:
+                # arbitrary limit of 20 for non-channel type to have a functionally significant number
+                max_members = len(allowed_users) if channel.channel_type == "channel" else min(20, len(allowed_users))
+                for user in random.sample(allowed_users, random.randrange(max_members)):
+                    members.append({"partner_id": user.partner_id.id, "channel_id": channel.id})
+        return res + self.env["discuss.channel.member"].create(members)

addons/mail/populate/discuss/mail_message.py

@@ -7,20 +7,25 @@ from odoo.tools import populate
 
 class Message(models.Model):
     _inherit = "mail.message"
-    _populate_dependencies = ["discuss.channel", "discuss.channel.member"] + MailMessage._populate_dependencies
 
-    def _populate_factories(self):
-        channel_ids = self.env.registry.populated_models["discuss.channel"]
+    @property
+    def _populate_dependencies(self):
+        return super()._populate_dependencies + ["discuss.channel", "discuss.channel.member"]
 
-        def get_author_id(values, counter, random):
-            channel = self.env[values["model"]].browse(values["res_id"])
-            channel_member_ids = [channel["id"] for channel in channel.channel_member_ids.read(["id"])]
-            return random.choice(channel_member_ids)
-
-        return [
-            ("body", populate.constant("message_body_{counter}")),
-            ("message_type", populate.constant("comment")),
-            ("model", populate.constant("discuss.channel")),
-            ("res_id", populate.randomize(channel_ids)),
-            ("author_id", populate.compute(get_author_id)),
-        ]
+    def _populate(self, size):
+        res = super()._populate(size)
+        random = populate.Random("mail.message in discuss")
+        channels = self.env["discuss.channel"].browse(self.env.registry.populated_models["discuss.channel"])
+        messages = []
+        for channel in channels.filtered(lambda channel: channel.channel_member_ids):
+            for counter in range(random.randrange({"small": 100, "medium": 1000, "large": 10000}[size])):
+                messages.append(
+                    {
+                        "author_id": random.choice(channel.channel_member_ids.partner_id).id,
+                        "body": f"message_body_{counter}",
+                        "message_type": "comment",
+                        "model": "discuss.channel",
+                        "res_id": channel.id,
+                    }
+                )
+        return res + self.env["mail.message"].create(messages)

addons/mail/populate/mail_message.py

@@ -6,32 +6,32 @@ from odoo.tools import populate
 
 class Message(models.Model):
     _inherit = "mail.message"
-    _populate_dependencies = ["res.partner"]
-    _populate_sizes = {"small": 1000, "medium": 10000, "large": 500000}
-
-    def _populate_factories(self):
-        thread_ids = self.env.registry.populated_models["res.partner"]
-        return [
-            ("body", populate.constant("message_body_{counter}")),
-            ("message_type", populate.constant("comment")),
-            ("model", populate.constant("res.partner")),
-            ("res_id", populate.randomize(thread_ids)),
-            ("author_id", populate.randomize(thread_ids)),
-        ]
+    _populate_dependencies = ["res.partner", "res.users"]
 
     def _populate(self, size):
-        partner = self.env.ref("base.user_admin").partner_id
-        # create 100 in the chatter of the res.partner admin
+        return super()._populate(size) + self._populate_threads(size, "res.partner")
+
+    def _populate_threads(self, size, model_name):
+        comment_subtype = self.env.ref('mail.mt_comment')
+        note_subtype = self.env.ref('mail.mt_note')
+        random = populate.Random("mail.message")
+        partners = self.env["res.partner"].browse(self.env.registry.populated_models["res.partner"])
+        threads = self.env[model_name].browse(self.env.registry.populated_models[model_name])
+        admin = self.env.ref("base.user_admin").partner_id
         messages = []
-        for counter in range(100):
-            messages.append(
-                {
-                    "body": f"message_body_{counter}",
-                    "message_type": "comment",
-                    "model": "res.partner",
-                    "res_id": partner.id,
-                    "author_id": partner.id,
-                }
-            )
-        self.env["mail.message"].create(messages)
-        return super()._populate(size)
+        for thread in threads:
+            for counter in range(random.randrange({"small": 10, "medium": 100, "large": 1000}[size])):
+                author = random.choice(admin + partners)
+                message_type = random.choice(["email", "comment"]) if author.partner_share else "comment"
+                subtype = comment_subtype if author.partner_share else random.choice(comment_subtype + note_subtype)
+                messages.append(
+                    {
+                        "author_id": author.id,
+                        "body": f"message_body_{counter}",
+                        "message_type": message_type,
+                        "model": "res.partner",
+                        "res_id": thread.id,
+                        "subtype_id": subtype.id,
+                    }
+                )
+        return self.env["mail.message"].create(messages)

addons/website_forum/populate/mail_message.py

@@ -10,60 +10,37 @@ _logger = logging.getLogger(__name__)
 
 class Message(models.Model):
     _inherit = 'mail.message'
-    _populate_dependencies = ['forum.post']
-    _populate_sizes = {'small': 100, 'medium': 1000, 'large': 90000}
 
-    def _populate_factories(self):
-        comment_subtype = self.env.ref('mail.mt_comment')
-        random = populate.Random('comments_on_forum_posts')
-
-        def get_author_id(iterator, *args):
-            user_ids = self.env.registry.populated_models['res.users']
-            author_ids = self.env['res.users'].search_read([('id', 'in', user_ids)], ['partner_id'])
-
-            for values in iterator:
-                author_id = random.choice(author_ids)
-                values.update(author_id=author_id['partner_id'][0])
-                yield values
-
-        def get_res_id_and_date(iterator, *args):
-            """Randomly assign messages to some populated posts and answers.
-            This makes sure these questions and answers get one to four comments.
-            Also define a date that is more recent than the post/answer's create_date
-            """
-            question_ids = self.env.registry.populated_models['forum.post']
-            posts = self.env['forum.post'].search_read(['|', ('id', 'in', question_ids), ('parent_id', 'in', question_ids)], ['id', 'create_date'])
-            sample_size = int(len(question_ids) * 0.7)
-            res_ids = random.sample(posts, sample_size)
-            nb_comments = 0
-            idx = 0
-            post_to_comment_id = None
-            hours = [_ for _ in range(1, 24)]
-
-            for values in iterator:
-                if nb_comments == 0:
-                    nb_comments = random.choices(*zip(*CP_WEIGHTS.items()))[0]
-                    idx += 1
-                    post_to_comment_id = res_ids[idx]['id']
-                nb_comments -= 1
-                values.update({
-                    'date': fields.Datetime.add(res_ids[idx]['create_date'], hours=random.choice(hours)),
-                    'res_id': post_to_comment_id
-                })
-                yield values
-
-        return [
-            ('author_id', get_author_id),
-            ('body', populate.constant('message_body_{counter}')),
-            ('message_type', populate.constant('comment')),
-            ('model', populate.constant('forum.post')),
-            ('_res_id_and_date', get_res_id_and_date),
-            ('subtype_id', populate.constant(comment_subtype.id)),
-        ]
+    @property
+    def _populate_dependencies(self):
+        return super()._populate_dependencies + ["res.users", "forum.post"]
 
     def _populate(self, size):
+        """Randomly assign messages to some populated posts and answers.
+        This makes sure these questions and answers get one to four comments.
+        Also define a date that is more recent than the post/answer's create_date
+        """
         records = super()._populate(size)
-
+        comment_subtype = self.env.ref('mail.mt_comment')
+        hours = [_ for _ in range(1, 24)]
+        random = populate.Random('comments_on_forum_posts')
+        users = self.env["res.users"].browse(self.env.registry.populated_models['res.users'])
+        vals_list = []
+        question_ids = self.env.registry.populated_models['forum.post']
+        posts = self.env['forum.post'].search(['|', ('id', 'in', question_ids), ('parent_id', 'in', question_ids)])
+        for post in random.sample(posts, int(len(question_ids) * 0.7)):
+            nb_comments = random.choices(*zip(*CP_WEIGHTS.items()))[0]
+            for counter in range(nb_comments):
+                vals_list.append({
+                    "author_id": random.choice(users.partner_id.ids),
+                    "body": f"message_body_{counter}",
+                    "date": fields.Datetime.add(post.create_date, hours=random.choice(hours)),
+                    "message_type": "comment",
+                    "model": "forum.post",
+                    "res_id": post.id,
+                    "subtype_id": comment_subtype.id,
+                })
+        messages = self.env["mail.message"].create(vals_list)
         _logger.info('mail.message: update comments create date and uid')
         _logger.info('forum.post: update last_activity_date for posts with comments and/or commented answers')
         query = """
@@ -101,6 +78,5 @@ class Message(models.Model):
               FROM updated_posts up
              WHERE up.parent_id = fp.id
     """
-        self.env.cr.execute(query, {'comment_ids': tuple(records.ids)})
-
-        return records
+        self.env.cr.execute(query, {'comment_ids': tuple(messages.ids)})
+        return records + messages

odoo/addons/base/populate/res_company.py

@@ -19,10 +19,11 @@ class Partner(models.Model):
     def _populate_factories(self):
         # Activate currency to avoid fail iterator
         (self.env.ref('base.USD') | self.env.ref('base.EUR')).active = True
+        last_id = self.env["res.company"].search([], order="id desc", limit=1).id
 
         # remaining: paperformat_id, parent_id, partner_id, font, report_header, external_report_layout_id, report_footer
         def get_name(values=None, counter=0, **kwargs):
-            return 'company_%s_%s' % (counter, self.env['res.currency'].browse(values['currency_id']).name)
+            return 'company_%s_%s' % (last_id + counter + 1, self.env['res.currency'].browse(values['currency_id']).name)
 
         active_currencies = self.env['res.currency'].search([('active', '=', True)]).ids
         return [

odoo/addons/base/populate/res_user.py

@@ -20,6 +20,8 @@ class Users(models.Model):
     _populate_dependencies = ["res.partner"]
 
     def _populate_factories(self):
+        last_id = self.env["res.users"].search([], order="id desc", limit=1).id
+
         def generate_partner_id(iterator, *args):
             partner_factories = self.env['res.partner']._populate_factories()
             partner_generator = populate.chain_factories(partner_factories, self._name)
@@ -30,13 +32,19 @@ class Users(models.Model):
         def get_company_ids(values, **kwargs):
             return [(6, 0, [values['company_id']])]
 
+        def compute_login(values=None, counter=0, **kwargs):
+            return f'user_login_{last_id + counter + 1}'
+
+        def compute_name(values=None, counter=0, **kwargs):
+            return f'user_{last_id + counter + 1}'
+
         return [
             ('active', populate.cartesian([True, False], [0.9, 0.1])),
             ('partner_id', generate_partner_id),
             ('company_id', populate.randomize(self.env.registry.populated_models['res.company'])),
             ('company_ids', populate.compute(get_company_ids)),
-            ('login', populate.constant('user_login_{counter}')),
-            ('name', populate.constant('user_{counter}')),
+            ('login', populate.compute(compute_login)),
+            ('name', populate.compute(compute_name)),
         ]
 
     def _populate(self, size):