From be7fced8064d7dcf232d925cb8eccd29ee37077f Mon Sep 17 00:00:00 2001 From: Valentin Chevalier Date: Fri, 30 Apr 2021 15:59:23 +0000 Subject: [PATCH] [ADD] product_images: Add a new module to fetch images using barcode Users will now have a quick way to add images on their products in batches, allowing fast visual identification of them. task-2518519 closes odoo/odoo#70198 Signed-off-by: Antoine Vandevenne (anv) --- .../base_setup/models/res_config_settings.py | 1 + .../views/res_config_settings_views.xml | 1 + .../views/res_config_settings_views.xml | 23 ++ addons/product_images/__init__.py | 13 + addons/product_images/__manifest__.py | 24 ++ addons/product_images/data/ir_cron_data.xml | 19 + addons/product_images/models/__init__.py | 5 + .../product_images/models/ir_cron_trigger.py | 41 +++ addons/product_images/models/product.py | 11 + .../models/res_config_settings.py | 16 + .../security/ir.model.access.csv | 2 + .../security/product_security.xml | 12 + .../views/res_config_settings_views.xml | 24 ++ addons/product_images/wizard/__init__.py | 3 + .../wizard/product_fetch_image_wizard.py | 328 ++++++++++++++++++ .../product_fetch_image_wizard_views.xml | 62 ++++ 16 files changed, 585 insertions(+) create mode 100644 addons/product_images/__init__.py create mode 100644 addons/product_images/__manifest__.py create mode 100644 addons/product_images/data/ir_cron_data.xml create mode 100644 addons/product_images/models/__init__.py create mode 100644 addons/product_images/models/ir_cron_trigger.py create mode 100644 addons/product_images/models/product.py create mode 100644 addons/product_images/models/res_config_settings.py create mode 100644 addons/product_images/security/ir.model.access.csv create mode 100644 addons/product_images/security/product_security.xml create mode 100644 addons/product_images/views/res_config_settings_views.xml create mode 100644 addons/product_images/wizard/__init__.py create mode 100644 addons/product_images/wizard/product_fetch_image_wizard.py create mode 100644 addons/product_images/wizard/product_fetch_image_wizard_views.xml diff --git a/addons/base_setup/models/res_config_settings.py b/addons/base_setup/models/res_config_settings.py index 5f6c98483ac..4b0271f146d 100644 --- a/addons/base_setup/models/res_config_settings.py +++ b/addons/base_setup/models/res_config_settings.py @@ -50,6 +50,7 @@ class ResConfigSettings(models.TransientModel): company_name = fields.Char(related="company_id.display_name", string="Company Name") company_informations = fields.Text(compute="_compute_company_informations") profiling_enabled_until = fields.Datetime("Profiling enabled until", config_parameter='base.profiling_enabled_until') + module_product_images = fields.Boolean("Get product pictures using barcode") def open_company(self): return { diff --git a/addons/base_setup/views/res_config_settings_views.xml b/addons/base_setup/views/res_config_settings_views.xml index c1559aa6bb8..53edfbeedfc 100644 --- a/addons/base_setup/views/res_config_settings_views.xml +++ b/addons/base_setup/views/res_config_settings_views.xml @@ -279,6 +279,7 @@ +
diff --git a/addons/product/views/res_config_settings_views.xml b/addons/product/views/res_config_settings_views.xml index 8398bd44bdb..0f53b3868bb 100644 --- a/addons/product/views/res_config_settings_views.xml +++ b/addons/product/views/res_config_settings_views.xml @@ -38,6 +38,29 @@
+ + + diff --git a/addons/product_images/__init__.py b/addons/product_images/__init__.py new file mode 100644 index 00000000000..2820019846b --- /dev/null +++ b/addons/product_images/__init__.py @@ -0,0 +1,13 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from . import models +from . import wizard + +from odoo import api, SUPERUSER_ID + + +def uninstall_hook(cr, registry): + env = api.Environment(cr, SUPERUSER_ID, {}) + ICP = env['ir.config_parameter'] + ICP.set_param('google.custom_search.cx', False) + ICP.set_param('google.custom_search.key', False) diff --git a/addons/product_images/__manifest__.py b/addons/product_images/__manifest__.py new file mode 100644 index 00000000000..f3f5183cd6f --- /dev/null +++ b/addons/product_images/__manifest__.py @@ -0,0 +1,24 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +{ + 'name': 'Product Images', + 'version': '1.0', + 'description': """ +Automatically set product images based on the barcode +===================================================== + +This module integrates with the Google Custom Search API to set images on products based on the +barcode. + """, + 'license': 'LGPL-3', + 'category': 'Technical', + 'depends': ['product'], + 'data': [ + 'data/ir_cron_data.xml', + 'security/product_security.xml', + 'security/ir.model.access.csv', + 'views/res_config_settings_views.xml', + 'wizard/product_fetch_image_wizard_views.xml', + ], + 'uninstall_hook': 'uninstall_hook', +} diff --git a/addons/product_images/data/ir_cron_data.xml b/addons/product_images/data/ir_cron_data.xml new file mode 100644 index 00000000000..af8f0a772d0 --- /dev/null +++ b/addons/product_images/data/ir_cron_data.xml @@ -0,0 +1,19 @@ + + + + + + Product Images: Get product images from Google + 9999 + months + -1 + True + + model._cron_fetch_image() + + + diff --git a/addons/product_images/models/__init__.py b/addons/product_images/models/__init__.py new file mode 100644 index 00000000000..fb2f7c79f23 --- /dev/null +++ b/addons/product_images/models/__init__.py @@ -0,0 +1,5 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from . import ir_cron_trigger +from . import product +from . import res_config_settings diff --git a/addons/product_images/models/ir_cron_trigger.py b/addons/product_images/models/ir_cron_trigger.py new file mode 100644 index 00000000000..f13e4c0dbd4 --- /dev/null +++ b/addons/product_images/models/ir_cron_trigger.py @@ -0,0 +1,41 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from odoo import _, api, models +from odoo.exceptions import ValidationError + + +class IrCronTrigger(models.Model): + _inherit = 'ir.cron.trigger' + + @api.constrains('cron_id') + def _check_image_cron_is_not_already_triggered(self): + """ Ensure that there is a maximum of one trigger at a time for `ir_cron_fetch_image`. + + This cron is triggered in an optimal way to retrieve fastly the images without blocking a + worker for a long amount of time. It fetches images in multiples batches to allow other + crons to run in between. The cron also schedules itself if there are remaining products to + be processed or if it encounters errors like a rate limit reached, a ConnectionTimeout, or + service unavailable. Multiple triggers at the same will trouble the rate limit management + and/or errors handling. More information in `product_fetch_image_wizard.py`. + + :return: None + :raise ValidationError: If the maximum number of coexisting triggers for + `ir_cron_fetch_image` is reached + """ + ir_cron_fetch_image = self.env.ref( + 'product_images.ir_cron_fetch_image', raise_if_not_found=False + ) + + if ir_cron_fetch_image and self.cron_id.id != ir_cron_fetch_image.id: + return + + cron_triggers_count = self.env['ir.cron.trigger'].search_count( + [('cron_id', '=', ir_cron_fetch_image.id)] + ) + # When the cron is automatically triggered, we must allow two triggers to exists at the same + # time: the one that triggered the cron and the one that will schedule another cron run. We + # check whether the cron was automatically triggered rather than manually triggered to cover + # the case where the admin would create an ir.cron.trigger manually. + max_coexisting_cron_triggers = 2 if self.env.context.get('automatically_triggered') else 1 + if cron_triggers_count > max_coexisting_cron_triggers: + raise ValidationError(_("This action is already scheduled. Please try again later.")) diff --git a/addons/product_images/models/product.py b/addons/product_images/models/product.py new file mode 100644 index 00000000000..5fae9ea01ad --- /dev/null +++ b/addons/product_images/models/product.py @@ -0,0 +1,11 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from odoo import fields, models + + +class ProductProduct(models.Model): + _inherit = "product.product" + + image_fetch_pending = fields.Boolean( + help="Whether an image must be fetched for this product. Handled by a cron.", + ) diff --git a/addons/product_images/models/res_config_settings.py b/addons/product_images/models/res_config_settings.py new file mode 100644 index 00000000000..79bce568b16 --- /dev/null +++ b/addons/product_images/models/res_config_settings.py @@ -0,0 +1,16 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from odoo import fields, models + + +class ResConfigSettings(models.TransientModel): + _inherit = 'res.config.settings' + + google_custom_search_key = fields.Char( + string="Google Custom Search API Key", + config_parameter='google.custom_search.key', + ) + google_pse_id = fields.Char( + string="The identifier of the Google Programmable Search Engine", + config_parameter='google.pse.id', + ) diff --git a/addons/product_images/security/ir.model.access.csv b/addons/product_images/security/ir.model.access.csv new file mode 100644 index 00000000000..8eb2a34655d --- /dev/null +++ b/addons/product_images/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_product_get_pic_barcode_manager,product.images.get.pic.barcode manager,model_product_fetch_image_wizard,base.group_system,1,1,1,0 diff --git a/addons/product_images/security/product_security.xml b/addons/product_images/security/product_security.xml new file mode 100644 index 00000000000..a23ebac9a80 --- /dev/null +++ b/addons/product_images/security/product_security.xml @@ -0,0 +1,12 @@ + + + + + + Product fetch image wizard: access only your own wizard + + + [('create_uid', '=', user.id)] + + + diff --git a/addons/product_images/views/res_config_settings_views.xml b/addons/product_images/views/res_config_settings_views.xml new file mode 100644 index 00000000000..27ecf9f1311 --- /dev/null +++ b/addons/product_images/views/res_config_settings_views.xml @@ -0,0 +1,24 @@ + + + + res.config.settings.view.form.inherit.product.images + res.config.settings + + + +
+
+
+
+ + + + diff --git a/addons/product_images/wizard/__init__.py b/addons/product_images/wizard/__init__.py new file mode 100644 index 00000000000..c4590ab3a0b --- /dev/null +++ b/addons/product_images/wizard/__init__.py @@ -0,0 +1,3 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +from . import product_fetch_image_wizard diff --git a/addons/product_images/wizard/product_fetch_image_wizard.py b/addons/product_images/wizard/product_fetch_image_wizard.py new file mode 100644 index 00000000000..d0ebe0ad035 --- /dev/null +++ b/addons/product_images/wizard/product_fetch_image_wizard.py @@ -0,0 +1,328 @@ +# Part of Odoo. See LICENSE file for full copyright and licensing details. + +import base64 +import logging +from datetime import timedelta + +import requests +from requests.exceptions import ConnectionError as RequestConnectionError + +from odoo import _, api, fields, models +from odoo.exceptions import UserError + +_logger = logging.getLogger(__name__) + + +class ProductFetchImageWizard(models.TransientModel): + _name = 'product.fetch.image.wizard' + _description = "Fetch product images from Google Images based on the product's barcode number." + + _session = requests.Session() + + @api.model + def default_get(self, fields_list): + # Check that the cron has not been deleted and raise an error if so + ir_cron_fetch_image = self.env.ref( + 'product_images.ir_cron_fetch_image', raise_if_not_found=False + ) + if not ir_cron_fetch_image: + raise UserError(_( + "The scheduled action \"Product Images: Get product images from Google\" has " + "been deleted. Please contact your administrator to have the action restored " + "or to reinstall the module \"product_images\"." + )) + + # Check that the cron is not already triggered and raise an error if so + cron_triggers_count = self.env['ir.cron.trigger'].search_count( + [('cron_id', '=', ir_cron_fetch_image.id)] + ) + if cron_triggers_count > 0: + raise UserError(_( + "A task to process products in the background is already running. Please try again" + "later." + )) + + # Check if API keys are set without retrieving the values to avoid leaking them + ICP = self.env['ir.config_parameter'] + google_pse_id_is_set = bool(ICP.get_param('google.pse.id')) + google_custom_search_key_is_set = bool(ICP.get_param('google.custom_search.key')) + if not (google_pse_id_is_set and google_custom_search_key_is_set): + raise google_pse_id_is_set(_( + "The API Key and Search Engine ID must be set in the General Settings." + )) + + # Compute default values + if self._context.get('active_model') == 'product.template': + product_ids = self.env['product.template'].browse( + self._context.get('active_ids') + ).product_variant_ids + else: + product_ids = self.env['product.product'].browse( + self._context.get('active_ids') + ) + nb_products_selected = len(product_ids) + products_to_process = product_ids.filtered(lambda p: not p.image_1920 and p.barcode) + nb_products_to_process = len(products_to_process) + nb_products_unable_to_process = nb_products_selected - nb_products_to_process + defaults = super().default_get(fields_list) + defaults.update( + products_to_process=products_to_process, + nb_products_selected=nb_products_selected, + nb_products_to_process=nb_products_to_process, + nb_products_unable_to_process=nb_products_unable_to_process, + ) + return defaults + + nb_products_selected = fields.Integer(string="Number of selected products", readonly=True) + products_to_process = fields.Many2many( + comodel_name='product.product', + help="The list of selected products that meet the criteria (have a barcode and no image)", + ) + nb_products_to_process = fields.Integer(string="Number of products to process", readonly=True) + nb_products_unable_to_process = fields.Integer( + string="Number of product unprocessable", readonly=True + ) + + def action_fetch_image(self): + """ Fetch the images of the first ten products and delegate the remaining to the cron. + + The first ten images are immediately fetched to improve the user experience. This way, they + can immediately browse the processed products and be assured that the task is running well. + Also, if any error occurs, it can be thrown to the user. Then, a cron job is triggered to be + run as soon as possible, unless the daily request limit has been reached. In that case, the + cron job is scheduled to run a day later. + + :return: A notification to inform the user about the outcome of the action + :rtype: dict + """ + self.products_to_process.image_fetch_pending = True # Flag products to process for the cron + + # Process the first 10 products immediately + matching_images_count = self._process_products(self._get_products_to_process(10)) + + if self._get_products_to_process(1): # Delegate remaining products to the cron + # Check that the cron has not been deleted and raise an error if so + ir_cron_fetch_image = self.env.ref( + 'product_images.ir_cron_fetch_image', raise_if_not_found=False + ) + if not ir_cron_fetch_image: + raise UserError(_( + "The scheduled action \"Product Images: Get product images from Google\" has " + "been deleted. Please contact your administrator to have the action restored " + "or to reinstall the module \"product_images\"." + )) + + # Check that the cron is not already triggered and create a new trigger if not + cron_triggers_count = self.env['ir.cron.trigger'].search_count( + [('cron_id', '=', ir_cron_fetch_image.id)] + ) + if cron_triggers_count == 0: + self.with_context(automatically_triggered=False)._trigger_fetch_images_cron() + message = _( + "Products are processed in the background. Images will be updated progressively." + ) + warning_type = 'success' + else: + message = _( + "%(matching_images_count)s matching images have been found for %(product_count)s " + "products.", + matching_images_count=matching_images_count, + product_count=len(self.products_to_process) + ) + warning_type = 'success' if matching_images_count > 0 else 'warning' + return { + 'type': 'ir.actions.client', + 'tag': 'display_notification', + 'params': { + 'title': _("Product images"), + 'type': warning_type, + 'message': message, + 'next': {'type': 'ir.actions.act_window_close'}, + } + } + + def _cron_fetch_image(self): + """ Fetch images of a list of products using their barcode. + + This method is called from a cron job. If the daily request limit is reached, the cron job + is scheduled to run again a day later. + + :return: None + """ + # Retrieve 100 products at a time to limit the run time and avoid reaching Google's default + # rate limit. + self._process_products(self._get_products_to_process(100)) + if self._get_products_to_process(1): + self.with_context(automatically_triggered=True)._trigger_fetch_images_cron( + fields.Datetime.now() + timedelta(minutes=1.0) + ) + + def _get_products_to_process(self, limit=10000): + """ Get the products that need to be processed and meet the criteria. + + The criteria are to have a barcode and no image. If `products_to_process` is not populated, + the DB is searched to find matching product records. + + :param int limit: The maximum number of records to return, defaulting to 10000 to match + Google's API default rate limit + :return: The products that meet the criteria + :rtype: recordset of `product.product` + """ + products_to_process = self.products_to_process or self.env['product.product'].search( + [('image_fetch_pending', '=', True)], limit=limit + ) + return products_to_process.filtered( + # p.image_fetch_pending needed for self.products_to_process's records that might already + # have been processed but not yet removed from the list when called from + # action_fetch_image. + lambda p: not p.image_1920 and p.barcode and p.image_fetch_pending + )[:limit] # Apply the limit after the filter with self.products_to_process for more results + + def _process_products(self, products_to_process): + """ Fetch an image from the Google Custom Search API for each product. + + We fetch the 10 first image URLs and save the first valid image. + + :param recordset products_to_process: The products for which an image must be fetched, as a + `product.product` recordset + :return: The number of products for which a matching image was found + :rtype: int + :raises UserError: If the project is misconfigured on Google's side + :raises UserError: If the API Key or Search Engine ID is incorrect + """ + if not products_to_process: + return 0 + + nb_service_unavailable_codes = 0 + nb_timeouts = 0 + for product in products_to_process: + # Fetch image URLs and handle eventual errors + try: + response = self._fetch_image_urls_from_google(product.barcode) + if response.status_code == requests.codes.forbidden: + raise UserError(_( + "The Custom Search API is not enabled in your Google project. Please visit " + "your Google Cloud Platform project page and enable it, then retry. If you " + "enabled this API recently, please wait a few minutes and retry." + )) + elif response.status_code == requests.codes.service_unavailable: + nb_service_unavailable_codes += 1 + if nb_service_unavailable_codes <= 3: # Temporary loss of service + continue # Let the image of this product be fetched by the next cron run + + # The service has not responded more han 3 times, stop trying for now and wait + # for the next cron run. + self.with_context(automatically_triggered=True)._trigger_fetch_images_cron( + fields.Datetime.now() + timedelta(hours=1.0) + ) + _logger.warning( + "received too many service_unavailable responses. delegating remaining " + "images to next cron run." + ) + break + elif response.status_code == requests.codes.too_many_requests: + self.with_context(automatically_triggered=True)._trigger_fetch_images_cron( + fields.Datetime.now() + timedelta(days=1.0) + ) + _logger.warning( + "search quota exceeded. delegating remaining images to next cron run." + ) + break + elif response.status_code == requests.codes.bad_request: + raise UserError(_( + "Your API Key or your Search Engine ID is incorrect." + )) + except (RequestConnectionError): + nb_timeouts += 1 + if nb_timeouts <= 3: # Temporary loss of service + continue # Let the image of this product be fetched by the next cron run + + # The service has not responded more han 3 times, stop trying for now and wait for + # the next cron run. + self.with_context(automatically_triggered=True)._trigger_fetch_images_cron( + fields.Datetime.now() + timedelta(hours=1.0) + ) + _logger.warning( + "encountered too many timeouts. delegating remaining images to next cron run." + ) + break + + # Fetch image and handle possible error + response_content = response.json() + if int(response_content['searchInformation']['totalResults']) > 0: + for item in response_content['items']: # Only populated if totalResults > 0 + try: + image = self._get_image_from_url(item['link']) + if image: + product.image_1920 = image + break # Stop at the first valid image + except ( + RequestConnectionError, + UserError, # Raised when the image couldn't be decoded as base64 + ): + pass # Move on to the next image + + product.image_fetch_pending = False + self.env.cr.commit() # Commit every image in case the cron is killed + + return len(products_to_process.filtered('image_1920')) + + def _fetch_image_urls_from_google(self, barcode): + """ Fetch the first 10 image URLs from the Google Custom Search API. + + :param string barcode: A product's barcode + :return: A response or None + :rtype: Response + """ + if not barcode: + return + + ICP = self.env['ir.config_parameter'] + return self._session.get( + url='https://customsearch.googleapis.com/customsearch/v1', + params={ + 'cx': ICP.get_param('google.pse.id').strip(), + 'safe': 'active', + 'searchType': 'image', + 'key': ICP.get_param('google.custom_search.key').strip(), + 'rights': 'cc_publicdomain,cc_attribute,cc_sharealike', + 'imgSize': 'large', + 'imgType': 'photo', + 'fields': 'searchInformation/totalResults,items(link)', + 'q': barcode, + } + ) + + def _get_image_from_url(self, url): + """ Retrieve an image from the URL. + + If the url contains 'x-raw-image:///', the request failed or the response header + 'Content-Type' does not contain 'image/', return None + + :param string url: url of an image + :return: The retrieved image or None + :rtype: bytes + """ + image = None + if 'x-raw-image:///' not in url: # Ignore images with incorrect link + response = self._session.get(url, timeout=5) + if response.status_code == requests.codes.ok \ + and 'image/' in response.headers['Content-Type']: # Ignore non-image results + image = base64.b64encode(response.content) + return image + + def _trigger_fetch_images_cron(self, at=None): + """ Create a trigger for the con `ir_cron_fetch_image`. + + By default the cron is scheduled to be executed as soon as possible but + the optional `at` argument may be given to delay the execution later + with a precision down to 1 minute. + + :param Optional[datetime.datetime] at: + When to execute the cron, at one moments in time instead of as soon as possible. + """ + self.env.ref('product_images.ir_cron_fetch_image')._trigger(at) + # If two `ir_cron_fetch_image` are triggered automatically, and the first one is not + # committed, the constrains will return a ValidationError and roll back to the last commit, + # leaving no `ir_cron_fetch_image` in the schedule. + self.env.cr.commit() diff --git a/addons/product_images/wizard/product_fetch_image_wizard_views.xml b/addons/product_images/wizard/product_fetch_image_wizard_views.xml new file mode 100644 index 00000000000..0e0140fd8ba --- /dev/null +++ b/addons/product_images/wizard/product_fetch_image_wizard_views.xml @@ -0,0 +1,62 @@ + + + + + product.fetch.image.wizard.view + product.fetch.image.wizard + +
+ +

+ You selected products, + of which will be processed. + + products will not be + processed because they either already have an image or their barcode + number is not set. + + + As only 10,000 products can be processed per day, the remaining will be + done tomorrow. + +

+

+ Please note that some images might not be royalty-free. You should not + publish these on your website. +

+
+
+ +
+ + + + + Get Pictures from Google Images + product.fetch.image.wizard + form + new + + + + + Get Pictures from Google Images + product.fetch.image.wizard + form + new + + + +