pengyuthon/odoo_source
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ADD] product_images: Add a new module to fetch images using barcode

Browse Source 
Users will now have a quick way to add images on their products in batches, allowing fast visual identification of them.

task-2518519

closes odoo/odoo#70198

Signed-off-by: Antoine Vandevenne (anv) <AntoineVDV@users.noreply.github.com>
This commit is contained in:
Valentin Chevalier
2021-08-13 15:28:22 +00:00
parent 0f4dfb4459
commit be7fced806
16 changed files with 585 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
addons/base_setup/models/res_config_settings.py
+1
View File
@@ -50,6 +50,7 @@ class ResConfigSettings(models.TransientModel):
company_name = fields.Char(related="company_id.display_name", string="Company Name")
company_informations = fields.Text(compute="_compute_company_informations")
profiling_enabled_until = fields.Datetime("Profiling enabled until", config_parameter='base.profiling_enabled_until')
module_product_images = fields.Boolean("Get product pictures using barcode")
def open_company(self):
return {
addons/base_setup/views/res_config_settings_views.xml
+1
View File
@@ -279,6 +279,7 @@
</div>
</div>
</div>
<div id="product_get_pic_setting"/>
<div class="col-12 col-lg-6 o_setting_box" id="google_spreadsheet_setting">
<div class="o_setting_left_pane">
<field name="module_google_spreadsheet" />
addons/product/views/res_config_settings_views.xml
+23
View File
@@ -38,6 +38,29 @@
</div>
</div>
</xpath>
<xpath expr="//div[@id='product_get_pic_setting']" position="replace">
<div class="col-12 col-lg-6 o_setting_box" id="product_get_pic_setting">
<div class="o_setting_left_pane">
<field name="module_product_images"/>
</div>
<div class="o_setting_right_pane">
<label for="module_product_images" string="Google Images"/>
<a href="https://www.odoo.com/documentation/15.0/applications/general/product_images.html"
title="Documentation" class="o_doc_link" target="_blank"/>
<div class="text-muted">
Get product pictures using Barcode
</div>
<div class="content-group mt16"
attrs="{'invisible': [('module_product_images','=',False)]}"
id="msg_module_product_images">
<div class="mt16 text-warning">
<strong>Save</strong> this page and come back
here to set up the feature.
</div>
</div>
</div>
</div>
</xpath>
</field>
</record>
</odoo>
addons/product_images/__init__.py
+13
View File
@@ -0,0 +1,13 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from . import models
from . import wizard
from odoo import api, SUPERUSER_ID
def uninstall_hook(cr, registry):
env = api.Environment(cr, SUPERUSER_ID, {})
ICP = env['ir.config_parameter']
ICP.set_param('google.custom_search.cx', False)
ICP.set_param('google.custom_search.key', False)
addons/product_images/__manifest__.py
+24
View File
@@ -0,0 +1,24 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
{
'name': 'Product Images',
'version': '1.0',
'description': """
Automatically set product images based on the barcode
=====================================================
This module integrates with the Google Custom Search API to set images on products based on the
barcode.
""",
'license': 'LGPL-3',
'category': 'Technical',
'depends': ['product'],
'data': [
'data/ir_cron_data.xml',
'security/product_security.xml',
'security/ir.model.access.csv',
'views/res_config_settings_views.xml',
'wizard/product_fetch_image_wizard_views.xml',
],
'uninstall_hook': 'uninstall_hook',
}
addons/product_images/data/ir_cron_data.xml
+19
View File
@@ -0,0 +1,19 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<!-- The following cron is trigger-only. When the user triggers it, the cron processes some
products to add an image from an external provider and schedules itself until all products
are processed. The cron has then no reason to be triggered anymore. Also, its trigger
should always be unique to be able to optimize the number of requests in accordance with
the rate limit that some image providers may apply. -->
<record id="ir_cron_fetch_image" model="ir.cron">
<field name="name">Product Images: Get product images from Google</field>
<field name="interval_number">9999</field>
<field name="interval_type">months</field>
<field name="numbercall">-1</field>
<field name="doall">True</field>
<field name="model_id" ref="model_product_fetch_image_wizard"/>
<field name="code">model._cron_fetch_image()</field>
</record>
</odoo>
addons/product_images/models/__init__.py
+5
View File
@@ -0,0 +1,5 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from . import ir_cron_trigger
from . import product
from . import res_config_settings
addons/product_images/models/ir_cron_trigger.py
+41
View File
@@ -0,0 +1,41 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from odoo import _, api, models
from odoo.exceptions import ValidationError
class IrCronTrigger(models.Model):
_inherit = 'ir.cron.trigger'
@api.constrains('cron_id')
def _check_image_cron_is_not_already_triggered(self):
""" Ensure that there is a maximum of one trigger at a time for `ir_cron_fetch_image`.
This cron is triggered in an optimal way to retrieve fastly the images without blocking a
worker for a long amount of time. It fetches images in multiples batches to allow other
crons to run in between. The cron also schedules itself if there are remaining products to
be processed or if it encounters errors like a rate limit reached, a ConnectionTimeout, or
service unavailable. Multiple triggers at the same will trouble the rate limit management
and/or errors handling. More information in `product_fetch_image_wizard.py`.
:return: None
:raise ValidationError: If the maximum number of coexisting triggers for
`ir_cron_fetch_image` is reached
"""
ir_cron_fetch_image = self.env.ref(
'product_images.ir_cron_fetch_image', raise_if_not_found=False
)
if ir_cron_fetch_image and self.cron_id.id != ir_cron_fetch_image.id:
return
cron_triggers_count = self.env['ir.cron.trigger'].search_count(
[('cron_id', '=', ir_cron_fetch_image.id)]
)
# When the cron is automatically triggered, we must allow two triggers to exists at the same
# time: the one that triggered the cron and the one that will schedule another cron run. We
# check whether the cron was automatically triggered rather than manually triggered to cover
# the case where the admin would create an ir.cron.trigger manually.
max_coexisting_cron_triggers = 2 if self.env.context.get('automatically_triggered') else 1
if cron_triggers_count > max_coexisting_cron_triggers:
raise ValidationError(_("This action is already scheduled. Please try again later."))
addons/product_images/models/product.py
+11
View File
@@ -0,0 +1,11 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from odoo import fields, models
class ProductProduct(models.Model):
_inherit = "product.product"
image_fetch_pending = fields.Boolean(
help="Whether an image must be fetched for this product. Handled by a cron.",
)
addons/product_images/models/res_config_settings.py
+16
View File
@@ -0,0 +1,16 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from odoo import fields, models
class ResConfigSettings(models.TransientModel):
_inherit = 'res.config.settings'
google_custom_search_key = fields.Char(
string="Google Custom Search API Key",
config_parameter='google.custom_search.key',
)
google_pse_id = fields.Char(
string="The identifier of the Google Programmable Search Engine",
config_parameter='google.pse.id',
)
addons/product_images/security/ir.model.access.csv
+2
View File
@@ -0,0 +1,2 @@
id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
access_product_get_pic_barcode_manager,product.images.get.pic.barcode manager,model_product_fetch_image_wizard,base.group_system,1,1,1,0
1 id name model_id:id group_id:id perm_read perm_write perm_create perm_unlink
2 access_product_get_pic_barcode_manager product.images.get.pic.barcode manager model_product_fetch_image_wizard base.group_system 1 1 1 0
addons/product_images/security/product_security.xml
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo noupdate="1">
<record model="ir.rule" id="product_images_product_fetch_image_wizard_rule">
<field name="name">
Product fetch image wizard: access only your own wizard
</field>
<field name="model_id" ref="model_product_fetch_image_wizard"/>
<field name="domain_force">[('create_uid', '=', user.id)]</field>
</record>
</odoo>
addons/product_images/views/res_config_settings_views.xml
+24
View File
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<record id="res_config_settings_view_form" model="ir.ui.view">
<field name="name">res.config.settings.view.form.inherit.product.images</field>
<field name="model">res.config.settings</field>
<field name="inherit_id" ref="base_setup.res_config_settings_view_form"/>
<field name="arch" type="xml">
<xpath expr="//div[@id='msg_module_product_images']" position="replace">
<div class="content-group"
attrs="{'invisible': [('module_product_images', '=', False)]}"
id="msg_module_product_images">
<div class="content-group mt16 row">
<label string="API Key" for="google_custom_search_key"
class="col-4 col-lg-4 o_light_label"/>
<field name="google_custom_search_key" nolabel="1"/>
<label string="Search Engine ID" for="google_pse_id"
class="col-4 col-lg-4 o_light_label"/>
<field name="google_pse_id" nolabel="1"/>
</div>
</div>
</xpath>
</field>
</record>
</odoo>
addons/product_images/wizard/__init__.py
+3
View File
@@ -0,0 +1,3 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
from . import product_fetch_image_wizard
addons/product_images/wizard/product_fetch_image_wizard.py
+328
View File
@@ -0,0 +1,328 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import base64
import logging
from datetime import timedelta
import requests
from requests.exceptions import ConnectionError as RequestConnectionError
from odoo import _, api, fields, models
from odoo.exceptions import UserError
_logger = logging.getLogger(__name__)
class ProductFetchImageWizard(models.TransientModel):
_name = 'product.fetch.image.wizard'
_description = "Fetch product images from Google Images based on the product's barcode number."
_session = requests.Session()
@api.model
def default_get(self, fields_list):
# Check that the cron has not been deleted and raise an error if so
ir_cron_fetch_image = self.env.ref(
'product_images.ir_cron_fetch_image', raise_if_not_found=False
)
if not ir_cron_fetch_image:
raise UserError(_(
"The scheduled action \"Product Images: Get product images from Google\" has "
"been deleted. Please contact your administrator to have the action restored "
"or to reinstall the module \"product_images\"."
))
# Check that the cron is not already triggered and raise an error if so
cron_triggers_count = self.env['ir.cron.trigger'].search_count(
[('cron_id', '=', ir_cron_fetch_image.id)]
)
if cron_triggers_count > 0:
raise UserError(_(
"A task to process products in the background is already running. Please try again"
"later."
))
# Check if API keys are set without retrieving the values to avoid leaking them
ICP = self.env['ir.config_parameter']
google_pse_id_is_set = bool(ICP.get_param('google.pse.id'))
google_custom_search_key_is_set = bool(ICP.get_param('google.custom_search.key'))
if not (google_pse_id_is_set and google_custom_search_key_is_set):
raise google_pse_id_is_set(_(
"The API Key and Search Engine ID must be set in the General Settings."
))
# Compute default values
if self._context.get('active_model') == 'product.template':
product_ids = self.env['product.template'].browse(
self._context.get('active_ids')
).product_variant_ids
else:
product_ids = self.env['product.product'].browse(
self._context.get('active_ids')
)
nb_products_selected = len(product_ids)
products_to_process = product_ids.filtered(lambda p: not p.image_1920 and p.barcode)
nb_products_to_process = len(products_to_process)
nb_products_unable_to_process = nb_products_selected - nb_products_to_process
defaults = super().default_get(fields_list)
defaults.update(
products_to_process=products_to_process,
nb_products_selected=nb_products_selected,
nb_products_to_process=nb_products_to_process,
nb_products_unable_to_process=nb_products_unable_to_process,
)
return defaults
nb_products_selected = fields.Integer(string="Number of selected products", readonly=True)
products_to_process = fields.Many2many(
comodel_name='product.product',
help="The list of selected products that meet the criteria (have a barcode and no image)",
)
nb_products_to_process = fields.Integer(string="Number of products to process", readonly=True)
nb_products_unable_to_process = fields.Integer(
string="Number of product unprocessable", readonly=True
)
def action_fetch_image(self):
""" Fetch the images of the first ten products and delegate the remaining to the cron.
The first ten images are immediately fetched to improve the user experience. This way, they
can immediately browse the processed products and be assured that the task is running well.
Also, if any error occurs, it can be thrown to the user. Then, a cron job is triggered to be
run as soon as possible, unless the daily request limit has been reached. In that case, the
cron job is scheduled to run a day later.
:return: A notification to inform the user about the outcome of the action
:rtype: dict
"""
self.products_to_process.image_fetch_pending = True # Flag products to process for the cron
# Process the first 10 products immediately
matching_images_count = self._process_products(self._get_products_to_process(10))
if self._get_products_to_process(1): # Delegate remaining products to the cron
# Check that the cron has not been deleted and raise an error if so
ir_cron_fetch_image = self.env.ref(
'product_images.ir_cron_fetch_image', raise_if_not_found=False
)
if not ir_cron_fetch_image:
raise UserError(_(
"The scheduled action \"Product Images: Get product images from Google\" has "
"been deleted. Please contact your administrator to have the action restored "
"or to reinstall the module \"product_images\"."
))
# Check that the cron is not already triggered and create a new trigger if not
cron_triggers_count = self.env['ir.cron.trigger'].search_count(
[('cron_id', '=', ir_cron_fetch_image.id)]
)
if cron_triggers_count == 0:
self.with_context(automatically_triggered=False)._trigger_fetch_images_cron()
message = _(
"Products are processed in the background. Images will be updated progressively."
)
warning_type = 'success'
else:
message = _(
"%(matching_images_count)s matching images have been found for %(product_count)s "
"products.",
matching_images_count=matching_images_count,
product_count=len(self.products_to_process)
)
warning_type = 'success' if matching_images_count > 0 else 'warning'
return {
'type': 'ir.actions.client',
'tag': 'display_notification',
'params': {
'title': _("Product images"),
'type': warning_type,
'message': message,
'next': {'type': 'ir.actions.act_window_close'},
}
}
def _cron_fetch_image(self):
""" Fetch images of a list of products using their barcode.
This method is called from a cron job. If the daily request limit is reached, the cron job
is scheduled to run again a day later.
:return: None
"""
# Retrieve 100 products at a time to limit the run time and avoid reaching Google's default
# rate limit.
self._process_products(self._get_products_to_process(100))
if self._get_products_to_process(1):
self.with_context(automatically_triggered=True)._trigger_fetch_images_cron(
fields.Datetime.now() + timedelta(minutes=1.0)
)
def _get_products_to_process(self, limit=10000):
""" Get the products that need to be processed and meet the criteria.
The criteria are to have a barcode and no image. If `products_to_process` is not populated,
the DB is searched to find matching product records.
:param int limit: The maximum number of records to return, defaulting to 10000 to match
Google's API default rate limit
:return: The products that meet the criteria
:rtype: recordset of `product.product`
"""
products_to_process = self.products_to_process or self.env['product.product'].search(
[('image_fetch_pending', '=', True)], limit=limit
)
return products_to_process.filtered(
# p.image_fetch_pending needed for self.products_to_process's records that might already
# have been processed but not yet removed from the list when called from
# action_fetch_image.
lambda p: not p.image_1920 and p.barcode and p.image_fetch_pending
)[:limit] # Apply the limit after the filter with self.products_to_process for more results
def _process_products(self, products_to_process):
""" Fetch an image from the Google Custom Search API for each product.
We fetch the 10 first image URLs and save the first valid image.
:param recordset products_to_process: The products for which an image must be fetched, as a
`product.product` recordset
:return: The number of products for which a matching image was found
:rtype: int
:raises UserError: If the project is misconfigured on Google's side
:raises UserError: If the API Key or Search Engine ID is incorrect
"""
if not products_to_process:
return 0
nb_service_unavailable_codes = 0
nb_timeouts = 0
for product in products_to_process:
# Fetch image URLs and handle eventual errors
try:
response = self._fetch_image_urls_from_google(product.barcode)
if response.status_code == requests.codes.forbidden:
raise UserError(_(
"The Custom Search API is not enabled in your Google project. Please visit "
"your Google Cloud Platform project page and enable it, then retry. If you "
"enabled this API recently, please wait a few minutes and retry."
))
elif response.status_code == requests.codes.service_unavailable:
nb_service_unavailable_codes += 1
if nb_service_unavailable_codes <= 3: # Temporary loss of service
continue # Let the image of this product be fetched by the next cron run
# The service has not responded more han 3 times, stop trying for now and wait
# for the next cron run.
self.with_context(automatically_triggered=True)._trigger_fetch_images_cron(
fields.Datetime.now() + timedelta(hours=1.0)
)
_logger.warning(
"received too many service_unavailable responses. delegating remaining "
"images to next cron run."
)
break
elif response.status_code == requests.codes.too_many_requests:
self.with_context(automatically_triggered=True)._trigger_fetch_images_cron(
fields.Datetime.now() + timedelta(days=1.0)
)
_logger.warning(
"search quota exceeded. delegating remaining images to next cron run."
)
break
elif response.status_code == requests.codes.bad_request:
raise UserError(_(
"Your API Key or your Search Engine ID is incorrect."
))
except (RequestConnectionError):
nb_timeouts += 1
if nb_timeouts <= 3: # Temporary loss of service
continue # Let the image of this product be fetched by the next cron run
# The service has not responded more han 3 times, stop trying for now and wait for
# the next cron run.
self.with_context(automatically_triggered=True)._trigger_fetch_images_cron(
fields.Datetime.now() + timedelta(hours=1.0)
)
_logger.warning(
"encountered too many timeouts. delegating remaining images to next cron run."
)
break
# Fetch image and handle possible error
response_content = response.json()
if int(response_content['searchInformation']['totalResults']) > 0:
for item in response_content['items']: # Only populated if totalResults > 0
try:
image = self._get_image_from_url(item['link'])
if image:
product.image_1920 = image
break # Stop at the first valid image
except (
RequestConnectionError,
UserError, # Raised when the image couldn't be decoded as base64
):
pass # Move on to the next image
product.image_fetch_pending = False
self.env.cr.commit() # Commit every image in case the cron is killed
return len(products_to_process.filtered('image_1920'))
def _fetch_image_urls_from_google(self, barcode):
""" Fetch the first 10 image URLs from the Google Custom Search API.
:param string barcode: A product's barcode
:return: A response or None
:rtype: Response
"""
if not barcode:
return
ICP = self.env['ir.config_parameter']
return self._session.get(
url='https://customsearch.googleapis.com/customsearch/v1',
params={
'cx': ICP.get_param('google.pse.id').strip(),
'safe': 'active',
'searchType': 'image',
'key': ICP.get_param('google.custom_search.key').strip(),
'rights': 'cc_publicdomain,cc_attribute,cc_sharealike',
'imgSize': 'large',
'imgType': 'photo',
'fields': 'searchInformation/totalResults,items(link)',
'q': barcode,
}
)
def _get_image_from_url(self, url):
""" Retrieve an image from the URL.
If the url contains 'x-raw-image:///', the request failed or the response header
'Content-Type' does not contain 'image/', return None
:param string url: url of an image
:return: The retrieved image or None
:rtype: bytes
"""
image = None
if 'x-raw-image:///' not in url: # Ignore images with incorrect link
response = self._session.get(url, timeout=5)
if response.status_code == requests.codes.ok \
and 'image/' in response.headers['Content-Type']: # Ignore non-image results
image = base64.b64encode(response.content)
return image
def _trigger_fetch_images_cron(self, at=None):
""" Create a trigger for the con `ir_cron_fetch_image`.
By default the cron is scheduled to be executed as soon as possible but
the optional `at` argument may be given to delay the execution later
with a precision down to 1 minute.
:param Optional[datetime.datetime] at:
When to execute the cron, at one moments in time instead of as soon as possible.
"""
self.env.ref('product_images.ir_cron_fetch_image')._trigger(at)
# If two `ir_cron_fetch_image` are triggered automatically, and the first one is not
# committed, the constrains will return a ValidationError and roll back to the last commit,
# leaving no `ir_cron_fetch_image` in the schedule.
self.env.cr.commit()
addons/product_images/wizard/product_fetch_image_wizard_views.xml
+62
View File
@@ -0,0 +1,62 @@
<?xml version="1.0" encoding="utf-8"?>
<odoo>
<record id="product_fetch_image_wizard_view_form" model="ir.ui.view">
<field name="name">product.fetch.image.wizard.view</field>
<field name="model">product.fetch.image.wizard</field>
<field name="arch" type="xml">
<form>
<sheet>
<p>
You selected <field name="nb_products_selected"/> products,
<field name="nb_products_to_process"/> of which will be processed.
<span attrs="{'invisible': [('nb_products_unable_to_process', '=', 0)]}">
<field name="nb_products_unable_to_process"/> products will not be
processed because they either already have an image or their barcode
number is not set.
</span>
<span attrs="{'invisible': [('nb_products_selected', '&lt;=', 10000)]}">
As only 10,000 products can be processed per day, the remaining will be
done tomorrow.
</span>
</p>
<p attrs="{'invisible': [('nb_products_to_process', '=', 0)]}">
Please note that some images might not be royalty-free. You should not
publish these on your website.
</p>
<footer>
<button name="action_fetch_image"
type="object"
string="Get Pictures"
class="oe_highlight"
attrs="{'invisible': [('nb_products_to_process', '=', 0)]}"/>
<button special="cancel"
string="Cancel"
attrs="{'invisible': [('nb_products_to_process', '=', 0)]}"/>
<button special="cancel"
string="Cancel"
class="oe_highlight"
attrs="{'invisible': [('nb_products_to_process', '!=', 0)]}"/>
</footer>
</sheet>
</form>
</field>
</record>
<record id="product_template_action_get_pic_with_barcode" model="ir.actions.act_window">
<field name="name">Get Pictures from Google Images</field>
<field name="res_model">product.fetch.image.wizard</field>
<field name="view_mode">form</field>
<field name="target">new</field>
<field name="binding_model_id" ref="product.model_product_template"/>
</record>
<record id="product_product_action_get_pic_with_barcode" model="ir.actions.act_window">
<field name="name">Get Pictures from Google Images</field>
<field name="res_model">product.fetch.image.wizard</field>
<field name="view_mode">form</field>
<field name="target">new</field>
<field name="binding_model_id" ref="product.model_product_product"/>
</record>
</odoo>