From 146d7b2886afaee2d54e6d903b89c1c948db1365 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 19 Jun 2014 18:09:01 +0200 Subject: [PATCH 01/14] [IMP] refactor assets bundle for future features --- openerp/addons/base/ir/ir_qweb.py | 150 +++++++++++++++++++----------- 1 file changed, 94 insertions(+), 56 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index f0d15fad609..d51d7cc79c6 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -20,7 +20,7 @@ from PIL import Image import openerp.http import openerp.tools -import openerp.tools.func +from openerp.tools.func import lazy_property import openerp.tools.lru from openerp.http import request from openerp.tools.safe_eval import safe_eval as eval @@ -993,6 +993,13 @@ def get_field_type(column, options): """ return options.get('widget', column._type) +class AssetNotFound(Exception): + def __init__(self, message=None, url=None): + if not message and url: + message = "Could not find asset '%s'" % url + Exception.__init__(self, message) + self.url = url + class AssetsBundle(object): cache = openerp.tools.lru.LRU(32) rx_css_import = re.compile("(@import[^;{]+;?)", re.M) @@ -1016,13 +1023,13 @@ class AssetsBundle(object): src = el.get('src') href = el.get('href') if el.tag == 'style': - self.stylesheets.append(StylesheetAsset(source=el.text)) + self.stylesheets.append(StylesheetAsset(self, inline=el.text)) elif el.tag == 'link' and el.get('rel') == 'stylesheet' and self.can_aggregate(href): - self.stylesheets.append(StylesheetAsset(url=href)) + self.stylesheets.append(StylesheetAsset(self, url=href)) elif el.tag == 'script' and not src: - self.javascripts.append(JavascriptAsset(source=el.text)) + self.javascripts.append(JavascriptAsset(self, inline=el.text)) elif el.tag == 'script' and self.can_aggregate(src): - self.javascripts.append(JavascriptAsset(url=src)) + self.javascripts.append(JavascriptAsset(self, url=src)) else: self.remains.append(lxml.html.tostring(el)) else: @@ -1035,7 +1042,9 @@ class AssetsBundle(object): def can_aggregate(self, url): return not urlparse(url).netloc and not url.startswith(('/web/css', '/web/js')) - def to_html(self, sep='\n ', css=True, js=True, debug=False): + def to_html(self, sep=None, css=True, js=True, debug=False): + if sep is None: + sep = '\n ' response = [] if debug: if css: @@ -1052,7 +1061,7 @@ class AssetsBundle(object): response.extend(self.remains) return sep + sep.join(response) - @openerp.tools.func.lazy_property + @lazy_property def last_modified(self): return max(itertools.chain( (asset.last_modified for asset in self.javascripts), @@ -1060,7 +1069,7 @@ class AssetsBundle(object): [datetime.datetime(1970, 1, 1)], )) - @openerp.tools.func.lazy_property + @lazy_property def checksum(self): checksum = hashlib.new('sha1') for asset in itertools.chain(self.javascripts, self.stylesheets): @@ -1098,47 +1107,65 @@ class AssetsBundle(object): return self.cache[key] class WebAsset(object): - def __init__(self, source=None, url=None): - self.source = source + def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID): + self.bundle = bundle + self.inline = inline self.url = url - self._irattach = None - self._content = None - self.filename = None - self.last_modified = None - if source: - self.last_modified = datetime.datetime(1970, 1, 1) - if url: - module = filter(None, self.url.split('/'))[0] + self.cr = cr if cr is not None else request.cr + self.uid = uid + self._filename = None + self._ir_attach = None + if not inline and not url: + raise Exception("A bundle should either be inlined or url linked") + + def stat(self): + if not (self.inline or self._filename or self._ir_attach): + addon = filter(None, self.url.split('/'))[0] try: # Test url against modules static assets - mpath = openerp.http.addons_manifest[module]['addons_path'] - self.filename = mpath + self.url.replace('/', os.path.sep) - self.last_modified = datetime.datetime.fromtimestamp(os.path.getmtime(self.filename)) + mpath = openerp.http.addons_manifest[addon]['addons_path'] + self._filename = mpath + self.url.replace('/', os.path.sep) except Exception: try: # Test url against ir.attachments domain = [('type', '=', 'binary'), ('url', '=', self.url)] - attach = request.registry['ir.attachment'].search_read(request.cr, SUPERUSER_ID, domain, ['__last_update', 'datas', 'mimetype'], context=request.context) - self._irattach = attach[0] - server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT - try: - self.last_modified = datetime.datetime.strptime(attach[0]['__last_update'], server_format + '.%f') - except ValueError: - self.last_modified = datetime.datetime.strptime(attach[0]['__last_update'], server_format) + attach = request.registry['ir.attachment'].search_read(self.cr, self.uid, domain, ['__last_update', 'datas', 'mimetype'], context=request.context) + self._ir_attach = attach[0] except Exception: - raise KeyError("Could not find asset '%s' for '%s' addon" % (self.url, module)) + raise AssetNotFound(url=self.url) - @openerp.tools.func.lazy_property + @lazy_property + def last_modified(self): + try: + self.stat() + if self._filename: + return datetime.datetime.fromtimestamp(os.path.getmtime(self._filename)) + elif self._ir_attach: + server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT + last_update = self._ir_attach['__last_update'] + try: + return datetime.datetime.strptime(last_update, server_format + '.%f') + except ValueError: + return datetime.datetime.strptime(last_update, server_format) + except Exception: + pass + return datetime.datetime(1970, 1, 1) + + @lazy_property def content(self): - if self.source: - return self.source - if self._irattach: - return self._irattach['datas'].decode('base64') - return self.get_content() + return self.inline or self._fetch_content() - def get_content(self): - with open(self.filename, 'rb') as fp: - return fp.read().decode('utf-8') + def _fetch_content(self): + """ Fetch content from file or database""" + self.stat() + try: + if self._filename: + with open(self._filename, 'rb') as fp: + return fp.read().decode('utf-8') + else: + return self._ir_attach['datas'].decode('base64') + except Exception: + raise AssetNotFound(url=self.url) def minify(self): return self.content @@ -1147,33 +1174,29 @@ class JavascriptAsset(WebAsset): def minify(self): return rjsmin(self.content) + def _fetch_content(self): + try: + return super(JavascriptAsset, self)._fetch_content() + except AssetNotFound, e: + return """ + console.error("Could not find javascript '%s' defined in bundle '%s'"); + """ % (e.url, self.bundle.xmlid) + def to_html(self): if self.url: return '' % self.url else: - return '' % self.source + return '' % self.inline class StylesheetAsset(WebAsset): rx_import = re.compile(r"""@import\s+('|")(?!'|"|/|https?://)""", re.U) rx_url = re.compile(r"""url\s*\(\s*('|"|)(?!'|"|/|https?://|data:)""", re.U) rx_sourceMap = re.compile(r'(/\*# sourceMappingURL=.*)', re.U) + rx_charset = re.compile(r'(@charset "[^"]+";)', re.U) - def _get_content(self): - with open(self.filename, 'rb') as fp: - firstline = fp.readline() - m = re.match(r'@charset "([^"]+)";', firstline) - if m: - encoding = m.group(1) - else: - encoding = "utf-8" - # "reinject" first line as it's not @charset - fp.seek(0) - - return fp.read().decode(encoding) - - def get_content(self): - content = self._get_content() - if self.url: + def _fetch_content(self): + try: + content = super(StylesheetAsset, self)._fetch_content() web_dir = os.path.dirname(self.url) content = self.rx_import.sub( @@ -1185,6 +1208,21 @@ class StylesheetAsset(WebAsset): r"url(\1%s/" % (web_dir,), content, ) + + # remove charset declarations, we only support utf-8 + content = self.rx_charset.sub('', content) + except AssetNotFound, e: + content = """ + body:before { + background: #ffc; + width: 100%%; + padding: 1em 2%%; + float: left; + clear: both; + font-size: 14px; + content: "Could not find stylesheet '%s' defined in bundle '%s'"; + } + """ % (e.url, self.bundle.xmlid) return content def minify(self): @@ -1201,7 +1239,7 @@ class StylesheetAsset(WebAsset): if self.url: return '' % self.url else: - return '' % self.source + return '' % self.inline def rjsmin(script): """ Minify js with a clever regex. From aa97aaa9d7ca7adc7ed46ee3f4805d30f5717b4e Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 24 Jun 2014 18:52:38 +0200 Subject: [PATCH 02/14] [ADD] Assets Bundle versioning and better cache handling An asset bundle is now versionned with the dates of the ir.ui.views that compose it and also with the dates of the files and ir.attachments linked inside the bundle. This new behavior is reflected in the bundle's lru cache managment. --- addons/web/controllers/main.py | 41 +++++++-------- openerp/addons/base/ir/ir_qweb.py | 76 +++++++++++++++++++--------- openerp/addons/base/ir/ir_ui_view.py | 17 +++++-- 3 files changed, 85 insertions(+), 49 deletions(-) diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 98f65075922..09416179725 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -49,6 +49,9 @@ else: env = jinja2.Environment(loader=loader, autoescape=True) env.filters["json"] = simplejson.dumps +# 1 week cache for asset bundles as advised by Google Page Speed +BUNDLE_MAXAGE = 60 * 60 * 24 * 7 + #---------------------------------------------------------- # OpenERP Web helpers #---------------------------------------------------------- @@ -547,37 +550,31 @@ class Home(http.Controller): def login(self, db, login, key, redirect="/web", **kw): return login_and_redirect(db, login, key, redirect_url=redirect) - @http.route('/web/js/', type='http', auth="public") - def js_bundle(self, xmlid, **kw): - # manifest backward compatible mode, to be removed - values = {'manifest_list': manifest_list} + @http.route([ + '/web/js/', + '/web/js//', + ], type='http', auth='public') + def js_bundle(self, xmlid, sha=None, **kw): try: - assets_html = request.render(xmlid, lazy=False, qcontext=values) + bundle = AssetsBundle(xmlid, debug=request.debug) except QWebTemplateNotFound: return request.not_found() - bundle = AssetsBundle(xmlid, assets_html, debug=request.debug) - response = request.make_response( - bundle.js(), [('Content-Type', 'application/javascript')]) + response = request.make_response(bundle.js(), [('Content-Type', 'application/javascript')]) + return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE) - # TODO: check that we don't do weird lazy overriding of __call__ which break body-removal - return make_conditional( - response, bundle.last_modified, bundle.checksum, max_age=60*60*24) - - @http.route('/web/css/', type='http', auth='public') - def css_bundle(self, xmlid, **kw): - values = {'manifest_list': manifest_list} # manifest backward compatible mode, to be removed + @http.route([ + '/web/css/', + '/web/css//', + ], type='http', auth='public') + def css_bundle(self, xmlid, sha=None, **kw): try: - assets_html = request.render(xmlid, lazy=False, qcontext=values) + bundle = AssetsBundle(xmlid, debug=request.debug) except QWebTemplateNotFound: return request.not_found() - bundle = AssetsBundle(xmlid, assets_html, debug=request.debug) - response = request.make_response( - bundle.css(), [('Content-Type', 'text/css')]) - - return make_conditional( - response, bundle.last_modified, bundle.checksum, max_age=60*60*24) + response = request.make_response(bundle.css(), [('Content-Type', 'text/css')]) + return make_conditional(response, bundle.last_modified, max_age=BUNDLE_MAXAGE) class WebClient(http.Controller): diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index d51d7cc79c6..87ba351cb9a 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -414,16 +414,15 @@ class QWeb(orm.AbstractModel): def render_tag_call_assets(self, element, template_attributes, generated_attributes, qwebcontext): """ This special 't-call' tag can be used in order to aggregate/minify javascript and css assets""" - name = template_attributes['call-assets'] - - # Backward compatibility hack for manifest usage - qwebcontext['manifest_list'] = openerp.addons.web.controllers.main.manifest_list - - d = qwebcontext.copy() - d.context['inherit_branding'] = False - content = self.render_tag_call( - element, {'call': name}, generated_attributes, d) - bundle = AssetsBundle(name, html=content) + if element.childNodes: + # An asset bundle is rendered in two differents contexts (when genereting html and + # when generating the bundle itself) so they must be qwebcontext free + # even '0' variable is forbidden + template = qwebcontext.get('__template__') + raise QWebException("t-call-assets cannot contain children nodes", template=template) + xmlid = template_attributes['call-assets'] + cr, uid, context = [getattr(qwebcontext, attr) for attr in ('cr', 'uid', 'context')] + bundle = AssetsBundle(xmlid, cr=cr, uid=uid, context=context, registry=self.pool) css = self.get_attr_bool(template_attributes.get('css'), default=True) js = self.get_attr_bool(template_attributes.get('js'), default=True) return bundle.to_html(css=css, js=js, debug=bool(qwebcontext.get('debug'))) @@ -1004,13 +1003,32 @@ class AssetsBundle(object): cache = openerp.tools.lru.LRU(32) rx_css_import = re.compile("(@import[^;{]+;?)", re.M) - def __init__(self, xmlid, html=None, debug=False): - self.debug = debug + def __init__(self, xmlid, debug=False, cr=None, uid=None, context=None, registry=None): self.xmlid = xmlid + self.debug = debug + self.cr = request.cr if cr is None else cr + self.uid = request.uid if uid is None else uid + self.context = request.context if context is None else context + self.registry = request.registry if registry is None else registry self.javascripts = [] self.stylesheets = [] self.remains = [] self._checksum = None + self._last_modified = datetime.datetime(1970, 1, 1) + + last_updates = [] + context = self.context.copy() + context.update( + inherit_branding=False, + collect_last_updates=last_updates, + ) + html = self.registry['ir.ui.view'].render(self.cr, self.uid, xmlid, context=context) + if last_updates: + # ir.ui.view are orm cached. If the bundle view is actually cached, we won't receive + # last_updates. In this case we consider it's already cached in self.cache and thus + # only the dates of the files/ir.attachements assets will be compared. + server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT + self._last_modified = datetime.datetime.strptime(max(last_updates), server_format) if html: self.parse(html) @@ -1055,9 +1073,9 @@ class AssetsBundle(object): response.append(jscript.to_html()) else: if css and self.stylesheets: - response.append('' % self.xmlid) + response.append('' % (self.xmlid, self.version)) if js and self.javascripts: - response.append('' % self.xmlid) + response.append('' % (self.xmlid, self.version)) response.extend(self.remains) return sep + sep.join(response) @@ -1066,9 +1084,13 @@ class AssetsBundle(object): return max(itertools.chain( (asset.last_modified for asset in self.javascripts), (asset.last_modified for asset in self.stylesheets), - [datetime.datetime(1970, 1, 1)], + [self._last_modified], )) + @lazy_property + def version(self): + return hashlib.sha1(str(self.last_modified)).hexdigest()[0:7] + @lazy_property def checksum(self): checksum = hashlib.new('sha1') @@ -1077,17 +1099,23 @@ class AssetsBundle(object): return checksum.hexdigest() def js(self): - key = 'js_' + self.checksum + key = 'js_%s' % self.xmlid + if key in self.cache and self.cache[key][0] != self.version: + # Invalidate cache on version mismach + self.cache.pop(key) if key not in self.cache: content =';\n'.join(asset.minify() for asset in self.javascripts) - self.cache[key] = content + self.cache[key] = (self.version, content) if self.debug: return "/*\n%s\n*/\n" % '\n'.join( [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] - return self.cache[key] + return self.cache[key][1] def css(self): - key = 'css_' + self.checksum + key = 'css_%s' % self.xmlid + if key in self.cache and self.cache[key][0] != self.version: + # Invalidate cache on version mismach + self.cache.pop(key) if key not in self.cache: content = '\n'.join(asset.minify() for asset in self.stylesheets) # move up all @import rules to the top @@ -1100,11 +1128,11 @@ class AssetsBundle(object): matches.append(content) content = u'\n'.join(matches) - self.cache[key] = content + self.cache[key] = (self.version, content) if self.debug: return "/*\n%s\n*/\n" % '\n'.join( [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] - return self.cache[key] + return self.cache[key][1] class WebAsset(object): def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID): @@ -1116,7 +1144,7 @@ class WebAsset(object): self._filename = None self._ir_attach = None if not inline and not url: - raise Exception("A bundle should either be inlined or url linked") + raise Exception("An asset should either be inlined or url linked") def stat(self): if not (self.inline or self._filename or self._ir_attach): @@ -1128,8 +1156,10 @@ class WebAsset(object): except Exception: try: # Test url against ir.attachments + fields = ['__last_update', 'datas', 'mimetype'] domain = [('type', '=', 'binary'), ('url', '=', self.url)] - attach = request.registry['ir.attachment'].search_read(self.cr, self.uid, domain, ['__last_update', 'datas', 'mimetype'], context=request.context) + ira = request.registry['ir.attachment'] + attach = ira.search_read(self.cr, self.uid, domain, fields, context=request.context) self._ir_attach = attach[0] except Exception: raise AssetNotFound(url=self.url) diff --git a/openerp/addons/base/ir/ir_ui_view.py b/openerp/addons/base/ir/ir_ui_view.py index 4018d779e53..b6fed6ddfbd 100644 --- a/openerp/addons/base/ir/ir_ui_view.py +++ b/openerp/addons/base/ir/ir_ui_view.py @@ -367,9 +367,13 @@ class view(osv.osv): ]) view_ids = self.search(cr, uid, conditions, context=context) - return [(view.arch, view.id) - for view in self.browse(cr, 1, view_ids, context) - if not (view.groups_id and user_groups.isdisjoint(view.groups_id))] + inheriting_views = [] + for view in self.browse(cr, 1, view_ids, context): + if not (view.groups_id and user_groups.isdisjoint(view.groups_id)): + inheriting_views.append((view.arch, view.id)) + if 'collect_last_updates' in context: + context['collect_last_updates'].append(view.write_date) + return inheriting_views def raise_view_error(self, cr, uid, message, view_id, context=None): view = self.browse(cr, uid, view_id, context) @@ -519,7 +523,7 @@ class view(osv.osv): if context is None: context = {} if root_id is None: root_id = source_id - sql_inherit = self.pool['ir.ui.view'].get_inheriting_views_arch(cr, uid, source_id, model, context=context) + sql_inherit = self.get_inheriting_views_arch(cr, uid, source_id, model, context=context) for (specs, view_id) in sql_inherit: specs_tree = etree.fromstring(specs.encode('utf-8')) if context.get('inherit_branding'): @@ -546,13 +550,18 @@ class view(osv.osv): v = v.inherit_id root_id = v.id + collect_last_updates = 'collect_last_updates' in context # arch and model fields are always returned if fields: fields = list(set(fields) | set(['arch', 'model'])) + if collect_last_updates and 'write_date' not in fields: + fields.append('write_date') # read the view arch [view] = self.read(cr, uid, [root_id], fields=fields, context=context) view_arch = etree.fromstring(view['arch'].encode('utf-8')) + if collect_last_updates: + context['collect_last_updates'].append(view['write_date']) if not v.inherit_id: arch_tree = view_arch else: From 653113094fb8c519385562beace7237e3bdb855b Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 26 Jun 2014 16:49:51 +0200 Subject: [PATCH 03/14] [ADD] Sass support in bundles --- addons/web/controllers/main.py | 4 +- openerp/addons/base/ir/ir_qweb.py | 126 ++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 28 deletions(-) diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 09416179725..df649461499 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -556,7 +556,7 @@ class Home(http.Controller): ], type='http', auth='public') def js_bundle(self, xmlid, sha=None, **kw): try: - bundle = AssetsBundle(xmlid, debug=request.debug) + bundle = AssetsBundle(xmlid) except QWebTemplateNotFound: return request.not_found() @@ -569,7 +569,7 @@ class Home(http.Controller): ], type='http', auth='public') def css_bundle(self, xmlid, sha=None, **kw): try: - bundle = AssetsBundle(xmlid, debug=request.debug) + bundle = AssetsBundle(xmlid) except QWebTemplateNotFound: return request.not_found() diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index 87ba351cb9a..388cd22e96c 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -9,9 +9,12 @@ import math import os import re import sys +import textwrap +import uuid import xml # FIXME use lxml and etree import itertools import lxml.html +from subprocess import Popen, PIPE from urlparse import urlparse import babel @@ -1000,18 +1003,28 @@ class AssetNotFound(Exception): self.url = url class AssetsBundle(object): + # Sass installation: + # + # sudo gem install sass compass bootstrap-sass + # + # If the following error is encountered: + # 'ERROR: Cannot load compass.' + # Use this: + # sudo gem install compass --pre + cmd_sass = ['sass', '--stdin', '-t', 'compressed', '--unix-newlines', '--compass', '-r', 'bootstrap-sass'] cache = openerp.tools.lru.LRU(32) rx_css_import = re.compile("(@import[^;{]+;?)", re.M) + rx_css_split = re.compile("\/\*\! ([a-f0-9-]+) \*\/") def __init__(self, xmlid, debug=False, cr=None, uid=None, context=None, registry=None): self.xmlid = xmlid - self.debug = debug self.cr = request.cr if cr is None else cr self.uid = request.uid if uid is None else uid self.context = request.context if context is None else context self.registry = request.registry if registry is None else registry self.javascripts = [] self.stylesheets = [] + self.css_errors = [] self.remains = [] self._checksum = None self._last_modified = datetime.datetime(1970, 1, 1) @@ -1040,10 +1053,17 @@ class AssetsBundle(object): elif isinstance(el, lxml.html.HtmlElement): src = el.get('src') href = el.get('href') + atype = el.get('type') if el.tag == 'style': - self.stylesheets.append(StylesheetAsset(self, inline=el.text)) + if atype == 'text/sass' or src.endswith('.sass'): + self.stylesheets.append(SassAsset(self, inline=el.text)) + else: + self.stylesheets.append(StylesheetAsset(self, inline=el.text)) elif el.tag == 'link' and el.get('rel') == 'stylesheet' and self.can_aggregate(href): - self.stylesheets.append(StylesheetAsset(self, url=href)) + if href.endswith('.sass') or atype == 'text/sass': + self.stylesheets.append(SassAsset(self, url=href)) + else: + self.stylesheets.append(StylesheetAsset(self, url=href)) elif el.tag == 'script' and not src: self.javascripts.append(JavascriptAsset(self, inline=el.text)) elif el.tag == 'script' and self.can_aggregate(src): @@ -1106,9 +1126,6 @@ class AssetsBundle(object): if key not in self.cache: content =';\n'.join(asset.minify() for asset in self.javascripts) self.cache[key] = (self.version, content) - if self.debug: - return "/*\n%s\n*/\n" % '\n'.join( - [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] return self.cache[key][1] def css(self): @@ -1117,7 +1134,13 @@ class AssetsBundle(object): # Invalidate cache on version mismach self.cache.pop(key) if key not in self.cache: + self.compile_css() content = '\n'.join(asset.minify() for asset in self.stylesheets) + + if self.css_errors: + msg = '\n'.join(self.css_errors) + content += self.css_message(msg.replace('\n', '\\A ')) + # move up all @import rules to the top matches = [] def push(matchobj): @@ -1128,14 +1151,57 @@ class AssetsBundle(object): matches.append(content) content = u'\n'.join(matches) + if self.css_errors: + return content self.cache[key] = (self.version, content) - if self.debug: - return "/*\n%s\n*/\n" % '\n'.join( - [asset.url for asset in self.javascripts if asset.url]) + self.cache[key] + return self.cache[key][1] + def css_message(self, message): + return """ + body:before { + background: #ffc; + width: 100%%; + font-size: 14px; + font-family: monospace; + white-space: pre; + content: "%s"; + } + """ % message.replace('"', '\\"') + + def compile_css(self): + # Css compilation is global because they are independant + sass = [asset for asset in self.stylesheets if isinstance(asset, SassAsset)] + if not sass: + return + source = '\n'.join([asset.get_source() for asset in sass]) + try: + compiler = Popen(self.cmd_sass, stdin=PIPE, stdout=PIPE, stderr=PIPE) + except Exception: + msg ="Could not find 'sass' program needed to compile sass/scss files" + _logger.error(msg) + self.css_errors.append(msg) + return + result = compiler.communicate(input=source) + if compiler.returncode: + msg = "Error while compiling Sass for bundle '%s':\n\n%s" % (self.xmlid, ''. join(result)) + _logger.warning(msg) + self.css_errors.append(msg) + return + fragments = self.rx_css_split.split(result[0].strip())[1:] + while fragments: + asset_id = fragments.pop(0) + asset = next(asset for asset in sass if asset.id == asset_id) + asset.content = fragments.pop(0) + if asset.url: + # TODO: write in ir.attachment for debug mode + pass + class WebAsset(object): + html_url = '%s' + def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID): + self.id = str(uuid.uuid4()) self.bundle = bundle self.inline = inline self.url = url @@ -1164,6 +1230,9 @@ class WebAsset(object): except Exception: raise AssetNotFound(url=self.url) + def to_html(): + raise NotImplementedError() + @lazy_property def last_modified(self): try: @@ -1200,9 +1269,13 @@ class WebAsset(object): def minify(self): return self.content + def with_header(self, content): + location = self.url or "Inline in bundle '%s'" % self.bundle.xmlid + return '\n/* %s */\n%s' % (location, content) + class JavascriptAsset(WebAsset): def minify(self): - return rjsmin(self.content) + return self.with_header(rjsmin(self.content)) def _fetch_content(self): try: @@ -1214,9 +1287,9 @@ class JavascriptAsset(WebAsset): def to_html(self): if self.url: - return '' % self.url + return '' % (self.html_url % self.url) else: - return '' % self.inline + return '' % self.content class StylesheetAsset(WebAsset): rx_import = re.compile(r"""@import\s+('|")(?!'|"|/|https?://)""", re.U) @@ -1242,17 +1315,9 @@ class StylesheetAsset(WebAsset): # remove charset declarations, we only support utf-8 content = self.rx_charset.sub('', content) except AssetNotFound, e: - content = """ - body:before { - background: #ffc; - width: 100%%; - padding: 1em 2%%; - float: left; - clear: both; - font-size: 14px; - content: "Could not find stylesheet '%s' defined in bundle '%s'"; - } - """ % (e.url, self.bundle.xmlid) + error = "Could not find stylesheet '%s' in bundle '%s'" % (e.url, self.bundle.xmlid) + self.bundle.css_errors.append(error) + return '' return content def minify(self): @@ -1263,13 +1328,22 @@ class StylesheetAsset(WebAsset): # space content = re.sub(r'\s+', ' ', content) content = re.sub(r' *([{}]) *', r'\1', content) - return content + return self.with_header(content) def to_html(self): if self.url: - return '' % self.url + return '' % (self.html_url % self.url) else: - return '' % self.inline + return '' % self.content + +class SassAsset(StylesheetAsset): + html_url = '%s.css' + + def minify(self): + return self.with_header(self.content) + + def get_source(self): + return "/*! %s */\n%s" % (self.id, textwrap.dedent(self.content)) def rjsmin(script): """ Minify js with a clever regex. From 36e5b418d863d58a019fa46c6eab00f08a8b3319 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 26 Jun 2014 17:40:30 +0200 Subject: [PATCH 04/14] Sass: pull up @imports and remove file imports --- openerp/addons/base/ir/ir_qweb.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index 388cd22e96c..d69535c8576 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1014,6 +1014,7 @@ class AssetsBundle(object): cmd_sass = ['sass', '--stdin', '-t', 'compressed', '--unix-newlines', '--compass', '-r', 'bootstrap-sass'] cache = openerp.tools.lru.LRU(32) rx_css_import = re.compile("(@import[^;{]+;?)", re.M) + rx_sass_import = re.compile("""(@import\s?['"]([^'"]+)['"])""") rx_css_split = re.compile("\/\*\! ([a-f0-9-]+) \*\/") def __init__(self, xmlid, debug=False, cr=None, uid=None, context=None, registry=None): @@ -1175,6 +1176,19 @@ class AssetsBundle(object): if not sass: return source = '\n'.join([asset.get_source() for asset in sass]) + + # move up all @import rules to the top and exclude file imports + imports = [] + def push(matchobj): + ref = matchobj.group(2) + line = '@import "%s"' % ref + if '.' not in ref and line not in imports and not ref.startswith(('.', '/', '~')): + imports.append(line) + return '' + source = re.sub(self.rx_sass_import, push, source) + imports.append(source) + source = u'\n'.join(imports) + try: compiler = Popen(self.cmd_sass, stdin=PIPE, stdout=PIPE, stderr=PIPE) except Exception: @@ -1184,7 +1198,8 @@ class AssetsBundle(object): return result = compiler.communicate(input=source) if compiler.returncode: - msg = "Error while compiling Sass for bundle '%s':\n\n%s" % (self.xmlid, ''. join(result)) + error = (''. join(result)).split('Load paths')[0] + msg = "Error while compiling Sass for bundle '%s':\n\n%s" % (self.xmlid, error.strip()) _logger.warning(msg) self.css_errors.append(msg) return From 0d1d225249ce79f900ed191e206045207bb6ac62 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 26 Jun 2014 18:18:12 +0200 Subject: [PATCH 05/14] Remove ir.ui.view last_update collection and use checksum on rendered bundle --- openerp/addons/base/ir/ir_qweb.py | 36 ++++++++++------------------ openerp/addons/base/ir/ir_ui_view.py | 15 +++--------- 2 files changed, 16 insertions(+), 35 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index d69535c8576..943675a3ebb 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1028,26 +1028,14 @@ class AssetsBundle(object): self.css_errors = [] self.remains = [] self._checksum = None - self._last_modified = datetime.datetime(1970, 1, 1) - last_updates = [] context = self.context.copy() - context.update( - inherit_branding=False, - collect_last_updates=last_updates, - ) - html = self.registry['ir.ui.view'].render(self.cr, self.uid, xmlid, context=context) - if last_updates: - # ir.ui.view are orm cached. If the bundle view is actually cached, we won't receive - # last_updates. In this case we consider it's already cached in self.cache and thus - # only the dates of the files/ir.attachements assets will be compared. - server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT - self._last_modified = datetime.datetime.strptime(max(last_updates), server_format) - if html: - self.parse(html) + context['inherit_branding'] = False + self.html = self.registry['ir.ui.view'].render(self.cr, self.uid, xmlid, context=context) + self.parse() - def parse(self, html): - fragments = lxml.html.fragments_fromstring(html) + def parse(self): + fragments = lxml.html.fragments_fromstring(self.html) for el in fragments: if isinstance(el, basestring): self.remains.append(el) @@ -1102,22 +1090,24 @@ class AssetsBundle(object): @lazy_property def last_modified(self): + """Returns last modified date of linked files""" return max(itertools.chain( (asset.last_modified for asset in self.javascripts), (asset.last_modified for asset in self.stylesheets), - [self._last_modified], )) @lazy_property def version(self): - return hashlib.sha1(str(self.last_modified)).hexdigest()[0:7] + return self.checksum[0:7] @lazy_property def checksum(self): - checksum = hashlib.new('sha1') - for asset in itertools.chain(self.javascripts, self.stylesheets): - checksum.update(asset.content.encode("utf-8")) - return checksum.hexdigest() + """ + Not really a full checksum. + We compute a SHA1 on the rendered bundle + max linked files last_modified date + """ + check = self.html + str(self.last_modified) + return hashlib.sha1(check).hexdigest() def js(self): key = 'js_%s' % self.xmlid diff --git a/openerp/addons/base/ir/ir_ui_view.py b/openerp/addons/base/ir/ir_ui_view.py index b6fed6ddfbd..5ca4b26b046 100644 --- a/openerp/addons/base/ir/ir_ui_view.py +++ b/openerp/addons/base/ir/ir_ui_view.py @@ -367,13 +367,9 @@ class view(osv.osv): ]) view_ids = self.search(cr, uid, conditions, context=context) - inheriting_views = [] - for view in self.browse(cr, 1, view_ids, context): - if not (view.groups_id and user_groups.isdisjoint(view.groups_id)): - inheriting_views.append((view.arch, view.id)) - if 'collect_last_updates' in context: - context['collect_last_updates'].append(view.write_date) - return inheriting_views + return [(view.arch, view.id) + for view in self.browse(cr, 1, view_ids, context) + if not (view.groups_id and user_groups.isdisjoint(view.groups_id))] def raise_view_error(self, cr, uid, message, view_id, context=None): view = self.browse(cr, uid, view_id, context) @@ -550,18 +546,13 @@ class view(osv.osv): v = v.inherit_id root_id = v.id - collect_last_updates = 'collect_last_updates' in context # arch and model fields are always returned if fields: fields = list(set(fields) | set(['arch', 'model'])) - if collect_last_updates and 'write_date' not in fields: - fields.append('write_date') # read the view arch [view] = self.read(cr, uid, [root_id], fields=fields, context=context) view_arch = etree.fromstring(view['arch'].encode('utf-8')) - if collect_last_updates: - context['collect_last_updates'].append(view['write_date']) if not v.inherit_id: arch_tree = view_arch else: From d7a7ac404dd6236dfb320e36832ed4b1ce3fe845 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Thu, 26 Jun 2014 18:51:55 +0200 Subject: [PATCH 06/14] Support for Sass in debug mode --- openerp/addons/base/ir/ir_qweb.py | 46 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index 943675a3ebb..4ff8ac967c2 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -30,7 +30,6 @@ from openerp.tools.safe_eval import safe_eval as eval from openerp.osv import osv, orm, fields from openerp.tools import html_escape as escape from openerp.tools.translate import _ -from openerp import SUPERUSER_ID _logger = logging.getLogger(__name__) @@ -1074,6 +1073,7 @@ class AssetsBundle(object): sep = '\n ' response = [] if debug: + self.compile_css() if css: for style in self.stylesheets: response.append(style.to_html()) @@ -1198,20 +1198,19 @@ class AssetsBundle(object): asset_id = fragments.pop(0) asset = next(asset for asset in sass if asset.id == asset_id) asset.content = fragments.pop(0) - if asset.url: - # TODO: write in ir.attachment for debug mode - pass class WebAsset(object): html_url = '%s' - def __init__(self, bundle, inline=None, url=None, cr=None, uid=SUPERUSER_ID): + def __init__(self, bundle, inline=None, url=None): self.id = str(uuid.uuid4()) self.bundle = bundle self.inline = inline self.url = url - self.cr = cr if cr is not None else request.cr - self.uid = uid + self.cr = bundle.cr + self.uid = bundle.uid + self.registry = bundle.registry + self.context = bundle.context self._filename = None self._ir_attach = None if not inline and not url: @@ -1229,8 +1228,8 @@ class WebAsset(object): # Test url against ir.attachments fields = ['__last_update', 'datas', 'mimetype'] domain = [('type', '=', 'binary'), ('url', '=', self.url)] - ira = request.registry['ir.attachment'] - attach = ira.search_read(self.cr, self.uid, domain, fields, context=request.context) + ira = self.registry['ir.attachment'] + attach = ira.search_read(self.cr, self.uid, domain, fields, context=self.context) self._ir_attach = attach[0] except Exception: raise AssetNotFound(url=self.url) @@ -1274,7 +1273,9 @@ class WebAsset(object): def minify(self): return self.content - def with_header(self, content): + def with_header(self, content=None): + if content is None: + content = self.content location = self.url or "Inline in bundle '%s'" % self.bundle.xmlid return '\n/* %s */\n%s' % (location, content) @@ -1294,7 +1295,7 @@ class JavascriptAsset(WebAsset): if self.url: return '' % (self.html_url % self.url) else: - return '' % self.content + return '' % self.with_header() class StylesheetAsset(WebAsset): rx_import = re.compile(r"""@import\s+('|")(?!'|"|/|https?://)""", re.U) @@ -1339,13 +1340,32 @@ class StylesheetAsset(WebAsset): if self.url: return '' % (self.html_url % self.url) else: - return '' % self.content + return '' % self.with_header() class SassAsset(StylesheetAsset): html_url = '%s.css' def minify(self): - return self.with_header(self.content) + return self.with_header() + + def to_html(self): + if self.url: + ira = self.registry['ir.attachment'] + url = self.html_url % self.url + domain = [('type', '=', 'binary'), ('url', '=', self.url)] + ira_id = ira.search(self.cr, self.uid, domain, context=self.context) + if ira_id: + # TODO: update only if needed + ira.write(self.cr, self.uid, [ira_id], {'datas': self.content}, context=self.context) + else: + ira.create(self.cr, self.uid, dict( + datas=self.content.encode('base64'), + mimetype='text/css', + type='binary', + name=url, + url=url, + ), context=self.context) + return super(SassAsset, self).to_html() def get_source(self): return "/*! %s */\n%s" % (self.id, textwrap.dedent(self.content)) From 74e0b948fa79a2ce2f7a4c911fe9b6111d23cad8 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Mon, 30 Jun 2014 14:45:41 +0200 Subject: [PATCH 07/14] Better error management, proper feedback on encoding errors --- openerp/addons/base/ir/ir_qweb.py | 78 ++++++++++++++++++------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index 9c41fab397a..ddcdb00116d 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1000,12 +1000,10 @@ def get_field_type(column, options): """ return options.get('widget', column._type) -class AssetNotFound(Exception): - def __init__(self, message=None, url=None): - if not message and url: - message = "Could not find asset '%s'" % url - Exception.__init__(self, message) - self.url = url +class AssetError(Exception): + pass +class AssetNotFound(AssetError): + pass class AssetsBundle(object): # Sass installation: @@ -1079,8 +1077,8 @@ class AssetsBundle(object): sep = '\n ' response = [] if debug: - self.compile_css() - if css: + if css and self.stylesheet: + self.compile_sass() for style in self.stylesheets: response.append(style.to_html()) if js: @@ -1089,7 +1087,7 @@ class AssetsBundle(object): else: if css and self.stylesheets: response.append('' % (self.xmlid, self.version)) - if js and self.javascripts: + if js: response.append('' % (self.xmlid, self.version)) response.extend(self.remains) return sep + sep.join(response) @@ -1131,7 +1129,7 @@ class AssetsBundle(object): # Invalidate cache on version mismach self.cache.pop(key) if key not in self.cache: - self.compile_css() + self.compile_sass() content = '\n'.join(asset.minify() for asset in self.stylesheets) if self.css_errors: @@ -1166,8 +1164,12 @@ class AssetsBundle(object): } """ % message.replace('"', '\\"') - def compile_css(self): - # Css compilation is global because they are independant + def compile_sass(self): + """ + Checks if the bundle contains any sass content, then compiles it to css. + Css compilation is done at the bundle level and not in the assets + because they are potentially interdependant. + """ sass = [asset for asset in self.stylesheets if isinstance(asset, SassAsset)] if not sass: return @@ -1188,23 +1190,32 @@ class AssetsBundle(object): try: compiler = Popen(self.cmd_sass, stdin=PIPE, stdout=PIPE, stderr=PIPE) except Exception: - msg ="Could not find 'sass' program needed to compile sass/scss files" + msg = "Could not find 'sass' program needed to compile sass/scss files" _logger.error(msg) self.css_errors.append(msg) return - result = compiler.communicate(input=source) + result = compiler.communicate(input=source.encode('utf-8')) if compiler.returncode: - error = (''. join(result)).split('Load paths')[0] - msg = "Error while compiling Sass for bundle '%s':\n\n%s" % (self.xmlid, error.strip()) - _logger.warning(msg) - self.css_errors.append(msg) + error = self.get_sass_error(result[1], source=source) + _logger.warning(error) + self.css_errors.append(error) return - fragments = self.rx_css_split.split(result[0].strip())[1:] + compiled = result[0].strip().decode('utf8') + fragments = self.rx_css_split.split(compiled)[1:] while fragments: asset_id = fragments.pop(0) asset = next(asset for asset in sass if asset.id == asset_id) asset.content = fragments.pop(0) + def get_sass_error(self, stderr, source=None): + # TODO: try to find out which asset the error belongs to + error = stderr.split('Load paths')[0].replace(' Use --trace for backtrace.', '') + error += "This error occured while compiling the bundle '%s' containing:" % self.xmlid + for asset in self.stylesheets: + if isinstance(asset, SassAsset): + error += '\n - %s' % (asset.url if asset.url else '') + return error + class WebAsset(object): html_url = '%s' @@ -1219,6 +1230,8 @@ class WebAsset(object): self.context = bundle.context self._filename = None self._ir_attach = None + name = '' if inline else url + self.name = "%s defined in bundle '%s'" % (name, bundle.xmlid) if not inline and not url: raise Exception("An asset should either be inlined or url linked") @@ -1238,7 +1251,7 @@ class WebAsset(object): attach = ira.search_read(self.cr, self.uid, domain, fields, context=self.context) self._ir_attach = attach[0] except Exception: - raise AssetNotFound(url=self.url) + raise AssetNotFound("Could not find %s" % self.name) def to_html(): raise NotImplementedError() @@ -1266,15 +1279,19 @@ class WebAsset(object): def _fetch_content(self): """ Fetch content from file or database""" - self.stat() try: + self.stat() if self._filename: with open(self._filename, 'rb') as fp: return fp.read().decode('utf-8') else: return self._ir_attach['datas'].decode('base64') - except Exception: - raise AssetNotFound(url=self.url) + except UnicodeDecodeError: + raise AssetError('%s is not utf-8 encoded.' % self.name) + except IOError: + raise AssetNotFound('File %s does not exist.' % self.name) + except: + raise AssetError('Could not get content for %s.' % self.name) def minify(self): return self.content @@ -1282,8 +1299,7 @@ class WebAsset(object): def with_header(self, content=None): if content is None: content = self.content - location = self.url or "Inline in bundle '%s'" % self.bundle.xmlid - return '\n/* %s */\n%s' % (location, content) + return '\n/* %s */\n%s' % (self.name, content) class JavascriptAsset(WebAsset): def minify(self): @@ -1292,10 +1308,8 @@ class JavascriptAsset(WebAsset): def _fetch_content(self): try: return super(JavascriptAsset, self)._fetch_content() - except AssetNotFound, e: - return """ - console.error("Could not find javascript '%s' defined in bundle '%s'"); - """ % (e.url, self.bundle.xmlid) + except AssetError, e: + return "console.error(%s);" % json.dumps(e.message) def to_html(self): if self.url: @@ -1326,9 +1340,8 @@ class StylesheetAsset(WebAsset): # remove charset declarations, we only support utf-8 content = self.rx_charset.sub('', content) - except AssetNotFound, e: - error = "Could not find stylesheet '%s' in bundle '%s'" % (e.url, self.bundle.xmlid) - self.bundle.css_errors.append(error) + except AssetError, e: + self.bundle.css_errors.append(e.message) return '' return content @@ -1374,6 +1387,7 @@ class SassAsset(StylesheetAsset): return super(SassAsset, self).to_html() def get_source(self): + # TODO: ensure same indentation level for all bundles return "/*! %s */\n%s" % (self.id, textwrap.dedent(self.content)) def rjsmin(script): From 45215a231d8773f71bafd061419f2ff1cbe13f10 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Mon, 30 Jun 2014 18:41:29 +0200 Subject: [PATCH 08/14] Fix indentation normalization --- openerp/addons/base/ir/ir_qweb.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index ddcdb00116d..e73305bf563 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1363,6 +1363,9 @@ class StylesheetAsset(WebAsset): class SassAsset(StylesheetAsset): html_url = '%s.css' + rx_indent = re.compile(r'^( +|\t+)', re.M) + indent = None + reindent = ' ' def minify(self): return self.with_header() @@ -1387,8 +1390,22 @@ class SassAsset(StylesheetAsset): return super(SassAsset, self).to_html() def get_source(self): - # TODO: ensure same indentation level for all bundles - return "/*! %s */\n%s" % (self.id, textwrap.dedent(self.content)) + content = textwrap.dedent(self.content) + + def fix_indent(m): + ind = m.group() + if self.indent is None: + self.indent = ind + if self.indent == self.reindent: + # Don't reindent the file if identation is the final one (reindent) + raise StopIteration() + return ind.replace(self.indent, self.reindent) + + try: + content = self.rx_indent.sub(fix_indent, content) + except StopIteration: + pass + return "/*! %s */\n%s" % (self.id, content) def rjsmin(script): """ Minify js with a clever regex. From 4f2b5f12fce4ad2dcd402905bdb7182875aea894 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Mon, 30 Jun 2014 19:17:47 +0200 Subject: [PATCH 09/14] Fix typo --- openerp/addons/base/ir/ir_qweb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index e73305bf563..b2fcd20d663 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1077,7 +1077,7 @@ class AssetsBundle(object): sep = '\n ' response = [] if debug: - if css and self.stylesheet: + if css and self.stylesheets: self.compile_sass() for style in self.stylesheets: response.append(style.to_html()) From 7990caaf02448d6c33e0d10b9afdaf9ee4eb05eb Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Mon, 30 Jun 2014 19:26:14 +0200 Subject: [PATCH 10/14] Fix another encoding issue --- openerp/addons/base/ir/ir_qweb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index b2fcd20d663..3a21cc4344a 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1381,7 +1381,7 @@ class SassAsset(StylesheetAsset): ira.write(self.cr, self.uid, [ira_id], {'datas': self.content}, context=self.context) else: ira.create(self.cr, self.uid, dict( - datas=self.content.encode('base64'), + datas=self.content.encode('utf8').encode('base64'), mimetype='text/css', type='binary', name=url, From 79195f18003019360b891f492696e3b5e97e8d01 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 1 Jul 2014 10:02:55 +0200 Subject: [PATCH 11/14] Add support for the @media attribut in css bundles --- openerp/addons/base/ir/ir_qweb.py | 38 +++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index 3a21cc4344a..a986de10448 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -14,6 +14,7 @@ import uuid import xml # FIXME use lxml and etree import itertools import lxml.html +import werkzeug from subprocess import Popen, PIPE from urlparse import urlparse @@ -1046,16 +1047,17 @@ class AssetsBundle(object): src = el.get('src') href = el.get('href') atype = el.get('type') + media = el.get('media') if el.tag == 'style': if atype == 'text/sass' or src.endswith('.sass'): - self.stylesheets.append(SassAsset(self, inline=el.text)) + self.stylesheets.append(SassAsset(self, inline=el.text, media=media)) else: - self.stylesheets.append(StylesheetAsset(self, inline=el.text)) + self.stylesheets.append(StylesheetAsset(self, inline=el.text, media=media)) elif el.tag == 'link' and el.get('rel') == 'stylesheet' and self.can_aggregate(href): if href.endswith('.sass') or atype == 'text/sass': - self.stylesheets.append(SassAsset(self, url=href)) + self.stylesheets.append(SassAsset(self, url=href, media=media)) else: - self.stylesheets.append(StylesheetAsset(self, url=href)) + self.stylesheets.append(StylesheetAsset(self, url=href, media=media)) elif el.tag == 'script' and not src: self.javascripts.append(JavascriptAsset(self, inline=el.text)) elif el.tag == 'script' and self.can_aggregate(src): @@ -1205,7 +1207,7 @@ class AssetsBundle(object): while fragments: asset_id = fragments.pop(0) asset = next(asset for asset in sass if asset.id == asset_id) - asset.content = fragments.pop(0) + asset._content = fragments.pop(0) def get_sass_error(self, stderr, source=None): # TODO: try to find out which asset the error belongs to @@ -1228,6 +1230,7 @@ class WebAsset(object): self.uid = bundle.uid self.registry = bundle.registry self.context = bundle.context + self._content = None self._filename = None self._ir_attach = None name = '' if inline else url @@ -1273,9 +1276,11 @@ class WebAsset(object): pass return datetime.datetime(1970, 1, 1) - @lazy_property + @property def content(self): - return self.inline or self._fetch_content() + if not self._content: + self._content = self.inline or self._fetch_content() + return self._content def _fetch_content(self): """ Fetch content from file or database""" @@ -1323,6 +1328,17 @@ class StylesheetAsset(WebAsset): rx_sourceMap = re.compile(r'(/\*# sourceMappingURL=.*)', re.U) rx_charset = re.compile(r'(@charset "[^"]+";)', re.U) + def __init__(self, *args, **kw): + self.media = kw.pop('media', None) + super(StylesheetAsset, self).__init__(*args, **kw) + + @property + def content(self): + content = super(StylesheetAsset, self).content + if self.media: + content = '@media %s { %s }' % (self.media, content) + return content + def _fetch_content(self): try: content = super(StylesheetAsset, self)._fetch_content() @@ -1356,10 +1372,12 @@ class StylesheetAsset(WebAsset): return self.with_header(content) def to_html(self): + media = (' media="%s"' % werkzeug.utils.escape(self.media)) if self.media else '' if self.url: - return '' % (self.html_url % self.url) + href = self.html_url % self.url + return '' % (href, media) else: - return '' % self.with_header() + return '' % (media, self.with_header()) class SassAsset(StylesheetAsset): html_url = '%s.css' @@ -1390,7 +1408,7 @@ class SassAsset(StylesheetAsset): return super(SassAsset, self).to_html() def get_source(self): - content = textwrap.dedent(self.content) + content = textwrap.dedent(self.inline or self._fetch_content()) def fix_indent(m): ind = m.group() From 861e2728237fa257b09477213926a1eda4f600ce Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 1 Jul 2014 14:22:04 +0200 Subject: [PATCH 12/14] Create and write ir.attachments with superuserid --- openerp/addons/base/ir/ir_qweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index a986de10448..ba4b452aa70 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1396,9 +1396,9 @@ class SassAsset(StylesheetAsset): ira_id = ira.search(self.cr, self.uid, domain, context=self.context) if ira_id: # TODO: update only if needed - ira.write(self.cr, self.uid, [ira_id], {'datas': self.content}, context=self.context) + ira.write(self.cr, openerp.SUPERUSER_ID, [ira_id], {'datas': self.content}, context=self.context) else: - ira.create(self.cr, self.uid, dict( + ira.create(self.cr, openerp.SUPERUSER_ID, dict( datas=self.content.encode('utf8').encode('base64'), mimetype='text/css', type='binary', From a0d2df72f6fc29eedda8ed690816910f156734c8 Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 1 Jul 2014 15:18:39 +0200 Subject: [PATCH 13/14] Rename sha to version in bundle controllers --- addons/web/controllers/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/addons/web/controllers/main.py b/addons/web/controllers/main.py index 5c02cbe0279..b74efc9c452 100644 --- a/addons/web/controllers/main.py +++ b/addons/web/controllers/main.py @@ -556,9 +556,9 @@ class Home(http.Controller): @http.route([ '/web/js/', - '/web/js//', + '/web/js//', ], type='http', auth='public') - def js_bundle(self, xmlid, sha=None, **kw): + def js_bundle(self, xmlid, version=None, **kw): try: bundle = AssetsBundle(xmlid) except QWebTemplateNotFound: @@ -569,9 +569,9 @@ class Home(http.Controller): @http.route([ '/web/css/', - '/web/css//', + '/web/css//', ], type='http', auth='public') - def css_bundle(self, xmlid, sha=None, **kw): + def css_bundle(self, xmlid, version=None, **kw): try: bundle = AssetsBundle(xmlid) except QWebTemplateNotFound: From 0be5fb7d91439265740561d92774412d09260f1b Mon Sep 17 00:00:00 2001 From: Fabien Meghazi Date: Tue, 1 Jul 2014 15:21:36 +0200 Subject: [PATCH 14/14] Fix sass attribute recognition --- openerp/addons/base/ir/ir_qweb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openerp/addons/base/ir/ir_qweb.py b/openerp/addons/base/ir/ir_qweb.py index ba4b452aa70..aa58ef6e05b 100644 --- a/openerp/addons/base/ir/ir_qweb.py +++ b/openerp/addons/base/ir/ir_qweb.py @@ -1044,8 +1044,8 @@ class AssetsBundle(object): if isinstance(el, basestring): self.remains.append(el) elif isinstance(el, lxml.html.HtmlElement): - src = el.get('src') - href = el.get('href') + src = el.get('src', '') + href = el.get('href', '') atype = el.get('type') media = el.get('media') if el.tag == 'style':