From 828b7aef2f66fbc15ed28e39e7adc9068e9c4fa7 Mon Sep 17 00:00:00 2001
From: Fabien Meghazi <fme@openerp.com>
Date: Thu, 15 May 2014 16:58:16 +0200
Subject: [PATCH 1/4] [WIP] sitemap

---
 addons/website/controllers/main.py         | 44 +++++++++++++++++-----
 addons/website/views/website_templates.xml |  2 +-
 2 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py
index be13b5c7f1f..fc8dca6d07f 100644
--- a/addons/website/controllers/main.py
+++ b/addons/website/controllers/main.py
@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*-
 import cStringIO
+from itertools import islice
 import json
 import logging
+import math
 import re
 
 from sys import maxint
@@ -22,6 +24,24 @@ logger = logging.getLogger(__name__)
 MAX_IMAGE_WIDTH, MAX_IMAGE_HEIGHT = IMAGE_LIMITS = (1024, 768)
 LOC_PER_SITEMAP = 45000
 
+import time
+
+from functools import wraps
+
+def timeit(f):
+    @wraps(f)
+    def wrapper(*args, **kw):
+        ts = time.time()
+        result = f(*args, **kw)
+        te = time.time()
+
+        print 'func:%r args:[%r, %r] took: %2.4f sec' % \
+          (f.__name__, args, kw, te-ts)
+        return result
+    return wrapper
+
+
+
 class Website(openerp.addons.web.controllers.main.Home):
     #------------------------------------------------------
     # View
@@ -67,15 +87,17 @@ class Website(openerp.addons.web.controllers.main.Home):
     def robots(self):
         return request.render('website.robots', {'url_root': request.httprequest.url_root}, mimetype='text/plain')
 
+    @timeit
     @http.route('/sitemap.xml', type='http', auth="public", website=True)
     def sitemap_xml_index(self):
-        return request.not_found() # Temporary disable sitemap
-        pages = list(request.website.enumerate_pages())
-        if len(pages)<=LOC_PER_SITEMAP:
-            return self.__sitemap_xml(pages, 0)
+        count = 0
+        for loc in request.website.enumerate_pages():
+            count += 1
+        if count <= LOC_PER_SITEMAP:
+            return self.__sitemap_xml(0)
         # Sitemaps must be split in several smaller files with a sitemap index
         values = {
-            'pages': range(len(pages)/LOC_PER_SITEMAP+1),
+            'pages': range(int(math.ceil(float(count) / LOC_PER_SITEMAP))),
             'url_root': request.httprequest.url_root
         }
         headers = {
@@ -85,18 +107,20 @@ class Website(openerp.addons.web.controllers.main.Home):
 
     @http.route('/sitemap-<int:page>.xml', type='http', auth="public", website=True)
     def sitemap_xml(self, page):
-        pages = list(request.website.enumerate_pages())
-        return self.__sitemap_xml(pages, page)
+        return self.__sitemap_xml(page)
 
-    def __sitemap_xml(self, pages, index=0):
+    @timeit
+    def __sitemap_xml(self, index=0):
+        start = index * LOC_PER_SITEMAP
+        locs = islice(request.website.enumerate_pages(), start , start + LOC_PER_SITEMAP)
         values = {
-            'pages': pages[index*LOC_PER_SITEMAP:(index+1)*LOC_PER_SITEMAP],
+            'locs': locs,
             'url_root': request.httprequest.url_root.rstrip('/')
         }
         headers = {
             'Content-Type': 'application/xml;charset=utf-8',
         }
-        return request.render('website.sitemap_xml', values, headers=headers)
+        return request.render('website.sitemap_xml', values, headers=headers).flatten()
 
     #------------------------------------------------------
     # Edit
diff --git a/addons/website/views/website_templates.xml b/addons/website/views/website_templates.xml
index b9f33ef1c6b..3611645cbf5 100644
--- a/addons/website/views/website_templates.xml
+++ b/addons/website/views/website_templates.xml
@@ -699,7 +699,7 @@ Sitemap: <t t-esc="url_root"/>sitemap.xml
 
 <template id="sitemap_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <url t-foreach="pages" t-as="page">
+  <url t-foreach="locs" t-as="page">
     <loc><t t-esc="url_root"/><t t-esc="page['loc']"/></loc><t t-if="page.get('lastmod', False)">
     <lastmod t-esc="page['lastmod']"/></t><t t-if="page.get('priority', False)">
     <priority t-esc="page['priority']"/></t><t t-if="page.get('changefreq', False)">

From 3d881b1a0c30e74e965511c5b64c98e42945cbc3 Mon Sep 17 00:00:00 2001
From: Fabien Meghazi <fme@openerp.com>
Date: Thu, 15 May 2014 17:26:37 +0200
Subject: [PATCH 2/4] [REM] debuggin stuff

---
 addons/website/controllers/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py
index fc8dca6d07f..b1e414c8a33 100644
--- a/addons/website/controllers/main.py
+++ b/addons/website/controllers/main.py
@@ -120,7 +120,7 @@ class Website(openerp.addons.web.controllers.main.Home):
         headers = {
             'Content-Type': 'application/xml;charset=utf-8',
         }
-        return request.render('website.sitemap_xml', values, headers=headers).flatten()
+        return request.render('website.sitemap_xml', values, headers=headers)
 
     #------------------------------------------------------
     # Edit

From f90a37ea70f67f2a090c63c8b8d606640e89f8f3 Mon Sep 17 00:00:00 2001
From: Fabien Meghazi <fme@openerp.com>
Date: Fri, 16 May 2014 15:31:26 +0200
Subject: [PATCH 3/4] [IMP] sitemap caches it's output in ir.attachment

---
 addons/website/controllers/main.py         | 109 +++++++++++----------
 addons/website/views/website_templates.xml |  16 +--
 2 files changed, 68 insertions(+), 57 deletions(-)

diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py
index b1e414c8a33..df958d969dd 100644
--- a/addons/website/controllers/main.py
+++ b/addons/website/controllers/main.py
@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import cStringIO
+import datetime
 from itertools import islice
 import json
 import logging
-import math
 import re
 
 from sys import maxint
@@ -13,8 +13,6 @@ import werkzeug.wrappers
 from PIL import Image
 
 import openerp
-from openerp.osv import fields
-from openerp.addons.website.models import website
 from openerp.addons.web import http
 from openerp.http import request, Response
 
@@ -23,24 +21,7 @@ logger = logging.getLogger(__name__)
 # Completely arbitrary limits
 MAX_IMAGE_WIDTH, MAX_IMAGE_HEIGHT = IMAGE_LIMITS = (1024, 768)
 LOC_PER_SITEMAP = 45000
-
-import time
-
-from functools import wraps
-
-def timeit(f):
-    @wraps(f)
-    def wrapper(*args, **kw):
-        ts = time.time()
-        result = f(*args, **kw)
-        te = time.time()
-
-        print 'func:%r args:[%r, %r] took: %2.4f sec' % \
-          (f.__name__, args, kw, te-ts)
-        return result
-    return wrapper
-
-
+SITEMAP_CACHE_TIME = datetime.timedelta(hours=6)
 
 class Website(openerp.addons.web.controllers.main.Home):
     #------------------------------------------------------
@@ -87,40 +68,66 @@ class Website(openerp.addons.web.controllers.main.Home):
     def robots(self):
         return request.render('website.robots', {'url_root': request.httprequest.url_root}, mimetype='text/plain')
 
-    @timeit
     @http.route('/sitemap.xml', type='http', auth="public", website=True)
     def sitemap_xml_index(self):
-        count = 0
-        for loc in request.website.enumerate_pages():
-            count += 1
-        if count <= LOC_PER_SITEMAP:
-            return self.__sitemap_xml(0)
-        # Sitemaps must be split in several smaller files with a sitemap index
-        values = {
-            'pages': range(int(math.ceil(float(count) / LOC_PER_SITEMAP))),
-            'url_root': request.httprequest.url_root
-        }
-        headers = {
-            'Content-Type': 'application/xml;charset=utf-8',
-        }
-        return request.render('website.sitemap_index_xml', values, headers=headers)
+        cr, uid, context = request.cr, openerp.SUPERUSER_ID, request.context
+        ira = request.registry['ir.attachment']
+        iuv = request.registry['ir.ui.view']
+        mimetype ='application/xml;charset=utf-8'
+        content = None
 
-    @http.route('/sitemap-<int:page>.xml', type='http', auth="public", website=True)
-    def sitemap_xml(self, page):
-        return self.__sitemap_xml(page)
+        def create_map(url, content):
+            ira.create(cr, uid, dict(
+                datas=content.encode('base64'),
+                mimetype=mimetype,
+                type='binary',
+                name=url,
+                url=url,
+            ), context=context)
 
-    @timeit
-    def __sitemap_xml(self, index=0):
-        start = index * LOC_PER_SITEMAP
-        locs = islice(request.website.enumerate_pages(), start , start + LOC_PER_SITEMAP)
-        values = {
-            'locs': locs,
-            'url_root': request.httprequest.url_root.rstrip('/')
-        }
-        headers = {
-            'Content-Type': 'application/xml;charset=utf-8',
-        }
-        return request.render('website.sitemap_xml', values, headers=headers)
+        sitemap = ira.search_read(cr, uid, [('url', '=' , '/sitemap.xml'), ('type', '=', 'binary')], ('datas', 'create_date'), context=context)
+        if sitemap:
+            # Check if stored version is still valid
+            server_format = openerp.tools.misc.DEFAULT_SERVER_DATETIME_FORMAT
+            create_date = datetime.datetime.strptime(sitemap[0]['create_date'], server_format)
+            delta = datetime.datetime.now() - create_date
+            if delta < SITEMAP_CACHE_TIME:
+                content = sitemap[0]['datas'].decode('base64')
+
+        if not content:
+            # Remove all sitemaps in ir.attachments as we're going to regenerated them
+            sitemap_ids = ira.search(cr, uid, [('url', '=like' , '/sitemap%.xml'), ('type', '=', 'binary')], context=context)
+            if sitemap_ids:
+                ira.unlink(cr, uid, sitemap_ids, context=context)
+
+            pages = 0
+            first_page = None
+            locs = request.website.enumerate_pages()
+            while True:
+                start = pages * LOC_PER_SITEMAP
+                loc_slice = islice(locs, start, start + LOC_PER_SITEMAP)
+                urls = iuv.render(cr, uid, 'website.sitemap_locs', dict(locs=loc_slice), context=context)
+                if urls.strip():
+                    page = iuv.render(cr, uid, 'website.sitemap_xml', dict(content=urls), context=context)
+                    if not first_page:
+                        first_page = page
+                    pages += 1
+                    create_map('/sitemap-%d.xml' % pages, page)
+                else:
+                    break
+            if not pages:
+                return request.not_found()
+            elif pages == 1:
+                content = first_page
+            else:
+                # Sitemaps must be split in several smaller files with a sitemap index
+                content = iuv.render(cr, uid, 'website.sitemap_index_xml', dict(
+                    pages=range(1, pages + 1),
+                    url_root=request.httprequest.url_root,
+                ), context=context)
+            create_map('/sitemap.xml', content)
+
+        return request.make_response(content, [('Content-Type', mimetype)])
 
     #------------------------------------------------------
     # Edit
diff --git a/addons/website/views/website_templates.xml b/addons/website/views/website_templates.xml
index 3611645cbf5..c322528764a 100644
--- a/addons/website/views/website_templates.xml
+++ b/addons/website/views/website_templates.xml
@@ -697,14 +697,18 @@ User-agent: *
 Sitemap: <t t-esc="url_root"/>sitemap.xml
 </template>
 
+<template id="sitemap_locs">
+    <url t-foreach="locs" t-as="page">
+        <loc><t t-esc="url_root"/><t t-esc="page['loc']"/></loc><t t-if="page.get('lastmod', False)">
+        <lastmod t-esc="page['lastmod']"/></t><t t-if="page.get('priority', False)">
+        <priority t-esc="page['priority']"/></t><t t-if="page.get('changefreq', False)">
+        <changefreq t-esc="page['changefreq']"/></t>
+    </url>
+</template>
+
 <template id="sitemap_xml">&lt;?xml version="1.0" encoding="UTF-8"?&gt;
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
-  <url t-foreach="locs" t-as="page">
-    <loc><t t-esc="url_root"/><t t-esc="page['loc']"/></loc><t t-if="page.get('lastmod', False)">
-    <lastmod t-esc="page['lastmod']"/></t><t t-if="page.get('priority', False)">
-    <priority t-esc="page['priority']"/></t><t t-if="page.get('changefreq', False)">
-    <changefreq t-esc="page['changefreq']"/></t>
-  </url>
+    <t t-raw="content"/>
 </urlset>
 </template>
 

From 42d8a90afd38f1bb0ed5b527e73321256ee702d7 Mon Sep 17 00:00:00 2001
From: Fabien Meghazi <fme@openerp.com>
Date: Mon, 19 May 2014 13:59:38 +0200
Subject: [PATCH 4/4] [IMP] minor changes to sitemap

---
 addons/website/controllers/main.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/addons/website/controllers/main.py b/addons/website/controllers/main.py
index df958d969dd..460f64147e8 100644
--- a/addons/website/controllers/main.py
+++ b/addons/website/controllers/main.py
@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 # Completely arbitrary limits
 MAX_IMAGE_WIDTH, MAX_IMAGE_HEIGHT = IMAGE_LIMITS = (1024, 768)
 LOC_PER_SITEMAP = 45000
-SITEMAP_CACHE_TIME = datetime.timedelta(hours=6)
+SITEMAP_CACHE_TIME = datetime.timedelta(hours=12)
 
 class Website(openerp.addons.web.controllers.main.Home):
     #------------------------------------------------------
@@ -76,7 +76,7 @@ class Website(openerp.addons.web.controllers.main.Home):
         mimetype ='application/xml;charset=utf-8'
         content = None
 
-        def create_map(url, content):
+        def create_sitemap(url, content):
             ira.create(cr, uid, dict(
                 datas=content.encode('base64'),
                 mimetype=mimetype,
@@ -112,7 +112,7 @@ class Website(openerp.addons.web.controllers.main.Home):
                     if not first_page:
                         first_page = page
                     pages += 1
-                    create_map('/sitemap-%d.xml' % pages, page)
+                    create_sitemap('/sitemap-%d.xml' % pages, page)
                 else:
                     break
             if not pages:
@@ -125,7 +125,7 @@ class Website(openerp.addons.web.controllers.main.Home):
                     pages=range(1, pages + 1),
                     url_root=request.httprequest.url_root,
                 ), context=context)
-            create_map('/sitemap.xml', content)
+            create_sitemap('/sitemap.xml', content)
 
         return request.make_response(content, [('Content-Type', mimetype)])