From 0eb99442224bb12bedc7150139796e8de64a7e1f Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Mon, 24 Mar 2014 09:50:46 +0100
Subject: [PATCH 01/26] [IMP] refactoring of readgroup method.  It should have
 the same functionality, but is much simpler.  This is necessary to implement
 eager groupby (orm.py)

bzr revid: ged@openerp.com-20140324085046-zpfpcostivf8454q
---
 openerp/osv/orm.py | 230 ++++++++++++++++++++-------------------------
 1 file changed, 100 insertions(+), 130 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 64fa06783ab..cc8e7aa066b 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2222,7 +2222,7 @@ class BaseModel(object):
                              self._name, order_part)
         return groupby_terms, orderby_terms
 
-    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context=None, orderby=False):
+    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False):
         """
         Get the list of records in list view grouped by the given ``groupby`` fields
 
@@ -2252,91 +2252,80 @@ class BaseModel(object):
                             * if user tries to bypass access rules for read on the requested object
 
         """
-        context = context or {}
         self.check_access_rights(cr, uid, 'read')
-        if not fields:
-            fields = self._columns.keys()
 
-        query = self._where_calc(cr, uid, domain, context=context)
-        self._apply_ir_rules(cr, uid, query, 'read', context=context)
-
-        # Take care of adding join(s) if groupby is an '_inherits'ed field
-        groupby_list = groupby
-        qualified_groupby_field = groupby
-        if groupby:
-            if isinstance(groupby, list):
-                groupby = groupby[0]
-            splitted_groupby = groupby.split(':')
-            if len(splitted_groupby) == 2:
-                groupby = splitted_groupby[0]
-                groupby_function = splitted_groupby[1]
-            else:
-                groupby_function = False
-            qualified_groupby_field = self._inherits_join_calc(groupby, query)
-
-        if groupby:
-            assert not groupby or groupby in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
-            groupby_def = self._columns.get(groupby) or (self._inherit_fields.get(groupby) and self._inherit_fields.get(groupby)[2])
-            assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
-
-        # TODO it seems fields_get can be replaced by _all_columns (no need for translation)
+        # Step 0 : preparing some useful variables
+        fields = fields or self._columns.keys()
         fget = self.fields_get(cr, uid, fields)
-        group_by_params = {}
-        select_terms = []
-        groupby_type = None
-        if groupby:
-            if fget.get(groupby):
-                groupby_type = fget[groupby]['type']
-                if groupby_type in ('date', 'datetime'):
-                    if groupby_function:
-                        interval = groupby_function
-                    else:
-                        interval = 'month'
+        if isinstance(groupby, basestring):
+            groupby = [groupby]
+        split_groupby = groupby[0].split(':') if groupby else None
+        first_groupby = split_groupby[0] if split_groupby else None
+        groupby_function = split_groupby[1] if split_groupby and len(split_groupby) == 2 else None
+        interval = groupby_function if groupby_function else 'month'
+        groupby_type = fget[first_groupby]['type'] if first_groupby else None
+        if groupby_type in ('date', 'datetime'):
+            dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby_type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
+            tz_convert = groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones
+            time_display_format = {
+                'day': 'dd MMM YYYY', 
+                'week': "'W'w YYYY", 
+                'month': 'MMMM YYYY', 
+                'quarter': 'QQQ YYYY', 
+                'year': 'YYYY'}[interval]
+            time_interval = {
+                'day': dateutil.relativedelta.relativedelta(months=3),
+                'week': datetime.timedelta(days=7),
+                'month': dateutil.relativedelta.relativedelta(months=1),
+                'quarter': dateutil.relativedelta.relativedelta(months=3),
+                'year': dateutil.relativedelta.relativedelta(years=1)}[interval]
 
-                    if interval == 'day':
-                        display_format = 'dd MMM YYYY' 
-                    elif interval == 'week':
-                        display_format = "'W'w YYYY"
-                    elif interval == 'month':
-                        display_format = 'MMMM YYYY'
-                    elif interval == 'quarter':
-                        display_format = 'QQQ YYYY'
-                    elif interval == 'year':
-                        display_format = 'YYYY'
+        query = self._where_calc(cr, uid, domain, context=context) 
 
-                    if groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones:
-                        # Convert groupby result to user TZ to avoid confusion!
-                        # PostgreSQL is compatible with all pytz timezone names, so we can use them
-                        # directly for conversion, starting with timestamps stored in UTC. 
-                        timezone = context.get('tz', 'UTC')
-                        qualified_groupby_field = "timezone('%s', timezone('UTC',%s))" % (timezone, qualified_groupby_field)
-                    qualified_groupby_field = "date_trunc('%s', %s)" % (interval, qualified_groupby_field)
-                elif groupby_type == 'boolean':
-                    qualified_groupby_field = "coalesce(%s,false)" % qualified_groupby_field
-                select_terms.append("%s as %s " % (qualified_groupby_field, groupby))
-            else:
+        # Step 1: security stuff 
+        # add relevant ir_rules to the where clause, perform some basic sanity checks
+        self._apply_ir_rules(cr, uid, query, 'read', context=context)
+        if first_groupby:
+            assert first_groupby in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
+            groupby_def = self._columns.get(first_groupby) or (self._inherit_fields.get(first_groupby) and self._inherit_fields.get(first_groupby)[2])
+            assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
+            if not (first_groupby in fget):
                 # Don't allow arbitrary values, as this would be a SQL injection vector!
                 raise except_orm(_('Invalid group_by'),
-                                 _('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(groupby,))
+                                 _('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(first_groupby,))
 
+        # Step 2: preparing the query:
+        # compute aggregated fields, format groupbys, adjust timezone, ...
         aggregated_fields = [
             f for f in fields
             if f not in ('id', 'sequence', groupby)
             if fget[f]['type'] in ('integer', 'float')
             if (f in self._all_columns and getattr(self._all_columns[f].column, '_classic_write'))]
-        for f in aggregated_fields:
-            group_operator = fget[f].get('group_operator', 'sum')
-            qualified_field = self._inherits_join_calc(f, query)
-            select_terms.append("%s(%s) AS %s" % (group_operator, qualified_field, f))
 
-        order = orderby or groupby or ''
-        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, groupby, qualified_groupby_field, query, groupby_type)
+        field_formatter = lambda f: (fget[f].get('group_operator', 'sum'), self._inherits_join_calc(f, query), f)
+        select_terms = ["%s(%s) AS %s" % field_formatter(f) for f in aggregated_fields]
+
+        qualified_groupby_field = self._inherits_join_calc(first_groupby, query) if first_groupby else None
+
+        if groupby_type in ('date', 'datetime'):
+            if tz_convert:
+                # Convert groupby result to user TZ to avoid confusion!
+                # PostgreSQL is compatible with all pytz timezone names, so we can use them
+                # directly for conversion, starting with timestamps stored in UTC. 
+                timezone = context.get('tz', 'UTC')
+                qualified_groupby_field = "timezone('%s', timezone('UTC',%s))" % (timezone, qualified_groupby_field)
+            qualified_groupby_field = "date_trunc('%s', %s)" % (interval, qualified_groupby_field)
+
+        if groupby_type == 'boolean':
+            qualified_groupby_field = "coalesce(%s,false)" % qualified_groupby_field
+
+        if first_groupby:
+                select_terms.append("%s as %s " % (qualified_groupby_field, first_groupby))
+
+        order = orderby or first_groupby or ''
+        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, first_groupby, qualified_groupby_field, query, groupby_type)
 
         from_clause, where_clause, where_clause_params = query.get_sql()
-        if len(groupby_list) < 2 and context.get('group_by_no_leaf'):
-            count_field = '_'
-        else:
-            count_field = groupby
 
         prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
         prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''
@@ -2352,7 +2341,7 @@ class BaseModel(object):
             %(offset)s
         """ % {
             'table': self._table,
-            'count_field': count_field,
+            'count_field': '_' if (len(groupby) < 2 and context.get('group_by_no_leaf')) else first_groupby,
             'extra_fields': prefix_terms(',', select_terms),
             'from': from_clause,
             'where': prefix_term('WHERE', where_clause),
@@ -2362,73 +2351,54 @@ class BaseModel(object):
             'offset': prefix_term('OFFSET', int(offset) if limit else None),
         }
         cr.execute(query, where_clause_params)
-        alldata = {}
-        fetched_data = cr.dictfetchall()
 
-        data_ids = []
-        for r in fetched_data:
-            for fld, val in r.items():
-                if val is None: r[fld] = False
-            alldata[r['id']] = r
-            data_ids.append(r['id'])
-            del r['id']
+        if not first_groupby:
+            return {r.pop('id'): r for r in cr.dictfetchall() }
 
-        if groupby:
-            data = self.read(cr, uid, data_ids, [groupby], context=context)
-            # restore order of the search as read() uses the default _order (this is only for groups, so the footprint of data should be small):
-            data_dict = dict((d['id'], d[groupby] ) for d in data)
-            result = [{'id': i, groupby: data_dict[i]} for i in data_ids]
-        else:
-            result = [{'id': i} for i in data_ids]
+        none_to_false = lambda record: {k: (False if v is None else v) for k,v in record.iteritems() }
+        fetched_data = map(none_to_false, cr.dictfetchall())
 
-        for d in result:
-            if groupby:
-                d['__domain'] = [(groupby, '=', alldata[d['id']][groupby] or False)] + domain
-                if not isinstance(groupby_list, (str, unicode)):
-                    if groupby or not context.get('group_by_no_leaf', False):
-                        d['__context'] = {'group_by': groupby_list[1:]}
-            if groupby and groupby in fget:
-                groupby_type = fget[groupby]['type']
-                if d[groupby] and groupby_type in ('date', 'datetime'):
-                    groupby_datetime = alldata[d['id']][groupby]
-                    if isinstance(groupby_datetime, basestring):
-                        _default = datetime.datetime(1970, 1, 1)    # force starts of month
-                        groupby_datetime = dateutil.parser.parse(groupby_datetime, default=_default)
-                    tz_convert = groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones
-                    if tz_convert:
-                        groupby_datetime =  pytz.timezone(context['tz']).localize(groupby_datetime)
-                    d[groupby] = babel.dates.format_date(
-                        groupby_datetime, format=display_format, locale=context.get('lang', 'en_US'))
-                    domain_dt_begin = groupby_datetime
-                    if interval == 'quarter':
-                        domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(months=3)
-                    elif interval == 'month':
-                        domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(months=1)
-                    elif interval == 'week':
-                        domain_dt_end = groupby_datetime + datetime.timedelta(days=7)
-                    elif interval == 'day':
-                        domain_dt_end = groupby_datetime + datetime.timedelta(days=1)
-                    else:
-                        domain_dt_end = groupby_datetime + dateutil.relativedelta.relativedelta(years=1)
-                    if tz_convert:
-                        # the time boundaries were all computed in the apparent TZ of the user,
-                        # so we need to convert them to UTC to have proper server-side values.
-                        domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
-                        domain_dt_end = domain_dt_end.astimezone(pytz.utc)
-                    dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby_type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
-                    d['__domain'] = [(groupby, '>=', domain_dt_begin.strftime(dt_format)),
-                                     (groupby, '<', domain_dt_end.strftime(dt_format))] + domain
-                del alldata[d['id']][groupby]
-            d.update(alldata[d['id']])
-            del d['id']
+        data_ids = [r['id'] for r in fetched_data]
+        data_dict = {d['id']: d[first_groupby] for d in self.read(cr, uid, data_ids, [first_groupby], context=context)} 
+        sorted_data = [{first_groupby: data_dict[id]} for id in data_ids]
 
-        if groupby and groupby in self._group_by_full:
-            result = self._read_group_fill_results(cr, uid, domain, groupby, groupby_list,
+        def format_result (fromquery, fromread):
+            result = {
+                '__domain': [(first_groupby, '=', fromquery[first_groupby] or False)] + domain,
+                '__context': {'group_by': groupby[1:]}
+            }
+            result.update(fromquery)
+            result.update(fromread)
+
+            if groupby_type in ('date', 'datetime') and fromquery[first_groupby]:
+                groupby_datetime = fromquery[first_groupby]
+                if isinstance(groupby_datetime, basestring):
+                    groupby_datetime = datetime.datetime.strptime(groupby_datetime, dt_format)
+                if tz_convert:
+                    groupby_datetime =  pytz.timezone(context['tz']).localize(groupby_datetime)
+                result[first_groupby] = babel.dates.format_date(
+                    groupby_datetime, format=time_display_format, locale=context.get('lang', 'en_US'))
+                domain_dt_begin = groupby_datetime
+                domain_dt_end = groupby_datetime + time_interval
+                if tz_convert:
+                    # the time boundaries were all computed in the apparent TZ of the user,
+                    # so we need to convert them to UTC to have proper server-side values.
+                    domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
+                    domain_dt_end = domain_dt_end.astimezone(pytz.utc)
+                result['__domain'] = [(first_groupby, '>=', domain_dt_begin.strftime(dt_format)),
+                                 (first_groupby, '<', domain_dt_end.strftime(dt_format))] + domain
+            del result['id']
+            return result
+
+        result = map(format_result, fetched_data, sorted_data)
+
+        if first_groupby in self._group_by_full:
+            result = self._read_group_fill_results(cr, uid, domain, first_groupby, groupby,
                                                    aggregated_fields, result, read_group_order=order,
                                                    context=context)
-
         return result
 
+
     def _inherits_join_add(self, current_model, parent_model_name, query):
         """
         Add missing table SELECT and JOIN clause to ``query`` for reaching the parent table (no duplicates)

From 30e787511f743648c7c86d932e215a854b7a5448 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Tue, 25 Mar 2014 17:11:04 +0100
Subject: [PATCH 02/26] [IMP] refactoring of readgroup method, to make it
 eager.  Still incomplete, but it is mostly working (orm.py)

bzr revid: ged@openerp.com-20140325161104-d0xgof4bcskh4ie6
---
 openerp/osv/orm.py | 217 +++++++++++++++++++++++++--------------------
 1 file changed, 119 insertions(+), 98 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index cc8e7aa066b..ac07a6f5bca 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -55,7 +55,7 @@ import traceback
 import types
 
 import babel.dates
-import dateutil.parser
+import dateutil.relativedelta
 import psycopg2
 from lxml import etree
 
@@ -2185,20 +2185,21 @@ class BaseModel(object):
                 r['__fold'] = folded.get(r[groupby] and r[groupby][0], False)
         return result
 
-    def _read_group_prepare(self, orderby, aggregated_fields, groupby, qualified_groupby_field, query, groupby_type=None):
+    def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query, fget):
         """
         Prepares the GROUP BY and ORDER BY terms for the read_group method. Adds the missing JOIN clause
         to the query if order should be computed against m2o field. 
         :param orderby: the orderby definition in the form "%(field)s %(order)s"
         :param aggregated_fields: list of aggregated fields in the query
         :param groupby: the current groupby field name
-        :param qualified_groupby_field: the fully qualified SQL name for the grouped field
+        :param qualified_field: the fully qualified SQL name for the grouped field
         :param osv.Query query: the query under construction
         :param groupby_type: the type of the grouped field
         :return: (groupby_terms, orderby_terms)
         """
         orderby_terms = []
-        groupby_terms = [qualified_groupby_field] if groupby else []
+        groupby_terms = [gb['qualified_field'] for gb in annotated_groupbys]
+        groupby_fields = [gb['field'] for gb in annotated_groupbys]
         if not orderby:
             return groupby_terms, orderby_terms    
 
@@ -2206,8 +2207,9 @@ class BaseModel(object):
         for order_part in orderby.split(','):
             order_split = order_part.split()
             order_field = order_split[0]
-            if order_field == groupby:
-                if groupby_type == 'many2one':
+            if order_field in groupby_fields:
+
+                if fget[order_field]['type'] == 'many2one':
                     order_clause = self._generate_order_by(order_part, query).replace('ORDER BY ', '')
                     if order_clause:
                         orderby_terms.append(order_clause)
@@ -2222,7 +2224,7 @@ class BaseModel(object):
                              self._name, order_part)
         return groupby_terms, orderby_terms
 
-    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False):
+    def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False, lazy=True):
         """
         Get the list of records in list view grouped by the given ``groupby`` fields
 
@@ -2242,6 +2244,9 @@ class BaseModel(object):
                              overriding the natural sort ordering of the
                              groups, see also :py:meth:`~osv.osv.osv.search`
                              (supported only for many2one fields currently)
+        :param bool lazy: if true, the results are only grouped by the first groupby and the 
+                remaining groupbys are put in the __context key.  If false, all the groupbys are
+                done in one call.
         :return: list of dictionaries(one dictionary for each record) containing:
 
                     * the values of fields grouped by the fields in ``groupby`` argument
@@ -2253,86 +2258,84 @@ class BaseModel(object):
 
         """
         self.check_access_rights(cr, uid, 'read')
-
-        # Step 0 : preparing some useful variables
+        query = self._where_calc(cr, uid, domain, context=context) 
         fields = fields or self._columns.keys()
         fget = self.fields_get(cr, uid, fields)
-        if isinstance(groupby, basestring):
-            groupby = [groupby]
-        split_groupby = groupby[0].split(':') if groupby else None
-        first_groupby = split_groupby[0] if split_groupby else None
-        groupby_function = split_groupby[1] if split_groupby and len(split_groupby) == 2 else None
-        interval = groupby_function if groupby_function else 'month'
-        groupby_type = fget[first_groupby]['type'] if first_groupby else None
-        if groupby_type in ('date', 'datetime'):
-            dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby_type == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
-            tz_convert = groupby_type == 'datetime' and context.get('tz') in pytz.all_timezones
-            time_display_format = {
-                'day': 'dd MMM YYYY', 
-                'week': "'W'w YYYY", 
-                'month': 'MMMM YYYY', 
-                'quarter': 'QQQ YYYY', 
-                'year': 'YYYY'}[interval]
-            time_interval = {
-                'day': dateutil.relativedelta.relativedelta(months=3),
-                'week': datetime.timedelta(days=7),
-                'month': dateutil.relativedelta.relativedelta(months=1),
-                'quarter': dateutil.relativedelta.relativedelta(months=3),
-                'year': dateutil.relativedelta.relativedelta(years=1)}[interval]
+        groupby = [groupby] if isinstance(groupby, basestring) else groupby
+        display_formats = {
+            'day': 'dd MMM YYYY', 
+            'week': "'W'w YYYY", 
+            'month': 'MMMM YYYY', 
+            'quarter': 'QQQ YYYY', 
+            'year': 'YYYY'
+        }
+        time_intervals = {
+            'day': dateutil.relativedelta.relativedelta(months=3),
+            'week': datetime.timedelta(days=7),
+            'month': dateutil.relativedelta.relativedelta(months=1),
+            'quarter': dateutil.relativedelta.relativedelta(months=3),
+            'year': dateutil.relativedelta.relativedelta(years=1)
+        }
 
-        query = self._where_calc(cr, uid, domain, context=context) 
+        def process_groupby (gb):
+            split = gb.split(':')
+            field_type = fget[split[0]]['type'] 
+            gb_function = split[1] if len(split) == 2 else None
+            temporal = field_type in ('date', 'datetime')
+            tz_convert = field_type == 'datetime' and context.get('tz') in pytz.all_timezones
+            qualified_field = self._inherits_join_calc(split[0], query)
+            if temporal:
+                if tz_convert:
+                    qualified_field = "timezone('%s', timezone('UTC',%s))" % (context.get('tz', 'UTC'), qualified_field)
+                qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field)
+            if field_type == 'boolean':
+                qualified_field = "coalesce(%s,false)" % qualified_field
+            return {
+                'field': split[0], 
+                'type': field_type, 
+                'display_format': display_formats[gb_function or 'month'] if temporal else None,
+                'interval': time_intervals[gb_function or 'month'] if temporal else None,                
+                'tz_convert': tz_convert,
+                'qualified_field': qualified_field
+            }
+
+        annotated_groupbys = map(process_groupby, groupby[:1] if lazy else groupby)
+        groupby_fields = [g['field'] for g in annotated_groupbys]
+        order = orderby or ','.join(groupby_fields)
+        groupby_dict = {gb['field']: gb for gb in annotated_groupbys}
 
-        # Step 1: security stuff 
-        # add relevant ir_rules to the where clause, perform some basic sanity checks
         self._apply_ir_rules(cr, uid, query, 'read', context=context)
-        if first_groupby:
-            assert first_groupby in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
-            groupby_def = self._columns.get(first_groupby) or (self._inherit_fields.get(first_groupby) and self._inherit_fields.get(first_groupby)[2])
+        for gb in groupby_fields:
+            assert gb in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
+            groupby_def = self._columns.get(gb) or (self._inherit_fields.get(gb) and self._inherit_fields.get(gb)[2])
             assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
-            if not (first_groupby in fget):
+            if not (gb in fget):
                 # Don't allow arbitrary values, as this would be a SQL injection vector!
                 raise except_orm(_('Invalid group_by'),
-                                 _('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(first_groupby,))
+                                 _('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(gb,))
 
-        # Step 2: preparing the query:
-        # compute aggregated fields, format groupbys, adjust timezone, ...
         aggregated_fields = [
             f for f in fields
-            if f not in ('id', 'sequence', groupby)
+            if f not in ('id', 'sequence')
+            if f not in groupby_fields
             if fget[f]['type'] in ('integer', 'float')
             if (f in self._all_columns and getattr(self._all_columns[f].column, '_classic_write'))]
 
         field_formatter = lambda f: (fget[f].get('group_operator', 'sum'), self._inherits_join_calc(f, query), f)
         select_terms = ["%s(%s) AS %s" % field_formatter(f) for f in aggregated_fields]
 
-        qualified_groupby_field = self._inherits_join_calc(first_groupby, query) if first_groupby else None
-
-        if groupby_type in ('date', 'datetime'):
-            if tz_convert:
-                # Convert groupby result to user TZ to avoid confusion!
-                # PostgreSQL is compatible with all pytz timezone names, so we can use them
-                # directly for conversion, starting with timestamps stored in UTC. 
-                timezone = context.get('tz', 'UTC')
-                qualified_groupby_field = "timezone('%s', timezone('UTC',%s))" % (timezone, qualified_groupby_field)
-            qualified_groupby_field = "date_trunc('%s', %s)" % (interval, qualified_groupby_field)
-
-        if groupby_type == 'boolean':
-            qualified_groupby_field = "coalesce(%s,false)" % qualified_groupby_field
-
-        if first_groupby:
-                select_terms.append("%s as %s " % (qualified_groupby_field, first_groupby))
-
-        order = orderby or first_groupby or ''
-        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, first_groupby, qualified_groupby_field, query, groupby_type)
+        for gb in annotated_groupbys:
+            select_terms.append("%s as %s " % (gb['qualified_field'], gb['field']))
 
+        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query, fget)
         from_clause, where_clause, where_clause_params = query.get_sql()
+        count_field = groupby_fields[0] if (len(groupby_fields) == 1 and context.get('group_by_no_leaf')) else '_'
 
         prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
         prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''
 
         query = """
-            SELECT min(%(table)s.id) AS id, count(%(table)s.id) AS %(count_field)s_count
-                   %(extra_fields)s
+            SELECT min(%(table)s.id) AS id, count(%(table)s.id) AS %(count_field)s_count %(extra_fields)s
             FROM %(from)s
             %(where)s
             %(groupby)s
@@ -2341,7 +2344,7 @@ class BaseModel(object):
             %(offset)s
         """ % {
             'table': self._table,
-            'count_field': '_' if (len(groupby) < 2 and context.get('group_by_no_leaf')) else first_groupby,
+            'count_field': count_field,
             'extra_fields': prefix_terms(',', select_terms),
             'from': from_clause,
             'where': prefix_term('WHERE', where_clause),
@@ -2352,53 +2355,71 @@ class BaseModel(object):
         }
         cr.execute(query, where_clause_params)
 
-        if not first_groupby:
+        if not groupby_fields:
             return {r.pop('id'): r for r in cr.dictfetchall() }
 
-        none_to_false = lambda record: {k: (False if v is None else v) for k,v in record.iteritems() }
-        fetched_data = map(none_to_false, cr.dictfetchall())
+        def prepare_data(key, value):
+            value = False if value is None else value
+            gb = groupby_dict.get(key)
+            if gb and gb['type'] in ('date', 'datetime') and value:
+                if isinstance(value, basestring):
+                    dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
+                    value = datetime.datetime.strptime(value, dt_format)
+                if gb['tz_convert']:
+                    value =  pytz.timezone(context['tz']).localize(value)
+            return value
 
-        data_ids = [r['id'] for r in fetched_data]
-        data_dict = {d['id']: d[first_groupby] for d in self.read(cr, uid, data_ids, [first_groupby], context=context)} 
-        sorted_data = [{first_groupby: data_dict[id]} for id in data_ids]
+        fetched_data = cr.dictfetchall()
 
-        def format_result (fromquery, fromread):
-            result = {
-                '__domain': [(first_groupby, '=', fromquery[first_groupby] or False)] + domain,
-                '__context': {'group_by': groupby[1:]}
-            }
-            result.update(fromquery)
-            result.update(fromread)
+        many2onefields = [gb['field'] for gb in annotated_groupbys if gb['type'] == 'many2one']
+        if many2onefields:
+            data_ids = [r['id'] for r in fetched_data]
+            data_dict = {d['id']: d for d in self.read(cr, uid, data_ids, many2onefields, context=context)} 
+            for d in fetched_data:
+                d.update(data_dict[d['id']])
 
-            if groupby_type in ('date', 'datetime') and fromquery[first_groupby]:
-                groupby_datetime = fromquery[first_groupby]
-                if isinstance(groupby_datetime, basestring):
-                    groupby_datetime = datetime.datetime.strptime(groupby_datetime, dt_format)
-                if tz_convert:
-                    groupby_datetime =  pytz.timezone(context['tz']).localize(groupby_datetime)
-                result[first_groupby] = babel.dates.format_date(
-                    groupby_datetime, format=time_display_format, locale=context.get('lang', 'en_US'))
-                domain_dt_begin = groupby_datetime
-                domain_dt_end = groupby_datetime + time_interval
-                if tz_convert:
-                    # the time boundaries were all computed in the apparent TZ of the user,
-                    # so we need to convert them to UTC to have proper server-side values.
+        data = map(lambda r: {k: prepare_data(k,v) for k,v in r.iteritems()}, fetched_data)
+
+        def get_domain(gb, value):
+            if gb['type'] in ('date', 'datetime') and value:
+                dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
+                domain_dt_begin = value
+                domain_dt_end = value + gb['interval']
+                if gb['tz_convert']:
                     domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
                     domain_dt_end = domain_dt_end.astimezone(pytz.utc)
-                result['__domain'] = [(first_groupby, '>=', domain_dt_begin.strftime(dt_format)),
-                                 (first_groupby, '<', domain_dt_end.strftime(dt_format))] + domain
+                return [(gb['field'], '>=', domain_dt_begin.strftime(dt_format)),
+                       (gb['field'], '<', domain_dt_end.strftime(dt_format))]
+            if gb['type'] == 'many2one' and value:
+                    value = value[0]
+            return [(gb['field'], '=', value)]
+
+        def format_result (fromquery):
+            domain_group = [dom for gb in annotated_groupbys for dom in get_domain(gb, fromquery[gb['field']])]
+            result = {
+                '__domain': domain_group + domain,
+                '__context': {'group_by': groupby[len(annotated_groupbys):]}
+            }
+            result.update(fromquery)
+            for k,v in result.iteritems():
+                gb = groupby_dict.get(k)
+                if gb and gb['type'] in ('date', 'datetime') and v:
+                    result[k] = babel.dates.format_date(v, format=gb['display_format'], locale=context.get('lang', 'en_US'))
             del result['id']
             return result
 
-        result = map(format_result, fetched_data, sorted_data)
+        for gb in groupby_fields:
+            if gb in self._group_by_full:
+                print "DING"
+        result = map(format_result, data)
 
-        if first_groupby in self._group_by_full:
-            result = self._read_group_fill_results(cr, uid, domain, first_groupby, groupby,
-                                                   aggregated_fields, result, read_group_order=order,
-                                                   context=context)
+        # TO DO!!!!!!!!!!!!
+        # if first_groupby in self._group_by_full:
+        #     result = self._read_group_fill_results(cr, uid, domain, first_groupby, groupby,
+        #                                            aggregated_fields, result, read_group_order=order,
+        #                                            context=context)
         return result
 
-
     def _inherits_join_add(self, current_model, parent_model_name, query):
         """
         Add missing table SELECT and JOIN clause to ``query`` for reaching the parent table (no duplicates)

From 8c93dcbf62f918c10d01a62db6c04d5c173e8761 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 09:38:00 +0200
Subject: [PATCH 03/26] [IMP] refactoring of readgroup method. The work is
 pretty much complete with this commit. Now, it correctly fills in empty group
 using _read_group_fill_results (orm.py)

bzr revid: ged@openerp.com-20140402073800-0ggxqfpyo2in8ozt
---
 openerp/osv/orm.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index ac07a6f5bca..45dc264c67c 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2115,7 +2115,7 @@ class BaseModel(object):
             pass
 
 
-    def _read_group_fill_results(self, cr, uid, domain, groupby, groupby_list, aggregated_fields,
+    def _read_group_fill_results(self, cr, uid, domain, groupby, remaining_groupbys, aggregated_fields,
                                  read_group_result, read_group_order=None, context=None):
         """Helper method for filling in empty groups for all possible values of
            the field being grouped by"""
@@ -2135,8 +2135,8 @@ class BaseModel(object):
 
         result_template = dict.fromkeys(aggregated_fields, False)
         result_template[groupby + '_count'] = 0
-        if groupby_list and len(groupby_list) > 1:
-            result_template['__context'] = {'group_by': groupby_list[1:]}
+        if remaining_groupbys:
+            result_template['__context'] = {'group_by': remaining_groupbys}
 
         # Merge the left_side (current results as dicts) with the right_side (all
         # possible values as m2o pairs). Both lists are supposed to be using the
@@ -2396,10 +2396,9 @@ class BaseModel(object):
 
         def format_result (fromquery):
             domain_group = [dom for gb in annotated_groupbys for dom in get_domain(gb, fromquery[gb['field']])]
-            result = {
-                '__domain': domain_group + domain,
-                '__context': {'group_by': groupby[len(annotated_groupbys):]}
-            }
+            result = { '__domain': domain_group + domain }
+            if len(groupby) - len(annotated_groupbys) >= 1:
+                result['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
             result.update(fromquery)
             for k,v in result.iteritems():
                 gb = groupby_dict.get(k)
@@ -2408,16 +2407,13 @@ class BaseModel(object):
             del result['id']
             return result
 
+        result = map(format_result, data)
+        
         for gb in groupby_fields:
             if gb in self._group_by_full:
-                print "DING"
-        result = map(format_result, data)
-
-        # TO DO!!!!!!!!!!!!
-        # if first_groupby in self._group_by_full:
-        #     result = self._read_group_fill_results(cr, uid, domain, first_groupby, groupby,
-        #                                            aggregated_fields, result, read_group_order=order,
-        #                                            context=context)
+                result = self._read_group_fill_results(cr, uid, domain, gb, groupby[len(annotated_groupbys):],
+                                                       aggregated_fields, result, read_group_order=order,
+                                                       context=context)
         return result
 
     def _inherits_join_add(self, current_model, parent_model_name, query):

From 95b2daf17b9f3301095851e855b85dcbe6fa955f Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 09:41:22 +0200
Subject: [PATCH 04/26] [IMP] modifies the web client to be able to use the new
 'lazy' parameter of the method readgroup (so now the client can requires a
 readgroup to be eager/lazy) (addon web)

bzr revid: ged@openerp.com-20140402074122-sf52hult1qkqos81
---
 addons/web/static/src/js/data.js | 44 ++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/addons/web/static/src/js/data.js b/addons/web/static/src/js/data.js
index 517afa7f5c2..413d0621011 100644
--- a/addons/web/static/src/js/data.js
+++ b/addons/web/static/src/js/data.js
@@ -27,6 +27,7 @@ instance.web.Query = instance.web.Class.extend({
         this._fields = fields;
         this._filter = [];
         this._context = {};
+        this._lazy = true;
         this._limit = false;
         this._offset = 0;
         this._order_by = [];
@@ -36,6 +37,7 @@ instance.web.Query = instance.web.Class.extend({
         var q = new instance.web.Query(this._model, this._fields);
         q._context = this._context;
         q._filter = this._filter;
+        q._lazy = this._lazy;
         q._limit = this._limit;
         q._offset = this._offset;
         q._order_by = this._order_by;
@@ -51,6 +53,7 @@ instance.web.Query = instance.web.Class.extend({
                 q._context = new instance.web.CompoundContext(
                         q._context, to_set.context);
                 break;
+            case 'lazy':
             case 'limit':
             case 'offset':
             case 'order_by':
@@ -140,6 +143,7 @@ instance.web.Query = instance.web.Class.extend({
             domain: this._model.domain(this._filter),
             context: ctx,
             offset: this._offset,
+            lazy: this._lazy,
             limit: this._limit,
             orderby: instance.web.serialize_sort(this._order_by) || false
         }).then(function (results) {
@@ -148,8 +152,9 @@ instance.web.Query = instance.web.Class.extend({
                 result.__context = result.__context || {};
                 result.__context.group_by = result.__context.group_by || [];
                 _.defaults(result.__context, ctx);
+                var grouping_fields = self._lazy ? [grouping[0]] : grouping;
                 return new instance.web.QueryGroup(
-                    self._model.name, grouping[0], result);
+                    self._model.name, grouping_fields, result);
             });
         });
     },
@@ -175,6 +180,18 @@ instance.web.Query = instance.web.Class.extend({
         if (!domain) { return this; }
         return this.clone({filter: domain});
     },
+    /**
+     * Creates a new query with the provided parameter lazy replacing the current
+     * query's own.
+     *
+     * @param {Boolean} lazy indicates if the read_group should return only the 
+     * first level of groupby records, or should return the records grouped by
+     * all levels at once (so, it makes only 1 db request).
+     * @returns {openerp.web.Query}
+     */
+    lazy: function (lazy) {
+        return this.clone({lazy: lazy});
+    },
     /**
      * Creates a new query with the provided limit replacing the current
      * query's own limit
@@ -213,7 +230,7 @@ instance.web.Query = instance.web.Class.extend({
 });
 
 instance.web.QueryGroup = instance.web.Class.extend({
-    init: function (model, grouping_field, read_group_group) {
+    init: function (model, grouping_fields, read_group_group) {
         // In cases where group_by_no_leaf and no group_by, the result of
         // read_group has aggregate fields but no __context or __domain.
         // Create default (empty) values for those so that things don't break
@@ -221,29 +238,40 @@ instance.web.QueryGroup = instance.web.Class.extend({
             {__context: {group_by: []}, __domain: []},
             read_group_group);
 
-        var raw_field = grouping_field && grouping_field.split(':')[0];
+        var raw_fields = _.map(grouping_fields, function (field) {
+            return field && field.split(':')[0];
+        });
+
+
         var aggregates = {};
         _(fixed_group).each(function (value, key) {
             if (key.indexOf('__') === 0
-                    || key === raw_field
-                    || key === raw_field + '_count') {
+                    || _.contains(raw_fields, key)
+                    || (key === raw_fields[0] + '_count')) {
                 return;
             }
             aggregates[key] = value || 0;
         });
 
+
         this.model = new instance.web.Model(
             model, fixed_group.__context, fixed_group.__domain);
 
-        var group_size = fixed_group[raw_field + '_count'] || fixed_group.__count || 0;
+        var group_size = fixed_group[raw_fields[0] + '_count'] || fixed_group.__count || 0;
         var leaf_group = fixed_group.__context.group_by.length === 0;
 
+        var value = (grouping_fields.length === 1) 
+                ? fixed_group[raw_fields[0]]
+                : _.map(raw_fields, function (field) { return fixed_group[field]; });
+        var grouped_on = (grouping_fields.length === 1) 
+                ? grouping_fields[0] 
+                : grouping_fields;
         this.attributes = {
             folded: !!(fixed_group.__fold),
-            grouped_on: grouping_field,
+            grouped_on: grouped_on,
             // if terminal group (or no group) and group_by_no_leaf => use group.__count
             length: group_size,
-            value: fixed_group[raw_field],
+            value: value,
             // A group is open-able if it's not a leaf in group_by_no_leaf mode
             has_children: !(leaf_group && fixed_group.__context['group_by_no_leaf']),
 

From c4d0299eca6941e4f38f86c7e8c4a3e9a5cf84a7 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 09:43:41 +0200
Subject: [PATCH 05/26] [IMP] complete rewrite of the data loading code in the
 graph view, using the lazy/eager feature of readgroup.  The code is simpler
 and more efficient: it only performs a fixed number of requests (well, the
 number is 'fixed' in the sense that it does not depend of the size of the
 data being displayed, but it still depends on the number of dimensions)
 (addon web_graph)

bzr revid: ged@openerp.com-20140402074341-8cleqizqculxmkys
---
 addons/web_graph/static/src/js/pivot_table.js | 406 ++++++++----------
 1 file changed, 182 insertions(+), 224 deletions(-)

diff --git a/addons/web_graph/static/src/js/pivot_table.js b/addons/web_graph/static/src/js/pivot_table.js
index f77d49f9a1a..e8a438b3fe5 100644
--- a/addons/web_graph/static/src/js/pivot_table.js
+++ b/addons/web_graph/static/src/js/pivot_table.js
@@ -82,7 +82,9 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
 
 	get_values: function (id1, id2, default_values) {
 		var cell = _.findWhere(this.cells, {x: Math.min(id1, id2), y: Math.max(id1, id2)});
-		return (cell !== undefined) ? cell.values : (default_values || new Array(this.measures.length));
+		return (cell !== undefined) ?
+                    cell.values :
+                    (default_values || new Array(this.measures.length));
 	},
 
 	// ----------------------------------------------------------------------
@@ -144,12 +146,16 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
     get_ancestors: function (header) {
         var self = this;
         if (!header.children) return [];
-        return  [].concat.apply([], _.map(header.children, function (c) {return self.get_ancestors_and_self(c); }));
+        return  [].concat.apply([], _.map(header.children, function (c) {
+            return self.get_ancestors_and_self(c);
+        }));
     },
 
     get_ancestors_and_self: function (header) {
         var self = this;
-        return [].concat.apply([header], _.map(header.children, function (c) { return self.get_ancestors_and_self(c); }));
+        return [].concat.apply([header], _.map(header.children, function (c) {
+            return self.get_ancestors_and_self(c);
+        }));
     },
 
 	get_total: function (header) {
@@ -205,54 +211,28 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
 	expand: function (header_id, groupby) {
         var self = this,
             header = this.get_header(header_id),
-			otherRoot = this.get_other_root(header),
-			fields = otherRoot.groupby.concat(this.measures);
+            other_root = this.get_other_root(header),
+            this_gb = [groupby.field],
+            other_gbs = _.pluck(other_root.groupby, 'field');
 
         if (header.path.length === header.root.groupby.length) {
             header.root.groupby.push(groupby);
         }
-        groupby = [groupby].concat(otherRoot.groupby);
-
-        return this.get_groups(groupby, fields, header.domain).then(function (groups) {
-            _.each(groups.reverse(), function (group) {
-                // make header
-                var child = self.make_header(group, header);
-                child.expanded = false;
-                header.children.splice(0,0, child);
-                header.root.headers.splice(header.root.headers.indexOf(header) + 1, 0, child);
-                // make cells
-                _.each(self.get_ancestors_and_self(group), function (data) {
-                    var values = _.map(self.measures, function (m) {
-                        return data.attributes.aggregates[m.field];
-                    });
-                    var other = _.find(otherRoot.headers, function (h) {
-                        if (header.root === self.cols) {
-                            return _.isEqual(data.path.slice(1), h.path);
-                        } else {
-                            return _.isEqual(_.rest(data.path), h.path);
-                            }
-                        });
-                    if (other) {
-                        self.add_cell(child.id, other.id, values);
-                    }
-                });
+        return this.perform_requests(this_gb, other_gbs, header.domain).then(function () {
+            var data = Array.prototype.slice.call(arguments);
+            data = data.slice(other_gbs.length + 1, data.length);
+            _.each(data, function (data_pt) {
+                self.make_headers_and_cell(
+                    data_pt, header.root.headers, other_root.headers, 1, header.path);
             });
             header.expanded = true;
+            header.children.forEach(function (child) {
+                child.expanded = false;
+                child.root = header.root;
+            });
         });
 	},
 
-	make_header: function (group, parent) {
-		var title = parent ? group.attributes.value : _t('Total');
-        return {
-			id: _.uniqueId(),
-			path: parent ? parent.path.concat(title) : [],
-			title: title,
-			children: [],
-			domain: parent ? group.model._domain : this.domain,
-			root: parent ? parent.root : undefined,
-		};
-	},
-
 	swap_axis: function () {
 		var temp = this.rows;
 		this.rows = this.cols;
@@ -262,206 +242,184 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
 	// ----------------------------------------------------------------------
 	// Data updating methods
 	// ----------------------------------------------------------------------
-    // Load the data from the db, using the method this.load_data
     // update_data will try to preserve the expand/not expanded status of each
     // column/row.  If you want to expand all, then set this.cols.headers/this.rows.headers 
     // to null before calling update_data.
-	update_data: function () {
-		var self = this;
+    update_data: function () {
+        var self = this;
+        return this.perform_requests().then (function () {
+            var data = Array.prototype.slice.call(arguments);
+            self.no_data = !data[0][0].attributes.length;
+            if (self.no_data) {
+                return;
+            }
+            var row_headers = [],
+                col_headers = [];
+            self.cells = [];
 
-		return this.load_data().then (function (result) {
-			if (result) {
-				self.no_data = false;
-				self[self.cols.headers ? 'update_headers' : 'expand_headers'](self.cols, result.col_headers);
-				self[self.rows.headers ? 'update_headers' : 'expand_headers'](self.rows, result.row_headers);
-			} else {
-				self.no_data = true;
-			}
-		});
-	},
+            var dim_col = self.cols.groupby.length,
+                i, j, index;
 
-	expand_headers: function (root, new_headers) {
-		root.headers = new_headers;
-		_.each(root.headers, function (header) {
-			header.root = root;
-			header.expanded = (header.children.length > 0);
-		});
-	},
-
-	update_headers: function (root, new_headers) {
-		_.each(root.headers, function (header) {
-			var corresponding_header = _.find(new_headers, function (h) {
-				return _.isEqual(h.path, header.path);
-			});
-			if (corresponding_header && header.expanded) {
-				corresponding_header.expanded = true;
-				_.each(corresponding_header.children, function (c) {
-					c.expanded = false;
-				});
-			}
-			if (corresponding_header && (!header.expanded)) {
-				corresponding_header.expanded = false;
-			}
-		});
-		var updated_headers = _.filter(new_headers, function (header) {
-			return (header.expanded !== undefined);
-		});
-		_.each(updated_headers, function (header) {
-			if (!header.expanded) {
-				header.children = [];
-			}
-			header.root = root;
-		});
-		root.headers = updated_headers;
-	},
-
-    // ----------------------------------------------------------------------
-    // Data loading methods
-    // ----------------------------------------------------------------------
-
-    // To obtain all the values required to draw the full table, we have to do 
-    // at least      2 + min(row.groupby.length, col.groupby.length)
-    // calls to readgroup. To simplify the code, we will always do 
-    // 2 + row.groupby.length calls. For example, if row.groupby = [r1, r2, r3] 
-    // and col.groupby = [c1, c2], then we will make the call with the following 
-    // groupbys: [r1,r2,r3], [c1,r1,r2,r3], [c1,c2,r1,r2,r3], [].
-    load_data: function () {
-        var self = this,
-            cols = this.cols.groupby,
-            rows = this.rows.groupby,
-            visible_fields = rows.concat(cols, self.measures);
-
-        if (this.measures.length === 0) {
-            return $.Deferred.resolve().promise();
-        }
-
-        var groupbys = _.map(_.range(cols.length + 1), function (i) {
-            return cols.slice(0, i).concat(rows);
-        });
-        groupbys.push([]);
-
-        var get_data_requests = _.map(groupbys, function (groupby) {
-            return self.get_groups(groupby, visible_fields, self.domain);
-        });
-
-        return $.when.apply(null, get_data_requests).then(function () {
-            var data = Array.prototype.slice.call(arguments),
-                row_data = data[0],
-                col_data = (cols.length !== 0) ? data[data.length - 2] : [],
-                has_data = data[data.length - 1][0];
-
-            return has_data && self.format_data(col_data, row_data, data);
+            for (i = 0; i < self.rows.groupby.length + 1; i++) {
+                for (j = 0; j < dim_col + 1; j++) {
+                    index = i*(dim_col + 1) + j;
+                    self.make_headers_and_cell(data[index], row_headers, col_headers, i);
+                }
+            }
+            self.set_headers(row_headers, self.rows);
+            self.set_headers(col_headers, self.cols);
         });
     },
 
-	get_groups: function (groupbys, fields, domain, path) {
-		var self = this,
-            groupby = (groupbys.length) ? groupbys[0] : [];
-        path = path || [];
+    make_headers_and_cell: function (data_pts, row_headers, col_headers, index, prefix) {
+        var self = this;
+        data_pts.forEach(function (data_pt) {
+            var row_value = (prefix || []).concat(data_pt.attributes.value.slice(0,index));
+            var col_value = data_pt.attributes.value.slice(index);
+            
+            var row = self.find_or_create_header(row_headers, row_value, data_pt);
+            var col = self.find_or_create_header(col_headers, col_value, data_pt);
 
-		return this._query_db(groupby, fields, domain, path).then(function (groups) {
-            if (groupbys.length > 1) {
-                var get_subgroups = $.when.apply(null, _.map(groups, function (group) {
-                    return self.get_groups(_.rest(groupbys), fields, group.model._domain, path.concat(group.attributes.value)).then(function (subgroups) {
-                        group.children = subgroups;
-                    });
-                }));
-                 return get_subgroups.then(function () {
-                     return groups;
-                 });
-            } else {
-                return groups;
+            var cell_value = _.map(self.measures, function (m) { 
+                return data_pt.attributes.aggregates[m.field];
+            });
+            self.cells.push({
+                x: Math.min(row.id, col.id), 
+                y: Math.max(row.id, col.id), 
+                values: cell_value
+            });            
+        });
+    },
+
+    make_header: function (values) {
+        return _.extend({
+            children: [], 
+            domain: this.domain, 
+            expanded: undefined, 
+            id: _.uniqueId(),
+            path: [],
+            root: undefined,
+            title: undefined
+        }, values || {});
+    },
+
+    find_or_create_header: function (headers, path, data_pt) {
+        var hdr = _.find(headers, function (header) { 
+            return _.isEqual(path, header.path);
+        });
+        if (hdr) {
+            return hdr;
+        }
+        if (!path.length) {
+            hdr = this.make_header({title: _t('Total')});
+            headers.push(hdr);
+            return hdr;
+        }
+        hdr = this.make_header({
+            path:path, 
+            domain:data_pt.model._domain,
+            title: _t(_.last(path))
+        });
+        parent = _.find(headers, function (header) {
+            return _.isEqual(header.path, _.initial(path, 1));
+        });
+
+        var previous = parent.children.length ? _.last(parent.children) : parent;
+        headers.splice(headers.indexOf(previous) + 1, 0, hdr);
+        parent.children.push(hdr);
+        return hdr;
+    },
+
+    perform_requests: function (group1, group2, domain) {
+        var self = this,
+            requests = [],
+            row_gbs = _.pluck(this.rows.groupby, 'field'),
+            col_gbs = _.pluck(this.cols.groupby, 'field'),
+            field_list = row_gbs.concat(col_gbs, _.pluck(this.measures, 'field')),
+            fields = field_list.map(function (f) { return self.raw_field(f); });
+
+        group1 = group1 || row_gbs;
+        group2 = group2 || col_gbs;
+
+        var i,j, groupbys;
+        for (i = 0; i < group1.length + 1; i++) {
+            for (j = 0; j < group2.length + 1; j++) {
+                groupbys = group1.slice(0,i).concat(group2.slice(0,j));
+                requests.push(self.get_groups(groupbys, fields, domain || self.domain));
             }
-		});
+        }
+        return $.when.apply(null, requests);
+    },
 
-	},
+    // set the 'expanded' status of new_headers more or less like root.headers, with root as root
+    set_headers: function(new_headers, root) {
+        if (root.headers) {
+            _.each(root.headers, function (header) {
+                var corresponding_header = _.find(new_headers, function (h) {
+                    return _.isEqual(h.path, header.path);
+                });
+                if (corresponding_header && header.expanded) {
+                    corresponding_header.expanded = true;
+                    _.each(corresponding_header.children, function (c) {
+                        c.expanded = false;
+                    });
+                }
+                if (corresponding_header && (!header.expanded)) {
+                    corresponding_header.expanded = false;
+                }
+            });
+            var updated_headers = _.filter(new_headers, function (header) {
+                return (header.expanded !== undefined);
+            });
+            _.each(updated_headers, function (header) {
+                header.root = root;
+            });
+            root.headers = updated_headers;
+        } else {
+            root.headers = new_headers;
+            _.each(root.headers, function (header) {
+                header.root = root;
+                header.expanded = (header.children.length > 0);
+            });
+        }
+        return new_headers;
+    },
 
-	_query_db: function (groupby, fields, domain, path) {
-		var self = this,
-            field_ids = _.without(_.pluck(fields, 'field'), '__count'),
-            fields = _.map(field_ids, function(f) { return self.raw_field(f); });
-
-		return this.model.query(field_ids)
-			.filter(domain)
-			.group_by(groupby.field)
-			.then(function (results) {
-				var groups = _.filter(results, function (group) {
-					return group.attributes.length > 0;
-				});
-				return _.map(groups, function (g) { return self.format_group(g, path); });
-			});
-	},
+    get_groups: function (groupbys, fields, domain) {
+        var self = this;
+        return this.model.query(_.without(fields, '__count'))
+            .filter(domain)
+            .lazy(false)
+            .group_by(groupbys)
+            .then(function (groups) {
+                return groups.map(function (group) {
+                    var attrs = group.attributes,
+                        grouped_on = attrs.grouped_on instanceof Array 
+                            ? attrs.grouped_on : [attrs.grouped_on],
+                        raw_grouped_on = grouped_on.map(function (f) {
+                            return self.raw_field(f);
+                        });
+                    if (grouped_on.length === 1) { attrs.value = [attrs.value]}
+                    attrs.value = _.range(grouped_on.length).map(function (i) {
+                        if (attrs.value[i] === false) { 
+                            return _t('Undefined');
+                        } else if (attrs.value[i] instanceof Array) {
+                            return attrs.value[i][1];
+                        }
+                        return attrs.value[i]
+                    });
+                    attrs.aggregates.__count = group.attributes.length;
+                    attrs.grouped_on = raw_grouped_on;
+                    return group;
+                });
+            });
+    },  
 
     // if field is a fieldname, returns field, if field is field_id:interval, retuns field_id
     raw_field: function (field) {
         return field.split(':')[0];
     },
 
-    // add the path to the group and sanitize the value...
-    format_group: function (group, current_path) {
-        var attrs = group.attributes,
-            value = attrs.value,
-            grouped_on = attrs.grouped_on ? this.raw_field(attrs.grouped_on) : false;
-
-        if (value === false) {
-            group.attributes.value = _t('Undefined');
-        } else if (grouped_on && this.fields[grouped_on].type === 'selection') {
-            var selection = this.fields[grouped_on].selection,
-                value_lookup = _.where(selection, {0:value}); 
-            group.attributes.value = value_lookup ? value_lookup[0][1] : _t('Undefined');
-        } else if (value instanceof Array) {
-            group.attributes.value = value[1];
-        }
-
-        group.path = (value !== undefined) ? (current_path || []).concat(group.attributes.value) : [];
-        group.attributes.aggregates.__count = group.attributes.length;
-
-        return group;
-    },
-
-	format_data: function (col_data, row_data, cell_data) {
-		var self = this,
-			dim_row = this.rows.groupby.length,
-			dim_col = this.cols.groupby.length,
-            col_headers = this.get_ancestors_and_self(this.make_headers(col_data, dim_col)),
-            row_headers = this.get_ancestors_and_self(this.make_headers(row_data, dim_row));
-
-        this.cells = [];
-		_.each(cell_data, function (data, index) {
-			self.make_cells(data, index, [], row_headers, col_headers);
-		}); // not pretty. make it more functional?
-
-		return {col_headers: col_headers, row_headers: row_headers};
-	},
-
-	make_headers: function (data, depth, parent) {
-		var self = this,
-            main = this.make_header(data, parent);
-
-		if (main.path.length < depth) {
-			main.children = _.map(data.children || data, function (data_pt) {
-				return self.make_headers (data_pt, depth, main);
-			});
-		}
-		return main;
-	},
-
-	make_cells: function (data, index, current_path, rows, cols) {
-		var self = this;
-		_.each(data, function (group) {
-			var attr = group.attributes,
-				path = attr.grouped_on ? current_path.concat(attr.value) : current_path,
-				values = _.map(self.measures, function (measure) { return attr.aggregates[measure.field]; }),
-                row = _.find(rows, function (header) { return _.isEqual(header.path, path.slice(index)); }),
-                col = _.find(cols, function (header) { return _.isEqual(header.path, path.slice(0, index)); });
-
-			self.add_cell(row.id, col.id, values);
-			if (group.children) {
-				self.make_cells (group.children, index, path, rows, cols);
-			}
-		});
-	},
-
 });
 
 })();

From c138b778df3055dad9e1add03deaf5b519e7581f Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 13:47:58 +0200
Subject: [PATCH 06/26] [FIX] correctly filters out empty groups in read_group
 results in the data loading code of graph view (addon web_graph)

bzr revid: ged@openerp.com-20140402114758-8qfdz8j76t2p2juo
---
 addons/web_graph/static/src/js/pivot_table.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/addons/web_graph/static/src/js/pivot_table.js b/addons/web_graph/static/src/js/pivot_table.js
index e8a438b3fe5..824b41a6b25 100644
--- a/addons/web_graph/static/src/js/pivot_table.js
+++ b/addons/web_graph/static/src/js/pivot_table.js
@@ -392,7 +392,9 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
             .lazy(false)
             .group_by(groupbys)
             .then(function (groups) {
-                return groups.map(function (group) {
+                return groups.filter(function (group) {
+                    return group.attributes.length > 0;
+                }).map(function (group) {
                     var attrs = group.attributes,
                         grouped_on = attrs.grouped_on instanceof Array 
                             ? attrs.grouped_on : [attrs.grouped_on],

From 9d8fa4d7201b5cc48d97b4eefa1610b7f6a63596 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 15:57:04 +0200
Subject: [PATCH 07/26] [FIX] only fills groups with empty groups in read_group
 when in lazy mode to prevent weird stuff happening with the
 read_group_fill_result algorithm.

bzr revid: ged@openerp.com-20140402135704-3d71gczgg362cezl
---
 openerp/osv/orm.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index f9ab9f423ef..9d267816ad5 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2417,9 +2417,12 @@ class BaseModel(object):
 
         result = map(format_result, data)
         
-        for gb in groupby_fields:
-            if gb in self._group_by_full:
-                result = self._read_group_fill_results(cr, uid, domain, gb, groupby[len(annotated_groupbys):],
+        if lazy and groupby_fields[0] in self._group_by_full:
+            # Right now, read_group only fill results in lazy mode (by default).
+            # If you need to have the empty groups in 'eager' mode, then the
+            # method _read_group_fill_results need to be completely reimplemented
+            # in a sane way 
+            result = self._read_group_fill_results(cr, uid, domain, groupby_fields[0], groupby[len(annotated_groupbys):],
                                                        aggregated_fields, result, read_group_order=order,
                                                        context=context)
         return result

From 729cd2ce023d8919aa1b250a4b51bf416cc12117 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Wed, 2 Apr 2014 17:08:17 +0200
Subject: [PATCH 08/26] [FIX] correct the expand functionality to avoid
 creating rows/cols in the other dimension of an unfolding action (addon
 web_graph)

bzr revid: ged@openerp.com-20140402150817-5s8drvhbznxq0hh9
---
 addons/web_graph/static/src/js/pivot_table.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/addons/web_graph/static/src/js/pivot_table.js b/addons/web_graph/static/src/js/pivot_table.js
index 824b41a6b25..9231765c485 100644
--- a/addons/web_graph/static/src/js/pivot_table.js
+++ b/addons/web_graph/static/src/js/pivot_table.js
@@ -223,7 +223,7 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
             data = data.slice(other_gbs.length + 1, data.length);
             _.each(data, function (data_pt) {
                 self.make_headers_and_cell(
-                    data_pt, header.root.headers, other_root.headers, 1, header.path);
+                    data_pt, header.root.headers, other_root.headers, 1, header.path, true);
             });
             header.expanded = true;
             header.children.forEach(function (child) {
@@ -271,12 +271,15 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
         });
     },
 
-    make_headers_and_cell: function (data_pts, row_headers, col_headers, index, prefix) {
+    make_headers_and_cell: function (data_pts, row_headers, col_headers, index, prefix, expand) {
         var self = this;
         data_pts.forEach(function (data_pt) {
             var row_value = (prefix || []).concat(data_pt.attributes.value.slice(0,index));
             var col_value = data_pt.attributes.value.slice(index);
             
+            if (expand && !_.find(col_headers, function (hdr) {return _.isEqual(col_value, hdr.path);})) {
+                return;
+            }          
             var row = self.find_or_create_header(row_headers, row_value, data_pt);
             var col = self.find_or_create_header(col_headers, col_value, data_pt);
 

From 0d4732b5c4194c7c25a13266dfdd1aed05e534eb Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Thu, 3 Apr 2014 11:40:50 +0200
Subject: [PATCH 09/26] [FIX] makes sure the internal data structure of pivot
 table is correct after update_data (the children were not correct when the
 tree of headers was not full, after an expand for example (addon web_graph)

bzr revid: ged@openerp.com-20140403094050-vjeq4dks86mejuub
---
 addons/web_graph/static/src/js/pivot_table.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/addons/web_graph/static/src/js/pivot_table.js b/addons/web_graph/static/src/js/pivot_table.js
index 9231765c485..309bb103fa8 100644
--- a/addons/web_graph/static/src/js/pivot_table.js
+++ b/addons/web_graph/static/src/js/pivot_table.js
@@ -219,8 +219,7 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
             header.root.groupby.push(groupby);
         }
         return this.perform_requests(this_gb, other_gbs, header.domain).then(function () {
-            var data = Array.prototype.slice.call(arguments);
-            data = data.slice(other_gbs.length + 1, data.length);
+            var data = Array.prototype.slice.call(arguments).slice(other_gbs.length + 1);
             _.each(data, function (data_pt) {
                 self.make_headers_and_cell(
                     data_pt, header.root.headers, other_root.headers, 1, header.path, true);
@@ -369,6 +368,7 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
                 }
                 if (corresponding_header && (!header.expanded)) {
                     corresponding_header.expanded = false;
+                    corresponding_header.children = [];
                 }
             });
             var updated_headers = _.filter(new_headers, function (header) {

From 75b66ed77c89f14cb4215f4a677298bdcfce3881 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Thu, 3 Apr 2014 11:43:16 +0200
Subject: [PATCH 10/26] [FIX] work around a read bug: false data are fetched
 when the same field is present multiple times as a field parameter.  The
 solution is to make sure the list many2onefields does not contain duplicate
 fields (orm.py)

bzr revid: ged@openerp.com-20140403094316-kmlliqbwd14ey68p
---
 openerp/osv/orm.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 9d267816ad5..d9340745b11 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2382,6 +2382,7 @@ class BaseModel(object):
         many2onefields = [gb['field'] for gb in annotated_groupbys if gb['type'] == 'many2one']
         if many2onefields:
             data_ids = [r['id'] for r in fetched_data]
+            many2onefields = list(set(many2onefields))
             data_dict = {d['id']: d for d in self.read(cr, uid, data_ids, many2onefields, context=context)} 
             for d in fetched_data:
                 d.update(data_dict[d['id']])
@@ -2416,7 +2417,7 @@ class BaseModel(object):
             return result
 
         result = map(format_result, data)
-        
+
         if lazy and groupby_fields[0] in self._group_by_full:
             # Right now, read_group only fill results in lazy mode (by default).
             # If you need to have the empty groups in 'eager' mode, then the

From 1aba5f08cf8697fbfe12b5694e609dfc174d7c03 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Thu, 3 Apr 2014 13:56:05 +0200
Subject: [PATCH 11/26] [FIX] fixes read_group so that it uses the correct
 _count attribute in the result dictionary (orm.py)

bzr revid: ged@openerp.com-20140403115605-tuboy66r9ews2u0e
---
 openerp/osv/orm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index d9340745b11..c916b40b6d8 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2337,7 +2337,7 @@ class BaseModel(object):
 
         groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query, fget)
         from_clause, where_clause, where_clause_params = query.get_sql()
-        count_field = groupby_fields[0] if (len(groupby_fields) == 1 and context.get('group_by_no_leaf')) else '_'
+        count_field = groupby_fields[0] if lazy and (len(groupby_fields) >= 2 or not context.get('group_by_no_leaf')) else '_'
 
         prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
         prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''

From 7dd2e6789d2bc7619b6d99dd65f64f133d401ff4 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Thu, 3 Apr 2014 16:29:23 +0200
Subject: [PATCH 12/26] [FIX] when using readgroup in eager mode, there is a
 possible name collision when grouping on several fields with different
 intervals, for ex, date_deadline:month, date_deadline:week.  This commit
 resolves the issue by changing the keys 'field' into 'field:interval' (for
 ex, date_deadline:month) in the results (orm.py)

bzr revid: ged@openerp.com-20140403142923-puiruf29tbi1pjj9
---
 openerp/osv/orm.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index c916b40b6d8..1c72001c275 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2207,7 +2207,7 @@ class BaseModel(object):
         """
         orderby_terms = []
         groupby_terms = [gb['qualified_field'] for gb in annotated_groupbys]
-        groupby_fields = [gb['field'] for gb in annotated_groupbys]
+        groupby_fields = [gb['groupby'] for gb in annotated_groupbys]
         if not orderby:
             return groupby_terms, orderby_terms    
 
@@ -2217,13 +2217,13 @@ class BaseModel(object):
             order_field = order_split[0]
             if order_field in groupby_fields:
 
-                if fget[order_field]['type'] == 'many2one':
+                if fget[order_field.split(':')[0]]['type'] == 'many2one':
                     order_clause = self._generate_order_by(order_part, query).replace('ORDER BY ', '')
                     if order_clause:
                         orderby_terms.append(order_clause)
                         groupby_terms += [order_term.split()[0] for order_term in order_clause.split(',')]
                 else:
-                    orderby_terms.append(order_part)
+                    orderby_terms.append('"%s"' % order_part)
             elif order_field in aggregated_fields:
                 orderby_terms.append(order_part)
             else:
@@ -2299,7 +2299,8 @@ class BaseModel(object):
             if field_type == 'boolean':
                 qualified_field = "coalesce(%s,false)" % qualified_field
             return {
-                'field': split[0], 
+                'field': split[0],
+                'groupby': gb,
                 'type': field_type, 
                 'display_format': display_formats[gb_function or 'month'] if temporal else None,
                 'interval': time_intervals[gb_function or 'month'] if temporal else None,                
@@ -2309,8 +2310,8 @@ class BaseModel(object):
 
         annotated_groupbys = map(process_groupby, groupby[:1] if lazy else groupby)
         groupby_fields = [g['field'] for g in annotated_groupbys]
-        order = orderby or ','.join(groupby_fields)
-        groupby_dict = {gb['field']: gb for gb in annotated_groupbys}
+        order = orderby or ','.join([g['groupby'] for g in annotated_groupbys])
+        groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
 
         self._apply_ir_rules(cr, uid, query, 'read', context=context)
         for gb in groupby_fields:
@@ -2333,7 +2334,7 @@ class BaseModel(object):
         select_terms = ["%s(%s) AS %s" % field_formatter(f) for f in aggregated_fields]
 
         for gb in annotated_groupbys:
-            select_terms.append("%s as %s " % (gb['qualified_field'], gb['field']))
+            select_terms.append('%s as "%s" ' % (gb['qualified_field'], gb['groupby']))
 
         groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query, fget)
         from_clause, where_clause, where_clause_params = query.get_sql()
@@ -2404,7 +2405,7 @@ class BaseModel(object):
             return [(gb['field'], '=', value)]
 
         def format_result (fromquery):
-            domain_group = [dom for gb in annotated_groupbys for dom in get_domain(gb, fromquery[gb['field']])]
+            domain_group = [dom for gb in annotated_groupbys for dom in get_domain(gb, fromquery[gb['groupby']])]
             result = { '__domain': domain_group + domain }
             if len(groupby) - len(annotated_groupbys) >= 1:
                 result['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
@@ -4421,7 +4422,7 @@ class BaseModel(object):
         return Query(tables, where_clause, where_params)
 
     def _check_qorder(self, word):
-        if not regex_order.match(word):
+        if not regex_order.match(word.split(':')[0]):
             raise except_orm(_('AccessError'), _('Invalid "order" specified. A valid "order" specification is a comma-separated list of valid field names (optionally followed by asc/desc for the direction)'))
         return True
 

From 9d915db2144be0d3b8e01340495f45b4187ceb76 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Thu, 3 Apr 2014 16:35:34 +0200
Subject: [PATCH 13/26] [FIX] read_group has been changed to return the results
 with field:interval as key (for example, date_deadline:week), to avoid name
 collisions.  This commit changes the web client to make sure it properly
 processes the new result format.  No change at all for queries that do not
 use the field:interval format (addon web)

bzr revid: ged@openerp.com-20140403143534-3jpugw9g8n966n9t
---
 addons/web/static/src/js/data.js | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/addons/web/static/src/js/data.js b/addons/web/static/src/js/data.js
index 413d0621011..9d11dbcf2a8 100644
--- a/addons/web/static/src/js/data.js
+++ b/addons/web/static/src/js/data.js
@@ -238,31 +238,26 @@ instance.web.QueryGroup = instance.web.Class.extend({
             {__context: {group_by: []}, __domain: []},
             read_group_group);
 
-        var raw_fields = _.map(grouping_fields, function (field) {
-            return field && field.split(':')[0];
-        });
-
-
+        var count_key = (grouping_fields[0] && grouping_fields[0].split(':')[0]) + '_count';
         var aggregates = {};
         _(fixed_group).each(function (value, key) {
             if (key.indexOf('__') === 0
-                    || _.contains(raw_fields, key)
-                    || (key === raw_fields[0] + '_count')) {
+                    || _.contains(grouping_fields, key)
+                    || (key === count_key)) {
                 return;
             }
             aggregates[key] = value || 0;
         });
 
-
         this.model = new instance.web.Model(
             model, fixed_group.__context, fixed_group.__domain);
 
-        var group_size = fixed_group[raw_fields[0] + '_count'] || fixed_group.__count || 0;
+        var group_size = fixed_group[count_key] || fixed_group.__count || 0;
         var leaf_group = fixed_group.__context.group_by.length === 0;
 
         var value = (grouping_fields.length === 1) 
-                ? fixed_group[raw_fields[0]]
-                : _.map(raw_fields, function (field) { return fixed_group[field]; });
+                ? fixed_group[grouping_fields[0]]
+                : _.map(grouping_fields, function (field) { return fixed_group[field]; });
         var grouped_on = (grouping_fields.length === 1) 
                 ? grouping_fields[0] 
                 : grouping_fields;

From f48ab8373bddd39951e15d8d2f204cc677e9be56 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Fri, 4 Apr 2014 11:19:47 +0200
Subject: [PATCH 14/26] [FIX] fixes a read_group test to use the correct
 field:interval key when grouping with an interval, and makes sure the sql
 request properly quote the order statement (without quoting DESC or ASC)
 (orm.py)

bzr revid: ged@openerp.com-20140404091947-z87yg2bc7htovj4q
---
 openerp/addons/base/tests/test_orm.py | 2 +-
 openerp/osv/orm.py                    | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/openerp/addons/base/tests/test_orm.py b/openerp/addons/base/tests/test_orm.py
index fcd81965b8c..9683874f676 100644
--- a/openerp/addons/base/tests/test_orm.py
+++ b/openerp/addons/base/tests/test_orm.py
@@ -134,7 +134,7 @@ class TestORM(common.TransactionCase):
             rg = self.partner.read_group(self.cr, self.uid, domain, ['date'], 'date' + ':' + interval)
             result = {}
             for r in rg:
-                result[r['date']] = set(self.partner.search(self.cr, self.uid, r['__domain']))
+                result[r['date:' + interval]] = set(self.partner.search(self.cr, self.uid, r['__domain']))
             return result
 
         self.assertEqual(len(read_group('day')), len(partners_by_day))
diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 1c72001c275..d8862591ed8 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2223,7 +2223,8 @@ class BaseModel(object):
                         orderby_terms.append(order_clause)
                         groupby_terms += [order_term.split()[0] for order_term in order_clause.split(',')]
                 else:
-                    orderby_terms.append('"%s"' % order_part)
+                    order = '"%s" %s' % (order_field, '' if len(order_split) == 1 else order_split[1])
+                    orderby_terms.append(order)
             elif order_field in aggregated_fields:
                 orderby_terms.append(order_part)
             else:

From eb41be40bfcc46eabaae6435b5703e9f23382f81 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Fri, 4 Apr 2014 13:42:16 +0200
Subject: [PATCH 15/26] [FIX] fixes an incorrect call to read_group (the
 groupby argument should be a list of groupbys and should not be a list with
 an empty string (addon gamification)

bzr revid: ged@openerp.com-20140404114216-35wqvkegnpnn7mil
---
 addons/gamification/models/goal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/addons/gamification/models/goal.py b/addons/gamification/models/goal.py
index 8e79c43c8a6..925b05de5ab 100644
--- a/addons/gamification/models/goal.py
+++ b/addons/gamification/models/goal.py
@@ -279,7 +279,7 @@ class gamification_goal(osv.Model):
 
                 if goal.definition_id.computation_mode == 'sum':
                     field_name = goal.definition_id.field_id.name
-                    res = obj.read_group(cr, uid, domain, [field_name], [''], context=context)
+                    res = obj.read_group(cr, uid, domain, [field_name], [field_name], context=context)
                     new_value = res and res[0][field_name] or 0.0
 
                 else:  # computation mode = count

From 870fed2bb9a320f3b859565bb7918f32361dced4 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Fri, 4 Apr 2014 13:45:43 +0200
Subject: [PATCH 16/26] [IMP] puts the closure 'process_groupby' in read_group
 in a separate method. (orm.py)

bzr revid: ged@openerp.com-20140404114543-nw7yx87h1dbkbhtf
---
 openerp/osv/orm.py | 57 ++++++++++++++++++++++++++--------------------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index d8862591ed8..5d09e3851af 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2233,6 +2233,31 @@ class BaseModel(object):
                              self._name, order_part)
         return groupby_terms, orderby_terms
 
+    def _read_group_process_groupby(self, gb, fget, query):
+        print "==="*20
+        print gb
+        split = gb.split(':')
+        field_type = fget[split[0]]['type'] 
+        gb_function = split[1] if len(split) == 2 else None
+        temporal = field_type in ('date', 'datetime')
+        tz_convert = field_type == 'datetime' and context.get('tz') in pytz.all_timezones
+        qualified_field = self._inherits_join_calc(split[0], query)
+        if temporal:
+            if tz_convert:
+                qualified_field = "timezone('%s', timezone('UTC',%s))" % (context.get('tz', 'UTC'), qualified_field)
+            qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field)
+        if field_type == 'boolean':
+            qualified_field = "coalesce(%s,false)" % qualified_field
+        return {
+            'field': split[0],
+            'groupby': gb,
+            'type': field_type, 
+            'display_format': display_formats[gb_function or 'month'] if temporal else None,
+            'interval': time_intervals[gb_function or 'month'] if temporal else None,                
+            'tz_convert': tz_convert,
+            'qualified_field': qualified_field
+        }
+
     def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False, lazy=True):
         """
         Get the list of records in list view grouped by the given ``groupby`` fields
@@ -2286,30 +2311,9 @@ class BaseModel(object):
             'year': dateutil.relativedelta.relativedelta(years=1)
         }
 
-        def process_groupby (gb):
-            split = gb.split(':')
-            field_type = fget[split[0]]['type'] 
-            gb_function = split[1] if len(split) == 2 else None
-            temporal = field_type in ('date', 'datetime')
-            tz_convert = field_type == 'datetime' and context.get('tz') in pytz.all_timezones
-            qualified_field = self._inherits_join_calc(split[0], query)
-            if temporal:
-                if tz_convert:
-                    qualified_field = "timezone('%s', timezone('UTC',%s))" % (context.get('tz', 'UTC'), qualified_field)
-                qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field)
-            if field_type == 'boolean':
-                qualified_field = "coalesce(%s,false)" % qualified_field
-            return {
-                'field': split[0],
-                'groupby': gb,
-                'type': field_type, 
-                'display_format': display_formats[gb_function or 'month'] if temporal else None,
-                'interval': time_intervals[gb_function or 'month'] if temporal else None,                
-                'tz_convert': tz_convert,
-                'qualified_field': qualified_field
-            }
-
-        annotated_groupbys = map(process_groupby, groupby[:1] if lazy else groupby)
+        groupby_list = groupby[:1] if lazy else groupby
+        annotated_groupbys = [self._read_group_process_groupby(gb, fget, query) 
+                                    for gb in groupby_list]
         groupby_fields = [g['field'] for g in annotated_groupbys]
         order = orderby or ','.join([g['groupby'] for g in annotated_groupbys])
         groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
@@ -2339,7 +2343,10 @@ class BaseModel(object):
 
         groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query, fget)
         from_clause, where_clause, where_clause_params = query.get_sql()
-        count_field = groupby_fields[0] if lazy and (len(groupby_fields) >= 2 or not context.get('group_by_no_leaf')) else '_'
+        if lazy and (len(groupby_fields) >= 2 or not context.get('group_by_no_leaf')):
+            count_field = groupby_fields[0] if len(groupby_fields) >= 1 else '_'
+        else:
+            count_field = '_'
 
         prefix_terms = lambda prefix, terms: (prefix + " " + ",".join(terms)) if terms else ''
         prefix_term = lambda prefix, term: ('%s %s' % (prefix, term)) if term else ''

From 39e69876c6109a110f61335ea20fb2ea6d26d475 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Fri, 4 Apr 2014 14:42:32 +0200
Subject: [PATCH 17/26] [IMP] moves some internal function in read_group to
 become class methods and slightly reorganize them (orm.py)

bzr revid: ged@openerp.com-20140404124232-dmyxuv1qf5kperet
---
 openerp/osv/orm.py | 132 ++++++++++++++++++++++++---------------------
 1 file changed, 70 insertions(+), 62 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 5d09e3851af..52fec2675a7 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2234,8 +2234,6 @@ class BaseModel(object):
         return groupby_terms, orderby_terms
 
     def _read_group_process_groupby(self, gb, fget, query):
-        print "==="*20
-        print gb
         split = gb.split(':')
         field_type = fget[split[0]]['type'] 
         gb_function = split[1] if len(split) == 2 else None
@@ -2243,6 +2241,20 @@ class BaseModel(object):
         tz_convert = field_type == 'datetime' and context.get('tz') in pytz.all_timezones
         qualified_field = self._inherits_join_calc(split[0], query)
         if temporal:
+            display_formats = {
+                'day': 'dd MMM YYYY', 
+                'week': "'W'w YYYY", 
+                'month': 'MMMM YYYY', 
+                'quarter': 'QQQ YYYY', 
+                'year': 'YYYY'
+            }
+            time_intervals = {
+                'day': dateutil.relativedelta.relativedelta(months=3),
+                'week': datetime.timedelta(days=7),
+                'month': dateutil.relativedelta.relativedelta(months=1),
+                'quarter': dateutil.relativedelta.relativedelta(months=3),
+                'year': dateutil.relativedelta.relativedelta(years=1)
+            }
             if tz_convert:
                 qualified_field = "timezone('%s', timezone('UTC',%s))" % (context.get('tz', 'UTC'), qualified_field)
             qualified_field = "date_trunc('%s', %s)" % (gb_function or 'month', qualified_field)
@@ -2258,6 +2270,58 @@ class BaseModel(object):
             'qualified_field': qualified_field
         }
 
+    def _read_group_prepare_data(self, key, value, groupby_dict):
+        """
+            Helper method to sanitize the data received by read_group. The None
+            values are converted to False, and the date/datetime are formatted,
+            and corrected according to the timezones.
+        """
+        value = False if value is None else value
+        gb = groupby_dict.get(key)
+        if gb and gb['type'] in ('date', 'datetime') and value:
+            if isinstance(value, basestring):
+                dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
+                value = datetime.datetime.strptime(value, dt_format)
+            if gb['tz_convert']:
+                value =  pytz.timezone(context['tz']).localize(value)
+        return value
+
+    def _read_group_get_domain(self, groupby, value):
+        """
+            Helper method to construct the domain corresponding to a groupby and 
+            a given value. This is mostly relevant for date/datetime.
+        """
+        if groupby['type'] in ('date', 'datetime') and value:
+            dt_format = DEFAULT_SERVER_DATETIME_FORMAT if groupby['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
+            domain_dt_begin = value
+            domain_dt_end = value + groupby['interval']
+            if groupby['tz_convert']:
+                domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
+                domain_dt_end = domain_dt_end.astimezone(pytz.utc)
+            return [(groupby['field'], '>=', domain_dt_begin.strftime(dt_format)),
+                   (groupby['field'], '<', domain_dt_end.strftime(dt_format))]
+        if groupby['type'] == 'many2one' and value:
+                value = value[0]
+        return [(groupby['field'], '=', value)]
+
+    def _read_group_format_result(self, data, annotated_groupbys, groupby, groupby_dict, domain, context):
+        """
+            Helper method to format the data contained in the dictianary data by 
+            adding the domain corresponding to its values, the groupbys in the 
+            context and by properly formatting the date/datetime values. 
+        """
+        domain_group = [dom for gb in annotated_groupbys for dom in self._read_group_get_domain(gb, data[gb['groupby']])]
+        result = { '__domain': domain_group + domain }
+        if len(groupby) - len(annotated_groupbys) >= 1:
+            result['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
+        result.update(data)
+        for k,v in result.iteritems():
+            gb = groupby_dict.get(k)
+            if gb and gb['type'] in ('date', 'datetime') and v:
+                result[k] = babel.dates.format_date(v, format=gb['display_format'], locale=context.get('lang', 'en_US'))
+        del result['id']
+        return result
+
     def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False, lazy=True):
         """
         Get the list of records in list view grouped by the given ``groupby`` fields
@@ -2296,21 +2360,6 @@ class BaseModel(object):
         fields = fields or self._columns.keys()
         fget = self.fields_get(cr, uid, fields)
         groupby = [groupby] if isinstance(groupby, basestring) else groupby
-        display_formats = {
-            'day': 'dd MMM YYYY', 
-            'week': "'W'w YYYY", 
-            'month': 'MMMM YYYY', 
-            'quarter': 'QQQ YYYY', 
-            'year': 'YYYY'
-        }
-        time_intervals = {
-            'day': dateutil.relativedelta.relativedelta(months=3),
-            'week': datetime.timedelta(days=7),
-            'month': dateutil.relativedelta.relativedelta(months=1),
-            'quarter': dateutil.relativedelta.relativedelta(months=3),
-            'year': dateutil.relativedelta.relativedelta(years=1)
-        }
-
         groupby_list = groupby[:1] if lazy else groupby
         annotated_groupbys = [self._read_group_process_groupby(gb, fget, query) 
                                     for gb in groupby_list]
@@ -2371,22 +2420,10 @@ class BaseModel(object):
             'offset': prefix_term('OFFSET', int(offset) if limit else None),
         }
         cr.execute(query, where_clause_params)
+        fetched_data = cr.dictfetchall()
 
         if not groupby_fields:
-            return {r.pop('id'): r for r in cr.dictfetchall() }
-
-        def prepare_data(key, value):
-            value = False if value is None else value
-            gb = groupby_dict.get(key)
-            if gb and gb['type'] in ('date', 'datetime') and value:
-                if isinstance(value, basestring):
-                    dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
-                    value = datetime.datetime.strptime(value, dt_format)
-                if gb['tz_convert']:
-                    value =  pytz.timezone(context['tz']).localize(value)
-            return value
-
-        fetched_data = cr.dictfetchall()
+            return {r.pop('id'): r for r in fetched_data}
 
         many2onefields = [gb['field'] for gb in annotated_groupbys if gb['type'] == 'many2one']
         if many2onefields:
@@ -2396,37 +2433,8 @@ class BaseModel(object):
             for d in fetched_data:
                 d.update(data_dict[d['id']])
 
-        data = map(lambda r: {k: prepare_data(k,v) for k,v in r.iteritems()}, fetched_data)
-
-        def get_domain(gb, value):
-            if gb['type'] in ('date', 'datetime') and value:
-                dt_format = DEFAULT_SERVER_DATETIME_FORMAT if gb['type'] == 'datetime' else DEFAULT_SERVER_DATE_FORMAT
-                domain_dt_begin = value
-                domain_dt_end = value + gb['interval']
-                if gb['tz_convert']:
-                    domain_dt_begin = domain_dt_begin.astimezone(pytz.utc)
-                    domain_dt_end = domain_dt_end.astimezone(pytz.utc)
-                return [(gb['field'], '>=', domain_dt_begin.strftime(dt_format)),
-                       (gb['field'], '<', domain_dt_end.strftime(dt_format))]
-            if gb['type'] == 'many2one' and value:
-                    value = value[0]
-            return [(gb['field'], '=', value)]
-
-        def format_result (fromquery):
-            domain_group = [dom for gb in annotated_groupbys for dom in get_domain(gb, fromquery[gb['groupby']])]
-            result = { '__domain': domain_group + domain }
-            if len(groupby) - len(annotated_groupbys) >= 1:
-                result['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
-            result.update(fromquery)
-            for k,v in result.iteritems():
-                gb = groupby_dict.get(k)
-                if gb and gb['type'] in ('date', 'datetime') and v:
-                    result[k] = babel.dates.format_date(v, format=gb['display_format'], locale=context.get('lang', 'en_US'))
-            del result['id']
-            return result
-
-        result = map(format_result, data)
-
+        data = map(lambda r: {k: self._read_group_prepare_data(k,v, groupby_dict) for k,v in r.iteritems()}, fetched_data)
+        result = [self._read_group_format_result(d, annotated_groupbys, groupby, groupby_dict, domain, context) for d in data]
         if lazy and groupby_fields[0] in self._group_by_full:
             # Right now, read_group only fill results in lazy mode (by default).
             # If you need to have the empty groups in 'eager' mode, then the

From 4d797222279e361bfe6db0fe45f3ff503b87c717 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Fri, 4 Apr 2014 16:30:09 +0200
Subject: [PATCH 18/26] [FIX] add missing semicolons, add a missing 'var'
 statement and remove trailing whitespace (addon web_graph)

bzr revid: ged@openerp.com-20140404143009-6uk7607bv4acbg4b
---
 addons/web_graph/static/src/js/pivot_table.js | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/addons/web_graph/static/src/js/pivot_table.js b/addons/web_graph/static/src/js/pivot_table.js
index 309bb103fa8..d0867ca6d1a 100644
--- a/addons/web_graph/static/src/js/pivot_table.js
+++ b/addons/web_graph/static/src/js/pivot_table.js
@@ -278,26 +278,26 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
             
             if (expand && !_.find(col_headers, function (hdr) {return _.isEqual(col_value, hdr.path);})) {
                 return;
-            }          
+            }
             var row = self.find_or_create_header(row_headers, row_value, data_pt);
             var col = self.find_or_create_header(col_headers, col_value, data_pt);
 
-            var cell_value = _.map(self.measures, function (m) { 
+            var cell_value = _.map(self.measures, function (m) {
                 return data_pt.attributes.aggregates[m.field];
             });
             self.cells.push({
-                x: Math.min(row.id, col.id), 
-                y: Math.max(row.id, col.id), 
+                x: Math.min(row.id, col.id),
+                y: Math.max(row.id, col.id),
                 values: cell_value
-            });            
+            });
         });
     },
 
     make_header: function (values) {
         return _.extend({
-            children: [], 
-            domain: this.domain, 
-            expanded: undefined, 
+            children: [],
+            domain: this.domain,
+            expanded: undefined,
             id: _.uniqueId(),
             path: [],
             root: undefined,
@@ -306,7 +306,7 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
     },
 
     find_or_create_header: function (headers, path, data_pt) {
-        var hdr = _.find(headers, function (header) { 
+        var hdr = _.find(headers, function (header) {
             return _.isEqual(path, header.path);
         });
         if (hdr) {
@@ -318,11 +318,11 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
             return hdr;
         }
         hdr = this.make_header({
-            path:path, 
+            path:path,
             domain:data_pt.model._domain,
             title: _t(_.last(path))
         });
-        parent = _.find(headers, function (header) {
+        var parent = _.find(headers, function (header) {
             return _.isEqual(header.path, _.initial(path, 1));
         });
 
@@ -399,26 +399,27 @@ openerp.web_graph.PivotTable = openerp.web.Class.extend({
                     return group.attributes.length > 0;
                 }).map(function (group) {
                     var attrs = group.attributes,
-                        grouped_on = attrs.grouped_on instanceof Array 
-                            ? attrs.grouped_on : [attrs.grouped_on],
+                        grouped_on = attrs.grouped_on instanceof Array ? attrs.grouped_on : [attrs.grouped_on],
                         raw_grouped_on = grouped_on.map(function (f) {
                             return self.raw_field(f);
                         });
-                    if (grouped_on.length === 1) { attrs.value = [attrs.value]}
+                    if (grouped_on.length === 1) {
+                        attrs.value = [attrs.value];
+                    }
                     attrs.value = _.range(grouped_on.length).map(function (i) {
-                        if (attrs.value[i] === false) { 
+                        if (attrs.value[i] === false) {
                             return _t('Undefined');
                         } else if (attrs.value[i] instanceof Array) {
                             return attrs.value[i][1];
                         }
-                        return attrs.value[i]
+                        return attrs.value[i];
                     });
                     attrs.aggregates.__count = group.attributes.length;
                     attrs.grouped_on = raw_grouped_on;
                     return group;
                 });
             });
-    },  
+    },
 
     // if field is a fieldname, returns field, if field is field_id:interval, retuns field_id
     raw_field: function (field) {

From b5dc53fea1be7afc22f2b41f84213f194b1efbf7 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Mon, 7 Apr 2014 11:10:28 +0200
Subject: [PATCH 19/26] [IMP] adds a test for the new eager/lazy functionality
 of read_group

bzr revid: ged@openerp.com-20140407091028-4ksvnqwvrvwaho58
---
 openerp/addons/base/tests/test_orm.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/openerp/addons/base/tests/test_orm.py b/openerp/addons/base/tests/test_orm.py
index 9683874f676..29a5a2880d1 100644
--- a/openerp/addons/base/tests/test_orm.py
+++ b/openerp/addons/base/tests/test_orm.py
@@ -141,6 +141,10 @@ class TestORM(common.TransactionCase):
         self.assertEqual(len(read_group('month')), len(partners_by_month))
         self.assertEqual(len(read_group('year')), len(partners_by_year))
 
+        rg = self.partner.read_group(self.cr, self.uid, [('id', 'in', all_partners)], 
+                        ['date'], ['date:month', 'date:day'], lazy=False)
+        self.assertEqual(len(rg), len(all_partners))
+
 
 class TestInherits(common.TransactionCase):
     """ test the behavior of the orm for models that use _inherits;

From 82825c4acd52c70248704c5d6f886e2a1bfee037 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Mon, 7 Apr 2014 15:02:23 +0200
Subject: [PATCH 20/26] [FIX] add context parameter to various helper methods
 to avoid stupid crash in read_group (orm.py)

bzr revid: ged@openerp.com-20140407130223-pdzge17omntpbh23
---
 openerp/osv/orm.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 52fec2675a7..10c855e4268 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2233,7 +2233,7 @@ class BaseModel(object):
                              self._name, order_part)
         return groupby_terms, orderby_terms
 
-    def _read_group_process_groupby(self, gb, fget, query):
+    def _read_group_process_groupby(self, gb, fget, query, context):
         split = gb.split(':')
         field_type = fget[split[0]]['type'] 
         gb_function = split[1] if len(split) == 2 else None
@@ -2270,7 +2270,7 @@ class BaseModel(object):
             'qualified_field': qualified_field
         }
 
-    def _read_group_prepare_data(self, key, value, groupby_dict):
+    def _read_group_prepare_data(self, key, value, groupby_dict, context):
         """
             Helper method to sanitize the data received by read_group. The None
             values are converted to False, and the date/datetime are formatted,
@@ -2361,7 +2361,7 @@ class BaseModel(object):
         fget = self.fields_get(cr, uid, fields)
         groupby = [groupby] if isinstance(groupby, basestring) else groupby
         groupby_list = groupby[:1] if lazy else groupby
-        annotated_groupbys = [self._read_group_process_groupby(gb, fget, query) 
+        annotated_groupbys = [self._read_group_process_groupby(gb, fget, query, context) 
                                     for gb in groupby_list]
         groupby_fields = [g['field'] for g in annotated_groupbys]
         order = orderby or ','.join([g['groupby'] for g in annotated_groupbys])
@@ -2433,7 +2433,7 @@ class BaseModel(object):
             for d in fetched_data:
                 d.update(data_dict[d['id']])
 
-        data = map(lambda r: {k: self._read_group_prepare_data(k,v, groupby_dict) for k,v in r.iteritems()}, fetched_data)
+        data = map(lambda r: {k: self._read_group_prepare_data(k,v, groupby_dict, context) for k,v in r.iteritems()}, fetched_data)
         result = [self._read_group_format_result(d, annotated_groupbys, groupby, groupby_dict, domain, context) for d in data]
         if lazy and groupby_fields[0] in self._group_by_full:
             # Right now, read_group only fill results in lazy mode (by default).

From b683b9c614512c6b3c186a488053dfb5f7bdc021 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Mon, 7 Apr 2014 15:39:31 +0200
Subject: [PATCH 21/26] [IMP] removes useless calls to field_get and replace
 them by self._all_columns in read_group method (orm.py).

bzr revid: ged@openerp.com-20140407133931-hrf9dmdj2fx050ww
---
 openerp/osv/orm.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 10c855e4268..bfa145fb4a6 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2193,7 +2193,7 @@ class BaseModel(object):
                 r['__fold'] = folded.get(r[groupby] and r[groupby][0], False)
         return result
 
-    def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query, fget):
+    def _read_group_prepare(self, orderby, aggregated_fields, annotated_groupbys, query):
         """
         Prepares the GROUP BY and ORDER BY terms for the read_group method. Adds the missing JOIN clause
         to the query if order should be computed against m2o field. 
@@ -2217,7 +2217,7 @@ class BaseModel(object):
             order_field = order_split[0]
             if order_field in groupby_fields:
 
-                if fget[order_field.split(':')[0]]['type'] == 'many2one':
+                if self._all_columns[order_field.split(':')[0]].column._type == 'many2one':
                     order_clause = self._generate_order_by(order_part, query).replace('ORDER BY ', '')
                     if order_clause:
                         orderby_terms.append(order_clause)
@@ -2233,9 +2233,9 @@ class BaseModel(object):
                              self._name, order_part)
         return groupby_terms, orderby_terms
 
-    def _read_group_process_groupby(self, gb, fget, query, context):
+    def _read_group_process_groupby(self, gb, query, context):
         split = gb.split(':')
-        field_type = fget[split[0]]['type'] 
+        field_type = self._all_columns[split[0]].column._type
         gb_function = split[1] if len(split) == 2 else None
         temporal = field_type in ('date', 'datetime')
         tz_convert = field_type == 'datetime' and context.get('tz') in pytz.all_timezones
@@ -2358,10 +2358,10 @@ class BaseModel(object):
         self.check_access_rights(cr, uid, 'read')
         query = self._where_calc(cr, uid, domain, context=context) 
         fields = fields or self._columns.keys()
-        fget = self.fields_get(cr, uid, fields)
+
         groupby = [groupby] if isinstance(groupby, basestring) else groupby
         groupby_list = groupby[:1] if lazy else groupby
-        annotated_groupbys = [self._read_group_process_groupby(gb, fget, query, context) 
+        annotated_groupbys = [self._read_group_process_groupby(gb, query, context) 
                                     for gb in groupby_list]
         groupby_fields = [g['field'] for g in annotated_groupbys]
         order = orderby or ','.join([g['groupby'] for g in annotated_groupbys])
@@ -2372,7 +2372,7 @@ class BaseModel(object):
             assert gb in fields, "Fields in 'groupby' must appear in the list of fields to read (perhaps it's missing in the list view?)"
             groupby_def = self._columns.get(gb) or (self._inherit_fields.get(gb) and self._inherit_fields.get(gb)[2])
             assert groupby_def and groupby_def._classic_write, "Fields in 'groupby' must be regular database-persisted fields (no function or related fields), or function fields with store=True"
-            if not (gb in fget):
+            if not (gb in self._all_columns):
                 # Don't allow arbitrary values, as this would be a SQL injection vector!
                 raise except_orm(_('Invalid group_by'),
                                  _('Invalid group_by specification: "%s".\nA group_by specification must be a list of valid fields.')%(gb,))
@@ -2381,16 +2381,16 @@ class BaseModel(object):
             f for f in fields
             if f not in ('id', 'sequence')
             if f not in groupby_fields
-            if fget[f]['type'] in ('integer', 'float')
-            if (f in self._all_columns and getattr(self._all_columns[f].column, '_classic_write'))]
+            if self._all_columns[f].column._type in ('integer', 'float')
+            if getattr(self._all_columns[f].column, '_classic_write')]
 
-        field_formatter = lambda f: (fget[f].get('group_operator', 'sum'), self._inherits_join_calc(f, query), f)
+        field_formatter = lambda f: (self._all_columns[f].column.group_operator or 'sum', self._inherits_join_calc(f, query), f)
         select_terms = ["%s(%s) AS %s" % field_formatter(f) for f in aggregated_fields]
 
         for gb in annotated_groupbys:
             select_terms.append('%s as "%s" ' % (gb['qualified_field'], gb['groupby']))
 
-        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query, fget)
+        groupby_terms, orderby_terms = self._read_group_prepare(order, aggregated_fields, annotated_groupbys, query)
         from_clause, where_clause, where_clause_params = query.get_sql()
         if lazy and (len(groupby_fields) >= 2 or not context.get('group_by_no_leaf')):
             count_field = groupby_fields[0] if len(groupby_fields) >= 1 else '_'

From 6aa003f1ad6585b56dc712289a6eec2d0d6e4775 Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Tue, 8 Apr 2014 09:38:13 +0200
Subject: [PATCH 22/26] [IMP] improves the helper function
 _read_group_format_result so that it is slightly faster (orm.py)

bzr revid: ged@openerp.com-20140408073813-8z23fun5274fvqaa
---
 openerp/osv/orm.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index bfa145fb4a6..0c5ba3c0467 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2311,16 +2311,16 @@ class BaseModel(object):
             context and by properly formatting the date/datetime values. 
         """
         domain_group = [dom for gb in annotated_groupbys for dom in self._read_group_get_domain(gb, data[gb['groupby']])]
-        result = { '__domain': domain_group + domain }
-        if len(groupby) - len(annotated_groupbys) >= 1:
-            result['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
-        result.update(data)
-        for k,v in result.iteritems():
+        for k,v in data.iteritems():
             gb = groupby_dict.get(k)
             if gb and gb['type'] in ('date', 'datetime') and v:
-                result[k] = babel.dates.format_date(v, format=gb['display_format'], locale=context.get('lang', 'en_US'))
-        del result['id']
-        return result
+                data[k] = babel.dates.format_date(v, format=gb['display_format'], locale=context.get('lang', 'en_US'))
+
+        data['__domain'] = domain_group + domain 
+        if len(groupby) - len(annotated_groupbys) >= 1:
+            data['__context'] = { 'group_by': groupby[len(annotated_groupbys):]}
+        del data['id']
+        return data
 
     def read_group(self, cr, uid, domain, fields, groupby, offset=0, limit=None, context={}, orderby=False, lazy=True):
         """

From 26e710fa2aad911ac756f2d16d748da68eb064a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thibault=20Delavall=C3=A9e?= <tde@openerp.com>
Date: Tue, 8 Apr 2014 10:16:48 +0200
Subject: [PATCH 23/26] [FIX] res_partner demo data: fixed email address, using
 a symbol not valid for an email address.

bzr revid: tde@openerp.com-20140408081648-5au34720s2m4izm6
---
 openerp/addons/base/res/res_partner_demo.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openerp/addons/base/res/res_partner_demo.xml b/openerp/addons/base/res/res_partner_demo.xml
index cffe87b098b..4b8a19a392e 100644
--- a/openerp/addons/base/res/res_partner_demo.xml
+++ b/openerp/addons/base/res/res_partner_demo.xml
@@ -438,7 +438,7 @@
             <field model="res.country" name="country_id" search="[('name','=','China')]"/>
             <field name="country_id" ref="base.cn"/>
             <field name="street">89 Dong Lu Road</field>
-            <field name="email">míng@yourcompany.example.com</field>
+            <field name="email">ming@yourcompany.example.com</field>
             <field name="phone">+86 215 069 5177</field>
             <field name="website">http://www.míng.com</field>
         </record>

From 568b8c76326b3dad3c9da12aae36a327f1ab6cfa Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Tue, 8 Apr 2014 10:20:29 +0200
Subject: [PATCH 24/26] [IMP] add and fix the documentation on some helper
 methods in read_group. (orm.py)

bzr revid: ged@openerp.com-20140408082029-pmdzkbak0zjvs8jw
---
 openerp/osv/orm.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index 0c5ba3c0467..d1c497b55a5 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2199,17 +2199,18 @@ class BaseModel(object):
         to the query if order should be computed against m2o field. 
         :param orderby: the orderby definition in the form "%(field)s %(order)s"
         :param aggregated_fields: list of aggregated fields in the query
-        :param groupby: the current groupby field name
-        :param qualified_field: the fully qualified SQL name for the grouped field
+        :param annotated_groupbys: list of dictionaries returned by _read_group_process_groupby
+                These dictionaries contains the qualified name of each groupby
+                (fully qualified SQL name for the corresponding field),
+                and the (non raw) field name.
         :param osv.Query query: the query under construction
-        :param groupby_type: the type of the grouped field
         :return: (groupby_terms, orderby_terms)
         """
         orderby_terms = []
         groupby_terms = [gb['qualified_field'] for gb in annotated_groupbys]
         groupby_fields = [gb['groupby'] for gb in annotated_groupbys]
         if not orderby:
-            return groupby_terms, orderby_terms    
+            return groupby_terms, orderby_terms
 
         self._check_qorder(orderby)
         for order_part in orderby.split(','):
@@ -2234,6 +2235,10 @@ class BaseModel(object):
         return groupby_terms, orderby_terms
 
     def _read_group_process_groupby(self, gb, query, context):
+        """
+            Helper method to collect important information about groupbys: raw
+            field name, type, time informations, qualified name, ...
+        """
         split = gb.split(':')
         field_type = self._all_columns[split[0]].column._type
         gb_function = split[1] if len(split) == 2 else None
@@ -2353,7 +2358,6 @@ class BaseModel(object):
         :rtype: [{'field_name_1': value, ...]
         :raise AccessError: * if user has no read rights on the requested object
                             * if user tries to bypass access rules for read on the requested object
-
         """
         self.check_access_rights(cr, uid, 'read')
         query = self._where_calc(cr, uid, domain, context=context) 

From 6d234c01bdc9f0386b04a0338408583d9313f4ad Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Tue, 8 Apr 2014 11:04:18 +0200
Subject: [PATCH 25/26] [IMP] change the regex matching orders in orm.py to
 accept field:interval.  This simplifies the code in _check_qorder (no need to
 compute the raw field name)

bzr revid: ged@openerp.com-20140408090418-5yjnfp6axumytmrq
---
 openerp/osv/orm.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index d1c497b55a5..0db95e7498d 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -75,7 +75,7 @@ _schema = logging.getLogger(__name__ + '.schema')
 # List of etree._Element subclasses that we choose to ignore when parsing XML.
 from openerp.tools import SKIPPED_ELEMENT_TYPES
 
-regex_order = re.compile('^( *([a-z0-9_]+|"[a-z0-9_]+")( *desc| *asc)?( *, *|))+$', re.I)
+regex_order = re.compile('^( *([a-z0-9:_]+|"[a-z0-9:_]+")( *desc| *asc)?( *, *|))+$', re.I)
 regex_object_name = re.compile(r'^[a-z0-9_.]+$')
 
 AUTOINIT_RECALCULATE_STORED_FIELDS = 1000
@@ -4442,7 +4442,7 @@ class BaseModel(object):
         return Query(tables, where_clause, where_params)
 
     def _check_qorder(self, word):
-        if not regex_order.match(word.split(':')[0]):
+        if not regex_order.match(word):
             raise except_orm(_('AccessError'), _('Invalid "order" specified. A valid "order" specification is a comma-separated list of valid field names (optionally followed by asc/desc for the direction)'))
         return True
 

From e143250b7a199c10ce5149183c000094447d374c Mon Sep 17 00:00:00 2001
From: Gery Debongnie <ged@openerp.com>
Date: Tue, 8 Apr 2014 14:24:17 +0200
Subject: [PATCH 26/26] [IMP] simplifies the readgroup code to select order by
 (orm.py)

bzr revid: ged@openerp.com-20140408122417-y75wp0plyloiamuy
---
 openerp/osv/orm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py
index d5c147dac73..7b11b9a2b3a 100644
--- a/openerp/osv/orm.py
+++ b/openerp/osv/orm.py
@@ -2373,7 +2373,7 @@ class BaseModel(object):
         annotated_groupbys = [self._read_group_process_groupby(gb, query, context) 
                                     for gb in groupby_list]
         groupby_fields = [g['field'] for g in annotated_groupbys]
-        order = orderby or ','.join([g['groupby'] for g in annotated_groupbys])
+        order = orderby or ','.join([g for g in groupby_list])
         groupby_dict = {gb['groupby']: gb for gb in annotated_groupbys}
 
         self._apply_ir_rules(cr, uid, query, 'read', context=context)