[IMP] core: smarter geoip

Maxmind offers multiple ip-geolocalization databases, historically we
have been using the City database which contains records on a
city-basis. Many years later it turns out we are primary using geoip to
know the country of the user. Geolocalization in the City database is
considered slow by our standard and we have been clever in order not to
geolocate each request by saving the info in the session.

On the other hand, the Country database that is offered by Maxmind is
much more lightweight and geoip using that country is considered a fast
operation by our standard.

In this work we make Odoo compatible with both the City and the Country
databases. Using multiple database at the same time, we can be smart and
only query each of the two on-demand. If a user ask for its country,
we'll use the fast Country db. If a user ask for its city/timezone we'll
use the slower City db.

By default it loads both database from the `/usr/share/GeoIP/` folder,
respectively the files `GeoLite2-City.mmdb` and `GeoLite2-Country.mmdb`,
you can provide alternative paths using the `--geoip-city-db` and
`--geoip-country-db` CLI options.

In the same mindset as #86015, geoip is still lazy. It is done on-demand
and the result is cached on the current request. The different with the
related PR is that as we know consider geoip to be fast, we no longer
cache the result in the session.

Task: 2848206
Part-of: odoo/odoo#91337

addons/website/tools.py

@@ -51,7 +51,7 @@ def MockRequest(
             sale_order_id=sale_order_id,
             website_sale_current_pl=website_sale_current_pl,
         ),
-        geoip={},
+        geoip=odoo.http.GeoIP('127.0.0.1'),
         db=env.registry.db_name,
         env=env,
         registry=env.registry,

debian/control

@@ -30,6 +30,7 @@ Depends:
  python3-decorator,
  python3-docutils,
  python3-freezegun,
+ python3-geoip2,
  python3-pil,
  python3-jinja2,
  python3-libsass,

odoo/addons/test_http/controllers.py

@@ -136,7 +136,15 @@ class TestHttp(http.Controller):
     # =====================================================
     @http.route('/test_http/geoip', type='http', auth='none')
     def geoip(self):
-        return str(request.geoip)
+        return json.dumps({
+            'city': request.geoip.city.name,
+            'country_code': request.geoip.country.iso_code or request.geoip.continent.code,
+            'country_name': request.geoip.country.name or request.geoip.continent.name,
+            'latitude': request.geoip.location.latitude,
+            'longitude': request.geoip.location.longitude,
+            'region': request.geoip.subdivisions[0].iso_code if request.geoip.subdivisions else None,
+            'time_zone': request.geoip.location.time_zone,
+        })
 
     @http.route('/test_http/save_session', type='http', auth='none')
     def touch(self):

odoo/addons/test_http/tests/test_common.py

@@ -13,11 +13,14 @@ class TestHttpBase(HttpCase):
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
+        geoip_resolver = MemoryGeoipResolver()
+        session_store = MemorySessionStore(session_class=Session)
+
         cls.addClassCleanup(lazy_property.reset_all, odoo.http.root)
         cls.classPatch(odoo.conf, 'server_wide_modules', ['base', 'web', 'test_http'])
-        lazy_property.reset_all(odoo.http.root)
-        cls.classPatch(odoo.http.root, 'session_store', MemorySessionStore(session_class=Session))
-        cls.classPatch(odoo.http.root, 'geoip_resolver', MemoryGeoipResolver())
+        cls.classPatch(odoo.http.root, 'session_store', session_store)
+        cls.classPatch(odoo.http.root, 'geoip_city_db', geoip_resolver)
+        cls.classPatch(odoo.http.root, 'geoip_country_db', geoip_resolver)
 
     def setUp(self):
         super().setUp()

odoo/addons/test_http/tests/test_misc.py

@@ -12,6 +12,7 @@ from odoo.tests.common import HOST, new_test_user, get_db_name
 from odoo.tools import config, file_path
 from odoo.addons.test_http.controllers import CT_JSON
 
+from odoo.addons.test_http.utils import TEST_IP
 from .test_common import TestHttpBase
 
 
@@ -93,6 +94,39 @@ class TestHttpMisc(TestHttpBase):
                 self.assertNotIn('error', res_rpc.keys(), res_rpc.get('error', {}).get('data', {}).get('message'))
                 self.assertIn(milky_way.name, res_rpc['result'], "QWeb template was correctly rendered")
 
+    def test_misc5_geoip(self):
+        res = self.nodb_url_open('/test_http/geoip')
+        res.raise_for_status()
+        self.assertEqual(res.json(), {
+            'city': None,
+            'country_code': None,
+            'country_name': None,
+            'latitude': None,
+            'longitude': None,
+            'region': None,
+            'time_zone': None,
+        })
+
+        # Fake client IP using proxy_mode and a forged X-Forwarded-For http header
+        headers = {
+            'Host': '',
+            'X-Forwarded-For': TEST_IP,
+            'X-Forwarded-Host': 'odoo.com',
+            'X-Forwarded-Proto': 'https'
+        }
+        with patch.dict('odoo.tools.config.options', {'proxy_mode': True}):
+            res = self.nodb_url_open('/test_http/geoip', headers=headers)
+            res.raise_for_status()
+            self.assertEqual(res.json(), {
+                'city': None,
+                'country_code': 'FR',
+                'country_name': 'France',
+                'latitude': 48.8582,
+                'longitude': 2.3387,
+                'region': None,
+                'time_zone': 'Europe/Paris',
+            })
+
 
 @tagged('post_install', '-at_install')
 class TestHttpCors(TestHttpBase):

odoo/addons/test_http/tests/test_session.py

@@ -39,7 +39,7 @@ class TestHttpSession(TestHttpBase):
     def test_session1_default_session(self):
         # The default session should not be saved on the filestore.
         with patch.object(odoo.http.root.session_store, 'save') as mock_save:
-            res = self.db_url_open('/test_http/greeting')
+            res = self.db_url_open('/test_http/geoip')
             res.raise_for_status()
             try:
                 mock_save.assert_not_called()
@@ -47,37 +47,6 @@ class TestHttpSession(TestHttpBase):
                 msg = f'save() was called with args: {mock_save.call_args}'
                 raise AssertionError(msg) from exc
 
-    def test_session2_geoip(self):
-        real_save = odoo.http.root.session_store.save
-        with patch.object(odoo.http.root.geoip_resolver, 'resolve') as mock_resolve,\
-             patch.object(odoo.http.root.session_store, 'save') as mock_save:
-            mock_resolve.return_value = GEOIP_ODOO_FARM_2
-            mock_save.side_effect = real_save
-
-            # Geoip is lazy: it should be computed only when necessary.
-            self.nodb_url_open('/test_http/greeting').raise_for_status()
-            mock_resolve.assert_not_called()
-
-            # Geoip is like the defaut session: the session should not
-            # be stored only due to geoip.
-            mock_resolve.reset_mock()
-            mock_save.reset_mock()
-            res = self.nodb_url_open('/test_http/geoip')
-            res.raise_for_status()
-            self.assertEqual(res.text, str(GEOIP_ODOO_FARM_2))
-            mock_save.assert_not_called()
-
-            # Geoip is cached on the session: we shouldn't geolocate the
-            # same ip multiple times.
-            mock_resolve.reset_mock()
-            mock_save.reset_mock()
-            self.nodb_url_open('/test_http/save_session').raise_for_status()
-            self.nodb_url_open('/test_http/geoip').raise_for_status()
-            res = self.nodb_url_open('/test_http/geoip')
-            res.raise_for_status()
-            self.assertEqual(res.text, str(GEOIP_ODOO_FARM_2))
-            mock_resolve.assert_called_once()
-
     def test_session3_logout_15_0_geoip(self):
         session = self.authenticate(None, None)
         session['db'] = 'idontexist'

odoo/addons/test_http/utils.py

@@ -1,15 +1,48 @@
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import geoip2.errors
+import geoip2.models
 from html.parser import HTMLParser
 from odoo.http import FilesystemSessionStore
 from odoo.tools._vendor.sessions import SessionStore
 
 
-class MemoryGeoipResolver:
-    def resolve(self, ip):
-        return {}
+TEST_IP = '192.0.2.42'  # 192.0.2.0/24 are reserved for documentation,
+                        # they are like example.com for ip addresses
+TEST_IP_GEOIP_CITY = geoip2.models.City(
+    {'continent': {'code': 'EU', 'geoname_id': 6255148, 'names': {'de': 'Europa', 'en': 'Europe', 'es': 'Europa', 'fr': 'Europe', 'ja': 'ヨーロッパ', 'pt-BR': 'Europa', 'ru': 'Европа', 'zh-CN': '欧洲'}},
+     'country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}},
+     'location': {'accuracy_radius': 500, 'latitude': 48.8582, 'longitude': 2.3387, 'time_zone': 'Europe/Paris'},
+     'registered_country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}},
+     'traits': {'ip_address': TEST_IP, 'prefix_len': 21},
+    }, ['en']
+)
+TEST_IP_GEOIP_COUNTRY = geoip2.models.Country(
+    {'continent': {'code': 'EU', 'geoname_id': 6255148, 'names': {'de': 'Europa', 'en': 'Europe', 'es': 'Europa', 'fr': 'Europe', 'ja': 'ヨーロッパ', 'pt-BR': 'Europa', 'ru': 'Европа', 'zh-CN': '欧洲'}},
+     'country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}},
+     'registered_country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}},
+     'traits': {'ip_address': TEST_IP, 'prefix_len': 21},
+    }, ['en']
+)
 
 
+class MemoryGeoipResolver:
+    def __init__(self):
+        self.country_db = {TEST_IP: TEST_IP_GEOIP_COUNTRY}
+        self.city_db = {TEST_IP: TEST_IP_GEOIP_CITY}
+
+    def country(self, ip):
+        record = self.country_db.get(ip)
+        if not record:
+            raise geoip2.errors.AddressNotFoundError(ip)
+        return record
+
+    def city(self, ip):
+        record = self.city_db.get(ip)
+        if not record:
+            raise geoip2.errors.AddressNotFoundError(ip)
+        return record
+
 class MemorySessionStore(SessionStore):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

odoo/http.py

@@ -137,6 +137,10 @@ from urllib.parse import urlparse
 from zlib import adler32
 
 import babel.core
+import geoip2.database
+import geoip2.models
+import geoip2.errors
+import maxminddb
 import psycopg2
 import werkzeug.datastructures
 import werkzeug.exceptions
@@ -166,7 +170,6 @@ from .modules.registry import Registry
 from .service import security, model as service_model
 from .tools import (config, consteq, date_utils, file_path, parse_version,
                     profiler, submap, unique, ustr,)
-from .tools.geoipresolver import GeoIPResolver
 from .tools.func import filter_kwargs, lazy_property
 from .tools.mimetypes import guess_mimetype
 from .tools._vendor import sessions
@@ -222,6 +225,12 @@ def get_default_session():
         'profile_params': None,
     }
 
+# Two empty objects used when the geolocalization failed. They have the
+# sames attributes as real countries/cities except that accessing them
+# evaluates to None.
+GEOIP_EMPTY_COUNTRY = geoip2.models.Country({})
+GEOIP_EMPTY_CITY = geoip2.models.City({})
+
 # The request mimetypes that transport JSON in their body.
 JSON_MIMETYPES = ('application/json', 'application/json-rpc')
 
@@ -997,6 +1006,102 @@ class Session(collections.abc.MutableMapping):
         self.is_dirty = True
 
 
+
+# =========================================================
+# GeoIP
+# =========================================================
+
+class GeoIP(collections.abc.Mapping):
+    """
+    Ip Geolocalization utility, determine information such as the
+    country or the timezone of the user based on their IP Address.
+
+    The instances share the same API as `:class:`geoip2.models.City`
+    <https://geoip2.readthedocs.io/en/latest/#geoip2.models.City>`_.
+
+    When the IP couldn't be geolocalized (missing database, bad address)
+    then an empty object is returned. This empty object can be used like
+    a regular one with the exception that all info are set None.
+
+    :param str ip: The IP Address to geo-localize
+
+    .. note:
+
+        The geoip info the the current request are available at
+        :attr:`~odoo.http.request.geoip`.
+
+    .. code-block:
+
+        >>> GeoIP('127.0.0.1').country.iso_code
+        >>> odoo_ip = socket.gethostbyname('odoo.com')
+        >>> GeoIP(odoo_ip).country.iso_code
+        'FR'
+    """
+
+    def __init__(self, ip):
+        self.ip = ip
+
+    @lazy_property
+    def _city_record(self):
+        try:
+            return root.geoip_city_db.city(self.ip)
+        except geoip2.errors.AddressNotFoundError:
+            return GEOIP_EMPTY_CITY
+
+    @lazy_property
+    def _country_record(self):
+        if '_city_record' in vars(self):
+            # the City class inherits from the Country class and the
+            # city record is in cache already, save a geolocalization
+            return self._city_record
+        try:
+            return root.geoip_country_db.country(self.ip)
+        except geoip2.errors.AddressNotFoundError:
+            return GEOIP_EMPTY_COUNTRY
+
+    def __getattr__(self, attr):
+        # Be smart and determine whether the attribute exists on the
+        # country object or on the city object.
+        if hasattr(GEOIP_EMPTY_COUNTRY, attr):
+            return getattr(self._country_record, attr)
+        if hasattr(GEOIP_EMPTY_CITY, attr):
+            return getattr(self._city_record, attr)
+        raise AttributeError(f"{self} has no attribute {attr!r}")
+
+    def __bool__(self):
+        return self.country_name is not None
+
+    # Old dict API, undocumented for now, will be deprecated some day
+    def __getitem__(self, item):
+        if item == 'country_name':
+            return self.country.name or self.continent.name
+
+        if item == 'country_code':
+            return self.country.iso_code or self.continent.code
+
+        if item == 'city':
+            return self.city.name
+
+        if item == 'latitude':
+            return self.location.latitude
+
+        if item == 'longitude':
+            return self.location.longitude
+
+        if item == 'region':
+            return self.subdivisions[0].iso_code if self.subdivisions else None
+
+        if item == 'time_zone':
+            return self.location.time_zone
+
+        raise KeyError(item)
+
+    def __iter__(self):
+        raise NotImplementedError("The dictionnary GeoIP API is deprecated.")
+
+    def __len__(self):
+        raise NotImplementedError("The dictionnary GeoIP API is deprecated.")
+
 # =========================================================
 # Request and Response
 # =========================================================
@@ -1137,6 +1242,7 @@ class Request:
         self.dispatcher = _dispatchers['http'](self)  # until we match
         #self.params = {}  # set by the Dispatcher
 
+        self.geoip = GeoIP(httprequest.remote_addr)
         self.registry = None
         self.env = None
 
@@ -1238,27 +1344,6 @@ class Request:
 
     _cr = cr
 
-    @property
-    def geoip(self):
-        """
-        Get the remote address geolocalisation.
-
-        When geolocalization is successful, the return value is a
-        dictionary whose format is:
-
-            {'city': str, 'country_code': str, 'country_name': str,
-             'latitude': float, 'longitude': float, 'region': str,
-             'time_zone': str}
-
-        When geolocalization fails, an empty dict is returned.
-        """
-        if '_geoip' not in self.session:
-            was_dirty = self.session.is_dirty
-            self.session._geoip = (self.registry['ir.http']._geoip_resolve()
-                                   if self.db else self._geoip_resolve())
-            self.session.is_dirty = was_dirty
-        return self.session._geoip
-
     # =====================================================
     # Helpers
     # =====================================================
@@ -1335,11 +1420,6 @@ class Request:
         except (ValueError, KeyError):
             return DEFAULT_LANG
 
-    def _geoip_resolve(self):
-        if not (root.geoip_resolver and self.httprequest.remote_addr):
-            return {}
-        return root.geoip_resolver.resolve(self.httprequest.remote_addr) or {}
-
     def get_http_params(self):
         """
         Extract key=value pairs from the query string and the forms
@@ -1487,10 +1567,8 @@ class Request:
             return
 
         if sess.should_rotate:
-            sess['_geoip'] = self.geoip
             root.session_store.rotate(sess, self.env)  # it saves
         elif sess.is_dirty:
-            sess['_geoip'] = self.geoip
             root.session_store.save(sess)
 
         # We must not set the cookie if the session id was specified
@@ -1920,18 +1998,33 @@ class Application:
         _logger.debug('HTTP sessions stored in: %s', path)
         return FilesystemSessionStore(path, session_class=Session, renew_missing=True)
 
-    @lazy_property
-    def geoip_resolver(self):
-        try:
-            return GeoIPResolver.open(config.get('geoip_database'))
-        except Exception as e:
-            _logger.warning('Cannot load GeoIP: %s', e)
-
     def get_db_router(self, db):
         if not db:
             return self.nodb_routing_map
         return request.registry['ir.http'].routing_map()
 
+    @lazy_property
+    def geoip_city_db(self):
+        try:
+            return geoip2.database.Reader(config['geoip_city_db'])
+        except (OSError, maxminddb.InvalidDatabaseError):
+            _logger.debug(
+                "Couldn't load Geoip City file at %s. IP Resolver disabled.",
+                config['geoip_city_db'], exc_info=True
+            )
+            return type('FakeReader', (), {
+                'city': lambda self, ip: GEOIP_EMPTY_CITY,
+                'country': lambda self, ip: GEOIP_EMPTY_COUNTRY
+            })()
+
+    @lazy_property
+    def geoip_country_db(self):
+        try:
+            return geoip2.database.Reader(config['geoip_country_db'])
+        except (OSError, maxminddb.InvalidDatabaseError) as exc:
+            _logger.debug("Couldn't load Geoip Country file (%s). Fallbacks on Geoip City.", exc,)
+            return self.geoip_city_db
+
     def set_csp(self, response):
         headers = response.headers
         if 'Content-Security-Policy' in headers:

odoo/tools/config.py

@@ -312,8 +312,10 @@ class configmanager(object):
                          type="int")
         group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true",
                          help="Try to enable the unaccent extension when creating new databases.")
-        group.add_option("--geoip-db", dest="geoip_database", my_default='/usr/share/GeoIP/GeoLite2-City.mmdb',
-                         help="Absolute path to the GeoIP database file.")
+        group.add_option("--geoip-city-db", "--geoip-db", dest="geoip_city_db", my_default='/usr/share/GeoIP/GeoLite2-City.mmdb',
+                         help="Absolute path to the GeoIP City database file.")
+        group.add_option("--geoip-country-db", dest="geoip_country_db", my_default='/usr/share/GeoIP/GeoLite2-Country.mmdb',
+                         help="Absolute path to the GeoIP Country database file.")
         parser.add_option_group(group)
 
         if os.name == 'posix':
@@ -449,7 +451,8 @@ class configmanager(object):
                 'db_maxconn', 'import_partial', 'addons_path', 'upgrade_path',
                 'syslog', 'without_demo', 'screencasts', 'screenshots',
                 'dbfilter', 'log_level', 'log_db',
-                'log_db_level', 'geoip_database', 'dev_mode', 'shell_interface'
+                'log_db_level', 'geoip_city_db', 'geoip_country_db', 'dev_mode',
+                'shell_interface',
         ]
 
         for arg in keys:
@@ -537,7 +540,7 @@ class configmanager(object):
             self.save()
 
         # normalize path options
-        for key in ['data_dir', 'logfile', 'pidfile', 'test_file', 'screencasts', 'screenshots', 'pg_path', 'translate_out', 'translate_in', 'geoip_database']:
+        for key in ['data_dir', 'logfile', 'pidfile', 'test_file', 'screencasts', 'screenshots', 'pg_path', 'translate_out', 'translate_in', 'geoip_city_db', 'geoip_country_db']:
             self.options[key] = self._normalize(self.options[key])
 
         conf.addons_paths = self.options['addons_path'].split(',')

odoo/tools/geoipresolver.py

@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-import os.path
-
-try:
-    import GeoIP    # Legacy
-except ImportError:
-    GeoIP = None
-
-try:
-    import geoip2
-    import geoip2.database
-except ImportError:
-    geoip2 = None
-
-class GeoIPResolver(object):
-    def __init__(self, fname):
-        self.fname = fname
-        try:
-            self._db = geoip2.database.Reader(fname)
-            self.version = 2
-        except Exception:
-            try:
-                self._db = GeoIP.open(fname, GeoIP.GEOIP_STANDARD)
-                self.version = 1
-                assert self._db.database_info is not None
-            except Exception:
-                raise ValueError('Invalid GeoIP database: %r' % fname)
-
-    def __del__(self):
-        if self.version == 2:
-            self._db.close()
-
-    @classmethod
-    def open(cls, fname):
-        if not GeoIP and not geoip2:
-            return None
-        if not os.path.exists(fname):
-            return None
-        return GeoIPResolver(fname)
-
-    def resolve(self, ip):
-        if self.version == 1:
-            return self._db.record_by_addr(ip) or {}
-        elif self.version == 2:
-            try:
-                r = self._db.city(ip)
-            except (ValueError, geoip2.errors.AddressNotFoundError):
-                return {}
-            # Compatibility with Legacy database.
-            # Some ips cannot be located to a specific country. Legacy DB used to locate them in
-            # continent instead of country. Do the same to not change behavior of existing code.
-            country, attr = (r.country, 'iso_code') if r.country.geoname_id else (r.continent, 'code')
-            return {
-                'city': r.city.name,
-                'country_code': getattr(country, attr),
-                'country_name': country.name,
-                'latitude': r.location.latitude,
-                'longitude': r.location.longitude,
-                'region': r.subdivisions[0].iso_code if r.subdivisions else None,
-                'time_zone': r.location.time_zone,
-            }
-
-    # compat
-    def record_by_addr(self, addr):
-        return self.resolve(addr)

requirements.txt

@@ -6,6 +6,7 @@ docutils==0.16
 ebaysdk==2.1.5
 freezegun==0.3.11; python_version < '3.8'
 freezegun==0.3.15; python_version >= '3.8'
+geoip2==2.9.0
 gevent==1.5.0 ; python_version == '3.7'
 gevent==20.9.0 ; python_version > '3.7' and python_version <= '3.9'
 gevent==21.8.0 ; python_version > '3.9'  # (Jammy)

setup.py

@@ -28,6 +28,7 @@ setup(
         'cryptography',
         'decorator',
         'docutils',
+        'geoip2',
         'gevent',
         'greenlet',
         'idna',

setup/package.dfdebian

@@ -27,6 +27,7 @@ RUN apt-get update -qq &&  \
         python3-dateutil \
         python3-decorator \
         python3-docutils \
+        python3-geoip2 \
         python3-gevent \
         python3-pil \
         python3-jinja2 \

setup/package.dffedora

@@ -22,6 +22,7 @@ RUN dnf update -d 0 -e 0 -y && \
         python3-devel \
         python3-docutils \
         python3-freezegun \
+        python3-geoip2 \
         python3-gevent \
         python3-greenlet \
         python3-idna \