diff --git a/addons/website/tools.py b/addons/website/tools.py index f2ffc673c6d..4f40c38aee7 100644 --- a/addons/website/tools.py +++ b/addons/website/tools.py @@ -51,7 +51,7 @@ def MockRequest( sale_order_id=sale_order_id, website_sale_current_pl=website_sale_current_pl, ), - geoip={}, + geoip=odoo.http.GeoIP('127.0.0.1'), db=env.registry.db_name, env=env, registry=env.registry, diff --git a/debian/control b/debian/control index 2016cd55b76..1303c669200 100644 --- a/debian/control +++ b/debian/control @@ -30,6 +30,7 @@ Depends: python3-decorator, python3-docutils, python3-freezegun, + python3-geoip2, python3-pil, python3-jinja2, python3-libsass, diff --git a/odoo/addons/test_http/controllers.py b/odoo/addons/test_http/controllers.py index 855a7526dac..242174ab1bc 100644 --- a/odoo/addons/test_http/controllers.py +++ b/odoo/addons/test_http/controllers.py @@ -136,7 +136,15 @@ class TestHttp(http.Controller): # ===================================================== @http.route('/test_http/geoip', type='http', auth='none') def geoip(self): - return str(request.geoip) + return json.dumps({ + 'city': request.geoip.city.name, + 'country_code': request.geoip.country.iso_code or request.geoip.continent.code, + 'country_name': request.geoip.country.name or request.geoip.continent.name, + 'latitude': request.geoip.location.latitude, + 'longitude': request.geoip.location.longitude, + 'region': request.geoip.subdivisions[0].iso_code if request.geoip.subdivisions else None, + 'time_zone': request.geoip.location.time_zone, + }) @http.route('/test_http/save_session', type='http', auth='none') def touch(self): diff --git a/odoo/addons/test_http/tests/test_common.py b/odoo/addons/test_http/tests/test_common.py index 41c9ed2ecaa..a8c72ae272f 100644 --- a/odoo/addons/test_http/tests/test_common.py +++ b/odoo/addons/test_http/tests/test_common.py @@ -13,11 +13,14 @@ class TestHttpBase(HttpCase): @classmethod def setUpClass(cls): super().setUpClass() + geoip_resolver = MemoryGeoipResolver() + session_store = MemorySessionStore(session_class=Session) + cls.addClassCleanup(lazy_property.reset_all, odoo.http.root) cls.classPatch(odoo.conf, 'server_wide_modules', ['base', 'web', 'test_http']) - lazy_property.reset_all(odoo.http.root) - cls.classPatch(odoo.http.root, 'session_store', MemorySessionStore(session_class=Session)) - cls.classPatch(odoo.http.root, 'geoip_resolver', MemoryGeoipResolver()) + cls.classPatch(odoo.http.root, 'session_store', session_store) + cls.classPatch(odoo.http.root, 'geoip_city_db', geoip_resolver) + cls.classPatch(odoo.http.root, 'geoip_country_db', geoip_resolver) def setUp(self): super().setUp() diff --git a/odoo/addons/test_http/tests/test_misc.py b/odoo/addons/test_http/tests/test_misc.py index d316c049784..577bafd2907 100644 --- a/odoo/addons/test_http/tests/test_misc.py +++ b/odoo/addons/test_http/tests/test_misc.py @@ -12,6 +12,7 @@ from odoo.tests.common import HOST, new_test_user, get_db_name from odoo.tools import config, file_path from odoo.addons.test_http.controllers import CT_JSON +from odoo.addons.test_http.utils import TEST_IP from .test_common import TestHttpBase @@ -93,6 +94,39 @@ class TestHttpMisc(TestHttpBase): self.assertNotIn('error', res_rpc.keys(), res_rpc.get('error', {}).get('data', {}).get('message')) self.assertIn(milky_way.name, res_rpc['result'], "QWeb template was correctly rendered") + def test_misc5_geoip(self): + res = self.nodb_url_open('/test_http/geoip') + res.raise_for_status() + self.assertEqual(res.json(), { + 'city': None, + 'country_code': None, + 'country_name': None, + 'latitude': None, + 'longitude': None, + 'region': None, + 'time_zone': None, + }) + + # Fake client IP using proxy_mode and a forged X-Forwarded-For http header + headers = { + 'Host': '', + 'X-Forwarded-For': TEST_IP, + 'X-Forwarded-Host': 'odoo.com', + 'X-Forwarded-Proto': 'https' + } + with patch.dict('odoo.tools.config.options', {'proxy_mode': True}): + res = self.nodb_url_open('/test_http/geoip', headers=headers) + res.raise_for_status() + self.assertEqual(res.json(), { + 'city': None, + 'country_code': 'FR', + 'country_name': 'France', + 'latitude': 48.8582, + 'longitude': 2.3387, + 'region': None, + 'time_zone': 'Europe/Paris', + }) + @tagged('post_install', '-at_install') class TestHttpCors(TestHttpBase): diff --git a/odoo/addons/test_http/tests/test_session.py b/odoo/addons/test_http/tests/test_session.py index c4163ae9a28..6a0ea900745 100644 --- a/odoo/addons/test_http/tests/test_session.py +++ b/odoo/addons/test_http/tests/test_session.py @@ -39,7 +39,7 @@ class TestHttpSession(TestHttpBase): def test_session1_default_session(self): # The default session should not be saved on the filestore. with patch.object(odoo.http.root.session_store, 'save') as mock_save: - res = self.db_url_open('/test_http/greeting') + res = self.db_url_open('/test_http/geoip') res.raise_for_status() try: mock_save.assert_not_called() @@ -47,37 +47,6 @@ class TestHttpSession(TestHttpBase): msg = f'save() was called with args: {mock_save.call_args}' raise AssertionError(msg) from exc - def test_session2_geoip(self): - real_save = odoo.http.root.session_store.save - with patch.object(odoo.http.root.geoip_resolver, 'resolve') as mock_resolve,\ - patch.object(odoo.http.root.session_store, 'save') as mock_save: - mock_resolve.return_value = GEOIP_ODOO_FARM_2 - mock_save.side_effect = real_save - - # Geoip is lazy: it should be computed only when necessary. - self.nodb_url_open('/test_http/greeting').raise_for_status() - mock_resolve.assert_not_called() - - # Geoip is like the defaut session: the session should not - # be stored only due to geoip. - mock_resolve.reset_mock() - mock_save.reset_mock() - res = self.nodb_url_open('/test_http/geoip') - res.raise_for_status() - self.assertEqual(res.text, str(GEOIP_ODOO_FARM_2)) - mock_save.assert_not_called() - - # Geoip is cached on the session: we shouldn't geolocate the - # same ip multiple times. - mock_resolve.reset_mock() - mock_save.reset_mock() - self.nodb_url_open('/test_http/save_session').raise_for_status() - self.nodb_url_open('/test_http/geoip').raise_for_status() - res = self.nodb_url_open('/test_http/geoip') - res.raise_for_status() - self.assertEqual(res.text, str(GEOIP_ODOO_FARM_2)) - mock_resolve.assert_called_once() - def test_session3_logout_15_0_geoip(self): session = self.authenticate(None, None) session['db'] = 'idontexist' diff --git a/odoo/addons/test_http/utils.py b/odoo/addons/test_http/utils.py index 3e26d821d02..9898c0cad35 100644 --- a/odoo/addons/test_http/utils.py +++ b/odoo/addons/test_http/utils.py @@ -1,15 +1,48 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. +import geoip2.errors +import geoip2.models from html.parser import HTMLParser from odoo.http import FilesystemSessionStore from odoo.tools._vendor.sessions import SessionStore -class MemoryGeoipResolver: - def resolve(self, ip): - return {} +TEST_IP = '192.0.2.42' # 192.0.2.0/24 are reserved for documentation, + # they are like example.com for ip addresses +TEST_IP_GEOIP_CITY = geoip2.models.City( + {'continent': {'code': 'EU', 'geoname_id': 6255148, 'names': {'de': 'Europa', 'en': 'Europe', 'es': 'Europa', 'fr': 'Europe', 'ja': 'ヨーロッパ', 'pt-BR': 'Europa', 'ru': 'Европа', 'zh-CN': '欧洲'}}, + 'country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}}, + 'location': {'accuracy_radius': 500, 'latitude': 48.8582, 'longitude': 2.3387, 'time_zone': 'Europe/Paris'}, + 'registered_country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}}, + 'traits': {'ip_address': TEST_IP, 'prefix_len': 21}, + }, ['en'] +) +TEST_IP_GEOIP_COUNTRY = geoip2.models.Country( + {'continent': {'code': 'EU', 'geoname_id': 6255148, 'names': {'de': 'Europa', 'en': 'Europe', 'es': 'Europa', 'fr': 'Europe', 'ja': 'ヨーロッパ', 'pt-BR': 'Europa', 'ru': 'Европа', 'zh-CN': '欧洲'}}, + 'country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}}, + 'registered_country': {'geoname_id': 3017382, 'is_in_european_union': True, 'iso_code': 'FR', 'names': {'de': 'Frankreich', 'en': 'France', 'es': 'Francia', 'fr': 'France', 'ja': 'フランス共和国', 'pt-BR': 'França', 'ru': 'Франция', 'zh-CN': '法国'}}, + 'traits': {'ip_address': TEST_IP, 'prefix_len': 21}, + }, ['en'] +) +class MemoryGeoipResolver: + def __init__(self): + self.country_db = {TEST_IP: TEST_IP_GEOIP_COUNTRY} + self.city_db = {TEST_IP: TEST_IP_GEOIP_CITY} + + def country(self, ip): + record = self.country_db.get(ip) + if not record: + raise geoip2.errors.AddressNotFoundError(ip) + return record + + def city(self, ip): + record = self.city_db.get(ip) + if not record: + raise geoip2.errors.AddressNotFoundError(ip) + return record + class MemorySessionStore(SessionStore): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/odoo/http.py b/odoo/http.py index e4922afed2a..187f5ae6bc8 100644 --- a/odoo/http.py +++ b/odoo/http.py @@ -137,6 +137,10 @@ from urllib.parse import urlparse from zlib import adler32 import babel.core +import geoip2.database +import geoip2.models +import geoip2.errors +import maxminddb import psycopg2 import werkzeug.datastructures import werkzeug.exceptions @@ -166,7 +170,6 @@ from .modules.registry import Registry from .service import security, model as service_model from .tools import (config, consteq, date_utils, file_path, parse_version, profiler, submap, unique, ustr,) -from .tools.geoipresolver import GeoIPResolver from .tools.func import filter_kwargs, lazy_property from .tools.mimetypes import guess_mimetype from .tools._vendor import sessions @@ -222,6 +225,12 @@ def get_default_session(): 'profile_params': None, } +# Two empty objects used when the geolocalization failed. They have the +# sames attributes as real countries/cities except that accessing them +# evaluates to None. +GEOIP_EMPTY_COUNTRY = geoip2.models.Country({}) +GEOIP_EMPTY_CITY = geoip2.models.City({}) + # The request mimetypes that transport JSON in their body. JSON_MIMETYPES = ('application/json', 'application/json-rpc') @@ -997,6 +1006,102 @@ class Session(collections.abc.MutableMapping): self.is_dirty = True + +# ========================================================= +# GeoIP +# ========================================================= + +class GeoIP(collections.abc.Mapping): + """ + Ip Geolocalization utility, determine information such as the + country or the timezone of the user based on their IP Address. + + The instances share the same API as `:class:`geoip2.models.City` + `_. + + When the IP couldn't be geolocalized (missing database, bad address) + then an empty object is returned. This empty object can be used like + a regular one with the exception that all info are set None. + + :param str ip: The IP Address to geo-localize + + .. note: + + The geoip info the the current request are available at + :attr:`~odoo.http.request.geoip`. + + .. code-block: + + >>> GeoIP('127.0.0.1').country.iso_code + >>> odoo_ip = socket.gethostbyname('odoo.com') + >>> GeoIP(odoo_ip).country.iso_code + 'FR' + """ + + def __init__(self, ip): + self.ip = ip + + @lazy_property + def _city_record(self): + try: + return root.geoip_city_db.city(self.ip) + except geoip2.errors.AddressNotFoundError: + return GEOIP_EMPTY_CITY + + @lazy_property + def _country_record(self): + if '_city_record' in vars(self): + # the City class inherits from the Country class and the + # city record is in cache already, save a geolocalization + return self._city_record + try: + return root.geoip_country_db.country(self.ip) + except geoip2.errors.AddressNotFoundError: + return GEOIP_EMPTY_COUNTRY + + def __getattr__(self, attr): + # Be smart and determine whether the attribute exists on the + # country object or on the city object. + if hasattr(GEOIP_EMPTY_COUNTRY, attr): + return getattr(self._country_record, attr) + if hasattr(GEOIP_EMPTY_CITY, attr): + return getattr(self._city_record, attr) + raise AttributeError(f"{self} has no attribute {attr!r}") + + def __bool__(self): + return self.country_name is not None + + # Old dict API, undocumented for now, will be deprecated some day + def __getitem__(self, item): + if item == 'country_name': + return self.country.name or self.continent.name + + if item == 'country_code': + return self.country.iso_code or self.continent.code + + if item == 'city': + return self.city.name + + if item == 'latitude': + return self.location.latitude + + if item == 'longitude': + return self.location.longitude + + if item == 'region': + return self.subdivisions[0].iso_code if self.subdivisions else None + + if item == 'time_zone': + return self.location.time_zone + + raise KeyError(item) + + def __iter__(self): + raise NotImplementedError("The dictionnary GeoIP API is deprecated.") + + def __len__(self): + raise NotImplementedError("The dictionnary GeoIP API is deprecated.") + # ========================================================= # Request and Response # ========================================================= @@ -1137,6 +1242,7 @@ class Request: self.dispatcher = _dispatchers['http'](self) # until we match #self.params = {} # set by the Dispatcher + self.geoip = GeoIP(httprequest.remote_addr) self.registry = None self.env = None @@ -1238,27 +1344,6 @@ class Request: _cr = cr - @property - def geoip(self): - """ - Get the remote address geolocalisation. - - When geolocalization is successful, the return value is a - dictionary whose format is: - - {'city': str, 'country_code': str, 'country_name': str, - 'latitude': float, 'longitude': float, 'region': str, - 'time_zone': str} - - When geolocalization fails, an empty dict is returned. - """ - if '_geoip' not in self.session: - was_dirty = self.session.is_dirty - self.session._geoip = (self.registry['ir.http']._geoip_resolve() - if self.db else self._geoip_resolve()) - self.session.is_dirty = was_dirty - return self.session._geoip - # ===================================================== # Helpers # ===================================================== @@ -1335,11 +1420,6 @@ class Request: except (ValueError, KeyError): return DEFAULT_LANG - def _geoip_resolve(self): - if not (root.geoip_resolver and self.httprequest.remote_addr): - return {} - return root.geoip_resolver.resolve(self.httprequest.remote_addr) or {} - def get_http_params(self): """ Extract key=value pairs from the query string and the forms @@ -1487,10 +1567,8 @@ class Request: return if sess.should_rotate: - sess['_geoip'] = self.geoip root.session_store.rotate(sess, self.env) # it saves elif sess.is_dirty: - sess['_geoip'] = self.geoip root.session_store.save(sess) # We must not set the cookie if the session id was specified @@ -1920,18 +1998,33 @@ class Application: _logger.debug('HTTP sessions stored in: %s', path) return FilesystemSessionStore(path, session_class=Session, renew_missing=True) - @lazy_property - def geoip_resolver(self): - try: - return GeoIPResolver.open(config.get('geoip_database')) - except Exception as e: - _logger.warning('Cannot load GeoIP: %s', e) - def get_db_router(self, db): if not db: return self.nodb_routing_map return request.registry['ir.http'].routing_map() + @lazy_property + def geoip_city_db(self): + try: + return geoip2.database.Reader(config['geoip_city_db']) + except (OSError, maxminddb.InvalidDatabaseError): + _logger.debug( + "Couldn't load Geoip City file at %s. IP Resolver disabled.", + config['geoip_city_db'], exc_info=True + ) + return type('FakeReader', (), { + 'city': lambda self, ip: GEOIP_EMPTY_CITY, + 'country': lambda self, ip: GEOIP_EMPTY_COUNTRY + })() + + @lazy_property + def geoip_country_db(self): + try: + return geoip2.database.Reader(config['geoip_country_db']) + except (OSError, maxminddb.InvalidDatabaseError) as exc: + _logger.debug("Couldn't load Geoip Country file (%s). Fallbacks on Geoip City.", exc,) + return self.geoip_city_db + def set_csp(self, response): headers = response.headers if 'Content-Security-Policy' in headers: diff --git a/odoo/tools/config.py b/odoo/tools/config.py index b7ddf5e8927..74d2c37a1fe 100644 --- a/odoo/tools/config.py +++ b/odoo/tools/config.py @@ -312,8 +312,10 @@ class configmanager(object): type="int") group.add_option("--unaccent", dest="unaccent", my_default=False, action="store_true", help="Try to enable the unaccent extension when creating new databases.") - group.add_option("--geoip-db", dest="geoip_database", my_default='/usr/share/GeoIP/GeoLite2-City.mmdb', - help="Absolute path to the GeoIP database file.") + group.add_option("--geoip-city-db", "--geoip-db", dest="geoip_city_db", my_default='/usr/share/GeoIP/GeoLite2-City.mmdb', + help="Absolute path to the GeoIP City database file.") + group.add_option("--geoip-country-db", dest="geoip_country_db", my_default='/usr/share/GeoIP/GeoLite2-Country.mmdb', + help="Absolute path to the GeoIP Country database file.") parser.add_option_group(group) if os.name == 'posix': @@ -449,7 +451,8 @@ class configmanager(object): 'db_maxconn', 'import_partial', 'addons_path', 'upgrade_path', 'syslog', 'without_demo', 'screencasts', 'screenshots', 'dbfilter', 'log_level', 'log_db', - 'log_db_level', 'geoip_database', 'dev_mode', 'shell_interface' + 'log_db_level', 'geoip_city_db', 'geoip_country_db', 'dev_mode', + 'shell_interface', ] for arg in keys: @@ -537,7 +540,7 @@ class configmanager(object): self.save() # normalize path options - for key in ['data_dir', 'logfile', 'pidfile', 'test_file', 'screencasts', 'screenshots', 'pg_path', 'translate_out', 'translate_in', 'geoip_database']: + for key in ['data_dir', 'logfile', 'pidfile', 'test_file', 'screencasts', 'screenshots', 'pg_path', 'translate_out', 'translate_in', 'geoip_city_db', 'geoip_country_db']: self.options[key] = self._normalize(self.options[key]) conf.addons_paths = self.options['addons_path'].split(',') diff --git a/odoo/tools/geoipresolver.py b/odoo/tools/geoipresolver.py deleted file mode 100644 index 2cd169524e6..00000000000 --- a/odoo/tools/geoipresolver.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import os.path - -try: - import GeoIP # Legacy -except ImportError: - GeoIP = None - -try: - import geoip2 - import geoip2.database -except ImportError: - geoip2 = None - -class GeoIPResolver(object): - def __init__(self, fname): - self.fname = fname - try: - self._db = geoip2.database.Reader(fname) - self.version = 2 - except Exception: - try: - self._db = GeoIP.open(fname, GeoIP.GEOIP_STANDARD) - self.version = 1 - assert self._db.database_info is not None - except Exception: - raise ValueError('Invalid GeoIP database: %r' % fname) - - def __del__(self): - if self.version == 2: - self._db.close() - - @classmethod - def open(cls, fname): - if not GeoIP and not geoip2: - return None - if not os.path.exists(fname): - return None - return GeoIPResolver(fname) - - def resolve(self, ip): - if self.version == 1: - return self._db.record_by_addr(ip) or {} - elif self.version == 2: - try: - r = self._db.city(ip) - except (ValueError, geoip2.errors.AddressNotFoundError): - return {} - # Compatibility with Legacy database. - # Some ips cannot be located to a specific country. Legacy DB used to locate them in - # continent instead of country. Do the same to not change behavior of existing code. - country, attr = (r.country, 'iso_code') if r.country.geoname_id else (r.continent, 'code') - return { - 'city': r.city.name, - 'country_code': getattr(country, attr), - 'country_name': country.name, - 'latitude': r.location.latitude, - 'longitude': r.location.longitude, - 'region': r.subdivisions[0].iso_code if r.subdivisions else None, - 'time_zone': r.location.time_zone, - } - - # compat - def record_by_addr(self, addr): - return self.resolve(addr) diff --git a/requirements.txt b/requirements.txt index 8028e0feeb1..9e34960fd0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ docutils==0.16 ebaysdk==2.1.5 freezegun==0.3.11; python_version < '3.8' freezegun==0.3.15; python_version >= '3.8' +geoip2==2.9.0 gevent==1.5.0 ; python_version == '3.7' gevent==20.9.0 ; python_version > '3.7' and python_version <= '3.9' gevent==21.8.0 ; python_version > '3.9' # (Jammy) diff --git a/setup.py b/setup.py index c9abe74cd4e..929fb49ba2f 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ setup( 'cryptography', 'decorator', 'docutils', + 'geoip2', 'gevent', 'greenlet', 'idna', diff --git a/setup/package.dfdebian b/setup/package.dfdebian index 6e110b66d11..7850130a385 100644 --- a/setup/package.dfdebian +++ b/setup/package.dfdebian @@ -27,6 +27,7 @@ RUN apt-get update -qq && \ python3-dateutil \ python3-decorator \ python3-docutils \ + python3-geoip2 \ python3-gevent \ python3-pil \ python3-jinja2 \ diff --git a/setup/package.dffedora b/setup/package.dffedora index c2a0149386c..bed19777cbd 100644 --- a/setup/package.dffedora +++ b/setup/package.dffedora @@ -22,6 +22,7 @@ RUN dnf update -d 0 -e 0 -y && \ python3-devel \ python3-docutils \ python3-freezegun \ + python3-geoip2 \ python3-gevent \ python3-greenlet \ python3-idna \