[IMP] requirements: resolve compatibility issues for 3.8 and 3.9

Only updates outdated requirements which actively cause issues:

* freezegun broken in 3.8 (removal of time.clock)
* xlrd broken in 3.8 (removal of time.clock)
* also monkeypatches xlrd.xlsx for 3.9 (removal of
  Element.getiterator, breaks because of defusedxml)
* jinja triggers DeprecationWarning in 3.8
* pillow triggers warning in 3.9
* lxml, greenlet don't compile in 3.9
* reportlab doesn't work in 3.9

New versions try to match those of Debian Bullseye.

Also adds a script to more easily compare dependency versions between
the requirements files and what's in various distributions (currently
supports checking against debian and ubuntu).

Furthermore updates warnings filtering:

* removes xlrd (mischeck was monkeypatched as noted above)
* removes setuptools (was for older versions, one would hope this
  isn't an issue anymore)
* adds babel: python-babel/babel#684 fixes the deprecation warning but
  is not part of any release yet
* ignores error related to `random.sample` on a set, this is a
  diagnostics bug because recordsets implement both Sequence and Set,
  and the stdlib checks for Set first (bpo-42470)

See #59980
Closes #61103

closes odoo/odoo#62510

X-original-commit: 648635deca67df09417ae55c6eb181c98524b74d
Signed-off-by: Xavier Morel (xmo) <xmo@odoo.com>

odoo/netsvc.py

@@ -128,12 +128,13 @@ def init_logger():
     # ignore deprecation warnings from invalid escape (there's a ton and it's
     # pretty likely a super low-value signal)
     warnings.filterwarnings('ignore', r'^invalid escape sequence \\.', category=DeprecationWarning)
+    # recordsets are both sequence and set so trigger warning despite no issue
+    warnings.filterwarnings('ignore', r'^Sampling from a set', category=DeprecationWarning, module='odoo')
     # ignore a bunch of warnings we can't really fix ourselves
     for module in [
-        'setuptools.depends',# older setuptools version using imp
+        'babel.util', # deprecated parser module, no release yet
         'zeep.loader',# zeep using defusedxml.lxml
         'reportlab.lib.rl_safe_eval',# reportlab importing ABC from collections
-        'xlrd/xlsx',# xlrd mischecks iter() on trees or something so calls deprecated getiterator() instead of iter()
         'ofxparse',# ofxparse importing ABC from collections
     ]:
         warnings.filterwarnings('ignore', category=DeprecationWarning, module=module)

odoo/tools/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+from . import _monkeypatches
 from . import pycompat
 from . import win32
 from . import appdirs

odoo/tools/_monkeypatches.py

@@ -0,0 +1,15 @@
+from xlrd import xlsx
+from lxml import etree
+
+# xlrd.xlsx supports defusedxml, defusedxml's etree interface is broken
+# (missing ElementTree and thus ElementTree.iter) which causes a fallback to
+# Element.getiterator(), triggering a warning before 3.9 and an error from 3.9.
+#
+# We have defusedxml installed because zeep has a hard dep on defused and
+# doesn't want to drop it (mvantellingen/python-zeep#1014).
+#
+# Ignore the check and set the relevant flags directly using lxml as we have a
+# hard dependency on it.
+xlsx.ET = etree
+xlsx.ET_has_iterparse = True
+xlsx.Element_has_iter = True

requirements.txt

@@ -4,30 +4,36 @@ decorator==4.3.0
 docutils==0.14
 ebaysdk==2.1.5
 feedparser==5.2.1
-freezegun==0.3.11
+freezegun==0.3.11; python_version < '3.8'
+freezegun==0.3.15; python_version >= '3.8'
 gevent==1.1.2 ; sys_platform != 'win32' and python_version < '3.7'
 gevent==1.5.0 ; python_version >= '3.7'
 gevent==1.4.0 ; sys_platform == 'win32' and python_version < '3.7'
 greenlet==0.4.10 ; python_version < '3.7'
-greenlet==0.4.15 ; python_version >= '3.7'
+greenlet==0.4.15 ; python_version == '3.7'
+greenlet==0.4.17 ; python_version > '3.7'
 html2text==2018.1.9
 idna==2.6
-Jinja2==2.10.1
+Jinja2==2.10.1; python_version < '3.8'
+# bullseye version, focal patched 2.10
+Jinja2==2.11.2; python_version >= '3.8'
 libsass==0.17.0
 lxml==3.7.1 ; sys_platform != 'win32' and python_version < '3.7'
-lxml==4.3.2 ; sys_platform != 'win32' and python_version >= '3.7'
+lxml==4.3.2 ; sys_platform != 'win32' and python_version == '3.7'
+lxml==4.6.1 ; sys_platform != 'win32' and python_version > '3.7'
 lxml ; sys_platform == 'win32'
 Mako==1.0.7
 MarkupSafe==1.1.0
 num2words==0.5.6
 ofxparse==0.19
 passlib==1.7.1
-Pillow==5.4.1 ; python_version < '3.7' or sys_platform != 'win32'
-Pillow==6.1.0 ; sys_platform == 'win32' and python_version >= '3.7'
+Pillow==5.4.1 ; python_version <= '3.7' and sys_platform != 'win32'
+Pillow==6.1.0 ; python_version <= '3.7' and sys_platform == 'win32'
+Pillow==8.0.1 ; python_version > '3.7'
 polib==1.1.0
 psutil==5.6.6
 psycopg2==2.7.7; sys_platform != 'win32' and python_version < '3.8'
-psycopg2==2.8.3; sys_platform == 'win32' or python_version >= '3.8'
+psycopg2==2.8.5; sys_platform == 'win32' or python_version >= '3.8'
 pydot==1.4.1
 python-ldap==3.1.0; sys_platform != 'win32'
 pyparsing==2.2.0
@@ -37,7 +43,8 @@ python-dateutil==2.7.3
 pytz==2019.1
 pyusb==1.0.2
 qrcode==6.1
-reportlab==3.5.13
+reportlab==3.5.13; python_version < '3.8'
+reportlab==3.5.55; python_version >= '3.8'
 requests==2.21.0
 zeep==3.2.0
 python-stdnum==1.8
@@ -45,5 +52,6 @@ vobject==0.9.6.1
 Werkzeug==0.16.1
 XlsxWriter==1.1.2
 xlwt==1.3.*
-xlrd==1.1.0
+xlrd==1.1.0; python_version < '3.8'
+xlrd==1.2.0; python_version >= '3.8'
 pypiwin32 ; sys_platform == 'win32'

setup/requirements-check.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python
+"""
+Checks versions from the requirements files against distribution-provided
+versions, taking distribution's Python version in account e.g. if checking
+against a release which bundles Python 3.5, checks the 3.5 version of
+requirements.
+
+* only shows requirements for which at least one release diverges from the
+  matching requirements version
+* empty cells mean that specific release matches its requirement (happens when
+  checking multiple releases: one of the other releases may mismatch the its
+  requirements necessating showing the row)
+
+Only handles the subset of requirements files we're currently using:
+* no version spec or strict equality
+* no extras
+* only sys_platform and python_version environment markers
+"""
+
+import argparse
+import gzip
+import itertools
+import json
+import operator
+import re
+import string
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from urllib.request import urlopen
+from sys import stdout, stderr
+from typing import Dict, List, Set, Optional, Any, Tuple
+
+Version = Tuple[int, ...]
+def parse_version(vstring: str) -> Optional[Version]:
+    if not vstring:
+        return None
+    return tuple(map(int, vstring.split('.')))
+
+# shared beween debian and ubuntu
+SPECIAL = {
+    'pytz': 'tz',
+    'libsass': 'libsass-python',
+}
+def unfuck(s: str) -> str:
+    """ Try to strip the garbage from the version string, just remove everything
+    following the first `+`, `~` or `-`
+    """
+    return re.match(r'''
+        (?:\d+:)? # debian crud prefix
+        (.*?) # the shit we actually want
+        (?:~|\+|-|\.dfsg)
+        .*
+    ''', s, flags=re.VERBOSE)[1]
+
+class Distribution(ABC):
+    def __init__(self, release):
+        self._release = release
+
+    @abstractmethod
+    def get_version(self, package: str) -> Optional[Version]:
+        ...
+
+    def __str__(self):
+        return f'{type(self).__name__.lower()} {self._release}'
+
+    @classmethod
+    def get(cls, name):
+        try:
+            return next(
+                c
+                for c in cls.__subclasses__()
+                if c.__name__.lower() == name
+            )
+        except StopIteration:
+            raise ValueError(f"Unknown distribution {name!r}")
+
+class Debian(Distribution):
+    def get_version(self, package):
+        """ Try to find which version of ``package`` is in Debian release {release}
+        """
+        package = SPECIAL.get(package, package)
+        # try the python prefix first: some packages have a native of foreign $X and
+        # either the bindings or a python equivalent at python-X, or just a name
+        # collision
+        for prefix in ['python-', '']:
+            res = json.load(urlopen(f'https://sources.debian.org/api/src/{prefix}{package}'))
+            if res.get('error') is None:
+                break
+        if res.get('error'):
+            return
+
+        return next(
+            parse_version(unfuck(distr['version']))
+            for distr in res['versions']
+            if distr['area'] == 'main'
+            if self._release in distr['suites']
+        )
+
+class Ubuntu(Distribution):
+    """ Ubuntu doesn't have an API, instead it has a huge text file
+    """
+    def __init__(self, release):
+        super().__init__(release)
+
+        self._packages = {}
+        # ideally we should request the proper Content-Encoding but PUC
+        # apparently does not care, and returns a somewhat funky
+        # content-encoding (x-gzip) anyway
+        data = gzip.open(
+            urlopen(f'https://packages.ubuntu.com/source/{release}/allpackages?format=txt.gz'),
+            mode='rt', encoding='utf-8'
+        )
+        for line in itertools.islice(data, 6, None): # first 6 lines is garbage header
+            # ignore the restricted, security, universe, multiverse tags
+            m = re.match(r'(\S+) \(([^)]+)\)', line.strip())
+            assert m, f"invalid line {line.strip()!r}"
+            self._packages[m[1]] = m[2]
+
+    def get_version(self, package):
+        package = SPECIAL.get(package, package)
+        for prefix in ['python3-', 'python-', '']:
+            v = self._packages.get(f'{prefix}{package}')
+            if v:
+                return parse_version(unfuck(v))
+        return None
+
+class Markers:
+    """ Simplistic RD parser for requirements env markers.
+
+    Evaluation of the env markers is so basic it goes to brunch in uggs.
+    """
+    def __init__(self, s=None):
+        self.rules = False
+        if s is not None:
+            self.rules, rest = self._parse_marker(s)
+            assert not rest
+
+    def evaluate(self, context: Dict[str, Any]) -> bool:
+        if not self.rules:
+            return True
+        return self._eval(self.rules, context)
+
+    def _eval(self, rule, context):
+        if rule[0] == 'OR':
+            return self._eval(rule[1], context) or self._eval(rule[2], context)
+        elif rule[0] == 'AND':
+            return self._eval(rule[1], context) and self._eval(rule[2], context)
+        elif rule[0] == 'ENV':
+            return context[rule[1]]
+        elif rule[0] == 'LIT':
+            return rule[1]
+        else:
+            op, var1, var2 = rule
+            var1 = self._eval(var1, context)
+            var2 = self._eval(var2, context)
+
+            # NOTE: currently doesn't follow PEP440 version matching at all
+            if op == '==': return var1 == var2
+            elif op == '!=': return var1 != var2
+            elif op == '<': return var1 < var2
+            elif op == '<=': return var1 <= var2
+            elif op == '>': return var1 > var2
+            elif op == '>=': return var1 >= var2
+            else:
+                raise NotImplementedError(f"Operator {op!r}")
+
+    def _parse_marker(self, s):
+        return self._parse_or(s)
+
+    def _parse_or(self, s):
+        sub1, rest = self._parse_and(s)
+        expr, n = re.subn(r'^\s*or\b', '', rest, count=1)
+        if not n:
+            return sub1, rest
+        sub2, rest = self._parse_and(expr)
+        return ('OR', sub1, sub2), rest
+
+    def _parse_and(self, s):
+        sub1, rest = self._parse_expr(s)
+        expr, n = re.subn(r'\s*and\b', '', rest, count=1)
+        if not n:
+            return sub1, rest
+        sub2, rest = self._parse_expr(expr)
+        return ('AND', sub1, sub2), rest
+
+    def _parse_expr(self, s):
+        expr, n = re.subn(r'^\s*\(', '', s, count=1)
+        if n:
+            sub, rest = self.parse_marker(expr)
+            rest, n = re.subn(r'\s*\)', '', rest, count=1)
+            assert n, f"expected closing parenthesis, found {rest}"
+            return sub, rest
+
+        var1, rest = self._parse_var(s)
+        op, rest = self._parse_op(rest)
+        var2, rest = self._parse_var(rest)
+        return (op, var1, var2), rest
+
+    def _parse_op(self, s):
+        m = re.match(r'''
+            \s*
+            (<= | < | != | >= | > | ~= | ===? | in \b | not \s+ in \b)
+            (.*)
+        ''', s, re.VERBOSE)
+        assert m, f"no operator in {s!r}"
+        return m.groups()
+
+    def _parse_var(self, s):
+        python_str = re.escape(string.printable.translate(str.maketrans({
+            '"': '',
+            "'": '',
+            '\\': '',
+            '-': '',
+        })))
+        m = re.match(fr'''
+            \s*
+            (:?
+                # TODO: add more envvars
+                (?P<env>python_version | os_name | sys_platform)
+              | " (?P<dquote>['{python_str}-]*) "
+              | ' (?P<squote>["{python_str}-]*) '
+            )
+            (?P<rest>.*)
+        ''', s, re.VERBOSE)
+        assert m, f"failed to find marker var in {s}"
+        if m['env']:
+            return ('ENV', m['env']), m['rest']
+        return ('LIT', m['dquote'] or m['squote'] or ''), m['rest']
+
+def parse_spec(line: str) -> (str, (Optional[str], Optional[str]), Markers):
+    """ Parse a requirements specification (a line of requirements)
+
+    Returns the package name, a version spec (operator and comparator) possibly
+    None and a Markers object.
+
+    Not correctly supported:
+
+    * version matching, not all operators are implemented and those which are
+      almost certainly don't match PEP 440
+
+    Not supported:
+
+    * url requirements
+    * multi-versions spec
+    * extras
+    * continuations
+
+    Full grammar is at https://www.python.org/dev/peps/pep-0508/#complete-grammar
+    """
+    # weirdly a distribution name can apparently start with a number
+    name, rest = re.match(r'([\w\d](?:[._-]*[\w\d]+)*)\s*(.*)', line.strip()).groups()
+    # skipping extras
+    version_cmp = version = None
+    versionspec = re.match(r'''
+        (< | <= | != | == | >= | > | ~= | ===)
+        \s*
+        ([\w\d_.*+!-]+)
+        \s*
+        (.*)
+    ''', rest, re.VERBOSE)
+    if versionspec:
+        version_cmp, version, rest = versionspec.groups()
+    markers = Markers()
+    if rest[:1] == ';':
+        markers = Markers(rest[1:])
+
+    return name, (version_cmp, version), markers
+
+def parse_requirements(reqpath: Path) -> Dict[str, List[Tuple[str, Markers]]]:
+    """ Parses a requirement file to a dict of {package: [(version, markers)]}
+
+    The env markers express *whether* that specific dep applies.
+    """
+    reqs = {}
+    for line in reqpath.open('r', encoding='utf-8'):
+        if line.isspace() or line.startswith('#'):
+            continue
+
+        name, (op, version), markers = parse_spec(line)
+        assert op is None or op == '==', f"unexpected version comparator {op}"
+        reqs.setdefault(name, []).append((version, markers))
+    return reqs
+
+def main(args):
+    checkers = [
+        Distribution.get(distro)(release)
+        for version in args.release
+        for (distro, release) in [version.split(':')]
+    ]
+
+    stderr.write(f"Fetch Python versions...\n")
+    pyvers = [
+        '.'.join(map(str, checker.get_version('python3-defaults')[:2]))
+        for checker in checkers
+    ]
+
+    uniq = sorted(v for v in set(pyvers))
+    table = [
+        ['']
+        + [f'req {v}' for v in uniq]
+        + [f'{checker._release} ({version})' for checker, version in zip(checkers, pyvers)]
+    ]
+
+    reqs = parse_requirements((Path.cwd() / __file__).parent.parent / 'requirements.txt')
+    tot = len(reqs) * len(checkers)
+
+    def progress(n=iter(range(tot+1))):
+        stderr.write(f"\rFetch requirements: {next(n)} / {tot}")
+
+    progress()
+    for req, options in reqs.items():
+        row = [req]
+        byver = {}
+        for pyver in uniq:
+            # FIXME: when multiple options apply, check which pip uses
+            #        (first-matching. best-matching, latest, ...)
+            for version, markers in options:
+                if markers.evaluate({
+                    'python_version': pyver,
+                    'sys_platform': 'linux',
+                }):
+                    byver[pyver] = version
+                    break
+            row.append(byver.get(pyver) or '')
+        # this requirement doesn't apply, ignore
+        if not byver:
+            # normally the progressbar is updated when processing each
+            # requirement against each checker, if the requirement doesn't apply
+            # to any checker we still need to consider the requirement fetched /
+            # resolved for each checker or our tally is incorrect
+            for _ in checkers:
+                progress()
+            continue
+
+        mismatch = False
+        for i, c in enumerate(checkers):
+            req_version = byver.get(pyvers[i], '')
+            check_version = '.'.join(map(str, c.get_version(req.lower()) or ['<missing>']))
+            progress()
+            if req_version != check_version:
+                row.append(check_version)
+                mismatch = True
+            else:
+                row.append('')
+
+        # only show row if one of the items diverges from requirement
+        if mismatch:
+            table.append(row)
+    stderr.write('\n')
+
+    # evaluate width of columns
+    sizes = [0] * (len(checkers) + len(uniq) + 1)
+    for row in table:
+        sizes = [
+            max(s, len(cell))
+            for s, cell in zip(sizes, row)
+        ]
+
+    # format table
+    for row in table:
+        stdout.write('| ')
+        for cell, width in zip(row, sizes):
+            stdout.write(f'{cell:<{width}} | ')
+        stdout.write('\n')
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        'release', nargs='+',
+        help="Release to check against, should use the format '{distro}:{release}' e.g. 'debian:sid'"
+    )
+    args = parser.parse_args()
+    main(args)