[FIX] web: pyjs: Tokenize float numbers in scientific notation

Steps to reproduce:

- Install Dashboard, Inventory apps
- Go to Inventory > Products
- Add a custom filter "Quantity on hand > 0.00001"
- Favorites > Add to my dashboard > Add
- Go to dashboard app

Issue:
JS Traceback is raised: `Expected "]", got "(name)"`

When executing the rpc call to `/board/add_to_dashboard`, the number
`0.00001`, which is passed as argument, is decoded as `1e-05` in the
python code and stored in the custom view.

When parsing the same view in the frontend, the tokenizer fails to
detect `1e-05` as a floating number and an error is thrown while parsing.

Solution:
Add regex for floating point numbers in scientific notation in `py.js`.
The regex accepts numbers such as:
- 1.2
- 12.
- .1
- 1e-02
- 1.2E-2
- 12.e+2
- .1E3

opw-3038707

closes odoo/odoo#111664

X-original-commit: d2d23e517fb81ab3779f6f0dbf0d035544c863d8
Signed-off-by: Samuel Degueldre <sad@odoo.com>
Signed-off-by: Stefan-Calin Crainiciuc (stcc) <stcc@odoo.com>

addons/web/static/lib/py.js/lib/py.js

@@ -263,8 +263,12 @@ var py = {};
 
         var DecNumber = '\\d+(L|l)?';
         var IntNumber = DecNumber;
-        var PointFloat = group('\\d+\\.\\d*', '\\.\\d+');
-        var FloatNumber = PointFloat;
+
+        var Exponent = '[eE][+-]?\\d+';
+        var PointFloat = group(`\\d+\\.\\d*(${Exponent})?`, `\\.\\d+(${Exponent})?`);
+        // Exponent not optional when no decimal point
+        var FloatNumber = group(PointFloat, `\\d+${Exponent}`);
+
         var Number = group(FloatNumber, IntNumber);
 
         var Operator = group("\\*\\*=?", ">>=?", "<<=?", "<>", "!=",

addons/web/static/src/core/py_js/py_tokenizer.js

@@ -210,8 +210,12 @@ const Name = "[a-zA-Z_]\\w*";
 const Whitespace = "[ \\f\\t]*";
 const DecNumber = "\\d+(L|l)?";
 const IntNumber = DecNumber;
-const PointFloat = group("\\d+\\.\\d*", "\\.\\d+");
-const FloatNumber = PointFloat;
+
+const Exponent = "[eE][+-]?\\d+";
+const PointFloat = group(`\\d+\\.\\d*(${Exponent})?`, `\\.\\d+(${Exponent})?`);
+// Exponent not optional when no decimal point
+const FloatNumber = group(PointFloat, `\\d+${Exponent}`);
+
 const Number = group(FloatNumber, IntNumber);
 const Operator = group("\\*\\*=?", ">>=?", "<<=?", "<>", "!=", "//=?", "[+\\-*/%&|^=<>]=?", "~");
 const Bracket = "[\\[\\]\\(\\)\\{\\}]";

addons/web/static/tests/core/py_js/py_tokenizer_tests.js

@@ -12,16 +12,51 @@ QUnit.module("py", {}, () => {
     });
 
     QUnit.test("can tokenize numbers", (assert) => {
+        /* Without exponent */
         assert.deepEqual(tokenize("1"), [{ type: 0 /* Number */, value: 1 }]);
         assert.deepEqual(tokenize("13"), [{ type: 0 /* Number */, value: 13 }]);
-        assert.deepEqual(tokenize("12.0"), [{ type: 0 /* Number */, value: 12 }]);
-        assert.deepEqual(tokenize("1.2"), [{ type: 0 /* Number */, value: 1.2 }]);
-        assert.deepEqual(tokenize("1.2"), [{ type: 0 /* Number */, value: 1.2 }]);
-        assert.deepEqual(tokenize(".42"), [{ type: 0 /* Number */, value: 0.42 }]);
         assert.deepEqual(tokenize("-1"), [
             { type: 2 /* Symbol */, value: "-" },
             { type: 0 /* Number */, value: 1 },
         ]);
+
+        /* With exponent */
+        assert.deepEqual(tokenize("1e2"), [{ type: 0 /* Number */, value: 100 }]);
+        assert.deepEqual(tokenize("13E+02"), [{ type: 0 /* Number */, value: 1300 }]);
+        assert.deepEqual(tokenize("15E-2"), [{ type: 0 /* Number */, value: 0.15 }]);
+        assert.deepEqual(tokenize("-30e+002"), [
+            { type: 2 /* Symbol */, value: "-" },
+            { type: 0 /* Number */, value: 3000 },
+        ]);
+    });
+
+    QUnit.test("can tokenize floats", (assert) => {
+        /* Without exponent */
+        assert.deepEqual(tokenize("12.0"), [{ type: 0 /* Number */, value: 12 }]);
+        assert.deepEqual(tokenize("1.2"), [{ type: 0 /* Number */, value: 1.2 }]);
+        assert.deepEqual(tokenize(".42"), [{ type: 0 /* Number */, value: 0.42 }]);
+        assert.deepEqual(tokenize("12."), [{type: 0 /* Number */, value: 12}]);
+        assert.deepEqual(tokenize("-1.23"), [
+            { type: 2 /* Symbol */, value: "-" },
+            { type: 0 /* Number */, value: 1.23 },
+        ]);
+
+        /* With exponent */
+        assert.deepEqual(tokenize("1234e-3"), [{type: 0 /* Number */, value: 1.234}]);
+        assert.deepEqual(tokenize("1.23E-03"), [{type: 0 /* Number */, value: 0.00123}]);
+        assert.deepEqual(tokenize('.23e-3'), [{type: 0 /* Number */, value: 0.00023}]);
+        assert.deepEqual(tokenize('23.e-03'), [{type: 0 /* Number */, value: 0.023}]);
+
+        assert.deepEqual(tokenize("12.1E2"), [{type: 0 /* Number */, value: 1210}]);
+        assert.deepEqual(tokenize("1.23e+03"), [{type: 0 /* Number */, value: 1230}]);
+        assert.deepEqual(tokenize('.23e2'), [{type: 0 /* Number */, value: 23}]);
+        assert.deepEqual(tokenize('15.E+02'), [{type: 0 /* Number */, value: 1500}]);
+
+        assert.deepEqual(tokenize("-23E02"), [
+            {type: 2 /* Symbol */, value: "-"},
+            {type: 0 /* Number */, value: 2300}
+        ]);
+
     });
 
     QUnit.test("can tokenize strings", (assert) => {