From 01715de04a5b4f92dab3d09cc4240dd4094a2ebf Mon Sep 17 00:00:00 2001
From: luvi <luvi@odoo.com>
Date: Wed, 5 Apr 2023 06:30:35 +0000
Subject: [PATCH] [IMP] base_import: improve date(time) formatting options

This commit improves the suggestions displayed to the user when editing
the date or datetime options in the sidebar of the Import action.
A tooltip is also displayed next to each label, to let the user know how
to format properly the date if he wants to write a custom one to match
his own needs.

Since commit [1], a legacy utils was imported to convert a moment format
to strftime. Since this file was legacy, it was needed to remove its import
from the import action and replace it with a newer utility function. Because
this function was not imported elsewhere, it could be removed completely.

In dates.js, a new function allow a user to write a date format in a
human readable format, while converting it to a valid Python strftime
format, to send back to the server the right date.

A test has been added for this improvement.

[1]: 2361f021c38550d9c488e073801a004ad7ccbffc

task-3248787

closes odoo/odoo#117750

Signed-off-by: Aaron Bohy (aab) <aab@odoo.com>
---
 .../import_data_sidepanel.xml                 |   7 ++
 addons/base_import/static/src/import_model.js | 113 ++++++++++++++----
 .../static/tests/import_action_tests.js       | 108 ++++++++++++-----
 addons/web/static/src/legacy/js/core/time.js  |  14 ---
 4 files changed, 176 insertions(+), 66 deletions(-)

diff --git a/addons/base_import/static/src/import_data_sidepanel/import_data_sidepanel.xml b/addons/base_import/static/src/import_data_sidepanel/import_data_sidepanel.xml
index addd92a4df2..de0337526a0 100644
--- a/addons/base_import/static/src/import_data_sidepanel/import_data_sidepanel.xml
+++ b/addons/base_import/static/src/import_data_sidepanel/import_data_sidepanel.xml
@@ -32,14 +32,21 @@
                 <div t-foreach="Object.entries(props.formattingOptions)" t-as="option" t-key="`${option[0]}-${option_index}`" class="o_import_options">
                     <label t-att-for="`${option[0]}-${option_index}`">
                         <t t-esc="option[1].label" />
+                        <sup t-if="option[1].help" class="text-info p-1" t-att-data-tooltip="option[1].help" t-esc="'?'" />
                     </label>
                     <input
                         t-if="option[1].type === 'input'"
                         t-att-id="`${option[0]}-${option_index}`"
+                        t-att-list="option[1].options ? `list-${ option_index }` : null"
                         t-attf-class="o_import_#{option[0]} o_import_dropdown w-100 mb-3"
                         t-att-value="option[1].value"
                         t-on-change="(ev) => this.setOptionValue(option[0], ev.target.value)"
                     />
+                    <datalist t-if="option[1].type === 'input' and option[1].options" t-att-id="`list-${ option_index }`">
+                        <option t-foreach="option[1].options" t-as="opt" t-key="opt_index" t-att-value="opt.value or opt" t-att-selected="opt.value ? opt.value === option[1].value : opt === option[1].value">
+                            <t t-esc="opt.label ? opt.label : opt" />
+                        </option>
+                    </datalist>
                     <select
                         t-elif="option[1].type === 'select'"
                         t-att-class="`o_select form-select w-100 mb-3 o_import_${option[0]} o_import_dropdown`"
diff --git a/addons/base_import/static/src/import_model.js b/addons/base_import/static/src/import_model.js
index ae3fbdd4ccc..1bfbfccde45 100644
--- a/addons/base_import/static/src/import_model.js
+++ b/addons/base_import/static/src/import_model.js
@@ -4,13 +4,61 @@ import { _t } from "@web/core/l10n/translation";
 import { registry } from "@web/core/registry";
 import { pick } from "@web/core/utils/objects";
 import { groupBy, sortBy } from "@web/core/utils/arrays";
+import { memoize } from "@web/core/utils/functions";
 import { sprintf } from "@web/core/utils/strings";
 import { useState } from "@odoo/owl";
 import { ImportBlockUI } from "./import_block_ui";
-import { moment_to_strftime_format } from "web.time";
 
 const mainComponentRegistry = registry.category("main_components");
 
+const strftimeFormatTable = {
+    d: "w",
+    DD: "d",
+    ddd: "a",
+    dddd: "A",
+    DDDD: "j",
+    ww: "U",
+    WW: "W",
+    mm: "M",
+    MM: "m",
+    MMM: "b",
+    MMMM: "B",
+    YYYY: "Y",
+    YY: "y",
+    ss: "S",
+    hh: "h",
+    HH: "H",
+    A: "p",
+};
+
+/**
+ * Convert a human readable format to Python strftime format. In case
+ * no corresponding format is supported, a similar fallback is given
+ * from the list of other supported formatting value.
+ *
+ * @param {string} value original Luxon format
+ * @returns {string} valid strftime format
+ */
+const humanToStrftimeFormat = memoize(function humanToStrftimeFormat(value) {
+    const regex = /(dddd|ddd|dd|d|mmmm|mmm|mm|ww|yyyy|yy|hh|ss|a)/gi;
+    return value.replace(regex, (value) => {
+        if (strftimeFormatTable[value]) {
+            return "%" + strftimeFormatTable[value];
+        }
+        return (
+            "%" +
+            (strftimeFormatTable[value.toLowerCase()] || strftimeFormatTable[value.toUpperCase()])
+        );
+    });
+});
+
+const strftimeToHumanFormat = memoize(function strftimeToHumanFormat(value) {
+    Object.entries(strftimeFormatTable).forEach(([k, v]) => {
+        value = value.replace(`%${v}`, k);
+    });
+    return value;
+});
+
 /**
  * -------------------------------------------------------------------------
  * Base Import Business Logic
@@ -90,6 +138,17 @@ export class BaseImportModel {
         return pick(this.importOptionsValues, ...Object.keys(this.formattingOptionsValues));
     }
 
+    /**
+     * This getter returns the current values pf the options, formatted to match the
+     * server API (date and datetime options should be Python strftime formatted)
+     */
+    get formattedImportOptions() {
+        const options = this.importOptions;
+        options.date_format = humanToStrftimeFormat(options.date_format);
+        options.datetime_format = humanToStrftimeFormat(options.datetime_format);
+        return options;
+    }
+
     get importOptions() {
         const tempImportOptions = {
             import_skip_records: [],
@@ -98,9 +157,7 @@ export class BaseImportModel {
             name_create_enabled_fields: {},
         };
         for (const [name, option] of Object.entries(this.importOptionsValues)) {
-            tempImportOptions[name] = ['date_format', 'datetime_format'].includes(name) && !option.value.includes('%')
-                ? moment_to_strftime_format(option.value)
-                : option.value;
+            tempImportOptions[name] = option.value;
         }
 
         for (const key in this.fieldsToHandle) {
@@ -252,15 +309,16 @@ export class BaseImportModel {
 
         const res = await this.orm.call("base_import.import", "parse_preview", [
             this.id,
-            this.importOptions,
+            this.formattedImportOptions,
         ]);
 
         if (!res.error) {
+            res.options.date_format = strftimeToHumanFormat(res.options.date_format);
+            res.options.datetime_format = strftimeToHumanFormat(res.options.datetime_format);
             this._onLoadSuccess(res);
         } else {
             this._onLoadError();
         }
-
         return { res, error: res.error };
     }
 
@@ -309,7 +367,12 @@ export class BaseImportModel {
     }
 
     async _executeImportStep(isTest, importRes) {
-        const importArgs = [this.id, importRes.fields, importRes.columns, this.importOptions];
+        const importArgs = [
+            this.id,
+            importRes.fields,
+            importRes.columns,
+            this.formattedImportOptions,
+        ];
         const { ids, messages, nextrow, name, error } = await this._callImport(isTest, importArgs);
 
         // Handle server errors
@@ -662,38 +725,38 @@ export class BaseImportModel {
                 label: _t("Text Delimiter:"),
                 type: "input",
                 value: '"',
-                options: "",
             },
             date_format: {
+                help: _t(
+                    "Use YYYY to represent the year, MM for the month and DD for the day. Include separators such as a dot, forward slash or dash. You can use a custom format in addition to the suggestions provided. Leave empty to let Odoo guess the format (recommended)"
+                ),
                 label: _t("Date Format:"),
-                type: "select",
+                type: "input",
                 value: "",
                 options: [
                     "YYYY-MM-DD",
-                    "DD/MM/YY",
-                    "DD/MM/YYYY",
-                    "DD-MM-YYYY",
-                    "DD-MMM-YY",
-                    "DD-MMM-YYYY",
-                    "MM/DD/YY",
-                    "MM/DD/YYYY",
-                    "MM-DD-YY",
-                    "MM-DD-YYYY",
-                    "DDMMYY",
-                    "DDMMYYYY",
-                    "YYMMDD",
-                    "YYYYMMDD",
-                    "YY/MM/DD",
                     "YYYY/MM/DD",
-                    "MMDDYY",
+                    "DD/MM/YYYY",
+                    "DDMMYYYY",
+                    "MM/DD/YYYY",
                     "MMDDYYYY",
                 ],
             },
             datetime_format: {
+                help: _t(
+                    "Use HH for hours in a 24h system, use II in conjonction with 'p' for a 12h system. You can use a custom format in addition to the suggestions provided. Leave empty to let Odoo guess the format (recommended)"
+                ),
                 label: _t("Datetime Format:"),
                 type: "input",
                 value: "",
-                options: "",
+                options: [
+                    "YYYY-MM-DD HH:mm:SS",
+                    "YYYY/MM/DD HH:mm:SS",
+                    "DD/MM/YYYY HH:mm:SS",
+                    "DDMMYYYY HH:mm:SS",
+                    "MM/DD/YYYY II:mm:SS p",
+                    "MMDDYYYY II:mm:SS p",
+                ],
             },
             float_thousand_separator: {
                 label: _t("Thousands Separator:"),
diff --git a/addons/base_import/static/tests/import_action_tests.js b/addons/base_import/static/tests/import_action_tests.js
index 4d73482348f..aa6ace453ff 100644
--- a/addons/base_import/static/tests/import_action_tests.js
+++ b/addons/base_import/static/tests/import_action_tests.js
@@ -1358,37 +1358,28 @@ QUnit.module("Base Import Tests", (hooks) => {
     });
 
     QUnit.test("Import view: date format should be converted to strftime", async function (assert) {
+        assert.expect(5);
         registerFakeHTTPService();
+        let parseCount = 0;
         await createImportAction({
             "base_import.import/parse_preview": async (route, args) => {
-                return parsePreview(args[1]);
+                parseCount++;
+                const response = await parsePreview(args[1]);
+                if (parseCount === 2) {
+                    assert.strictEqual(
+                        response.options.date_format,
+                        "%Y%m%d",
+                        "server sends back a strftime formatted date"
+                    );
+                }
+                return response;
             },
             "base_import.import/execute_import": (route, args) => {
                 assert.step("execute_import");
-                assert.deepEqual(
-                    args[3],
-                    {
-                        advanced: true,
-                        date_format: "%Y%m%d",
-                        datetime_format: "",
-                        encoding: "",
-                        fallback_values: {},
-                        float_decimal_separator: ".",
-                        float_thousand_separator: ",",
-                        has_headers: true,
-                        import_set_empty_fields: [],
-                        import_skip_records: [],
-                        keep_matches: false,
-                        limit: 2000,
-                        name_create_enabled_fields: {},
-                        quoting: '"',
-                        separator: "",
-                        sheet: "Template",
-                        sheets: ["Template", "Template 2"],
-                        skip: 0,
-                        tracking_disable: true,
-                    },
-                    "options are defaulted as expected"
+                assert.strictEqual(
+                    args[3].date_format,
+                    "%Y%m%d",
+                    "date is converted to strftime as expected during the import"
                 );
                 return executeImport(args);
             },
@@ -1397,9 +1388,18 @@ QUnit.module("Base Import Tests", (hooks) => {
         // Set and trigger the change of a file for the input
         const file = new File(["fake_file"], "fake_file.csv", { type: "text/plain" });
         await editInput(target, "input[type='file']", file);
-        await editSelect(target, ".o_import_date_format[name='date_format-3']", "YYYYMMDD");
-        await click(target.querySelector(".o_control_panel button:first-child"));
+        await editInput(target, ".o_import_date_format#date_format-3", "YYYYMMDD");
+
+        // Parse the file again with the updated date format to check that
+        // the format is correctly formatted in the UI
+        await click(target.querySelector(".o_import_formatting button"));
+        await click(target.querySelector(".o_control_panel button:nth-child(2)"));
         assert.verifySteps(["execute_import"]);
+        assert.strictEqual(
+            target.querySelector(".o_import_date_format").value,
+            "YYYYMMDD",
+            "UI displays the human formatted date"
+        );
     });
 
     QUnit.test("Import action: field selection has a clear button", async function (assert) {
@@ -1428,4 +1428,58 @@ QUnit.module("Base Import Tests", (hooks) => {
             "field has been unselected"
         );
     });
+
+    QUnit.test(
+        "Import a CSV: formatting options for date and datetime options",
+        async function (assert) {
+            registerFakeHTTPService((route, params) => {
+                assert.strictEqual(route, "/base_import/set_file");
+                assert.strictEqual(
+                    params.ufile[0].name,
+                    "fake_file.csv",
+                    "file is correctly uploaded to the server"
+                );
+            });
+            await createImportAction();
+
+            // Set and trigger the change of a file for the input
+            const file = new File(["fake_file"], "fake_file.csv", { type: "text/plain" });
+            await editInput(target, "input[type='file']", file);
+            assert.strictEqual(
+                target.querySelector(".o_import_date_format").list.id,
+                "list-3",
+                "a datalist is associated to the date input"
+            );
+            assert.containsOnce(
+                target.querySelector(".o_import_date_format").previousElementSibling,
+                "sup",
+                "a tooltip is displayed on the label of the option"
+            );
+            assert.strictEqual(
+                target.querySelector(".o_import_date_format").list.options[0].value,
+                "YYYY-MM-DD",
+                "an option for datetime has the right value"
+            );
+            assert.strictEqual(
+                target.querySelector(".o_import_datetime_format").list.id,
+                "list-4",
+                "a datalist is associated to the datetime input"
+            );
+            assert.containsOnce(
+                target.querySelector(".o_import_datetime_format").previousElementSibling,
+                "sup",
+                "a tooltip is displayed on the label of the option"
+            );
+            assert.strictEqual(
+                target.querySelector(".o_import_datetime_format").list.options[0].value,
+                "YYYY-MM-DD HH:mm:SS",
+                "an option for datetime has the right value"
+            );
+            assert.containsOnce(
+                target,
+                ".o_import_action .o_import_data_content",
+                "content panel is visible"
+            );
+        }
+    );
 });
diff --git a/addons/web/static/src/legacy/js/core/time.js b/addons/web/static/src/legacy/js/core/time.js
index 1234681ba6e..641608fc0db 100644
--- a/addons/web/static/src/legacy/js/core/time.js
+++ b/addons/web/static/src/legacy/js/core/time.js
@@ -238,18 +238,6 @@ function strftime_to_moment_format (value) {
     return _normalize_format_cache[value];
 }
 
-/**
- * Convert moment.js format to python strftime
- *
- * @param {String} value original format
- */
-function moment_to_strftime_format(value) {
-    var regex = /(MMMM|DDDD|dddd|YYYY|MMM|ddd|mm|ss|ww|WW|MM|YY|hh|HH|DD|A|d)/g;
-    return value.replace(regex, function(val){
-        return '%'+inverse_normalize_format_table[val];
-    });
-}
-
 var _normalize_format_cache = {};
 var normalize_format_table = {
     // Python strftime to moment.js conversion table
@@ -277,7 +265,6 @@ var normalize_format_table = {
     'x': 'MM/DD/YY',
     'X': 'HH:mm:ss'
 };
-var inverse_normalize_format_table = _.invert(normalize_format_table);
 
 /**
  * Get date format of the user's language
@@ -340,7 +327,6 @@ export default {
     auto_str_to_date: auto_str_to_date,
     auto_date_to_str: auto_date_to_str,
     strftime_to_moment_format: strftime_to_moment_format,
-    moment_to_strftime_format: moment_to_strftime_format,
     getLangDateFormat: getLangDateFormat,
     getLangTimeFormat: getLangTimeFormat,
     getLangDateFormatWoZero: getLangDateFormatWoZero,