odoo_source/addons/base_import/tests/test_csv_magic.py

# -*- coding: utf-8 -*-
"""
Tests for various autodetection magics for CSV imports
"""
import codecs

from odoo.tests import common

class ImportCase(common.TransactionCase):
    def _make_import(self, contents):
        return self.env['base_import.import'].create({
            'res_model': 'base_import.tests.models.complex',
            'file_name': 'f',
            'file_type': 'text/csv',
            'file': contents,
        })


class TestEncoding(ImportCase):
    """
    create + parse_preview -> check result options
    """

    def _check_text(self, text, encodings, **options):
        options.setdefault('quoting', '"')
        options.setdefault('separator', '\t')
        test_text = "text\tnumber\tdate\tdatetime\n%s\t1.23.45,67\t\t\n" % text
        for encoding in ['utf-8', 'utf-16', 'utf-32', *encodings]:
            if isinstance(encoding, tuple):
                encoding, es = encoding
            else:
                es = [encoding]
            preview = self._make_import(
                test_text.encode(encoding)).parse_preview(dict(options))

            self.assertIsNone(preview.get('error'))
            guessed = preview['options']['encoding']
            self.assertIsNotNone(guessed)
            self.assertIn(
                codecs.lookup(guessed).name, [
                    codecs.lookup(e).name
                    for e in es
                ]
            )

    def test_autodetect_encoding(self):
        """ Check that import preview can detect & return encoding
        """
        self._check_text("Iñtërnâtiônàlizætiøn", [('iso-8859-1', ['iso-8859-1', 'iso-8859-2'])])

        self._check_text("やぶら小路の藪柑子。海砂利水魚の、食う寝る処に住む処、パイポパイポ パイポのシューリンガン。", ['eucjp', 'shift_jis', 'iso2022_jp'])

        self._check_text("대통령은 제4항과 제5항의 규정에 의하여 확정된 법률을 지체없이 공포하여야 한다, 탄핵의 결정.", ['euc_kr', 'iso2022_kr'])

    # + control in widget
    def test_override_detection(self):
        """ ensure an explicitly specified encoding is not overridden by the
        auto-detection
        """
        s = "Iñtërnâtiônàlizætiøn".encode('utf-8')
        r = self._make_import(b'text\n' + s)\
            .parse_preview({
            'quoting': '"',
            'separator': '\t',
            'encoding': 'iso-8859-1',
        })
        self.assertIsNone(r.get('error'))
        self.assertEqual(r['options']['encoding'], 'iso-8859-1')
        self.assertEqual(r['preview'], [['text'], [s.decode('iso-8859-1')]])

class TestFileSeparator(ImportCase):

    def setUp(self):
        super().setUp()
        self.imp = self._make_import(
"""c|f
a|1
b|2
c|3
d|4
""")

    def test_explicit_success(self):
        r = self.imp.parse_preview({
            'separator': '|',
            'headers': True,
            'quoting': '"',
        })
        self.assertIsNone(r.get('error'))
        self.assertEqual(r['headers'], ['c', 'f'])
        self.assertEqual(r['preview'], [
            ['a', '1'],
            ['b', '2'],
            ['c', '3'],
            ['d', '4'],
        ])
        self.assertEqual(r['options']['separator'], '|')

    def test_explicit_fail(self):
        """ Don't protect user against making mistakes
        """
        r = self.imp.parse_preview({
            'separator': ',',
            'headers': True,
            'quoting': '"',
        })
        self.assertIsNone(r.get('error'))
        self.assertEqual(r['headers'], ['c|f'])
        self.assertEqual(r['preview'], [
            ['a|1'],
            ['b|2'],
            ['c|3'],
            ['d|4'],
        ])
        self.assertEqual(r['options']['separator'], ',')

    def test_guess_ok(self):
        r = self.imp.parse_preview({
            'separator': '',
            'headers': True,
            'quoting': '"',
        })
        self.assertIsNone(r.get('error'))
        self.assertEqual(r['headers'], ['c', 'f'])
        self.assertEqual(r['preview'], [
            ['a', '1'],
            ['b', '2'],
            ['c', '3'],
            ['d', '4'],
        ])
        self.assertEqual(r['options']['separator'], '|')

    def test_noguess(self):
        """ If the guesser has no idea what the separator is, it defaults to
        "," but should not set that value
        """
        imp = self._make_import('c\na\nb\nc\nd')
        r = imp.parse_preview({
            'separator': '',
            'headers': True,
            'quoting': '"',
        })
        self.assertIsNone(r.get('error'))
        self.assertEqual(r['headers'], ['c'])
        self.assertEqual(r['preview'], [
            ['a'],
            ['b'],
            ['c'],
            ['d'],
        ])
        self.assertEqual(r['options']['separator'], '')

class TestNumberSeparators(common.TransactionCase):
    def test_parse_float(self):
        w = self.env['base_import.import'].create({
            'res_model': 'base_import.tests.models.float',
        })
        data = w._parse_import_data(
            [
                ['1.62'], ['-1.62'], ['+1.62'], ['  +1.62  '], ['(1.62)'],
                ["1'234'567,89"], ["1.234.567'89"]
            ],
            ['value'], {}
        )
        self.assertEqual(
            [d[0] for d in data],
            ['1.62', '-1.62', '+1.62', '+1.62', '-1.62',
             '1234567.89', '1234567.89']
        )