From d7c6033f99bdae6392e5ae96af32193e6fe27b06 Mon Sep 17 00:00:00 2001
From: dbkosky <dako@odoo.com>
Date: Thu, 14 Apr 2022 15:08:02 +0000
Subject: [PATCH] [FIX] l10n_it_edi: format_alphanumeric

The Italian edi system accepts utf-8 encoded documents, but actually the
contents of the fields have to be latin-1 encoded. Format-alphanumeric
should remove any non-latin1 characters and replace them with a ?

Several fields to are also truncated before the function is applied.
This is so that these fields better match the specification. This will
help to prevent edi rejections in the future.

A test is also included to test with a few lines that should / should
not be adapted in by the format_alphanumeric function. Along with the
addition of the test, the test partner italian_partner_a's is_company
field is changed to True, since the partner is a company. The expected
xml has been altered to match the changes to partner_a.

closes odoo/odoo#89494

Task-id: 2826424
X-original-commit: 712758fc2505abfa271636caaa8ba38436086f32
Signed-off-by: Josse Colpaert <jco@odoo.com>
---
 .../l10n_it_edi/data/invoice_it_template.xml  | 48 ++++++-------
 addons/l10n_it_edi/models/account_invoice.py  |  4 ++
 .../expected_xmls/IT00470550013_basis.xml     |  3 +-
 .../l10n_it_edi_sdicoop/tests/test_edi_xml.py | 70 +++++++++++++++++++
 4 files changed, 99 insertions(+), 26 deletions(-)

diff --git a/addons/l10n_it_edi/data/invoice_it_template.xml b/addons/l10n_it_edi/data/invoice_it_template.xml
index daf07495a4e..d028fb698d5 100644
--- a/addons/l10n_it_edi/data/invoice_it_template.xml
+++ b/addons/l10n_it_edi/data/invoice_it_template.xml
@@ -8,18 +8,18 @@
                 <CodiceArticolo t-if="line.product_id.barcode">
                     <!--2.2.1.3-->
                     <CodiceTipo>EAN</CodiceTipo>
-                    <CodiceValore t-esc="line.product_id.barcode"/>
+                    <CodiceValore t-esc="format_alphanumeric(line.product_id.barcode)"/>
                 </CodiceArticolo>
                  <CodiceArticolo t-if="line.product_id.default_code">
                     <CodiceTipo>INTERNAL</CodiceTipo>
-                    <CodiceValore t-esc="line.product_id.default_code"/>
+                    <CodiceValore t-esc="format_alphanumeric(line.product_id.default_code)"/>
                 </CodiceArticolo>
                 <Descrizione>
-                    <t t-esc="line.name[:1000]"/>
+                    <t t-esc="format_alphanumeric(line.name[:1000])"/>
                     <t t-if="not line.name" t-esc="'NO NAME'"/>
                 </Descrizione>
                 <Quantita t-esc="format_numbers(line.quantity)"/>
-                <UnitaMisura t-if="line.product_uom_id.category_id != env.ref('uom.product_uom_categ_unit')" t-esc="line.product_uom_id.name"/>
+                <UnitaMisura t-if="line.product_uom_id.category_id != env.ref('uom.product_uom_categ_unit')"  t-esc="format_alphanumeric(line.product_uom_id.name)"/>
                 <PrezzoUnitario t-esc="'%.06f' % (line.price_subtotal / (( 1 - (line.discount or 0.0) / 100.0) * line.quantity) if line.quantity and line.discount != 100.0 else line.price_unit)"/>
                 <ScontoMaggiorazione t-if="line.discount != 0">
                     <!-- [2.2.1.10] -->
@@ -36,9 +36,9 @@
 
         <template id="account_invoice_it_FatturaPA_sede">
             <Sede>
-                <Indirizzo><t t-if="partner.street" t-esc="partner.street"/> <t t-if="partner.street2" t-esc="partner.street2"/></Indirizzo>
+                <Indirizzo><t t-if="partner.street" t-esc="format_alphanumeric(partner.street)"/> <t t-if="partner.street2" t-esc="format_alphanumeric(partner.street2)"/></Indirizzo>
                 <CAP><t t-if="partner.country_id.code != 'IT'" t-esc="'00000'"/><t t-else="" t-esc="partner.zip"/></CAP>
-                <Comune t-esc="partner.city"/>
+                <Comune t-esc="format_alphanumeric(partner.city)"/>
                 <Provincia t-if="partner.country_id.code == 'IT'" t-esc="partner.state_id.code"/>
                 <Nazione t-esc="partner.country_id.code"/>
             </Sede>
@@ -54,14 +54,14 @@
                             <IdPaese t-esc="get_vat_country(record.company_id.vat)"/>
                             <IdCodice t-esc="record.company_id.l10n_it_codice_fiscale or get_vat_number(record.company_id.vat)"/>
                         </IdTrasmittente>
-                        <ProgressivoInvio t-esc="record.name.replace('/','')[-10:]"/>
+                        <ProgressivoInvio t-esc="format_alphanumeric(record.name.replace('/','')[-10:])"/>
                         <FormatoTrasmissione t-esc="formato_trasmissione"/>
                         <CodiceDestinatario t-if="record.commercial_partner_id.l10n_it_pa_index and record.commercial_partner_id.country_id.code == 'IT'" t-esc="record.commercial_partner_id.l10n_it_pa_index.upper()"/>
                         <CodiceDestinatario t-if="not record.commercial_partner_id.l10n_it_pa_index and record.commercial_partner_id.country_id.code == 'IT'" t-esc="'0000000'"/>
                         <CodiceDestinatario t-if="record.commercial_partner_id.country_id.code != 'IT'" t-esc="'XXXXXXX'"/>
                         <ContattiTrasmittente>
-                            <Telefono t-if="format_phone(record.company_id.partner_id.phone)" t-esc="format_phone(record.company_id.partner_id.phone)"/>
-                            <Telefono t-if="not format_phone(record.company_id.partner_id.phone) and format_phone(record.company_id.partner_id.mobile)" t-esc="format_phone(record.company_id.partner_id.mobile)"/>
+                            <Telefono t-if="format_phone(record.company_id.partner_id.phone)" t-esc="format_alphanumeric(format_phone(record.company_id.partner_id.phone))"/>
+                            <Telefono t-if="not format_phone(record.company_id.partner_id.phone) and format_phone(record.company_id.partner_id.mobile)" t-esc="format_alphanumeric(format_phone(record.company_id.partner_id.mobile))"/>
                             <Email t-if="record.company_id.email" t-esc="record.company_id.email"/>
                         </ContattiTrasmittente>
                         <PECDestinatario t-if="record.commercial_partner_id.l10n_it_pec_email" t-esc="record.commercial_partner_id.l10n_it_pec_email"/>
@@ -74,7 +74,7 @@
                             </IdFiscaleIVA>
                             <CodiceFiscale t-if="record.company_id.l10n_it_codice_fiscale" t-esc="record.company_id.l10n_it_codice_fiscale"/>
                             <Anagrafica>
-                                <Denominazione t-esc="record.company_id.partner_id.display_name"/>
+                                <Denominazione t-esc="format_alphanumeric(record.company_id.partner_id.display_name[:80])"/>
                             </Anagrafica>
                             <RegimeFiscale t-esc="record.company_id.l10n_it_tax_system"/>
                         </DatiAnagrafici>
@@ -84,7 +84,7 @@
                         <IscrizioneREA t-if="record.company_id.l10n_it_has_eco_index">
                             <!--1.2.4-->
                             <Ufficio t-esc="record.company_id.l10n_it_eco_index_office.code"/>
-                            <NumeroREA t-esc="record.company_id.l10n_it_eco_index_number"/>
+                            <NumeroREA t-esc="format_alphanumeric(record.company_id.l10n_it_eco_index_number)"/>
                             <CapitaleSociale t-if="record.company_id.l10n_it_eco_index_share_capital != 0" t-esc="format_numbers_two(record.company_id.l10n_it_eco_index_share_capital)"/>
                             <SocioUnico t-if="record.company_id.l10n_it_eco_index_sole_shareholder != 'NO'" t-esc="record.company_id.l10n_it_eco_index_sole_shareholder"/>
                             <StatoLiquidazione t-esc="record.company_id.l10n_it_eco_index_liquidation_state"/>
@@ -99,9 +99,9 @@
                             </IdFiscaleIVA>
                             <CodiceFiscale t-if="record.company_id.l10n_it_tax_representative_partner_id.l10n_it_codice_fiscale" t-esc="record.company_id.l10n_it_tax_representative_partner_id.l10n_it_codice_fiscale"/>
                             <Anagrafica>
-                                <Denominazione t-if="record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="record.company_id.l10n_it_tax_representative_partner_id.display_name"/>
-                                <Nome t-if="not record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="' '.join(record.company_id.l10n_it_tax_representative_partner_id.name.split()[:1])"/>
-                                <Cognome t-if="not record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="' '.join(record.company_id.l10n_it_tax_representative_partner_id.name.split()[1:])"/>
+                                <Denominazione t-if="record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="format_alphanumeric(record.company_id.l10n_it_tax_representative_partner_id.display_name[:80])"/>
+                                <Nome t-if="not record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="format_alphanumeric(' '.join(record.company_id.l10n_it_tax_representative_partner_id.name.split()[:1])[:60])"/>
+                                <Cognome t-if="not record.company_id.l10n_it_tax_representative_partner_id.is_company" t-esc="format_alphanumeric(' '.join(record.company_id.l10n_it_tax_representative_partner_id.name.split()[1:])[:60])"/>
                             </Anagrafica>
                         </DatiAnagrafici>
                     </RappresentanteFiscale>
@@ -122,9 +122,9 @@
                             <CodiceFiscale t-if="not record.commercial_partner_id.vat" t-esc="record.commercial_partner_id.l10n_it_codice_fiscale"/>
                             <CodiceFiscale t-if="not record.commercial_partner_id.vat and not record.commercial_partner_id.l10n_it_codice_fiscale" t-esc="99999999999"/>
                             <Anagrafica>
-                                <Denominazione t-if="record.commercial_partner_id.is_company" t-esc="record.commercial_partner_id.display_name"/>
-                                <Nome t-if="not record.commercial_partner_id.is_company" t-esc="' '.join(record.commercial_partner_id.name.split()[:1])"/>
-                                <Cognome t-if="not record.commercial_partner_id.is_company" t-esc="' '.join(record.commercial_partner_id.name.split()[1:])"/>
+                                <Denominazione t-if="record.commercial_partner_id.is_company" t-esc="format_alphanumeric(record.commercial_partner_id.display_name[:80])"/>
+                                <Nome t-if="not record.commercial_partner_id.is_company" t-esc="format_alphanumeric(' '.join(record.commercial_partner_id.name.split()[:1])[:60])"/>
+                                <Cognome t-if="not record.commercial_partner_id.is_company" t-esc="format_alphanumeric(' '.join(record.commercial_partner_id.name.split()[1:])[:60])"/>
                             </Anagrafica>
                         </DatiAnagrafici>
                         <t t-call="l10n_it_edi.account_invoice_it_FatturaPA_sede">
@@ -139,7 +139,7 @@
                             <TipoDocumento t-esc="document_type"/>
                             <Divisa t-esc="currency.name"/>
                             <Data t-esc="format_date(record.invoice_date)"/>
-                            <Numero t-esc="record.name[-20:]"/>
+                            <Numero t-esc="format_alphanumeric(record.name[-20:])"/>
                             <DatiBollo t-if="record.l10n_it_stamp_duty">
                                 <!--2.1.1.6-->
                                 <BolloVirtuale>SI</BolloVirtuale>
@@ -147,11 +147,11 @@
                             </DatiBollo>
                         </DatiGeneraliDocumento>
                         <DatiOrdineAcquisto t-if="record.ref">
-                            <IdDocumento t-esc="record.ref[:20]" />
+                            <IdDocumento t-esc="format_alphanumeric(record.ref[:20])"/>
                         </DatiOrdineAcquisto>
                         <DatiDDT t-if="record.l10n_it_ddt_id">
                             <!--2.1.8-->
-                            <NumeroDDT t-esc="record.l10n_it_ddt_id.name"/>
+                            <NumeroDDT t-esc="format_alphanumeric(record.l10n_it_ddt_id.name[-20:])"/>
                             <DataDDT t-esc="format_date(record.l10n_it_ddt_id.date)"/>
                         </DatiDDT>
                     </DatiGenerali>
@@ -172,7 +172,7 @@
                                 <ImponibileImporto t-esc="format_monetary(abs(tax_dict['base_amount']), currency)"/>
                                 <Imposta t-esc="format_monetary(abs(tax_dict['tax_amount']), currency)"/>
                                 <EsigibilitaIVA t-if="not tax_dict['tax'].l10n_it_has_exoneration or tax_dict['tax'].l10n_it_kind_exoneration=='N6'" t-esc="tax_dict['tax'].l10n_it_vat_due_date"/>
-                                <RiferimentoNormativo t-if="tax_dict['tax'].l10n_it_has_exoneration" t-esc="tax_dict['tax'].l10n_it_law_reference"/>
+                                <RiferimentoNormativo t-if="tax_dict['tax'].l10n_it_has_exoneration" t-esc="format_alphanumeric(tax_dict['tax'].l10n_it_law_reference[:100])"/>
                             </DatiRiepilogo>
                         </t>
                     </DatiBeniServizi>
@@ -185,15 +185,15 @@
                                 <ModalitaPagamento>MP05</ModalitaPagamento>
                                 <DataScadenzaPagamento t-esc="format_date(payment.date_maturity)"/>
                                 <ImportoPagamento t-esc="format_monetary(abs(payment.price_total), currency)"/>
-                                <IstitutoFinanziario t-if="company_bank_account.bank_id" t-esc="company_bank_account.bank_id.name[:80]"/>
+                                <IstitutoFinanziario t-if="company_bank_account.bank_id" t-esc="format_alphanumeric(company_bank_account.bank_id.name[:80])"/>
                                 <IBAN t-if="company_bank_account.acc_type == 'iban'" t-esc="company_bank_account.sanitized_acc_number"/>
                                 <BIC t-if="company_bank_account.acc_type == 'bank' and company_bank_account.bank_id.bic" t-esc="company_bank_account.bank_id.bic"/>
-                                <CodicePagamento t-esc="record.payment_reference[:60]"/>
+                                <CodicePagamento t-esc="format_alphanumeric(record.payment_reference[:60])"/>
                             </DettaglioPagamento>
                         </t>
                     </DatiPagamento>
                     <Allegati t-if="pdf">
-                        <NomeAttachment t-esc="pdf_name"/>
+                        <NomeAttachment t-esc="format_alphanumeric(pdf_name[:60])"/>
                         <FormatoAttachment>PDF</FormatoAttachment>
                         <Attachment t-esc="pdf"/>
                     </Allegati>
diff --git a/addons/l10n_it_edi/models/account_invoice.py b/addons/l10n_it_edi/models/account_invoice.py
index e5d40b4c1fa..6f9412a513a 100644
--- a/addons/l10n_it_edi/models/account_invoice.py
+++ b/addons/l10n_it_edi/models/account_invoice.py
@@ -142,6 +142,9 @@ class AccountMove(models.Model):
                 return True
             return False
 
+        def format_alphanumeric(text_to_convert):
+            return text_to_convert.encode('latin-1', 'replace').decode('latin-1') if text_to_convert else False
+
         formato_trasmissione = "FPA12" if self._is_commercial_partner_pa() else "FPR12"
 
         if self.move_type == 'out_invoice':
@@ -179,6 +182,7 @@ class AccountMove(models.Model):
             'format_numbers': format_numbers,
             'format_numbers_two': format_numbers_two,
             'format_phone': format_phone,
+            'format_alphanumeric': format_alphanumeric,
             'discount_type': discount_type,
             'get_vat_number': get_vat_number,
             'get_vat_country': get_vat_country,
diff --git a/addons/l10n_it_edi_sdicoop/tests/expected_xmls/IT00470550013_basis.xml b/addons/l10n_it_edi_sdicoop/tests/expected_xmls/IT00470550013_basis.xml
index 56c49b87938..7930605299e 100644
--- a/addons/l10n_it_edi_sdicoop/tests/expected_xmls/IT00470550013_basis.xml
+++ b/addons/l10n_it_edi_sdicoop/tests/expected_xmls/IT00470550013_basis.xml
@@ -37,8 +37,7 @@
           <IdCodice>00465840031</IdCodice>
         </IdFiscaleIVA>
         <Anagrafica>
-          <Nome>Alessi</Nome>
-          <Cognome></Cognome>
+          <Denominazione>Alessi</Denominazione>
         </Anagrafica>
       </DatiAnagrafici>
       <Sede>
diff --git a/addons/l10n_it_edi_sdicoop/tests/test_edi_xml.py b/addons/l10n_it_edi_sdicoop/tests/test_edi_xml.py
index 4aa2bb3ef15..f70f7ff57a0 100644
--- a/addons/l10n_it_edi_sdicoop/tests/test_edi_xml.py
+++ b/addons/l10n_it_edi_sdicoop/tests/test_edi_xml.py
@@ -55,6 +55,7 @@ class TestItEdi(AccountEdiTestCommon):
             'zip': '28887',
             'city': 'Milan',
             'company_id': cls.company.id,
+            'is_company': True,
         })
 
         cls.standard_line = {
@@ -124,6 +125,27 @@ class TestItEdi(AccountEdiTestCommon):
             ],
         })
 
+        cls.non_latin_and_latin_invoice = cls.env['account.move'].with_company(cls.company).create({
+            'move_type': 'out_invoice',
+            'invoice_date': datetime.date(2022, 3, 24),
+            'partner_id': cls.italian_partner_a.id,
+            'partner_bank_id': cls.test_bank.id,
+            'invoice_line_ids': [
+                (0, 0, {
+                    **cls.standard_line,
+                    'name': 'ʢ◉ᴥ◉ʡ',
+                    }),
+                (0, 0, {
+                    **cls.standard_line,
+                    'name': '–-',
+                    }),
+                (0, 0, {
+                    **cls.standard_line,
+                    'name': 'this should be the same as it was',
+                    }),
+                ],
+            })
+
         # We create this because we are unable to post without a proxy user existing
         cls.proxy_user = cls.env['account_edi_proxy_client.user'].create({
             'id_client': 'l10n_it_edi_sdicoop_test',
@@ -137,6 +159,7 @@ class TestItEdi(AccountEdiTestCommon):
         cls.price_included_invoice._post()
         cls.partial_discount_invoice._post()
         cls.full_discount_invoice._post()
+        cls.non_latin_and_latin_invoice._post()
 
         cls.edi_basis_xml = cls._get_test_file_content('IT00470550013_basis.xml')
 
@@ -316,3 +339,50 @@ class TestItEdi(AccountEdiTestCommon):
             ''')
         invoice_etree = self.with_applied_xpath(invoice_etree, "<xpath expr='.//Allegati' position='replace'/>")
         self.assertXmlTreeEqual(invoice_etree, expected_etree)
+
+    @freeze_time('2020-03-24')
+    def test_non_latin_and_latin_inovice(self):
+        invoice_etree = etree.fromstring(self.non_latin_and_latin_invoice._export_as_xml())
+        expected_etree = self.with_applied_xpath(
+            etree.fromstring(self.edi_basis_xml),
+            '''
+            <xpath expr="//FatturaElettronicaBody//DatiBeniServizi" position="replace">
+            <DatiBeniServizi>
+              <DettaglioLinee>
+                <NumeroLinea>1</NumeroLinea>
+                <Descrizione>?????</Descrizione>
+                <Quantita>1.00</Quantita>
+                <PrezzoUnitario>800.400000</PrezzoUnitario>
+                <PrezzoTotale>800.40</PrezzoTotale>
+                <AliquotaIVA>22.00</AliquotaIVA>
+              </DettaglioLinee>
+              <DettaglioLinee>
+                <NumeroLinea>2</NumeroLinea>
+                <Descrizione>?-</Descrizione>
+                <Quantita>1.00</Quantita>
+                <PrezzoUnitario>800.400000</PrezzoUnitario>
+                <PrezzoTotale>800.40</PrezzoTotale>
+                <AliquotaIVA>22.00</AliquotaIVA>
+              </DettaglioLinee>
+              <DettaglioLinee>
+                <NumeroLinea>3</NumeroLinea>
+                <Descrizione>this should be the same as it was</Descrizione>
+                <Quantita>1.00</Quantita>
+                <PrezzoUnitario>800.400000</PrezzoUnitario>
+                <PrezzoTotale>800.40</PrezzoTotale>
+                <AliquotaIVA>22.00</AliquotaIVA>
+              </DettaglioLinee>
+              <DatiRiepilogo>
+                <AliquotaIVA>22.00</AliquotaIVA>
+                <ImponibileImporto>2401.20</ImponibileImporto>
+                <Imposta>528.27</Imposta>
+                <EsigibilitaIVA>I</EsigibilitaIVA>
+              </DatiRiepilogo>
+            </DatiBeniServizi>
+            </xpath>
+            <xpath expr="//DettaglioPagamento//ImportoPagamento" position="inside">
+              2929.47
+            </xpath>
+            ''')
+        invoice_etree = self.with_applied_xpath(invoice_etree, "<xpath expr='.//Allegati' position='replace'/>")
+        self.assertXmlTreeEqual(invoice_etree, expected_etree)