diff --git a/addons/attachment_indexation/models/ir_attachment.py b/addons/attachment_indexation/models/ir_attachment.py index dcc662a3c5d..11152b6c2ea 100644 --- a/addons/attachment_indexation/models/ir_attachment.py +++ b/addons/attachment_indexation/models/ir_attachment.py @@ -2,14 +2,16 @@ # Part of Odoo. See LICENSE file for full copyright and licensing details. import io import logging -import PyPDF2 import xml.dom.minidom import zipfile +from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter +from pdfminer.converter import TextConverter +from pdfminer.pdfpage import PDFPage from odoo import api, models _logger = logging.getLogger(__name__) -FTYPES = ['docx', 'pptx', 'xlsx', 'opendoc'] +FTYPES = ['docx', 'pptx', 'xlsx', 'opendoc', 'pdf'] def textToString(element): buff = u"" @@ -91,17 +93,19 @@ class IrAttachment(models.Model): def _index_pdf(self, bin_data): '''Index PDF documents''' - - # extractText gives very bad results for indexing, hence we don't index PDF anymore. A - # better alternative is probably PDFMiner.six, but not for stable. - # See POC at https://github.com/odoo/odoo/pull/27568. buf = u"" if bin_data.startswith(b'%PDF-'): f = io.BytesIO(bin_data) try: - pdf = PyPDF2.PdfFileReader(f, overwriteWarnings=False) - for page in pdf.pages: - buf += page.extractText() + resource_manager = PDFResourceManager() + with io.StringIO() as content, TextConverter(resource_manager, content) as device: + logging.getLogger("pdfminer").setLevel(logging.CRITICAL) + interpreter = PDFPageInterpreter(resource_manager, device) + + for page in PDFPage.get_pages(f): + interpreter.process_page(page) + + buf = content.getvalue() except Exception: pass return buf diff --git a/addons/attachment_indexation/tests/__init__.py b/addons/attachment_indexation/tests/__init__.py new file mode 100644 index 00000000000..d76ba91978d --- /dev/null +++ b/addons/attachment_indexation/tests/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import test_indexation diff --git a/addons/attachment_indexation/tests/files/test_content.pdf b/addons/attachment_indexation/tests/files/test_content.pdf new file mode 100644 index 00000000000..062e1e6e2f6 Binary files /dev/null and b/addons/attachment_indexation/tests/files/test_content.pdf differ diff --git a/addons/attachment_indexation/tests/test_indexation.py b/addons/attachment_indexation/tests/test_indexation.py new file mode 100644 index 00000000000..9f5fa0eca15 --- /dev/null +++ b/addons/attachment_indexation/tests/test_indexation.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from odoo.tests.common import TransactionCase, tagged +import os + +directory = os.path.dirname(__file__) + + +@tagged('post_install', '-at_install') +class TestCaseIndexation(TransactionCase): + + def test_attachment_pdf_indexation(self): + with open(os.path.join(directory, 'files', 'test_content.pdf'), 'rb') as file: + pdf = file.read() + text = self.env['ir.attachment']._index(pdf, 'application/pdf') + self.assertEqual(text, 'TestContent!!\x0c', 'the index content should be correct') diff --git a/requirements.txt b/requirements.txt index f6057683875..2507a3d10fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ mock==2.0.0 num2words==0.5.6 ofxparse==0.19 passlib==1.7.1 +pdfminer.six==20181108 Pillow==5.4.1 ; python_version < '3.7' or sys_platform != 'win32' Pillow==6.1.0 ; sys_platform == 'win32' and python_version >= '3.7' polib==1.1.0