diff --git a/openerp/addons/base/ir/ir_cron.py b/openerp/addons/base/ir/ir_cron.py index 790b4a43de0..4bf99cf9934 100644 --- a/openerp/addons/base/ir/ir_cron.py +++ b/openerp/addons/base/ir/ir_cron.py @@ -136,12 +136,13 @@ class ir_cron(osv.osv): except Exception, e: self._handle_callback_exception(cr, uid, model_name, method_name, args, job_id, e) - def _process_job(self, cr, job): + def _process_job(self, job_cr, job, cron_cr): """ Run a given job taking care of the repetition. - The cursor has a lock on the job (aquired by _acquire_job()). - + :param job_cr: cursor to use to execute the job, safe to commit/rollback :param job: job to be run (as a dictionary). + :param cron_cr: cursor holding lock on the cron job row, to use to update the next exec date, + must not be committed/rolled back! """ try: now = datetime.now() @@ -153,19 +154,19 @@ class ir_cron(osv.osv): if numbercall > 0: numbercall -= 1 if not ok or job['doall']: - self._callback(cr, job['user_id'], job['model'], job['function'], job['args'], job['id']) + self._callback(job_cr, job['user_id'], job['model'], job['function'], job['args'], job['id']) if numbercall: nextcall += _intervalTypes[job['interval_type']](job['interval_number']) ok = True addsql = '' if not numbercall: addsql = ', active=False' - cr.execute("UPDATE ir_cron SET nextcall=%s, numbercall=%s"+addsql+" WHERE id=%s", + cron_cr.execute("UPDATE ir_cron SET nextcall=%s, numbercall=%s"+addsql+" WHERE id=%s", (nextcall.strftime(DEFAULT_SERVER_DATETIME_FORMAT), numbercall, job['id'])) finally: - cr.commit() - cr.close() + job_cr.commit() + cron_cr.commit() @classmethod def _acquire_job(cls, db_name): @@ -181,44 +182,14 @@ class ir_cron(osv.osv): """ db = openerp.sql_db.db_connect(db_name) cr = db.cursor() + jobs = [] try: # Careful to compare timestamps with 'UTC' - everything is UTC as of v6.1. cr.execute("""SELECT * FROM ir_cron WHERE numbercall != 0 AND active AND nextcall <= (now() at time zone 'UTC') ORDER BY priority""") - for job in cr.dictfetchall(): - task_cr = db.cursor() - try: - # Try to grab an exclusive lock on the job row from within the task transaction - acquired_lock = False - task_cr.execute("""SELECT * - FROM ir_cron - WHERE id=%s - FOR UPDATE NOWAIT""", - (job['id'],), log_exceptions=False) - acquired_lock = True - except psycopg2.OperationalError, e: - if e.pgcode == '55P03': - # Class 55: Object not in prerequisite state; 55P03: lock_not_available - _logger.debug('Another process/thread is already busy executing job `%s`, skipping it.', job['name']) - continue - else: - # Unexpected OperationalError - raise - finally: - if not acquired_lock: - # we're exiting due to an exception while acquiring the lot - task_cr.close() - - # Got the lock on the job row, run its code - _logger.debug('Starting job `%s`.', job['name']) - openerp.modules.registry.RegistryManager.check_registry_signaling(db_name) - registry = openerp.pooler.get_pool(db_name) - registry[cls._name]._process_job(task_cr, job) - openerp.modules.registry.RegistryManager.signal_caches_change(db_name) - return True - + jobs = cr.dictfetchall() except psycopg2.ProgrammingError, e: if e.pgcode == '42P01': # Class 42 — Syntax Error or Access Rule Violation; 42P01: undefined_table @@ -228,12 +199,43 @@ class ir_cron(osv.osv): raise except Exception: _logger.warning('Exception in cron:', exc_info=True) - finally: - cr.commit() cr.close() - return False + for job in jobs: + lock_cr = db.cursor() + try: + # Try to grab an exclusive lock on the job row from within the task transaction + lock_cr.execute("""SELECT * + FROM ir_cron + WHERE id=%s + FOR UPDATE NOWAIT""", + (job['id'],), log_exceptions=False) + + # Got the lock on the job row, run its code + _logger.debug('Starting job `%s`.', job['name']) + job_cr = db.cursor() + try: + openerp.modules.registry.RegistryManager.check_registry_signaling(db_name) + registry = openerp.pooler.get_pool(db_name) + registry[cls._name]._process_job(job_cr, job, lock_cr) + openerp.modules.registry.RegistryManager.signal_caches_change(db_name) + except Exception: + _logger.exception('Unexpected exception while processing cron job %r', job) + finally: + job_cr.close() + + except psycopg2.OperationalError, e: + if e.pgcode == '55P03': + # Class 55: Object not in prerequisite state; 55P03: lock_not_available + _logger.debug('Another process/thread is already busy executing job `%s`, skipping it.', job['name']) + continue + else: + # Unexpected OperationalError + raise + finally: + # we're exiting due to an exception while acquiring the lock + lock_cr.close() def _try_lock(self, cr, uid, ids, context=None): """Try to grab a dummy exclusive write-lock to the rows with the given ids, diff --git a/openerp/addons/base/ir/ir_model.py b/openerp/addons/base/ir/ir_model.py index ff43a15692f..ed6f843f898 100644 --- a/openerp/addons/base/ir/ir_model.py +++ b/openerp/addons/base/ir/ir_model.py @@ -199,7 +199,7 @@ class ir_model(osv.osv): def instanciate(self, cr, user, model, context=None): class x_custom_model(osv.osv): - pass + _custom = True x_custom_model._name = model x_custom_model._module = False a = x_custom_model.create_instance(self.pool, cr) diff --git a/openerp/addons/base/res/res_partner_view.xml b/openerp/addons/base/res/res_partner_view.xml index d1a803ce2d8..c9215763c3e 100644 --- a/openerp/addons/base/res/res_partner_view.xml +++ b/openerp/addons/base/res/res_partner_view.xml @@ -177,7 +177,6 @@ diff --git a/openerp/netsvc.py b/openerp/netsvc.py index 37de4dd5b8b..533b1659191 100644 --- a/openerp/netsvc.py +++ b/openerp/netsvc.py @@ -36,6 +36,11 @@ import time import types from pprint import pformat +try: + import psutil +except ImportError: + psutil = None + # TODO modules that import netsvc only for things from loglevels must be changed to use loglevels. from loglevels import * import tools @@ -273,6 +278,9 @@ def dispatch_rpc(service_name, method, params): rpc_response_flag = rpc_response.isEnabledFor(logging.DEBUG) if rpc_request_flag or rpc_response_flag: start_time = time.time() + start_rss, start_vms = 0, 0 + if psutil: + start_rss, start_vms = psutil.Process(os.getpid()).get_memory_info() if rpc_request and rpc_response_flag: log(rpc_request,logging.DEBUG,'%s.%s'%(service_name,method), replace_request_password(params)) @@ -282,10 +290,14 @@ def dispatch_rpc(service_name, method, params): if rpc_request_flag or rpc_response_flag: end_time = time.time() + end_rss, end_vms = 0, 0 + if psutil: + end_rss, end_vms = psutil.Process(os.getpid()).get_memory_info() + logline = '%s.%s time:%.3fs mem: %sk -> %sk (diff: %sk)' % (service_name, method, end_time - start_time, start_vms / 1024, end_vms / 1024, (end_vms - start_vms)/1024) if rpc_response_flag: - log(rpc_response,logging.DEBUG,'%s.%s time:%.3fs '%(service_name,method,end_time - start_time), result) + log(rpc_response,logging.DEBUG, logline, result) else: - log(rpc_request,logging.DEBUG,'%s.%s time:%.3fs '%(service_name,method,end_time - start_time), replace_request_password(params), depth=1) + log(rpc_request,logging.DEBUG, logline, replace_request_password(params), depth=1) return result except openerp.exceptions.AccessError: diff --git a/openerp/osv/orm.py b/openerp/osv/orm.py index 6d78d7fd26c..6056e405b78 100644 --- a/openerp/osv/orm.py +++ b/openerp/osv/orm.py @@ -629,7 +629,8 @@ class MetaModel(type): self._module = module_name # Remember which models to instanciate for this module. - self.module_to_models.setdefault(self._module, []).append(self) + if not self._custom: + self.module_to_models.setdefault(self._module, []).append(self) # Definition of log access columns, automatically added to models if @@ -666,6 +667,7 @@ class BaseModel(object): _name = None _columns = {} _constraints = [] + _custom = False _defaults = {} _rec_name = None _parent_name = 'parent_id' @@ -942,7 +944,8 @@ class BaseModel(object): # managed by the metaclass. module_model_list = MetaModel.module_to_models.setdefault(cls._module, []) if cls not in module_model_list: - module_model_list.append(cls) + if not cls._custom: + module_model_list.append(cls) # Since we don't return an instance here, the __init__ # method won't be called. diff --git a/openerp/release.py b/openerp/release.py index e66d924906b..50bcd0c81e1 100644 --- a/openerp/release.py +++ b/openerp/release.py @@ -30,7 +30,7 @@ RELEASE_LEVELS_DISPLAY = {ALPHA: ALPHA, # properly comparable using normal operarors, for example: # (6,1,0,'beta',0) < (6,1,0,'candidate',1) < (6,1,0,'candidate',2) # (6,1,0,'candidate',2) < (6,1,0,'final',0) < (6,1,2,'final',0) -version_info = (7, 0, 0, ALPHA, 0) +version_info = (7, 0, 0, FINAL, 0) version = '.'.join(map(str, version_info[:2])) + RELEASE_LEVELS_DISPLAY[version_info[3]] + str(version_info[4] or '') serie = major_version = '.'.join(map(str, version_info[:2])) diff --git a/openerp/report/report_sxw.py b/openerp/report/report_sxw.py index e0731b819f5..b1f1e3d5891 100644 --- a/openerp/report/report_sxw.py +++ b/openerp/report/report_sxw.py @@ -617,9 +617,7 @@ class report_sxw(report_rml, preprocess.report): create_doc = self.generators[mime_type] odt = etree.tostring(create_doc(rml_dom, rml_parser.localcontext), encoding='utf-8', xml_declaration=True) - sxw_z = zipfile.ZipFile(sxw_io, mode='a') - sxw_z.writestr('content.xml', odt) - sxw_z.writestr('meta.xml', meta) + sxw_contents = {'content.xml':odt, 'meta.xml':meta} if report_xml.header: #Add corporate header/footer @@ -638,12 +636,25 @@ class report_sxw(report_rml, preprocess.report): rml_parser._add_header(odt) odt = etree.tostring(odt, encoding='utf-8', xml_declaration=True) - sxw_z.writestr('styles.xml', odt) + sxw_contents['styles.xml'] = odt finally: rml_file.close() - sxw_z.close() - final_op = sxw_io.getvalue() + + #created empty zip writing sxw contents to avoid duplication + sxw_out = StringIO.StringIO() + sxw_out_zip = zipfile.ZipFile(sxw_out, mode='w') + sxw_template_zip = zipfile.ZipFile (sxw_io, 'r') + for item in sxw_template_zip.infolist(): + if item.filename not in sxw_contents: + buffer = sxw_template_zip.read(item.filename) + sxw_out_zip.writestr(item.filename, buffer) + for item_filename, buffer in sxw_contents.iteritems(): + sxw_out_zip.writestr(item_filename, buffer) + sxw_template_zip.close() + sxw_out_zip.close() + final_op = sxw_out.getvalue() sxw_io.close() + sxw_out.close() return final_op, mime_type def create_single_html2html(self, cr, uid, ids, data, report_xml, context=None): diff --git a/openerp/service/workers.py b/openerp/service/workers.py index 0a54a80b963..ed77b28cf1b 100644 --- a/openerp/service/workers.py +++ b/openerp/service/workers.py @@ -355,12 +355,17 @@ class WorkerCron(Worker): time.sleep(interval) def process_work(self): + rpc_request = logging.getLogger('openerp.netsvc.rpc.request') + rpc_request_flag = rpc_request.isEnabledFor(logging.DEBUG) _logger.debug("WorkerCron (%s) polling for jobs", self.pid) if config['db_name']: db_names = config['db_name'].split(',') else: db_names = openerp.netsvc.ExportService._services['db'].exp_list(True) for db_name in db_names: + if rpc_request_flag: + start_time = time.time() + start_rss, start_vms = psutil.Process(os.getpid()).get_memory_info() while True: # acquired = openerp.addons.base.ir.ir_cron.ir_cron._acquire_job(db_name) # TODO why isnt openerp.addons.base defined ? @@ -371,7 +376,12 @@ class WorkerCron(Worker): # dont keep cursors in multi database mode if len(db_names) > 1: openerp.sql_db.close_db(db_name) - # TODO Each job should be considered as one request instead of each db + if rpc_request_flag: + end_time = time.time() + end_rss, end_vms = psutil.Process(os.getpid()).get_memory_info() + logline = '%s time:%.3fs mem: %sk -> %sk (diff: %sk)' % (db_name, end_time - start_time, start_vms / 1024, end_vms / 1024, (end_vms - start_vms)/1024) + _logger.debug("WorkerCron (%s) %s", self.pid, logline) + # TODO Each job should be considered as one request instead of each run self.request_count += 1 def start(self): diff --git a/openerp/tests/test_mail.py b/openerp/tests/test_mail.py index b3c11a13bfe..ddb8fc4a195 100644 --- a/openerp/tests/test_mail.py +++ b/openerp/tests/test_mail.py @@ -43,6 +43,47 @@ test12

test link """ +EDI_LIKE_HTML_SOURCE = """
+

Hello ${object.partner_id.name},

+

A new invoice is available for you:

+

+   REFERENCES
+   Invoice number: ${object.number}
+   Invoice total: ${object.amount_total} ${object.currency_id.name}
+   Invoice date: ${object.date_invoice}
+   Order reference: ${object.origin}
+   Your contact: ${object.user_id.name} +

+
+

It is also possible to directly pay with Paypal:

+ + + +
+

If you have any question, do not hesitate to contact us.

+

Thank you for choosing ${object.company_id.name or 'us'}!

+
+
+
+

+ ${object.company_id.name}

+
+
+ + ${object.company_id.street}
+ ${object.company_id.street2}
+ ${object.company_id.zip} ${object.company_id.city}
+ ${object.company_id.state_id and ('%s, ' % object.company_id.state_id.name) or ''} ${object.company_id.country_id.name or ''}
+ +
+ Phone:  ${object.company_id.phone} +
+ +
+
""" + TEXT_MAIL1 = """I contact you about our meeting for tomorrow. Here is the schedule I propose: 9 AM: brainstorming about our new amazing business app 9.45 AM: summary @@ -126,23 +167,85 @@ bert.tartopoils@miam.miam class TestSanitizer(unittest2.TestCase): """ Test the html sanitizer that filters html to remove unwanted attributes """ - def test_simple(self): - x = "yop" - self.assertEqual(x, html_sanitize(x)) + def test_basic_sanitizer(self): + cases = [ + ("yop", "

yop

"), # simple + ("lala

yop

xxx", "

lala

yop

xxx
"), # trailing text + ("Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci", + u"

Merci à l'intérêt pour notre produit.nous vous contacterons bientôt. Merci

"), # unicode + ] + for content, expected in cases: + html = html_sanitize(content) + self.assertEqual(html, expected, 'html_sanitize is broken') - def test_trailing_text(self): - x = 'lala

yop

xxx' - self.assertEqual(x, html_sanitize(x)) + def test_evil_malicious_code(self): + # taken from https://www.owasp.org/index.php/XSS_Filter_Evasion_Cheat_Sheet#Tests + cases = [ + (""), # no quotes and semicolons + (""), # UTF-8 Unicode encoding + (""), # hex encoding + (""), # embedded carriage return + (""), # embedded newline + (""), # embedded tab + (""), # embedded encoded tab + (""), # spaces and meta-characters + ("\">"), # malformed tag + (""), # non-alpha-non-digits + (""), # non-alpha-non-digits + ("<"), # extraneous open brackets + (" in test_mail) + try: + cleaner = clean.Cleaner(page_structure=True, style=False, safe_attrs_only=False, forms=False, kill_tags=tags_to_kill, remove_tags=tags_to_remove) + cleaned = cleaner.clean_html(src) + except TypeError, e: + # lxml.clean version < 2.3.1 does not have a kill_tags attribute + # to remove in 2014 + cleaner = clean.Cleaner(page_structure=True, style=False, safe_attrs_only=False, forms=False, remove_tags=tags_to_kill+tags_to_remove) + cleaned = cleaner.clean_html(src) + except: + _logger.warning('html_sanitize failed to parse %s' % (src)) + cleaned = '

Impossible to parse

' + return cleaned #----------------------------------------------------------