Xavier-Do
b2b36524c2
Performances from a general point of view can be difficult to track.
This commit proposes to improve logs in two ways:
The current logs only use the sql_counter, wich will only be updated
when a cursor is closed. In a test-enable install, this counter
is actually the queries of the tests wince the install cursor is
open untill the end. The first fix is to use bot sql_counter and
sql_log_count to have total queries untill now on closed cursor,
but also the current number of queries of the current cursor.
This means that the new log format will be
{nb} modules loaded in {time}, {loading_querie} (+{test_cr_queries}) queries
instead of
{nb} modules loaded in {time}, {tests_cr__queries}queries
Nothe that in the current version, {nb} is actually the total number of
loaded modules until now.
This commit also add an equivalent end log by module and change the
loglevel of module start on install (mainly usefull if an error occurs
before anything else is logged hidding the module causing this error.)
A cleaner runbot logger is also added, in order to be abble to call
_logger.runbot( instead of _logger.log(25. This will clarify the purpose
of such a log level.
closes odoo/odoo#47283
Signed-off-by: Christophe Monniez (moc) <moc@odoo.com>
109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
|
|
|
import logging
|
|
import time
|
|
|
|
import lxml.html
|
|
from werkzeug import urls
|
|
|
|
import odoo
|
|
import re
|
|
|
|
from odoo.addons.base.tests.common import HttpCaseWithUserDemo
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
@odoo.tests.common.tagged('post_install', '-at_install', 'crawl')
|
|
class Crawler(HttpCaseWithUserDemo):
|
|
""" Test suite crawling an Odoo CMS instance and checking that all
|
|
internal links lead to a 200 response.
|
|
|
|
If a username and a password are provided, authenticates the user before
|
|
starting the crawl
|
|
"""
|
|
|
|
def setUp(self):
|
|
super(Crawler, self).setUp()
|
|
|
|
if hasattr(self.env['res.partner'], 'grade_id'):
|
|
# Create at least one published parter, so that /partners doesn't
|
|
# return a 404
|
|
grade = self.env['res.partner.grade'].create({
|
|
'name': 'A test grade',
|
|
'website_published': True,
|
|
})
|
|
self.env['res.partner'].create({
|
|
'name': 'A Company for /partners',
|
|
'is_company': True,
|
|
'grade_id': grade.id,
|
|
'website_published': True,
|
|
})
|
|
|
|
def crawl(self, url, seen=None, msg=''):
|
|
if seen is None:
|
|
seen = set()
|
|
|
|
url_slug = re.sub(r"[/](([^/=?&]+-)?[0-9]+)([/]|$)", '/<slug>/', url)
|
|
url_slug = re.sub(r"([^/=?&]+)=[^/=?&]+", '\g<1>=param', url_slug)
|
|
if url_slug in seen:
|
|
return seen
|
|
else:
|
|
seen.add(url_slug)
|
|
|
|
_logger.info("%s %s", msg, url)
|
|
r = self.url_open(url)
|
|
code = r.status_code
|
|
self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))
|
|
|
|
if r.headers['Content-Type'].startswith('text/html'):
|
|
doc = lxml.html.fromstring(r.content)
|
|
for link in doc.xpath('//a[@href]'):
|
|
href = link.get('href')
|
|
|
|
parts = urls.url_parse(href)
|
|
# href with any fragment removed
|
|
href = parts.replace(fragment='').to_url()
|
|
|
|
# FIXME: handle relative link (not parts.path.startswith /)
|
|
if parts.netloc or \
|
|
not parts.path.startswith('/') or \
|
|
parts.path == '/web' or\
|
|
parts.path.startswith('/web/') or \
|
|
parts.path.startswith('/en_US/') or \
|
|
(parts.scheme and parts.scheme not in ('http', 'https')):
|
|
continue
|
|
|
|
self.crawl(href, seen, msg)
|
|
return seen
|
|
|
|
def test_10_crawl_public(self):
|
|
t0 = time.time()
|
|
t0_sql = self.registry.test_cr.sql_log_count
|
|
seen = self.crawl('/', msg='Anonymous Coward')
|
|
count = len(seen)
|
|
duration = time.time() - t0
|
|
sql = self.registry.test_cr.sql_log_count - t0_sql
|
|
_logger.runbot("public crawled %s urls in %.2fs %s queries, %.3fs %.2fq per request, ", count, duration, sql, duration / count, float(sql) / count)
|
|
|
|
def test_20_crawl_demo(self):
|
|
t0 = time.time()
|
|
t0_sql = self.registry.test_cr.sql_log_count
|
|
self.authenticate('demo', 'demo')
|
|
seen = self.crawl('/', msg='demo')
|
|
count = len(seen)
|
|
duration = time.time() - t0
|
|
sql = self.registry.test_cr.sql_log_count - t0_sql
|
|
_logger.runbot("demo crawled %s urls in %.2fs %s queries, %.3fs %.2fq per request", count, duration, sql, duration / count, float(sql) / count)
|
|
|
|
def test_30_crawl_admin(self):
|
|
t0 = time.time()
|
|
t0_sql = self.registry.test_cr.sql_log_count
|
|
self.authenticate('admin', 'admin')
|
|
seen = self.crawl('/', msg='admin')
|
|
count = len(seen)
|
|
duration = time.time() - t0
|
|
sql = self.registry.test_cr.sql_log_count - t0_sql
|
|
_logger.runbot("admin crawled %s urls in %.2fs %s queries, %.3fs %.2fq per request", count, duration, sql, duration / count, float(sql) / count)
|