[IMP] website: test_crawl dont follow external link

In case a controller with a relative url return an absolute link
the current crawler follow the redirect and all external link of this
external url too.

Now we don't follow redirection if it is on an other netloc that the
current one.

Part of https://github.com/odoo/odoo/pull/38950
task-2087641

addons/website/tests/test_crawl.py

@@ -53,7 +53,15 @@ class Crawler(HttpCaseWithUserDemo):
             seen.add(url_slug)
 
         _logger.info("%s %s", msg, url)
-        r = self.url_open(url)
+        r = self.url_open(url, allow_redirects=False)
+        if r.status_code in (301, 302):
+            # check local redirect to avoid fetch externals pages
+            new_url = r.headers.get('Location')
+            current_url = r.url
+            if urls.url_parse(new_url).netloc != urls.url_parse(current_url).netloc:
+                return seen
+            r = self.url_open(new_url)
+
         code = r.status_code
         self.assertIn(code, range(200, 300), "%s Fetching %s returned error response (%d)" % (msg, url, code))
 

odoo/tests/common.py

@@ -1220,13 +1220,13 @@ class HttpCase(TransactionCase):
         self.opener = requests.Session()
         self.opener.cookies['session_id'] = self.session_id
 
-    def url_open(self, url, data=None, files=None, timeout=10, headers=None):
+    def url_open(self, url, data=None, files=None, timeout=10, headers=None, allow_redirects=True):
         self.env['base'].flush()
         if url.startswith('/'):
             url = "http://%s:%s%s" % (HOST, odoo.tools.config['http_port'], url)
         if data or files:
-            return self.opener.post(url, data=data, files=files, timeout=timeout, headers=headers)
-        return self.opener.get(url, timeout=timeout, headers=headers)
+            return self.opener.post(url, data=data, files=files, timeout=timeout, headers=headers, allow_redirects=allow_redirects)
+        return self.opener.get(url, timeout=timeout, headers=headers, allow_redirects=allow_redirects)
 
     def _wait_remaining_requests(self, timeout=10):