[IMP] web: default robots.txt

Web lacked a default route for /robots.txt meaning that if you didn't
installed website (which comes with a full fledged robots.txt) you would
get a 404 page not found.

closes odoo/odoo#127402

Signed-off-by: Julien Castiaux (juc) <juc@odoo.com>

addons/web/controllers/home.py

@@ -154,3 +154,7 @@ class Home(http.Controller):
         headers = [('Content-Type', 'application/json'),
                    ('Cache-Control', 'no-store')]
         return request.make_response(data, headers)
+
+    @http.route(['/robots.txt'], type='http', auth="none")
+    def robots(self, **kwargs):
+        return "User-agent: *\nDisallow: /\n"

odoo/addons/test_http/tests/test_misc.py

@@ -136,6 +136,9 @@ class TestHttpMisc(TestHttpBase):
             self.assertEqual(res.status_code, 200)
             self.assertEqual(res.text, file.getvalue())
 
+    def test_misc7_robotstxt(self):
+        self.nodb_url_open('/robots.txt').raise_for_status()
+
 @tagged('post_install', '-at_install')
 class TestHttpCors(TestHttpBase):
     def test_cors0_http_default(self):