Merge pull request #2234 from alphagov/robots-txt

Update robots.txt to ensure review app isn’t indexed

app/app.js

@@ -54,6 +54,16 @@ module.exports = (options) => {
     next()
   })
 
+  // Ensure robots are still able to crawl the pages.
+  //
+  // This might seem like a mistake, but it's not. If a page is blocked by
+  // robots.txt, the crawler will never see the noindex directive, and so the
+  // page can still appear in search results.
+  app.get('/robots.txt', function (req, res) {
+    res.type('text/plain')
+    res.send('User-agent: *\nAllow: /')
+  })
+
   // Set up middleware to serve static assets
   app.use('/public', express.static(configPaths.public))
 
@@ -195,10 +205,5 @@ module.exports = (options) => {
   // Full page example views
   require('./full-page-examples.js')(app)
 
-  app.get('/robots.txt', function (req, res) {
-    res.type('text/plain')
-    res.send('User-agent: *\nDisallow: /')
-  })
-
   return app
 }

app/app.test.js

@@ -66,6 +66,15 @@ describe(`http://localhost:${PORT}`, () => {
     })
   })
 
+  describe('/robots.txt', () => {
+    it('should allow crawling by robots', done => {
+      requestPath('/robots.txt', (err, res) => {
+        expect(res.body).toMatch(/^Allow: \/$/m)
+        done(err)
+      })
+    })
+  })
+
   describe('/examples/template-custom', () => {
     const templatePath = '/examples/template-custom'