feat: add robots.txt response

src/index.js

@@ -15,6 +15,7 @@
 import { GoogleLogger } from './google-logger';
 import { CoralogixLogger } from './coralogix-logger';
 import { CoralogixErrorLogger } from './coralogix-error-logger';
+import { respondRobots } from './robots.js';
 
 function respondError(message, status, e, req) {
   const headers = new Headers();
@@ -44,6 +45,9 @@ function hashCode(s) {
 
 async function main(req) {
   try {
+    if (req.method === 'GET' && req.url.startsWith('/robots.txt')) {
+      return respondRobots(req);
+    }
     const body = req.method === 'GET'
       ? JSON.parse(new URL(req.url).searchParams.get('data'))
       : await req.json();

src/robots.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2022 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-env serviceworker */
+export function respondRobots() {
+  return new Response(`
+User-agent: *
+Disallow: /`, {
+    headers: {
+      'content-type': 'text/plain',
+    },
+  });
+}

test/post-deploy.test.js

@@ -47,6 +47,14 @@ describe('Helix RUM Collector Post-Deploy Tests', () => {
     expect(response).to.have.status(201);
   });
 
+  it('robots.txt denies everything', async () => {
+    const response = await chai.request(`https://${domain}`)
+      .get('/robots.txt');
+    expect(response).to.have.status(200);
+    // eslint-disable-next-line no-unused-expressions
+    expect(response).to.be.text;
+  });
+
   it('Missing id returns 400', async () => {
     const response = await chai.request(`https://${domain}`)
       .post('/')