Read from the new log service (store.sd)

stjohnjohnson · stjohnjohnson · commit 52c0c6c071ad · 2016-10-15T10:39:25.000-07:00
- Add a local dockerfile (for testing)
- Remove "extra" users from admin access
- Add Swarm plugin
- Remove snyk as it's done via GitHub webhooks now
- Allow overriding the whitelist via environment
diff --git a/.dockerignore b/.dockerignore
@@ -2,4 +2,3 @@ node_modules/
 test/
 features/
 artifacts/
-kubernetes/
diff --git a/Dockerfile.local b/Dockerfile.local
@@ -0,0 +1,18 @@
+FROM node:6
+
+# Create our application directory
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Install Screwdriver API
+COPY package.json /usr/src/app/
+RUN npm install --production
+
+# Copy everything else
+COPY . /usr/src/app
+
+# Expose the web service port
+EXPOSE 8080
+
+# Run the service
+CMD [ "npm", "start" ]
diff --git a/bin/server b/bin/server
@@ -1,25 +1,12 @@
 #!/usr/bin/env node
-'use strict';
 
+'use strict';
 
+/* eslint-disable import/no-dynamic-require */
 const winston = require('winston');
 const config = require('config');
 const server = require('../');
-
-// Setup Datastore
-const datastoreConfig = config.get('datastore');
-const DatastorePlugin = require(`screwdriver-datastore-${datastoreConfig.plugin}`);
-const datastore = new DatastorePlugin(datastoreConfig[datastoreConfig.plugin] || {});
-
-// Setup Executor
-const executorConfig = config.get('executor');
-const ExecutorPlugin = require(`screwdriver-executor-${executorConfig.plugin}`);
-const executor = new ExecutorPlugin(executorConfig[executorConfig.plugin] || {});
-
-// Source Code Plugin
-const scmConfig = config.get('scm');
-const ScmPlugin = require(`screwdriver-scm-${scmConfig.plugin}`);
-const scm = new ScmPlugin(scmConfig[scmConfig.plugin] || {});
+const hoek = require('hoek');
 
 // Setup Authentication
 const authConfig = config.get('auth');
@@ -33,6 +20,26 @@ const gitHubSecret = config.get('webhooks.github.secret');
 // Special urls for things like the UI
 const ecosystem = config.get('ecosystem');
 
+ecosystem.api = httpdConfig.uri;
+
+// Setup Datastore
+const datastoreConfig = config.get('datastore');
+const DatastorePlugin = require(`screwdriver-datastore-${datastoreConfig.plugin}`);
+const datastore = new DatastorePlugin(hoek.applyToDefaults({ ecosystem },
+    (datastoreConfig[datastoreConfig.plugin] || {})));
+
+// Setup Executor
+const executorConfig = config.get('executor');
+const ExecutorPlugin = require(`screwdriver-executor-${executorConfig.plugin}`);
+const executor = new ExecutorPlugin(hoek.applyToDefaults({ ecosystem },
+    (executorConfig[executorConfig.plugin] || {})));
+
+// Source Code Plugin
+const scmConfig = config.get('scm');
+const ScmPlugin = require(`screwdriver-scm-${scmConfig.plugin}`);
+const scm = new ScmPlugin(hoek.applyToDefaults({ ecosystem },
+    (scmConfig[scmConfig.plugin] || {})));
+
 // Setup Model Factories
 const Models = require('screwdriver-models');
 const pipelineFactory = Models.PipelineFactory.getInstance({
@@ -76,7 +83,7 @@ datastore.setup()
         }
     }))
     .then(instance => winston.info('Server running at %s', instance.info.uri))
-    .catch(err => {
+    .catch((err) => {
         winston.error(err);
         process.exit(1);
     });
diff --git a/config/custom-environment-variables.yaml b/config/custom-environment-variables.yaml
@@ -27,6 +27,12 @@ auth:
     # A flag to set if the server is running over https.
     # Used as a flag for the OAuth flow
     https: IS_HTTPS
+    whitelist:
+        __name: SECRET_WHITELIST
+        __format: json
+    admins:
+        __name: SECRET_ADMINS
+        __format: json
 
 httpd:
     # Port to listen on
@@ -60,15 +66,21 @@ datastore:
         secretAccessKey: DATASTORE_DYNAMODB_SECRET
 
 executor:
+    plugin: EXECUTOR_PLUGIN
     k8s:
         # The host or IP of the kubernetes cluster
         host: K8S_HOST
         # The jwt token used for authenticating kubernetes requests
         token: K8S_TOKEN
         # Launcher container tag to use
         launchVersion: LAUNCH_VERSION
-        # Log Service container tag to use
-        logVersion: LOG_VERSION
+    s3m:
+        # Configuration of Docker
+        docker:
+            __name: EXECUTOR_S3M_DOCKER
+            __format: json
+        # Launcher container tag to use
+        launchVersion: LAUNCH_VERSION
 
 
 webhooks:
@@ -79,5 +91,7 @@ webhooks:
 ecosystem:
     # URL for the User Interface
     ui: ECOSYSTEM_UI
+    # Externally routable URL for the Artifact Store
+    store: ECOSYSTEM_STORE
     # Badge service (needs to add a status and color)
     badges: ECOSYSTEM_BADGES
diff --git a/config/default.yaml b/config/default.yaml
@@ -52,27 +52,23 @@ auth:
         - jpcollins
     admins:
         - FenrirUnbound
-        - Filbird
         - cynthiax
         - d2lam
         - dvdizon
         - jer
         - minz1027
-        - nicolaifsf
-        - nkatzman
         - outofjungle
         - petey
-        - shruthi-venkateswaran
         - stjohnjohnson
         - tkyi
 
 httpd:
     # Port to listen on
-    port: 8080
+    port: 80
     # Host to listen on (set to localhost to only accept connections from this machine)
     host: 0.0.0.0
     # Externally routable URI (usually your load balancer or CNAME)
-    uri: http://localhost:8080
+    uri: http://localhost:80
     # SSL Support
     tls: false
         # If you want SSL, you can easily add it by replacing `tls: false` with an object that
@@ -88,25 +84,32 @@ datastore:
     sequelize:
         # Type of server to talk to
         dialect: sqlite
+        # More arguments here:
+        # http://docs.sequelizejs.com/en/latest/api/sequelize/
     dynamodb:
         # Prefix to table names
         prefix: ""
-        # AWS Access Key ID
-        accessKeyId: WHAT-A-LEGITIMATE-LOOKING-KEY-ID
-        # AWS Secret Access Key
-        secretAccessKey: TOTALLY-REAL-LOOKING-AWS-KEYS
+        # AWS Access Keys (required)
+        # accessKeyId:
+        # secretAccessKey:
 
 executor:
     plugin: k8s
     k8s:
         # The host or IP of the kubernetes cluster
         host: kubernetes
         # The jwt token used for authenticating kubernetes requests
-        token: NOT-A-REAL-JWT-TOKEN
-        # Launcher container tag to use
-        launchVersion: latest
-        # Log Service container tag to use
-        logVersion: latest
+        # Loaded from /var/run/secrets/kubernetes.io/serviceaccount/token by default
+        # token:
+
+        # Container tags to use
+        launchVersion: stable
+    s3m:
+        # Dockerode configuration https://github.com/apocas/dockerode#getting-started
+        docker: {}
+
+        # Container tags to use
+        launchVersion: stable
 
 scm:
     plugin: github
@@ -117,9 +120,9 @@ webhooks:
         secret: SUPER-SECRET-SIGNING-THING
 
 ecosystem:
-    # URL for the User Interface
+    # Externally routable URL for the User Interface
     ui: https://cd.screwdriver.cd
-    # URL for the Artifact Store
-    store: https://s3-us-west-2.amazonaws.com/logs.screwdriver.cd
+    # Externally routable URL for the Artifact Store
+    store: https://store.screwdriver.cd
     # Badge service (needs to add a status and color)
     badges: https://img.shields.io/badge/build-{{status}}-{{color}}.svg
diff --git a/package.json b/package.json
@@ -4,7 +4,7 @@
   "description": "API server for the Screwdriver CD service",
   "main": "index.js",
   "scripts": {
-    "pretest": "eslint . && snyk test",
+    "pretest": "eslint .",
     "test": "jenkins-mocha --recursive",
     "start": "./bin/server",
     "functional": "cucumber-js --format=pretty ./features/server.feature ./features/secrets.feature ./features/git-flow.feature ./features/authorization.feature"
@@ -80,7 +80,8 @@
     "screwdriver-data-schema": "^13.0.0",
     "screwdriver-datastore-dynamodb": "^3.0.0",
     "screwdriver-datastore-sequelize": "^1.0.2",
-    "screwdriver-executor-k8s": "^8.0.0",
+    "screwdriver-executor-k8s": "^9.0.0",
+    "screwdriver-executor-s3m": "^1.0.0",
     "screwdriver-models": "^15.0.0",
     "screwdriver-scm-github": "^1.0.1",
     "tinytim": "^0.1.1",
@@ -104,7 +105,6 @@
     "npm-auto-version": "^1.0.0",
     "request": "^2.73.0",
     "sinon": "~1.17.4",
-    "sinon-as-promised": "^4.0.2",
-    "snyk": "^1.17.5"
+    "sinon-as-promised": "^4.0.2"
   }
 }
diff --git a/plugins/builds/steps/logs.js b/plugins/builds/steps/logs.js
@@ -4,6 +4,7 @@ const boom = require('boom');
 const schema = require('screwdriver-data-schema');
 const request = require('request');
 const ndjson = require('ndjson');
+const MAX_LINES = 100;
 
 module.exports = config => ({
     method: 'GET',
@@ -31,15 +32,18 @@ module.exports = config => ({
                         throw boom.notFound('Step does not exist');
                     }
 
-                    const isNotDone = stepModel.code === undefined;
                     const isNotStarted = stepModel.startTime === undefined;
-                    const url = `${config.ecosystem.store}/${buildId}/${stepName}`;
                     const output = [];
 
                     if (isNotStarted) {
                         return reply(output).header('X-More-Data', 'false');
                     }
 
+                    const page = Math.floor(req.query.from / MAX_LINES);
+                    const isDone = stepModel.code !== undefined;
+                    const url = `${config.ecosystem.store}/v1/builds/`
+                        + `${buildId}/${stepName}/log.${page}`;
+
                     return request
                         // Load NDJson from S3 bucket
                         .get(url)
@@ -52,9 +56,10 @@ module.exports = config => ({
                                 output.push(line);
                             }
                         })
-                        // Set header X-More-Data: true if step not done
+                        // Set header X-More-Data: false if lines < MAX_LINES && step done
                         .on('end', () =>
-                            reply(output).header('X-More-Data', isNotDone.toString()));
+                            reply(output).header('X-More-Data',
+                                (!(output.length < MAX_LINES && isDone)).toString()));
                 })
                 .catch(err => reply(boom.wrap(err)));
         },
diff --git a/test/plugins/builds.test.js b/test/plugins/builds.test.js
diff --git a/test/plugins/data/step.long.log.ndjson b/test/plugins/data/step.long.log.ndjson
diff --git a/test/plugins/data/step.long2.log.ndjson b/test/plugins/data/step.long2.log.ndjson

-Original file line number
+Diff line change
     let secretAccessMock;
     let plugin;
     let server;
 -    const logBaseUrl = 'http://example.com/screwdriver-logs';
 +    const logBaseUrl = 'https://store.screwdriver.cd';
     before(() => {
         mockery.enable({
         ];
         beforeEach(() => {
 -            nock('http://example.com')
 -                .get(`/screwdriver-logs/${id}/${step}`)
 -                .replyWithFile(200, `${__dirname}/data/step.log.ndjson`);
             nock.disableNetConnect();
         });
             const buildMock = getMockBuilds(testBuild);
             buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.0`)
 +                .replyWithFile(200, `${__dirname}/data/step.log.ndjson`);
             return server.inject(`/builds/${id}/steps/${step}/logs`).then((reply) => {
                 assert.equal(reply.statusCode, 200);
             });
         });
 +        it('returns more-data for a step that is over but max lines', () => {
 +            const buildMock = getMockBuilds(testBuild);
++
 +            buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.0`)
 +                .replyWithFile(200, `${__dirname}/data/step.long.log.ndjson`);
++
 +            return server.inject(`/builds/${id}/steps/${step}/logs`).then((reply) => {
 +                assert.equal(reply.statusCode, 200);
 +                assert.equal(reply.result.length, 100);
 +                assert.propertyVal(reply.headers, 'x-more-data', 'true');
 +            });
 +        });
++
 +        it('returns from second page', () => {
 +            const buildMock = getMockBuilds(testBuild);
++
 +            buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.1`)
 +                .replyWithFile(200, `${__dirname}/data/step.long2.log.ndjson`);
++
 +            return server.inject(`/builds/${id}/steps/${step}/logs?from=100`).then((reply) => {
 +                assert.equal(reply.statusCode, 200);
 +                assert.equal(reply.result.length, 2);
 +                assert.propertyVal(reply.headers, 'x-more-data', 'false');
 +            });
 +        });
++
 +        it('returns from second empty page', () => {
 +            const buildMock = getMockBuilds(testBuild);
++
 +            buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.1`)
 +                .reply(200, '');
++
 +            return server.inject(`/builds/${id}/steps/${step}/logs?from=100`).then((reply) => {
 +                assert.equal(reply.statusCode, 200);
 +                assert.equal(reply.result.length, 0);
 +                assert.propertyVal(reply.headers, 'x-more-data', 'false');
 +            });
 +        });
++
         it('returns correct lines after a given line', () => {
             const buildMock = getMockBuilds(testBuild);
             buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.0`)
 +                .replyWithFile(200, `${__dirname}/data/step.log.ndjson`);
             return server.inject(`/builds/${id}/steps/${step}/logs?from=2`).then((reply) => {
                 assert.equal(reply.statusCode, 200);
             const buildMock = getMockBuilds(testBuild);
             buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/${step}/log.0`)
 +                .replyWithFile(200, `${__dirname}/data/step.log.ndjson`);
             return server.inject(`/builds/${id}/steps/publish/logs`).then((reply) => {
                 assert.equal(reply.statusCode, 200);
         it('returns empty array on invalid data', () => {
             const buildMock = getMockBuilds(testBuild);
 -            nock('http://example.com')
 -                .get(`/screwdriver-logs/${id}/test`)
 -                .reply(200, '<invalid JSON>\n<more bad JSON>');
+-
             buildFactoryMock.get.withArgs(id).resolves(buildMock);
 +            nock('https://store.screwdriver.cd')
 +                .get(`/v1/builds/${id}/test/log.0`)
 +                .reply(200, '<invalid JSON>\n<more bad JSON>');
             return server.inject(`/builds/${id}/steps/test/logs`).then((reply) => {
                 assert.equal(reply.statusCode, 200);
-Original file line number
+Diff line change
 +{"t":1476390518243,"m":"Building stuff","n":0}
 +{"t":1476390518244,"m":"Building stuff","n":1}
 +{"t":1476390518244,"m":"Building stuff","n":2}
 +{"t":1476390518244,"m":"Building stuff","n":3}
 +{"t":1476390518244,"m":"Building stuff","n":4}
 +{"t":1476390518244,"m":"Building stuff","n":5}
 +{"t":1476390518244,"m":"Building stuff","n":6}
 +{"t":1476390518244,"m":"Building stuff","n":7}
 +{"t":1476390518244,"m":"Building stuff","n":8}
 +{"t":1476390518244,"m":"Building stuff","n":9}
 +{"t":1476390518244,"m":"Building stuff","n":10}
 +{"t":1476390518244,"m":"Building stuff","n":11}
 +{"t":1476390518244,"m":"Building stuff","n":12}
 +{"t":1476390518244,"m":"Building stuff","n":13}
 +{"t":1476390518244,"m":"Building stuff","n":14}
 +{"t":1476390518244,"m":"Building stuff","n":15}
 +{"t":1476390518244,"m":"Building stuff","n":16}
 +{"t":1476390518244,"m":"Building stuff","n":17}
 +{"t":1476390518244,"m":"Building stuff","n":18}
 +{"t":1476390518244,"m":"Building stuff","n":19}
 +{"t":1476390518244,"m":"Building stuff","n":20}
 +{"t":1476390518244,"m":"Building stuff","n":21}
 +{"t":1476390518244,"m":"Building stuff","n":22}
 +{"t":1476390518244,"m":"Building stuff","n":23}
 +{"t":1476390518244,"m":"Building stuff","n":24}
 +{"t":1476390518244,"m":"Building stuff","n":25}
 +{"t":1476390518244,"m":"Building stuff","n":26}
 +{"t":1476390518244,"m":"Building stuff","n":27}
 +{"t":1476390518244,"m":"Building stuff","n":28}
 +{"t":1476390518244,"m":"Building stuff","n":29}
 +{"t":1476390518245,"m":"Building stuff","n":30}
 +{"t":1476390518245,"m":"Building stuff","n":31}
 +{"t":1476390518245,"m":"Building stuff","n":32}
 +{"t":1476390518245,"m":"Building stuff","n":33}
 +{"t":1476390518245,"m":"Building stuff","n":34}
 +{"t":1476390518245,"m":"Building stuff","n":35}
 +{"t":1476390518245,"m":"Building stuff","n":36}
 +{"t":1476390518245,"m":"Building stuff","n":37}
 +{"t":1476390518245,"m":"Building stuff","n":38}
 +{"t":1476390518245,"m":"Building stuff","n":39}
 +{"t":1476390518245,"m":"Building stuff","n":40}
 +{"t":1476390518245,"m":"Building stuff","n":41}
 +{"t":1476390518245,"m":"Building stuff","n":42}
 +{"t":1476390518245,"m":"Building stuff","n":43}
 +{"t":1476390518245,"m":"Building stuff","n":44}
 +{"t":1476390518245,"m":"Building stuff","n":45}
 +{"t":1476390518245,"m":"Building stuff","n":46}
 +{"t":1476390518245,"m":"Building stuff","n":47}
 +{"t":1476390518245,"m":"Building stuff","n":48}
 +{"t":1476390518245,"m":"Building stuff","n":49}
 +{"t":1476390518245,"m":"Building stuff","n":50}
 +{"t":1476390518245,"m":"Building stuff","n":51}
 +{"t":1476390518245,"m":"Building stuff","n":52}
 +{"t":1476390518245,"m":"Building stuff","n":53}
 +{"t":1476390518245,"m":"Building stuff","n":54}
 +{"t":1476390518245,"m":"Building stuff","n":55}
 +{"t":1476390518245,"m":"Building stuff","n":56}
 +{"t":1476390518245,"m":"Building stuff","n":57}
 +{"t":1476390518245,"m":"Building stuff","n":58}
 +{"t":1476390518245,"m":"Building stuff","n":59}
 +{"t":1476390518245,"m":"Building stuff","n":60}
 +{"t":1476390518245,"m":"Building stuff","n":61}
 +{"t":1476390518245,"m":"Building stuff","n":62}
 +{"t":1476390518245,"m":"Building stuff","n":63}
 +{"t":1476390518245,"m":"Building stuff","n":64}
 +{"t":1476390518245,"m":"Building stuff","n":65}
 +{"t":1476390518245,"m":"Building stuff","n":66}
 +{"t":1476390518245,"m":"Building stuff","n":67}
 +{"t":1476390518245,"m":"Building stuff","n":68}
 +{"t":1476390518245,"m":"Building stuff","n":69}
 +{"t":1476390518245,"m":"Building stuff","n":70}
 +{"t":1476390518245,"m":"Building stuff","n":71}
 +{"t":1476390518245,"m":"Building stuff","n":72}
 +{"t":1476390518245,"m":"Building stuff","n":73}
 +{"t":1476390518245,"m":"Building stuff","n":74}
 +{"t":1476390518245,"m":"Building stuff","n":75}
 +{"t":1476390518246,"m":"Building stuff","n":76}
 +{"t":1476390518246,"m":"Building stuff","n":77}
 +{"t":1476390518246,"m":"Building stuff","n":78}
 +{"t":1476390518246,"m":"Building stuff","n":79}
 +{"t":1476390518246,"m":"Building stuff","n":80}
 +{"t":1476390518246,"m":"Building stuff","n":81}
 +{"t":1476390518246,"m":"Building stuff","n":82}
 +{"t":1476390518246,"m":"Building stuff","n":83}
 +{"t":1476390518246,"m":"Building stuff","n":84}
 +{"t":1476390518246,"m":"Building stuff","n":85}
 +{"t":1476390518246,"m":"Building stuff","n":86}
 +{"t":1476390518246,"m":"Building stuff","n":87}
 +{"t":1476390518246,"m":"Building stuff","n":88}
 +{"t":1476390518246,"m":"Building stuff","n":89}
 +{"t":1476390518246,"m":"Building stuff","n":90}
 +{"t":1476390518246,"m":"Building stuff","n":91}
 +{"t":1476390518246,"m":"Building stuff","n":92}
 +{"t":1476390518246,"m":"Building stuff","n":93}
 +{"t":1476390518246,"m":"Building stuff","n":94}
 +{"t":1476390518246,"m":"Building stuff","n":95}
 +{"t":1476390518246,"m":"Building stuff","n":96}
 +{"t":1476390518246,"m":"Building stuff","n":97}
 +{"t":1476390518246,"m":"Building stuff","n":98}
 +{"t":1476390518247,"m":"Building stuff","n":99}
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+{"t":1476390518247,"m":"Building stuff","n":100}`
	`2`	`+{"t":1476390518248,"m":"Done!","n":101}`