Skip to content

Commit 394e2f1

Browse files
tkyitkyi
authoredSep 30, 2024··
feat(1749): Add endpoint to download all zipped artifacts (#3199)
·
v8.0.9v7.0.203
1 parent ffd02ab commit 394e2f1

File tree

6 files changed

+234
-1
lines changed

6 files changed

+234
-1
lines changed
 

‎package.json

+2
Original file line numberDiff line numberDiff line change
@@ -83,10 +83,12 @@
8383
"@hapi/inert": "^7.0.0",
8484
"@hapi/vision": "^7.0.0",
8585
"@promster/hapi": "^14.0.0",
86+
"archiver": "^7.0.1",
8687
"async": "^3.2.4",
8788
"badge-maker": "^3.3.1",
8889
"config": "^3.3.8",
8990
"dayjs": "^1.11.7",
91+
"got": "^11.8.3",
9092
"hapi-auth-bearer-token": "^8.0.0",
9193
"hapi-auth-jwt2": "^10.4.0",
9294
"hapi-rate-limit": "^7.1.0",

‎plugins/auth/token.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ module.exports = () => ({
4040

4141
const build = await buildFactory.get(request.params.buildId);
4242
const job = await jobFactory.get(build.jobId);
43-
const pipeline = pipelineFactory.get(job.pipelineId);
43+
const pipeline = await pipelineFactory.get(job.pipelineId);
4444

4545
profile = request.server.plugins.auth.generateProfile({
4646
username: request.params.buildId,

‎plugins/builds/artifacts/getAll.js

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
'use strict';
2+
3+
const archiver = require('archiver');
4+
const boom = require('@hapi/boom');
5+
const request = require('got');
6+
const joi = require('joi');
7+
const jwt = require('jsonwebtoken');
8+
const logger = require('screwdriver-logger');
9+
const { PassThrough } = require('stream');
10+
const schema = require('screwdriver-data-schema');
11+
const { v4: uuidv4 } = require('uuid');
12+
const idSchema = schema.models.build.base.extract('id');
13+
14+
15+
module.exports = config => ({
16+
method: 'GET',
17+
path: '/builds/{id}/artifacts',
18+
options: {
19+
description: 'Get a zipped file including all build artifacts',
20+
notes: 'Redirects to store with proper token',
21+
tags: ['api', 'builds', 'artifacts'],
22+
auth: {
23+
strategies: ['session', 'token'],
24+
scope: ['user', 'build', 'pipeline']
25+
},
26+
27+
handler: async (req, h) => {
28+
const { name: artifact, id: buildId } = req.params;
29+
const { credentials } = req.auth;
30+
const { canAccessPipeline } = req.server.plugins.pipelines;
31+
const { buildFactory, eventFactory } = req.server.app;
32+
33+
return buildFactory.get(buildId)
34+
.then(build => {
35+
if (!build) {
36+
throw boom.notFound('Build does not exist');
37+
}
38+
39+
return eventFactory.get(build.eventId);
40+
})
41+
.then(event => {
42+
if (!event) {
43+
throw boom.notFound('Event does not exist');
44+
}
45+
46+
return canAccessPipeline(credentials, event.pipelineId, 'pull', req.server.app);
47+
})
48+
.then(async () => {
49+
const token = jwt.sign({
50+
buildId, artifact, scope: ['user']
51+
}, config.authConfig.jwtPrivateKey, {
52+
algorithm: 'RS256',
53+
expiresIn: '10m',
54+
jwtid: uuidv4()
55+
});
56+
const baseUrl = `${config.ecosystem.store}/v1/builds/${buildId}/ARTIFACTS`;
57+
58+
// Fetch the manifest
59+
const manifest = await request({
60+
url: `${baseUrl}/manifest.txt?token=${token}`,
61+
method: 'GET'
62+
}).text();
63+
const manifestArray = manifest.trim().split('\n');
64+
65+
// Create a stream and set up archiver
66+
const archive = archiver('zip', { zlib: { level: 9 } });
67+
// PassThrough stream to make archiver readable by Hapi
68+
const passThrough = new PassThrough();
69+
70+
archive.on('error', (err) => {
71+
logger.error('Archiver error:', err);
72+
passThrough.emit('error', err); // Propagate the error to the PassThrough stream
73+
});
74+
passThrough.on('error', (err) => {
75+
logger.error('PassThrough stream error:', err);
76+
});
77+
78+
// Pipe the archive to PassThrough so it can be sent as a response
79+
archive.pipe(passThrough);
80+
81+
// Fetch the artifact files and append to the archive
82+
try {
83+
for (const file of manifestArray) {
84+
if (file) {
85+
const fileStream = request.stream(`${baseUrl}/${file}?token=${token}&type=download`);
86+
87+
fileStream.on('error', (err) => {
88+
logger.error(`Error downloading file: ${file}`, err);
89+
archive.emit('error', err);
90+
});
91+
92+
archive.append(fileStream, { name: file });
93+
}
94+
}
95+
// Finalize the archive after all files are appended
96+
archive.finalize();
97+
} catch (err) {
98+
logger.error('Error while streaming artifact files:', err);
99+
archive.emit('error', err);
100+
}
101+
102+
// Respond with the PassThrough stream (which is now readable by Hapi)
103+
return h.response(passThrough)
104+
.type('application/zip')
105+
.header('Content-Disposition', 'attachment; filename="SD_ARTIFACTS.zip"');
106+
})
107+
.catch(err => {
108+
throw err;
109+
});
110+
},
111+
validate: {
112+
params: joi.object({
113+
id: idSchema
114+
})
115+
}
116+
}
117+
});

‎plugins/builds/index.js

+2
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ const createRoute = require('./create');
1010
const stepGetRoute = require('./steps/get');
1111
const listStepsRoute = require('./steps/list');
1212
const artifactGetRoute = require('./artifacts/get');
13+
const artifactGetAllRoute = require('./artifacts/getAll');
1314
const artifactUnzipRoute = require('./artifacts/unzip');
1415
const stepUpdateRoute = require('./steps/update');
1516
const stepLogsRoute = require('./steps/logs');
@@ -508,6 +509,7 @@ const buildsPlugin = {
508509
tokenRoute(),
509510
metricsRoute(),
510511
artifactGetRoute(options),
512+
artifactGetAllRoute(options),
511513
artifactUnzipRoute()
512514
]);
513515
}

‎test/plugins/builds.test.js

+93
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
const urlLib = require('url');
44
const { assert } = require('chai');
55
const sinon = require('sinon');
6+
const fs = require('fs');
67
const hapi = require('@hapi/hapi');
78
const rewiremock = require('rewiremock/node');
89
const hoek = require('@hapi/hoek');
@@ -13,6 +14,7 @@ const testBuildWithSteps = require('./data/buildWithSteps.json');
1314
const testBuildsStatuses = require('./data/buildsStatuses.json');
1415
const testSecrets = require('./data/secrets.json');
1516
const testWorkflowGraphWithStages = require('./data/workflowGraphWithStages.json');
17+
const testManifest = fs.readFileSync(`${__dirname}/data/manifest.txt`);
1618
const rewireBuildsIndex = rewire('../../plugins/builds/triggers/helpers.js');
1719
/* eslint-disable no-underscore-dangle */
1820

@@ -6849,6 +6851,97 @@ describe('build plugin test', () => {
68496851
});
68506852
});
68516853

6854+
describe('GET /builds/{id}/artifacts', () => {
6855+
const id = 12345;
6856+
const buildMock = {
6857+
id: 123,
6858+
eventId: 1234
6859+
};
6860+
const eventMock = {
6861+
id: 1234,
6862+
pipelineId: 12345
6863+
};
6864+
const pipelineMock = {
6865+
id: 12345,
6866+
scmRepo: {
6867+
private: false
6868+
}
6869+
};
6870+
const expectedHeaders = {
6871+
'content-type': 'application/zip',
6872+
'content-disposition': 'attachment; filename="SD_ARTIFACTS.zip"',
6873+
'cache-control': 'no-cache'
6874+
};
6875+
let options;
6876+
6877+
beforeEach(() => {
6878+
options = {
6879+
url: `/builds/${id}/artifacts`,
6880+
auth: {
6881+
credentials: {
6882+
username: 'foo',
6883+
scope: ['user']
6884+
},
6885+
strategy: ['token']
6886+
}
6887+
};
6888+
buildFactoryMock.get.resolves(buildMock);
6889+
eventFactoryMock.get.resolves(eventMock);
6890+
pipelineFactoryMock.get.resolves(pipelineMock);
6891+
6892+
nock(logBaseUrl)
6893+
.persist()
6894+
.defaultReplyHeaders(expectedHeaders)
6895+
.get('/v1/builds/12345/ARTIFACTS/manifest.txt?token=sign')
6896+
.reply(200, testManifest);
6897+
nock(logBaseUrl)
6898+
.persist()
6899+
.defaultReplyHeaders(expectedHeaders)
6900+
.get(/\/v1\/builds\/12345\/ARTIFACTS\/.+?token=sign&type=download/)
6901+
.reply(200, testManifest);
6902+
});
6903+
6904+
it('returns 200 for a zipped artifact request', () => {
6905+
return server.inject(options).then(reply => {
6906+
assert.equal(reply.statusCode, 200);
6907+
assert.match(reply.headers, expectedHeaders);
6908+
});
6909+
});
6910+
6911+
it('returns 404 when build does not exist', () => {
6912+
buildFactoryMock.get.resolves(null);
6913+
6914+
return server.inject(options).then(reply => {
6915+
assert.equal(reply.statusCode, 404);
6916+
assert.equal(reply.result.message, 'Build does not exist');
6917+
});
6918+
});
6919+
6920+
it('returns 404 when event does not exist', () => {
6921+
eventFactoryMock.get.resolves(null);
6922+
6923+
return server.inject(options).then(reply => {
6924+
assert.equal(reply.statusCode, 404);
6925+
assert.equal(reply.result.message, 'Event does not exist');
6926+
});
6927+
});
6928+
6929+
it('returns 500 for server error', () => {
6930+
nock.disableNetConnect();
6931+
nock.cleanAll();
6932+
nock.enableNetConnect();
6933+
nock(logBaseUrl)
6934+
.defaultReplyHeaders(expectedHeaders)
6935+
.get('/v1/builds/12345/ARTIFACTS/manifest.txt?token=sign')
6936+
.reply(502);
6937+
6938+
return server.inject(options).then(reply => {
6939+
assert.equal(reply.statusCode, 500);
6940+
assert.equal(reply.result.message, 'An internal server error occurred');
6941+
});
6942+
});
6943+
});
6944+
68526945
describe('GET /builds/{id}/artifacts/{artifact}', () => {
68536946
const id = 12345;
68546947
const artifact = 'manifest';

‎test/plugins/data/manifest.txt

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
./steps.json
2+
./environment.json
3+
./artifacts/dog.jpeg
4+
./artifacts/dog.jpeg.tar
5+
./artifacts/dog.jpeg.tar.gz
6+
./artifacts/dog.jpeg.tgz
7+
./artifacts/example.gif
8+
./artifacts/fox-dog.jpg
9+
./artifacts/index.html
10+
./artifacts/jump.png
11+
./artifacts/jump.png.zip
12+
./artifacts/makefile
13+
./artifacts/sample-mov-file.mov
14+
./artifacts/sample-mp4-file.mp4
15+
./artifacts/sample.c
16+
./artifacts/sample.cpp
17+
./artifacts/screwdriver.yaml
18+
./artifacts/tox.ini
19+
./manifest.txt

0 commit comments

Comments
 (0)
Please sign in to comment.