Skip to content

Commit 546e653

Browse files
gerkaigerkai
committedOct 31, 2018
git-node: initial implementation of git node wpt (#294)
* git: initial implementation of git node wpt * support git node wpt all
1 parent 8c5ca82 commit 546e653

15 files changed

+467
-24
lines changed
 

‎.eslintignore

+1
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
.nyc_output
33
coverage
44
node_modules
5+
lib/wpt/templates

‎components/git/wpt.js

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
'use strict';
2+
3+
const yargs = require('yargs');
4+
const Request = require('../../lib/request');
5+
const CLI = require('../../lib/cli');
6+
const auth = require('../../lib/auth');
7+
const { WPTUpdater, HarnessUpdater } = require('../../lib/wpt');
8+
const { runPromise } = require('../../lib/run');
9+
10+
// TODO: read this from test/wpt/status/*.json
11+
const SUPPORTED_TESTS = ['url', 'console', 'encoding'];
12+
function builder(yargs) {
13+
return yargs
14+
.positional('name', {
15+
describe: 'Subset of the WPT to update, e.g. \'url\'',
16+
type: 'string',
17+
choices: ['all', 'harness'].concat(SUPPORTED_TESTS)
18+
})
19+
.options({
20+
nodedir: {
21+
describe: 'Path to the node.js project directory',
22+
type: 'string',
23+
default: '.'
24+
}
25+
});
26+
}
27+
28+
async function main(argv) {
29+
const { name, nodedir } = argv;
30+
const cli = new CLI();
31+
const credentials = await auth({
32+
github: true
33+
});
34+
const request = new Request(credentials);
35+
36+
const updaters = [];
37+
if (name === 'all') {
38+
updaters.push(new HarnessUpdater(cli, request, nodedir));
39+
for (const item of SUPPORTED_TESTS) {
40+
updaters.push(new WPTUpdater(item, cli, request, nodedir));
41+
}
42+
} else if (SUPPORTED_TESTS.includes(name)) {
43+
updaters.push(new WPTUpdater(name, cli, request, nodedir));
44+
} else if (name === 'harness') {
45+
updaters.push(new HarnessUpdater(cli, request, nodedir));
46+
} else {
47+
yargs.showHelp();
48+
return;
49+
}
50+
51+
for (const updater of updaters) {
52+
await updater.update();
53+
}
54+
updaters[0].updateLicense();
55+
}
56+
57+
function handler(argv) {
58+
runPromise(main(argv));
59+
}
60+
61+
module.exports = {
62+
command: 'wpt <name>',
63+
describe: 'Updates WPT suite',
64+
builder,
65+
handler
66+
};

‎docs/git-node.md

+17
Original file line numberDiff line numberDiff line change
@@ -221,3 +221,20 @@ V8 to `baseDir`.
221221
##### `--verbose`
222222

223223
Enable verbose output.
224+
225+
### `git node wpt`
226+
227+
Update or patch the Web Platform Tests in core.
228+
The updated files are placed under `./test/fixtures/wpt` by default. In addition
229+
to the assets, this also updates:
230+
231+
- `./test/fixtures/wpt/versions.json`
232+
- `./test/fixtures/wpt/README.md`
233+
- `./test/fixtures/wpt/LICENSE.md`
234+
235+
#### Example
236+
237+
```
238+
$ cd /path/to/node/project
239+
$ git node wpt url # Will update test/fixtures/wpt/url and related files
240+
```

‎lib/ci/ci_result_parser.js

+3-12
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ const {
2525
parseJobFromURL,
2626
CI_TYPES
2727
} = require('./ci_type_parser');
28+
const {
29+
flatten
30+
} = require('../utils');
2831
const qs = require('querystring');
2932
const _ = require('lodash');
3033
const chalk = require('chalk');
@@ -83,18 +86,6 @@ function getNodeName(url) {
8386
return parts[parts.length - 3];
8487
}
8588

86-
function flatten(arr) {
87-
let result = [];
88-
for (const item of arr) {
89-
if (Array.isArray(item)) {
90-
result = result.concat(flatten(item));
91-
} else {
92-
result.push(item);
93-
}
94-
}
95-
return result;
96-
}
97-
9889
class Job {
9990
constructor(cli, request, path, tree) {
10091
this.cli = cli;

‎lib/ci/ci_type_parser.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
const { parsePRFromURL } = require('../links');
44
const PRData = require('../pr_data');
5-
const { ascending } = require('../comp');
5+
const { ascending } = require('../utils');
66

77
const CI_URL_RE = /\/\/ci\.nodejs\.org(\S+)/mg;
88
const CI_DOMAIN = 'ci.nodejs.org';

‎lib/comp.js

-9
This file was deleted.

‎lib/github/tree.js

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
'use strict';
2+
3+
const COMMIT_QUERY = 'LastCommit';
4+
const TREE_QUERY = 'TreeEntries';
5+
const { flatten } = require('../utils');
6+
7+
class GitHubTree {
8+
constructor(cli, request, argv) {
9+
this.cli = cli;
10+
this.request = request;
11+
12+
this.owner = argv.owner;
13+
this.repo = argv.repo;
14+
this.branch = argv.branch;
15+
16+
if (argv.path.endsWith('/')) {
17+
this.path = argv.path.slice(0, argv.path - 1);
18+
} else {
19+
this.path = argv.path;
20+
}
21+
22+
this.lastCommit = argv.commit || null;
23+
}
24+
25+
get repoUrl() {
26+
const base = 'https://github.com';
27+
const { owner, repo } = this;
28+
return `${base}/${owner}/${repo}`;
29+
}
30+
31+
async _getLastCommit() {
32+
const { request, owner, repo, branch, path } = this;
33+
const data = await request.gql(COMMIT_QUERY, {
34+
owner,
35+
repo,
36+
branch,
37+
path
38+
});
39+
return data.repository.ref.target.history.nodes[0].oid;
40+
}
41+
42+
/**
43+
* @returns {string} the hash of the last commit in the tree
44+
*/
45+
async getLastCommit() {
46+
if (this.lastCommit) {
47+
return this.lastCommit;
48+
}
49+
this.lastCommit = await this._getLastCommit();
50+
return this.lastCommit;
51+
}
52+
53+
getPermanentUrl() {
54+
if (!this.lastCommit) {
55+
throw new Error('Call await tree.getLastCommit() first');
56+
}
57+
const commit = this.lastCommit;
58+
return `${this.repoUrl}/tree/${commit.slice(0, 10)}/${this.path}`;
59+
}
60+
61+
async text(assetPath) {
62+
await this.getLastCommit();
63+
const url = this.getAssetUrl(assetPath);
64+
return this.request.text(url);
65+
}
66+
67+
/**
68+
* Get the url of an asset. If the assetPath starts with `/`,
69+
* it will be treated as an absolute path and the
70+
* the path of the tree will not be prefixed in the url.
71+
* @param {string} assetPath
72+
*/
73+
getAssetUrl(assetPath) {
74+
if (!this.lastCommit) {
75+
throw new Error('Call await tree.getLastCommit() first');
76+
}
77+
const base = 'https://raw.githubusercontent.com';
78+
const { owner, repo, lastCommit, path } = this;
79+
let prefix = `${base}/${owner}/${repo}/${lastCommit}`;
80+
if (assetPath.startsWith('/')) { // absolute
81+
return `${prefix}/${assetPath}`;
82+
} else {
83+
return `${prefix}/${path}/${assetPath}`;
84+
}
85+
}
86+
87+
/**
88+
* Get a list of files inside the tree (recursively).
89+
* The returned file names will be relative to the path of the tree,
90+
* e.g. `url/resources/data.json` in a tree with `url` as path
91+
* will be `resources/data.json`
92+
*
93+
* @returns {{name: string, type: string}[]}
94+
*/
95+
async getFiles(path) {
96+
if (!path) {
97+
path = this.path;
98+
}
99+
let lastCommit = this.lastCommit;
100+
if (!lastCommit) {
101+
lastCommit = await this.getLastCommit();
102+
}
103+
const { request, owner, repo } = this;
104+
105+
const expression = `${lastCommit}:${path}`;
106+
this.cli.updateSpinner(`Querying files for ${path}`);
107+
const data = await request.gql(TREE_QUERY, {
108+
owner,
109+
repo,
110+
expression
111+
});
112+
const files = data.repository.object.entries;
113+
114+
const dirs = files.filter((file) => file.type === 'tree');
115+
const nondirs = files.filter((file) => file.type !== 'tree');
116+
117+
if (dirs.length) {
118+
const expanded = await Promise.all(
119+
dirs.map((dir) =>
120+
this.getFiles(`${path}/${dir.name}`)
121+
.then(files => files.map(
122+
({ name, type }) => ({ name: `${dir.name}/${name}`, type })
123+
))
124+
)
125+
);
126+
return nondirs.concat(flatten(expanded));
127+
} else {
128+
return nondirs;
129+
}
130+
}
131+
132+
getCacheKey() {
133+
const { branch, owner, repo, path } = this;
134+
return `tree-${owner}-${repo}-${branch}-${clean(path)}`;
135+
}
136+
}
137+
138+
function clean(path) {
139+
if (!path) {
140+
return '';
141+
}
142+
return path.replace('/', '-');
143+
}
144+
145+
// Uncomment this when testing to avoid extra network costs
146+
// const Cache = require('../cache');
147+
// const treeCache = new Cache();
148+
149+
// treeCache.wrap(GitHubTree, {
150+
// _getLastCommit() {
151+
// return { key: `${this.getCacheKey()}-commit`, ext: '.json' };
152+
// },
153+
// getFiles(path) {
154+
// return {
155+
// key: `${this.getCacheKey()}-${clean(path)}-files`,
156+
// ext: '.json'
157+
// };
158+
// },
159+
// text(assetPath) {
160+
// return { key: `${this.getCacheKey()}-${clean(assetPath)}`, ext: '.txt' };
161+
// }
162+
// });
163+
// treeCache.enable();
164+
165+
module.exports = GitHubTree;

‎lib/queries/LastCommit.gql

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
query LastCommit($owner: String!, $repo: String!, $branch: String!, $path: String) {
2+
repository(owner: $owner, name: $repo) {
3+
ref(qualifiedName: $branch) {
4+
target {
5+
... on Commit {
6+
history(first: 1, path: $path) {
7+
nodes {
8+
oid
9+
messageHeadline
10+
}
11+
}
12+
}
13+
}
14+
}
15+
}
16+
}

‎lib/queries/TreeEntries.gql

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
query TreeEntries($owner: String!, $repo: String!, $expression: String!) {
2+
repository(owner: $owner, name: $repo) {
3+
object(expression: $expression) {
4+
... on Tree {
5+
entries {
6+
name
7+
type
8+
}
9+
}
10+
}
11+
}
12+
}

‎lib/reviews.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ const {
33
PENDING, COMMENTED, APPROVED, CHANGES_REQUESTED, DISMISSED
44
} = require('./review_state');
55
const { isCollaborator } = require('./collaborators');
6-
const { ascending } = require('./comp');
6+
const { ascending } = require('./utils');
77
const LGTM_RE = /^lgtm\W?$/i;
88
const FROM_REVIEW = 'review';
99
const FROM_COMMENT = 'comment';

‎lib/utils.js

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
'use strict';
2+
3+
exports.ascending = function(a, b) {
4+
return a < b ? -1 : 1;
5+
};
6+
7+
exports.descending = function(a, b) {
8+
return a > b ? -1 : 1;
9+
};
10+
11+
function flatten(arr) {
12+
let result = [];
13+
for (const item of arr) {
14+
if (Array.isArray(item)) {
15+
result = result.concat(flatten(item));
16+
} else {
17+
result.push(item);
18+
}
19+
}
20+
return result;
21+
}
22+
exports.flatten = flatten;

‎lib/wpt/harness.js

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
'use strict';
2+
3+
const {
4+
WPTUpdater
5+
} = require('./');
6+
7+
class HarnessUpdater extends WPTUpdater {
8+
constructor(cli, request, nodedir) {
9+
super('resources', cli, request, nodedir);
10+
}
11+
12+
async update() {
13+
const harnessPath = this.fixtures(this.path, 'testharness.js');
14+
this.cli.startSpinner(`Downloading ${harnessPath}...`);
15+
await this.pullTextFile(this.fixtures(this.path), 'testharness.js');
16+
this.cli.stopSpinner(`Downloaded ${harnessPath}`);
17+
const lastCommit = this.tree.lastCommit;
18+
await this.updateVersions({
19+
harness: { commit: lastCommit, path: 'resources' }
20+
});
21+
}
22+
}
23+
24+
module.exports = HarnessUpdater;

‎lib/wpt/index.js

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
'use strict';
2+
3+
const GitHubTree = require('../github/tree');
4+
const path = require('path');
5+
const { writeFile, readJson, writeJson, readFile } = require('../file');
6+
const _ = require('lodash');
7+
8+
class WPTUpdater {
9+
constructor(path, cli, request, nodedir) {
10+
this.path = path;
11+
this.nodedir = nodedir;
12+
13+
this.cli = cli;
14+
this.request = request;
15+
this.treeParams = {
16+
owner: 'web-platform-tests',
17+
repo: 'wpt',
18+
branch: 'master',
19+
path
20+
};
21+
this.tree = new GitHubTree(cli, request, this.treeParams);
22+
this.assets = [];
23+
}
24+
25+
templates(...args) {
26+
return _.template(readFile(path.join(__dirname, 'templates', ...args)));
27+
}
28+
29+
fixtures(...args) {
30+
return path.join(this.nodedir, 'test', 'fixtures', 'wpt', ...args);
31+
}
32+
33+
// If filepath starts with '/', the path is relative to WPT project root,
34+
// otherwise it's relative to the path of this updater
35+
async pullTextFile(dest, filepath) {
36+
const content = await this.tree.text(filepath);
37+
const filename = path.join(dest, filepath);
38+
writeFile(filename, content);
39+
this.cli.updateSpinner(`Downloaded ${filename}`);
40+
}
41+
42+
async pullAllAssets() {
43+
const fixtures = this.fixtures(this.path);
44+
this.cli.separator(`Writing test fixtures to ${fixtures}...`);
45+
46+
this.cli.startSpinner('Querying asset list...');
47+
const assets = this.assets = await this.tree.getFiles();
48+
this.cli.stopSpinner(
49+
`Downloaded asset list, ${assets.length} files to write.`
50+
);
51+
52+
this.cli.startSpinner('Pulling assets...');
53+
await Promise.all(assets.map(
54+
(asset) => this.pullTextFile(fixtures, asset.name)
55+
));
56+
this.cli.stopSpinner(`Downloaded ${assets.length} assets.`);
57+
58+
return assets;
59+
}
60+
61+
getTreeUrl(path, commit) {
62+
const params = Object.assign({}, this.treeParams, { commit, path });
63+
const tree = new GitHubTree(this.cli, this.request, params);
64+
return tree.getPermanentUrl();
65+
}
66+
67+
/**
68+
* @param {string} nodedir
69+
* @param {Object<string, {commit: string, path: string}>} updated
70+
*/
71+
async updateVersions(updated) {
72+
const versionsPath = this.fixtures('versions.json');
73+
const readmePath = this.fixtures('README.md');
74+
75+
this.cli.startSpinner('Updating versions.json ...');
76+
const versions = readJson(versionsPath);
77+
Object.assign(versions, updated);
78+
writeJson(versionsPath, versions);
79+
this.cli.stopSpinner(`Updated ${versionsPath}`);
80+
81+
const urlMap = Object.keys(versions).map(
82+
(key) => [key, this.getTreeUrl(versions[key].path, versions[key].commit)]
83+
);
84+
85+
this.cli.startSpinner('Updating README ...');
86+
const readme = this.templates('README.md')({map: urlMap});
87+
writeFile(readmePath, readme);
88+
this.cli.stopSpinner(`Updated ${readmePath}`);
89+
}
90+
91+
async updateLicense() {
92+
this.cli.startSpinner('Updating license...');
93+
await this.pullTextFile(this.fixtures(), '/LICENSE.md');
94+
this.cli.stopSpinner(`Updated ${this.fixtures('LICENSE.md')}.`);
95+
}
96+
97+
async update() {
98+
await this.pullAllAssets();
99+
const lastCommit = await this.tree.getLastCommit();
100+
await this.updateVersions({
101+
[this.path]: {
102+
commit: lastCommit,
103+
path: this.path
104+
}
105+
});
106+
}
107+
}
108+
109+
class HarnessUpdater extends WPTUpdater {
110+
constructor(cli, request, nodedir) {
111+
super('resources', cli, request, nodedir);
112+
}
113+
114+
async update() { // override
115+
const harnessPath = this.fixtures(this.path, 'testharness.js');
116+
this.cli.startSpinner(`Downloading ${harnessPath}...`);
117+
await this.pullTextFile(this.fixtures(this.path), 'testharness.js');
118+
this.cli.stopSpinner(`Downloaded ${harnessPath}`);
119+
const lastCommit = this.tree.lastCommit;
120+
await this.updateVersions({
121+
harness: { commit: lastCommit, path: 'resources' }
122+
});
123+
}
124+
}
125+
126+
module.exports = {
127+
WPTUpdater,
128+
HarnessUpdater
129+
};

‎lib/wpt/templates/README.md

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Web Platform Test Fixtures
2+
3+
The fixtures here are pulled from https://github.com/web-platform-tests/wpt
4+
using [`git node wpt`](https://github.com/nodejs/node-core-utils/blob/master/docs/git-node.md#git-node-wpt)
5+
6+
7+
Last update:
8+
<% for (const [path, url] of map) { %>
9+
- <%= path %>: <%= url %><% } %>

‎test/unit/comp.test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
'use strict';
22

33
const assert = require('assert');
4-
const { ascending, descending } = require('../../lib/comp');
4+
const { ascending, descending } = require('../../lib/utils');
55

66
const arr = [
77
'2017-10-30T15:47:52Z',

0 commit comments

Comments
 (0)
Please sign in to comment.