Skip to content

Commit 4e0cef1

Browse files
panvapanva
committed
test: add WPTRunner support for variants and generating WPT reports
PR-URL: #46498 Reviewed-By: Yagiz Nizipli <[email protected]> Reviewed-By: Richard Lau <[email protected]> Backport-PR-URL: #46768
1 parent 908c4df commit 4e0cef1

File tree

2 files changed

+191
-41
lines changed

2 files changed

+191
-41
lines changed

‎Makefile

+6
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,12 @@ test-message: test-build
573573
test-wpt: all
574574
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
575575

576+
.PHONY: test-wpt-report
577+
test-wpt-report:
578+
$(RM) -r out/wpt
579+
mkdir -p out/wpt
580+
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
581+
576582
.PHONY: test-simple
577583
test-simple: | cctest # Depends on 'all'.
578584
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential

‎test/common/wpt.js

+185-41
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,119 @@ const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
88
const events = require('events');
9+
const os = require('os');
910
const { inspect } = require('util');
1011
const { Worker } = require('worker_threads');
1112

13+
function getBrowserProperties() {
14+
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
15+
const release = /^\d+\.\d+\.\d+$/.test(version);
16+
const browser = {
17+
browser_channel: release ? 'stable' : 'experimental',
18+
browser_version: version,
19+
};
20+
21+
return browser;
22+
}
23+
24+
/**
25+
* Return one of three expected values
26+
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
27+
*/
28+
function getOs() {
29+
switch (os.type()) {
30+
case 'Linux':
31+
return 'linux';
32+
case 'Darwin':
33+
return 'mac';
34+
case 'Windows_NT':
35+
return 'win';
36+
default:
37+
throw new Error('Unsupported os.type()');
38+
}
39+
}
40+
41+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
42+
function sanitizeUnpairedSurrogates(str) {
43+
return str.replace(
44+
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
45+
function(_, low, prefix, high) {
46+
let output = prefix || ''; // Prefix may be undefined
47+
const string = low || high; // Only one of these alternates can match
48+
for (let i = 0; i < string.length; i++) {
49+
output += codeUnitStr(string[i]);
50+
}
51+
return output;
52+
});
53+
}
54+
55+
function codeUnitStr(char) {
56+
return 'U+' + char.charCodeAt(0).toString(16);
57+
}
58+
59+
class WPTReport {
60+
constructor() {
61+
this.results = [];
62+
this.time_start = Date.now();
63+
}
64+
65+
addResult(name, status) {
66+
const result = {
67+
test: name,
68+
status,
69+
subtests: [],
70+
addSubtest(name, status, message) {
71+
const subtest = {
72+
status,
73+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
74+
name: sanitizeUnpairedSurrogates(name),
75+
};
76+
if (message) {
77+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
78+
subtest.message = sanitizeUnpairedSurrogates(message);
79+
}
80+
this.subtests.push(subtest);
81+
return subtest;
82+
},
83+
};
84+
this.results.push(result);
85+
return result;
86+
}
87+
88+
write() {
89+
this.time_end = Date.now();
90+
this.results = this.results.filter((result) => {
91+
return result.status === 'SKIP' || result.subtests.length !== 0;
92+
}).map((result) => {
93+
const url = new URL(result.test, 'http://wpt');
94+
url.pathname = url.pathname.replace(/\.js$/, '.html');
95+
result.test = url.href.slice(url.origin.length);
96+
return result;
97+
});
98+
99+
if (fs.existsSync('out/wpt/wptreport.json')) {
100+
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
101+
this.results = [...prev.results, ...this.results];
102+
this.time_start = prev.time_start;
103+
this.time_end = Math.max(this.time_end, prev.time_end);
104+
this.run_info = prev.run_info;
105+
} else {
106+
/**
107+
* Return required and some optional properties
108+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
109+
*/
110+
this.run_info = {
111+
product: 'node.js',
112+
...getBrowserProperties(),
113+
revision: process.env.WPT_REVISION || 'unknown',
114+
os: getOs(),
115+
};
116+
}
117+
118+
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
119+
}
120+
}
121+
12122
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
13123
// TODO: get rid of this half-baked harness in favor of the one
14124
// pulled from WPT
@@ -313,6 +423,10 @@ class WPTRunner {
313423
this.unexpectedFailures = [];
314424

315425
this.scriptsModifier = null;
426+
427+
if (process.env.WPT_REPORT != null) {
428+
this.report = new WPTReport();
429+
}
316430
}
317431

318432
/**
@@ -339,18 +453,27 @@ class WPTRunner {
339453
this.scriptsModifier = modifier;
340454
}
341455

342-
get fullInitScript() {
343-
if (this.initScript === null && this.dummyGlobalThisScript === null) {
456+
fullInitScript(hasSubsetScript, locationSearchString) {
457+
let { initScript } = this;
458+
if (hasSubsetScript || locationSearchString) {
459+
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
460+
}
461+
462+
if (locationSearchString) {
463+
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
464+
}
465+
466+
if (initScript === null && this.dummyGlobalThisScript === null) {
344467
return null;
345468
}
346469

347-
if (this.initScript === null) {
470+
if (initScript === null) {
348471
return this.dummyGlobalThisScript;
349472
} else if (this.dummyGlobalThisScript === null) {
350-
return this.initScript;
473+
return initScript;
351474
}
352475

353-
return `${this.dummyGlobalThisScript}\n\n//===\n${this.initScript}`;
476+
return `${this.dummyGlobalThisScript}\n\n//===\n${initScript}`;
354477
}
355478

356479
/**
@@ -398,15 +521,20 @@ class WPTRunner {
398521
for (const spec of queue) {
399522
const testFileName = spec.filename;
400523
const content = spec.getContent();
401-
const meta = spec.title = this.getMeta(content);
524+
const meta = spec.meta = this.getMeta(content);
402525

403526
const absolutePath = spec.getAbsolutePath();
404527
const relativePath = spec.getRelativePath();
405528
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
406529
const scriptsToRun = [];
530+
let hasSubsetScript = false;
531+
407532
// Scripts specified with the `// META: script=` header
408533
if (meta.script) {
409534
for (const script of meta.script) {
535+
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
536+
hasSubsetScript = true;
537+
}
410538
const obj = {
411539
filename: this.resource.toRealFilePath(relativePath, script),
412540
code: this.resource.read(relativePath, script, false)
@@ -423,33 +551,43 @@ class WPTRunner {
423551
this.scriptsModifier?.(obj);
424552
scriptsToRun.push(obj);
425553

426-
const workerPath = path.join(__dirname, 'wpt/worker.js');
427-
const worker = new Worker(workerPath, {
428-
execArgv: this.flags,
429-
workerData: {
430-
testRelativePath: relativePath,
431-
wptRunner: __filename,
432-
wptPath: this.path,
433-
initScript: this.fullInitScript,
434-
harness: {
435-
code: fs.readFileSync(harnessPath, 'utf8'),
436-
filename: harnessPath,
554+
/**
555+
* Example test with no META variant
556+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
557+
*
558+
* Example test with multiple META variants
559+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
560+
*/
561+
for (const variant of meta.variant || ['']) {
562+
const workerPath = path.join(__dirname, 'wpt/worker.js');
563+
const worker = new Worker(workerPath, {
564+
execArgv: this.flags,
565+
workerData: {
566+
testRelativePath: relativePath,
567+
wptRunner: __filename,
568+
wptPath: this.path,
569+
initScript: this.fullInitScript(hasSubsetScript, variant),
570+
harness: {
571+
code: fs.readFileSync(harnessPath, 'utf8'),
572+
filename: harnessPath,
573+
},
574+
scriptsToRun,
437575
},
438-
scriptsToRun,
439-
},
440-
});
441-
this.workers.set(testFileName, worker);
442-
443-
worker.on('message', (message) => {
444-
switch (message.type) {
445-
case 'result':
446-
return this.resultCallback(testFileName, message.result);
447-
case 'completion':
448-
return this.completionCallback(testFileName, message.status);
449-
default:
450-
throw new Error(`Unexpected message from worker: ${message.type}`);
451-
}
452-
});
576+
});
577+
this.workers.set(testFileName, worker);
578+
579+
let reportResult;
580+
worker.on('message', (message) => {
581+
switch (message.type) {
582+
case 'result':
583+
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
584+
return this.resultCallback(testFileName, message.result, reportResult);
585+
case 'completion':
586+
return this.completionCallback(testFileName, message.status);
587+
default:
588+
throw new Error(`Unexpected message from worker: ${message.type}`);
589+
}
590+
});
453591

454592
worker.on('error', (err) => {
455593
if (!this.inProgress.has(testFileName)) {
@@ -470,7 +608,8 @@ class WPTRunner {
470608
this.inProgress.delete(testFileName);
471609
});
472610

473-
await events.once(worker, 'exit').catch(() => {});
611+
await events.once(worker, 'exit').catch(() => {});
612+
}
474613
}
475614

476615
process.on('exit', () => {
@@ -529,6 +668,8 @@ class WPTRunner {
529668
}
530669
}
531670

671+
this.report?.write();
672+
532673
const ran = total - skipped;
533674
const passed = ran - expectedFailures - failures.length;
534675
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
@@ -552,8 +693,7 @@ class WPTRunner {
552693

553694
getTestTitle(filename) {
554695
const spec = this.specMap.get(filename);
555-
const title = spec.meta && spec.meta.title;
556-
return title ? `${filename} : ${title}` : filename;
696+
return spec.meta?.title || filename;
557697
}
558698

559699
// Map WPT test status to strings
@@ -579,14 +719,14 @@ class WPTRunner {
579719
* @param {string} filename
580720
* @param {Test} test The Test object returned by WPT harness
581721
*/
582-
resultCallback(filename, test) {
722+
resultCallback(filename, test, reportResult) {
583723
const status = this.getTestStatus(test.status);
584724
const title = this.getTestTitle(filename);
585725
console.log(`---- ${title} ----`);
586726
if (status !== kPass) {
587-
this.fail(filename, test, status);
727+
this.fail(filename, test, status, reportResult);
588728
} else {
589-
this.succeed(filename, test, status);
729+
this.succeed(filename, test, status, reportResult);
590730
}
591731
}
592732

@@ -634,11 +774,12 @@ class WPTRunner {
634774
}
635775
}
636776

637-
succeed(filename, test, status) {
777+
succeed(filename, test, status, reportResult) {
638778
console.log(`[${status.toUpperCase()}] ${test.name}`);
779+
reportResult?.addSubtest(test.name, 'PASS');
639780
}
640781

641-
fail(filename, test, status) {
782+
fail(filename, test, status, reportResult) {
642783
const spec = this.specMap.get(filename);
643784
const expected = spec.failedTests.includes(test.name);
644785
if (expected) {
@@ -654,6 +795,9 @@ class WPTRunner {
654795
const command = `${process.execPath} ${process.execArgv}` +
655796
` ${require.main.filename} ${filename}`;
656797
console.log(`Command: ${command}\n`);
798+
799+
reportResult?.addSubtest(test.name, 'FAIL', test.message);
800+
657801
this.addTestResult(filename, {
658802
name: test.name,
659803
expected,
@@ -683,7 +827,7 @@ class WPTRunner {
683827
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
684828
const key = parts[1];
685829
const value = parts[2];
686-
if (key === 'script') {
830+
if (key === 'script' || key === 'variant') {
687831
if (result[key]) {
688832
result[key].push(value);
689833
} else {

0 commit comments

Comments
 (0)