Skip to content

Commit 083faaa

Browse files
rwjbluerwjblue
authored
Merge pull request #793 from glimmerjs/upgrade-simple-html-tokenizer
Upgrade simple-html-tokenizer
2 parents 0e18d30 + f2fbdb5 commit 083faaa

File tree

5 files changed

+216
-113
lines changed

5 files changed

+216
-113
lines changed

‎build/broccoli/build-tests.js

+21-24
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ module.exports = function(tsTree, jsTree, packagesTree) {
3333
let nodeTests = transpileNodeTestsToCommonJS(jsTree);
3434

3535
return merge([browserTests, nodeTests]);
36-
}
36+
};
3737

3838
function transpileBrowserTestsToAMD(tsTree, jsTree) {
3939
let testTree = funnel(jsTree, {
@@ -58,16 +58,12 @@ function transpileBrowserTestsToAMD(tsTree, jsTree) {
5858

5959
function transpileNodeTestsToCommonJS(jsTree) {
6060
let testTree = funnel(jsTree, {
61-
include: [
62-
'@glimmer/**/test/**/*-node-test.js'
63-
]
61+
include: ['@glimmer/**/test/**/*-node-test.js']
6462
});
6563

6664
return babel(testTree, {
6765
sourceMaps: 'inline',
68-
plugins: [
69-
'transform-es2015-modules-commonjs'
70-
]
66+
plugins: ['transform-es2015-modules-commonjs']
7167
});
7268
}
7369

@@ -80,9 +76,7 @@ function generateTSLintTests(tsTree) {
8076

8177
function includeGlimmerAMD(packages) {
8278
let libAMD = funnel(packages, {
83-
include: [
84-
'@glimmer/*/dist/amd/es5/*.js'
85-
]
79+
include: ['@glimmer/*/dist/amd/es5/*.js']
8680
});
8781

8882
return concat(libAMD, {
@@ -91,10 +85,12 @@ function includeGlimmerAMD(packages) {
9185
}
9286

9387
function includeVendorDependencies() {
94-
let simpleHTMLTokenizer = funnel('node_modules/simple-html-tokenizer/dist/es6', {
95-
include: ['*.js'],
96-
destDir: 'simple-html-tokenizer'
97-
});
88+
let simpleHTMLTokenizer = funnel(
89+
'node_modules/simple-html-tokenizer/dist/es6',
90+
{
91+
destDir: 'simple-html-tokenizer'
92+
}
93+
);
9894

9995
let simpleDOM = new Rollup('node_modules/simple-dom/lib', {
10096
rollup: {
@@ -106,13 +102,18 @@ function includeVendorDependencies() {
106102
}
107103
});
108104

109-
let transpiled = transpileES6(merge([simpleHTMLTokenizer, handlebarsInlinedTrees.compiler, simpleDOM]), 'test-dependencies', {
110-
avoidDefine: false
111-
});
105+
let transpiled = transpileES6(
106+
merge([simpleHTMLTokenizer, handlebarsInlinedTrees.compiler, simpleDOM]),
107+
'test-dependencies',
108+
{
109+
avoidDefine: false
110+
}
111+
);
112112

113113
return concat(transpiled, {
114+
inputFiles: ['**/*.js'],
114115
outputFile: 'assets/vendor.js'
115-
})
116+
});
116117
}
117118

118119
function includeTestHarness() {
@@ -122,19 +123,15 @@ function includeTestHarness() {
122123

123124
let loaderPath = path.parse(require.resolve('loader.js'));
124125
let loader = funnel(loaderPath.dir, {
125-
files: [ loaderPath.base ],
126+
files: [loaderPath.base],
126127
destDir: '/assets'
127128
});
128129

129130
let qunit = funnel(path.join(require.resolve('qunitjs'), '..'), {
130131
destDir: 'assets/'
131132
});
132133

133-
let harnessTrees = [
134-
html,
135-
loader,
136-
qunit
137-
];
134+
let harnessTrees = [html, loader, qunit];
138135

139136
return merge(harnessTrees);
140137
}

‎package.json

+3-5
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,13 @@
2727
"yarn:unlink": "node bin/yarn-link-all --unlink"
2828
},
2929
"ember-addon": {
30-
"paths": [
31-
"build/symlink-dependencies"
32-
]
30+
"paths": ["build/symlink-dependencies"]
3331
},
3432
"dependencies": {
3533
"babel-plugin-debug-macros": "^0.1.11",
3634
"handlebars": "^4.0.6",
3735
"simple-dom": "^0.3.0",
38-
"simple-html-tokenizer": "^0.4.1"
36+
"simple-html-tokenizer": "0.5.0"
3937
},
4038
"devDependencies": {
4139
"@glimmer/build": "^0.8.2",
@@ -86,4 +84,4 @@
8684
"internal": ":house: Internal"
8785
}
8886
}
89-
}
87+
}

‎packages/@glimmer/syntax/lib/parser.ts

+35-30
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import {
22
EventedTokenizer,
33
EntityParser,
44
HTML5NamedCharRefs as namedCharRefs
5-
} from "simple-html-tokenizer";
6-
import { Program } from "./types/nodes";
7-
import * as AST from "./types/nodes";
5+
} from 'simple-html-tokenizer';
6+
import { Program } from './types/nodes';
7+
import * as AST from './types/nodes';
88
import * as HandlebarsAST from './types/handlebars-ast';
99
import { Option } from '@glimmer/interfaces';
1010
import { assert, expect } from '@glimmer/util';
@@ -33,46 +33,49 @@ export interface Attribute {
3333
valueStartColumn: number;
3434
}
3535

36-
export class Parser {
36+
export abstract class Parser {
3737
protected elementStack: Element[] = [];
3838
private source: string[];
3939
public currentAttribute: Option<Attribute> = null;
40-
public currentNode: Option<AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'>> = null;
40+
public currentNode: Option<
41+
AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'>
42+
> = null;
4143
public tokenizer = new EventedTokenizer(this, entityParser);
4244

4345
constructor(source: string) {
44-
this.tokenizer.states.tagOpen = function(this: EventedTokenizer) {
45-
let char = this.consume();
46-
if (char === "!") {
47-
this['state'] = 'markupDeclaration';
48-
} else if (char === "/") {
49-
this['state'] = 'endTagOpen';
50-
} else if (/[A-Za-z]/.test(char)) {
51-
this['state'] = 'tagName';
52-
this['delegate'].beginStartTag();
53-
this['delegate'].appendToTagName(char);
54-
}
55-
};
56-
57-
this.tokenizer.states.endTagOpen = function(this: EventedTokenizer) {
58-
let char = this.consume();
59-
if (/[A-Za-z]/.test(char)) {
60-
this['state'] = 'tagName';
61-
this['delegate'].beginEndTag();
62-
this['delegate'].appendToTagName(char);
63-
}
64-
};
65-
6646
this.source = source.split(/(?:\r\n?|\n)/g);
6747
}
6848

49+
abstract reset(): void;
50+
abstract finishData(): void;
51+
abstract tagOpen(): void;
52+
abstract beginData(): void;
53+
abstract appendToData(char: string): void;
54+
abstract beginStartTag(): void;
55+
abstract appendToTagName(char: string): void;
56+
abstract beginAttribute(): void;
57+
abstract appendToAttributeName(char: string): void;
58+
abstract beginAttributeValue(quoted: boolean): void;
59+
abstract appendToAttributeValue(char: string): void;
60+
abstract finishAttributeValue(): void;
61+
abstract markTagAsSelfClosing(): void;
62+
abstract beginEndTag(): void;
63+
abstract finishTag(): void;
64+
abstract beginComment(): void;
65+
abstract appendToCommentData(char: string): void;
66+
abstract finishComment(): void;
67+
abstract reportSyntaxError(error: string): void;
68+
6969
get currentAttr(): Attribute {
7070
return expect(this.currentAttribute, 'expected attribute');
7171
}
7272

7373
get currentTag(): Tag<'StartTag' | 'EndTag'> {
7474
let node = this.currentNode;
75-
assert(node && (node.type === 'StartTag' || node.type === 'EndTag'), 'expected tag');
75+
assert(
76+
node && (node.type === 'StartTag' || node.type === 'EndTag'),
77+
'expected tag'
78+
);
7679
return node as Tag<'StartTag' | 'EndTag'>;
7780
}
7881

@@ -98,7 +101,6 @@ export class Parser {
98101
let node = this.currentNode;
99102
assert(node && node.type === 'TextNode', 'expected a text node');
100103
return node as AST.TextNode;
101-
102104
}
103105

104106
acceptNode(node: HandlebarsAST.Program): Program;
@@ -111,7 +113,10 @@ export class Parser {
111113
return this.elementStack[this.elementStack.length - 1];
112114
}
113115

114-
sourceForNode(node: HandlebarsAST.Node, endNode?: { loc: HandlebarsAST.SourceLocation }): string {
116+
sourceForNode(
117+
node: HandlebarsAST.Node,
118+
endNode?: { loc: HandlebarsAST.SourceLocation }
119+
): string {
115120
let firstLine = node.loc.start.line - 1;
116121
let currentLine = firstLine - 1;
117122
let firstColumn = node.loc.start.column;

0 commit comments

Comments
 (0)