Upgrade simple-html-tokenizer

build/broccoli/build-tests.js

@@ -33,7 +33,7 @@ module.exports = function(tsTree, jsTree, packagesTree) {
   let nodeTests = transpileNodeTestsToCommonJS(jsTree);
 
   return merge([browserTests, nodeTests]);
-}
+};
 
 function transpileBrowserTestsToAMD(tsTree, jsTree) {
   let testTree = funnel(jsTree, {
@@ -58,16 +58,12 @@ function transpileBrowserTestsToAMD(tsTree, jsTree) {
 
 function transpileNodeTestsToCommonJS(jsTree) {
   let testTree = funnel(jsTree, {
-    include: [
-      '@glimmer/**/test/**/*-node-test.js'
-    ]
+    include: ['@glimmer/**/test/**/*-node-test.js']
   });
 
   return babel(testTree, {
     sourceMaps: 'inline',
-    plugins: [
-      'transform-es2015-modules-commonjs'
-    ]
+    plugins: ['transform-es2015-modules-commonjs']
   });
 }
 
@@ -80,9 +76,7 @@ function generateTSLintTests(tsTree) {
 
 function includeGlimmerAMD(packages) {
   let libAMD = funnel(packages, {
-    include: [
-      '@glimmer/*/dist/amd/es5/*.js'
-    ]
+    include: ['@glimmer/*/dist/amd/es5/*.js']
   });
 
   return concat(libAMD, {
@@ -91,10 +85,12 @@ function includeGlimmerAMD(packages) {
 }
 
 function includeVendorDependencies() {
-  let simpleHTMLTokenizer = funnel('node_modules/simple-html-tokenizer/dist/es6', {
-    include: ['*.js'],
-    destDir: 'simple-html-tokenizer'
-  });
+  let simpleHTMLTokenizer = funnel(
+    'node_modules/simple-html-tokenizer/dist/es6',
+    {
+      destDir: 'simple-html-tokenizer'
+    }
+  );
 
   let simpleDOM = new Rollup('node_modules/simple-dom/lib', {
     rollup: {
@@ -106,13 +102,18 @@ function includeVendorDependencies() {
     }
   });
 
-  let transpiled = transpileES6(merge([simpleHTMLTokenizer, handlebarsInlinedTrees.compiler, simpleDOM]), 'test-dependencies', {
-    avoidDefine: false
-  });
+  let transpiled = transpileES6(
+    merge([simpleHTMLTokenizer, handlebarsInlinedTrees.compiler, simpleDOM]),
+    'test-dependencies',
+    {
+      avoidDefine: false
+    }
+  );
 
   return concat(transpiled, {
+    inputFiles: ['**/*.js'],
     outputFile: 'assets/vendor.js'
-  })
+  });
 }
 
 function includeTestHarness() {
@@ -122,19 +123,15 @@ function includeTestHarness() {
 
   let loaderPath = path.parse(require.resolve('loader.js'));
   let loader = funnel(loaderPath.dir, {
-    files: [ loaderPath.base ],
+    files: [loaderPath.base],
     destDir: '/assets'
   });
 
   let qunit = funnel(path.join(require.resolve('qunitjs'), '..'), {
     destDir: 'assets/'
   });
 
-  let harnessTrees = [
-    html,
-    loader,
-    qunit
-  ];
+  let harnessTrees = [html, loader, qunit];
 
   return merge(harnessTrees);
 }

package.json

@@ -27,15 +27,13 @@
     "yarn:unlink": "node bin/yarn-link-all --unlink"
   },
   "ember-addon": {
-    "paths": [
-      "build/symlink-dependencies"
-    ]
+    "paths": ["build/symlink-dependencies"]
   },
   "dependencies": {
     "babel-plugin-debug-macros": "^0.1.11",
     "handlebars": "^4.0.6",
     "simple-dom": "^0.3.0",
-    "simple-html-tokenizer": "^0.4.1"
+    "simple-html-tokenizer": "0.5.0"
   },
   "devDependencies": {
     "@glimmer/build": "^0.8.2",
@@ -86,4 +84,4 @@
       "internal": ":house: Internal"
     }
   }
-}
+}

packages/@glimmer/syntax/lib/parser.ts

@@ -2,9 +2,9 @@ import {
   EventedTokenizer,
   EntityParser,
   HTML5NamedCharRefs as namedCharRefs
-} from "simple-html-tokenizer";
-import { Program } from "./types/nodes";
-import * as AST from "./types/nodes";
+} from 'simple-html-tokenizer';
+import { Program } from './types/nodes';
+import * as AST from './types/nodes';
 import * as HandlebarsAST from './types/handlebars-ast';
 import { Option } from '@glimmer/interfaces';
 import { assert, expect } from '@glimmer/util';
@@ -33,46 +33,49 @@ export interface Attribute {
   valueStartColumn: number;
 }
 
-export class Parser {
+export abstract class Parser {
   protected elementStack: Element[] = [];
   private source: string[];
   public currentAttribute: Option<Attribute> = null;
-  public currentNode: Option<AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'>> = null;
+  public currentNode: Option<
+    AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'>
+  > = null;
   public tokenizer = new EventedTokenizer(this, entityParser);
 
   constructor(source: string) {
-    this.tokenizer.states.tagOpen = function(this: EventedTokenizer) {
-      let char = this.consume();
-      if (char === "!") {
-        this['state'] = 'markupDeclaration';
-      } else if (char === "/") {
-        this['state'] = 'endTagOpen';
-      } else if (/[A-Za-z]/.test(char)) {
-        this['state'] = 'tagName';
-        this['delegate'].beginStartTag();
-        this['delegate'].appendToTagName(char);
-      }
-    };
-
-    this.tokenizer.states.endTagOpen = function(this: EventedTokenizer) {
-      let char = this.consume();
-      if (/[A-Za-z]/.test(char)) {
-        this['state'] = 'tagName';
-        this['delegate'].beginEndTag();
-        this['delegate'].appendToTagName(char);
-      }
-    };
-
     this.source = source.split(/(?:\r\n?|\n)/g);
   }
 
+  abstract reset(): void;
+  abstract finishData(): void;
+  abstract tagOpen(): void;
+  abstract beginData(): void;
+  abstract appendToData(char: string): void;
+  abstract beginStartTag(): void;
+  abstract appendToTagName(char: string): void;
+  abstract beginAttribute(): void;
+  abstract appendToAttributeName(char: string): void;
+  abstract beginAttributeValue(quoted: boolean): void;
+  abstract appendToAttributeValue(char: string): void;
+  abstract finishAttributeValue(): void;
+  abstract markTagAsSelfClosing(): void;
+  abstract beginEndTag(): void;
+  abstract finishTag(): void;
+  abstract beginComment(): void;
+  abstract appendToCommentData(char: string): void;
+  abstract finishComment(): void;
+  abstract reportSyntaxError(error: string): void;
+
   get currentAttr(): Attribute {
     return expect(this.currentAttribute, 'expected attribute');
   }
 
   get currentTag(): Tag<'StartTag' | 'EndTag'> {
     let node = this.currentNode;
-    assert(node && (node.type === 'StartTag' || node.type === 'EndTag'), 'expected tag');
+    assert(
+      node && (node.type === 'StartTag' || node.type === 'EndTag'),
+      'expected tag'
+    );
     return node as Tag<'StartTag' | 'EndTag'>;
   }
 
@@ -98,7 +101,6 @@ export class Parser {
     let node = this.currentNode;
     assert(node && node.type === 'TextNode', 'expected a text node');
     return node as AST.TextNode;
-
   }
 
   acceptNode(node: HandlebarsAST.Program): Program;
@@ -111,7 +113,10 @@ export class Parser {
     return this.elementStack[this.elementStack.length - 1];
   }
 
-  sourceForNode(node: HandlebarsAST.Node, endNode?: { loc: HandlebarsAST.SourceLocation }): string {
+  sourceForNode(
+    node: HandlebarsAST.Node,
+    endNode?: { loc: HandlebarsAST.SourceLocation }
+  ): string {
     let firstLine = node.loc.start.line - 1;
     let currentLine = firstLine - 1;
     let firstColumn = node.loc.start.column;