Module Syntax Support (#536)

* Add primary module support

Add primary module support for browsers.

* format moduleLoader.js

* support imports in interpreter and transpiler

* fix minor problems

* fix minor problems

* fix that imports scanning would break variables

* fix library cannot use global object

* fix libraries using global object

* add error handling

* remove trashes

* fix format

* Update package.json and yarn.lock

Update package.json and yarn.lock

* add tests for transformImportDeclarations

* file format

* Add error handling for modules

* format file

* change term 'hoist' to 'declare'

* file format

* add tests for loadIIFEModule

* add tests for single import declaration

* change 'loadIIFEModuleText' to 'loadModuleText'

Author: Hou-Rui

.vscode/launch.json

@@ -0,0 +1,37 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Source REPL",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/dist/repl/repl.js",
+            "console": "integratedTerminal",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/dist/**/*.js"
+            ]
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Source REPL for test.js",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/dist/repl/repl.js",
+            "args": ["~/test.js"],
+            "console": "integratedTerminal",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/dist/**/*.js"
+            ]
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "restructuredtext.confPath": "",
+    "restructuredtext.linter.disabled": true
+}

package.json

@@ -25,7 +25,8 @@
     "jest-html-reporter": "^2.8.2",
     "lodash": "^4.17.13",
     "node-getopt": "^0.3.2",
-    "source-map": "^0.7.3"
+    "source-map": "^0.7.3",
+    "xmlhttprequest-ts": "^1.0.1"
   },
   "main": "dist/index",
   "types": "dist/index",

src/errors/errors.ts

@@ -20,6 +20,38 @@ export class InterruptedError extends RuntimeSourceError {
   }
 }
 
+export class ModuleNotFound extends RuntimeSourceError {
+  constructor(public moduleName: string, node?: es.Node) {
+    super(node)
+  }
+
+  public explain() {
+    return `Module "${this.moduleName}" not found.`
+  }
+
+  public elaborate() {
+    return `
+      You should check your Internet connection, and ensure you have used the correct module path.
+    `
+  }
+}
+
+export class ModuleInternalError extends RuntimeSourceError {
+  constructor(public moduleName: string, node?: es.Node) {
+    super(node)
+  }
+
+  public explain() {
+    return `Error(s) occured when executing the module "${this.moduleName}".`
+  }
+
+  public elaborate() {
+    return `
+      You may need to contact with the author for this module to fix this error.
+    `
+  }
+}
+
 export class ExceptionError implements SourceError {
   public type = ErrorType.RUNTIME
   public severity = ErrorSeverity.ERROR

src/interpreter/interpreter.ts

@@ -9,6 +9,7 @@ import { conditionalExpression, literal, primitive } from '../utils/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
 import Closure from './closure'
+import { loadIIFEModule } from '../modules/moduleLoader'
 
 class BreakValue {}
 
@@ -96,9 +97,9 @@ const handleRuntimeError = (context: Context, error: RuntimeSourceError): never
   throw error
 }
 
-const HOISTED_BUT_NOT_YET_ASSIGNED = Symbol('Used to implement hoisting')
+const DECLARED_BUT_NOT_YET_ASSIGNED = Symbol('Used to implement hoisting')
 
-function hoistIdentifier(context: Context, name: string, node: es.Node) {
+function declareIdentifier(context: Context, name: string, node: es.Node) {
   const environment = currentEnvironment(context)
   if (environment.head.hasOwnProperty(name)) {
     const descriptors = Object.getOwnPropertyDescriptors(environment.head)
@@ -108,27 +109,30 @@ function hoistIdentifier(context: Context, name: string, node: es.Node) {
       new errors.VariableRedeclaration(node, name, descriptors[name].writable)
     )
   }
-  environment.head[name] = HOISTED_BUT_NOT_YET_ASSIGNED
+  environment.head[name] = DECLARED_BUT_NOT_YET_ASSIGNED
   return environment
 }
 
-function hoistVariableDeclarations(context: Context, node: es.VariableDeclaration) {
+function declareVariables(context: Context, node: es.VariableDeclaration) {
   for (const declaration of node.declarations) {
-    hoistIdentifier(context, (declaration.id as es.Identifier).name, node)
+    declareIdentifier(context, (declaration.id as es.Identifier).name, node)
   }
 }
 
-function hoistFunctionsAndVariableDeclarationsIdentifiers(
-  context: Context,
-  node: es.BlockStatement
-) {
+function declareImports(context: Context, node: es.ImportDeclaration) {
+  for (const declaration of node.specifiers) {
+    declareIdentifier(context, declaration.local.name, node)
+  }
+}
+
+function declareFunctionsAndVariables(context: Context, node: es.BlockStatement) {
   for (const statement of node.body) {
     switch (statement.type) {
       case 'VariableDeclaration':
-        hoistVariableDeclarations(context, statement)
+        declareVariables(context, statement)
         break
       case 'FunctionDeclaration':
-        hoistIdentifier(context, (statement.id as es.Identifier).name, statement)
+        declareIdentifier(context, (statement.id as es.Identifier).name, statement)
         break
     }
   }
@@ -137,7 +141,7 @@ function hoistFunctionsAndVariableDeclarationsIdentifiers(
 function defineVariable(context: Context, name: string, value: Value, constant = false) {
   const environment = context.runtime.environments[0]
 
-  if (environment.head[name] !== HOISTED_BUT_NOT_YET_ASSIGNED) {
+  if (environment.head[name] !== DECLARED_BUT_NOT_YET_ASSIGNED) {
     return handleRuntimeError(
       context,
       new errors.VariableRedeclaration(context.runtime.nodes[0]!, name, !constant)
@@ -176,7 +180,7 @@ const getVariable = (context: Context, name: string) => {
   let environment: Environment | null = context.runtime.environments[0]
   while (environment) {
     if (environment.head.hasOwnProperty(name)) {
-      if (environment.head[name] === HOISTED_BUT_NOT_YET_ASSIGNED) {
+      if (environment.head[name] === DECLARED_BUT_NOT_YET_ASSIGNED) {
         return handleRuntimeError(
           context,
           new errors.UnassignedVariable(name, context.runtime.nodes[0])
@@ -195,7 +199,7 @@ const setVariable = (context: Context, name: string, value: any) => {
   let environment: Environment | null = context.runtime.environments[0]
   while (environment) {
     if (environment.head.hasOwnProperty(name)) {
-      if (environment.head[name] === HOISTED_BUT_NOT_YET_ASSIGNED) {
+      if (environment.head[name] === DECLARED_BUT_NOT_YET_ASSIGNED) {
         break
       }
       const descriptors = Object.getOwnPropertyDescriptors(environment.head)
@@ -266,7 +270,7 @@ function* reduceIf(
 export type Evaluator<T extends es.Node> = (node: T, context: Context) => IterableIterator<Value>
 
 function* evaluateBlockSatement(context: Context, node: es.BlockStatement) {
-  hoistFunctionsAndVariableDeclarationsIdentifiers(context, node)
+  declareFunctionsAndVariables(context, node)
   let result
   for (const statement of node.body) {
     result = yield* evaluate(statement, context)
@@ -412,7 +416,7 @@ export const evaluators: { [nodeType: string]: Evaluator<es.Node> } = {
     const testNode = node.test!
     const updateNode = node.update!
     if (initNode.type === 'VariableDeclaration') {
-      hoistVariableDeclarations(context, initNode)
+      declareVariables(context, initNode)
     }
     yield* actualValue(initNode, context)
 
@@ -427,7 +431,7 @@ export const evaluators: { [nodeType: string]: Evaluator<es.Node> } = {
       pushEnvironment(context, environment)
       for (const name in loopEnvironment.head) {
         if (loopEnvironment.head.hasOwnProperty(name)) {
-          hoistIdentifier(context, name, node)
+          declareIdentifier(context, name, node)
           defineVariable(context, name, loopEnvironment.head[name], true)
         }
       }
@@ -600,6 +604,17 @@ export const evaluators: { [nodeType: string]: Evaluator<es.Node> } = {
     return result
   },
 
+  ImportDeclaration: function*(node: es.ImportDeclaration, context: Context) {
+    const moduleName = node.source.value as string
+    const neededSymbols = node.specifiers.map(spec => spec.local.name)
+    const module = loadIIFEModule(moduleName)
+    declareImports(context, node)
+    for (const name of neededSymbols) {
+      defineVariable(context, name, module[name], true);
+    }
+    return undefined
+  },
+
   Program: function*(node: es.BlockStatement, context: Context) {
     context.numberOfOuterEnvironments += 1
     const environment = createBlockEnvironment(context, 'programEnvironment')

src/modules/__tests__/moduleLoader.ts

@@ -0,0 +1,21 @@
+/**
+ * @jest-environment node
+ */
+
+import { loadModuleText, loadIIFEModule } from '../moduleLoader'
+import { ModuleNotFound, ModuleInternalError } from '../../errors/errors'
+import { stripIndent } from '../../utils/formatters'
+
+test('Try loading a non-existing module', () => {
+  const moduleName = '_non_existing_dir/_non_existing_file'
+  expect(() => loadModuleText(moduleName)).toThrow(ModuleNotFound)
+})
+
+test('Try executing a wrongly implemented module', () => {
+  // A module in wrong format
+  const path = '_mock_dir/_mock_file'
+  const wrongModuleText = stripIndent`
+    export function es6_function(params) {}
+  `
+  expect(() => loadIIFEModule(path, wrongModuleText)).toThrow(ModuleInternalError)
+})

src/modules/moduleLoader.ts

@@ -0,0 +1,29 @@
+import { ModuleNotFound, ModuleInternalError } from '../errors/errors'
+import { XMLHttpRequest as NodeXMLHttpRequest } from 'xmlhttprequest-ts'
+const HttpRequest = typeof window === 'undefined' ? NodeXMLHttpRequest : XMLHttpRequest
+
+// TODO: Change this URL to actual Backend URL
+const BACKEND_STATIC_URL = 'http://ec2-54-169-81-133.ap-southeast-1.compute.amazonaws.com/static'
+
+export function loadModuleText(path: string) {
+  const scriptPath = `${BACKEND_STATIC_URL}/${path}.js`
+  const req = new HttpRequest()
+  req.open('GET', scriptPath, false)
+  req.send(null)
+  if (req.status !== 200 && req.status !== 304) {
+    throw new ModuleNotFound(`module ${path} not found.`)
+  }
+  return req.responseText
+}
+
+/* tslint:disable */
+export function loadIIFEModule(path: string, moduleText?: string) {
+  try {
+    if (moduleText === undefined) {
+      moduleText = loadModuleText(path)
+    }
+    return eval(moduleText) as object
+  } catch (_error) {
+    throw new ModuleInternalError(path)
+  }
+}

src/transpiler/__tests__/modules.ts

@@ -0,0 +1,28 @@
+import { transformSingleImportDeclaration, transformImportDeclarations } from '../transpiler'
+import { stripIndent } from '../../utils/formatters'
+import { parse } from '../../parser/parser'
+import { mockContext } from '../../mocks/context'
+import { ImportDeclaration, Identifier } from 'estree'
+
+test('Transform single import decalration', () => {
+  const code = `import { foo, bar } from "test/one_module";`
+  const context = mockContext(4)
+  const program = parse(code, context)!
+  const result = transformSingleImportDeclaration(123, program.body[0] as ImportDeclaration)
+  const names = result.map(decl => (decl.declarations[0].id as Identifier).name)
+  expect(names[0]).toStrictEqual('foo')
+  expect(names[1]).toStrictEqual('bar')
+})
+
+test('Transform import decalrations variable decalarations', () => {
+  const code = stripIndent`
+    import { foo } from "test/one_module";
+    import { bar } from "test/another_module";
+    foo(bar);
+  `
+  const context = mockContext(4)
+  const program = parse(code, context)!
+  transformImportDeclarations(program)
+  expect(program.body[0].type).toBe('VariableDeclaration')
+  expect(program.body[1].type).toBe('VariableDeclaration')
+})