greatly simplify variable access/storage in transpiler (#968)

* greatly simplify variable access/storage in transpiler

* fix repl

* fix eslint

* Fix console.log warnings

Co-authored-by: Martin Henz <[email protected]>

Author: openorclose

src/createContext.ts

@@ -10,7 +10,7 @@ import * as misc from './stdlib/misc'
 import * as parser from './stdlib/parser'
 import * as stream from './stdlib/stream'
 import { streamPrelude } from './stdlib/stream.prelude'
-import { Context, CustomBuiltIns, Environment, Value, Variant } from './types'
+import { Context, CustomBuiltIns, Environment, NativeStorage, Value, Variant } from './types'
 import * as operators from './utils/operators'
 import * as gpu_lib from './gpu/lib'
 import { stringify } from './utils/stringify'
@@ -112,11 +112,13 @@ export const createGlobalEnvironment = (): Environment => ({
   head: {}
 })
 
-const createNativeStorage = () => ({
-  globals: { variables: new Map(), previousScope: null },
+const createNativeStorage = (): NativeStorage => ({
+  builtins: new Map(),
+  previousProgramsIdentifiers: new Set(),
   operators: new Map(Object.entries(operators)),
   gpu: new Map(Object.entries(gpu_lib)),
-  maxExecTime: JSSLANG_PROPERTIES.maxExecTime
+  maxExecTime: JSSLANG_PROPERTIES.maxExecTime,
+  evaller: null
 })
 
 export const createEmptyContext = <T>(
@@ -171,10 +173,7 @@ export const defineSymbol = (context: Context, name: string, value: Value) => {
     writable: false,
     enumerable: true
   })
-  context.nativeStorage.globals!.variables.set(name, {
-    kind: 'const',
-    getValue: () => value
-  })
+  context.nativeStorage.builtins.set(name, value)
   const typeEnv = context.typeEnvironment[0]
   // if the global type env doesn't already have the imported symbol,
   // we set it to a type var T that can typecheck with anything.

src/index.ts

@@ -489,7 +489,6 @@ export async function runInContext(
     }
     let transpiled
     let sourceMapJson: RawSourceMap | undefined
-    let lastStatementSourceMapJson: RawSourceMap | undefined
     try {
       appendModulesToContext(program, context)
       // Mutates program
@@ -502,11 +501,7 @@ export async function runInContext(
           break
       }
 
-      const temp = transpile(program, context, false)
-      // some issues with formatting and semicolons and tslint so no destructure
-      transpiled = temp.transpiled
-      sourceMapJson = temp.codeMap
-      lastStatementSourceMapJson = temp.evalMap
+      ;({ transpiled, codeMap: sourceMapJson } = transpile(program, context))
       let value = await sandboxedEval(transpiled, context.nativeStorage, context.moduleParams)
       if (context.variant === 'lazy') {
         value = forceIt(value)
@@ -544,24 +539,20 @@ export async function runInContext(
       }
       const line = Number(match![1])
       const column = Number(match![2])
-      return SourceMapConsumer.with(
-        line === 1 ? lastStatementSourceMapJson! : sourceMapJson!,
-        null,
-        consumer => {
-          const {
-            line: originalLine,
-            column: originalColumn,
-            name
-          } = consumer.originalPositionFor({
-            line,
-            column
-          })
-          context.errors.push(
-            convertNativeErrorToSourceError(error, originalLine, originalColumn, name)
-          )
-          return resolvedErrorPromise
-        }
-      )
+      return SourceMapConsumer.with(sourceMapJson!, null, consumer => {
+        const {
+          line: originalLine,
+          column: originalColumn,
+          name
+        } = consumer.originalPositionFor({
+          line,
+          column
+        })
+        context.errors.push(
+          convertNativeErrorToSourceError(error, originalLine, originalColumn, name)
+        )
+        return resolvedErrorPromise
+      })
     }
   } else {
     let it = evaluate(program, context)

src/repl/transpiler.ts

@@ -29,7 +29,7 @@ function transpileCode(chapter = 1, variant: Variant = 'default', code = '', pre
   if (pretranspile) {
     return generate(program)
   } else {
-    return transpile(program as Program, context, false).transpiled
+    return transpile(program as Program, context).transpiled
   }
 }
 

src/transpiler/__tests__/__snapshots__/transpiled-code.ts.snap

@@ -13,6 +13,10 @@ export const sandboxedEval: Evaler = new Function(
   NATIVE_STORAGE_ID,
   MODULE_PARAMS_ID,
   `
-return eval(code)
+  if (${NATIVE_STORAGE_ID}.evaller === null) {
+    return eval(code);
+  } else {
+    return ${NATIVE_STORAGE_ID}.evaller(code);
+  }
 `
 ) as Evaler

src/transpiler/evalContainer.ts

@@ -78,16 +78,18 @@ export interface ConstWrapper {
   getValue: () => Value
 }
 
-export interface Globals {
-  variables: Map<string, ValueWrapper>
-  previousScope: Globals | null
-}
-
 export interface NativeStorage {
-  globals: Globals | null
+  builtins: Map<string, Value>
+  previousProgramsIdentifiers: Set<string>
   operators: Map<string, (...operands: Value[]) => Value>
   gpu: Map<string, (...operands: Value[]) => Value>
   maxExecTime: number
+  evaller: null | ((program: string) => Value)
+  /*
+  the first time evaller is used, it must be used directly like `eval(code)` to inherit
+  surrounding scope, so we cannot set evaller to `eval` directly. subsequent assignments to evaller will
+  close in the surrounding values, so no problem
+   */
 }
 
 export interface Context<T = any> {

src/transpiler/transpiler.ts

@@ -115,9 +115,10 @@ async function testInContext(code: string, options: TestOptions): Promise<TestRe
     const nativeTestContext = createTestContext(options)
     let pretranspiled: string = ''
     let transpiled: string = ''
-    let parsed
-    try {
-      parsed = parse(code, nativeTestContext)!
+    const parsed = parse(code, nativeTestContext)!
+    if (parsed === undefined) {
+      pretranspiled = 'parseError'
+    } else {
       // Mutates program
       switch (options.variant) {
         case 'gpu':
@@ -132,15 +133,11 @@ async function testInContext(code: string, options: TestOptions): Promise<TestRe
       try {
         transpiled = transpile(parsed, nativeTestContext, true).transpiled
         // replace declaration of builtins since they're repetitive
-        transpiled = transpiled.replace(
-          /\n  const \w+ = globals\.(previousScope.)+variables.get\("\w+"\)\.getValue\(\);/g,
-          ''
-        )
+        transpiled = transpiled.replace(/\n  const \w+ = nativeStorage\..*;/g, '')
+        transpiled = transpiled.replace(/\n\s*const \w+ = .*\.operators\..*;/g, '')
       } catch {
         transpiled = 'parseError'
       }
-    } catch {
-      pretranspiled = 'parseError'
     }
     const nativeResult = getTestResult(
       nativeTestContext,

src/types.ts

@@ -18,15 +18,9 @@ export function getUniqueId(usedIdentifiers: Set<string>, uniqueId = 'unique') {
 }
 
 export function getIdentifiersInNativeStorage(nativeStorage: NativeStorage) {
-  const identifiers = new Set<string>()
-  let variableScope = nativeStorage.globals
-  while (variableScope !== null) {
-    for (const name of variableScope.variables.keys()) {
-      identifiers.add(name)
-    }
-    variableScope = variableScope.previousScope
-  }
-  return identifiers
+  const used = new Set(...nativeStorage.builtins.keys())
+  nativeStorage.previousProgramsIdentifiers.forEach(id => used.add(id))
+  return used
 }
 
 export function getIdentifiersInProgram(program: es.Program) {

src/utils/testing.ts

@@ -1770,7 +1770,7 @@ export function runWithProgram(p: Program, context: Context): any {
   // setup externalBuiltins
   // certain functions are imported from cadet-frontend
   // so import them first every time
-  const externals = context.nativeStorage.globals!.variables
+  const externals = context.nativeStorage.builtins
   if (externals.size > 0) {
     EXTERNAL_PRIMITIVES.forEach(func => extractExternalBuiltin(func, externals))
   }
@@ -1834,5 +1834,5 @@ const externalFunctions = new Map<number, any>()
 function extractExternalBuiltin(func: [string, number], externals: Map<string, any>) {
   const name = func[0]
   const opcode = func[1]
-  externalFunctions.set(opcode, externals.get(name).getValue())
+  externalFunctions.set(opcode, externals.get(name))
 }