Further Enhancements to the Module System

.gitignore

@@ -6,6 +6,7 @@ node_modules
 dist/
 .idea/
 coverage/
+yarn-error.log
 
 # emacs backup files
 *~

jest.setup.ts

@@ -0,0 +1,16 @@
+import { SourceMapConsumer } from "source-map"
+
+jest.mock('lodash', () => ({
+  ...jest.requireActual('lodash'),
+  memoize: jest.fn((x: any) => x),
+}))
+
+jest.mock('./src/modules/moduleLoaderAsync')
+jest.mock('./src/modules/moduleLoader')
+
+// @ts-ignore
+SourceMapConsumer.initialize({
+  'lib/mappings.wasm': 'https://unpkg.com/source-map@0.7.3/lib/mappings.wasm'
+})
+
+global.fetch = jest.fn()

package.json

@@ -115,6 +115,9 @@
       "/src/scm-slang",
       "/src/py-slang/"
     ],
+    "setupFilesAfterEnv": [
+      "<rootDir>/jest.setup.ts"
+    ],
     "reporters": [
       "default",
       [

src/ec-evaluator/interpreter.ts

@@ -19,7 +19,7 @@ import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader'
 import { ModuleFunctions } from '../modules/moduleTypes'
 import { checkEditorBreakpoints } from '../stdlib/inspector'
 import { Context, ContiguousArrayElements, Result, Value } from '../types'
-import * as ast from '../utils/astCreator'
+import * as ast from '../utils/ast/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
 import * as instr from './instrCreator'

src/ec-evaluator/utils.ts

@@ -6,7 +6,7 @@ import * as errors from '../errors/errors'
 import { RuntimeSourceError } from '../errors/runtimeSourceError'
 import Closure from '../interpreter/closure'
 import { Environment, Frame, Value } from '../types'
-import * as ast from '../utils/astCreator'
+import * as ast from '../utils/ast/astCreator'
 import * as instr from './instrCreator'
 import { Agenda } from './interpreter'
 import { AgendaItem, AppInstr, AssmtInstr, Instr, InstrType } from './types'

src/errors/localImportErrors.ts

@@ -1,5 +1,5 @@
 import { UNKNOWN_LOCATION } from '../constants'
-import { nonAlphanumericCharEncoding } from '../localImports/filePaths'
+import { nonAlphanumericCharEncoding } from '../modules/preprocessor/filePaths'
 import { ErrorSeverity, ErrorType, SourceError } from '../types'
 
 export abstract class InvalidFilePathError implements SourceError {

src/finder.ts

@@ -18,7 +18,7 @@ import {
   FullWalkerCallback,
   recursive,
   WalkerCallback
-} from './utils/walkers'
+} from './utils/ast/walkers'
 
 // Finds the innermost node that matches the given location
 export function findIdentifierNode(

src/gpu/gpu.ts

@@ -1,6 +1,6 @@
 import * as es from 'estree'
 
-import * as create from '../utils/astCreator'
+import * as create from '../utils/ast/astCreator'
 import { getIdentifiersInProgram } from '../utils/uniqueIds'
 import GPUTransformer from './transfomer'
 

src/gpu/transfomer.ts

@@ -1,7 +1,7 @@
 import * as es from 'estree'
 
-import * as create from '../utils/astCreator'
-import { ancestor, make, simple } from '../utils/walkers'
+import * as create from '../utils/ast/astCreator'
+import { ancestor, make, simple } from '../utils/ast/walkers'
 import GPUBodyVerifier from './verification/bodyVerifier'
 import GPULoopVerifier from './verification/loopVerifier'
 

src/gpu/verification/bodyVerifier.ts

@@ -1,6 +1,6 @@
 import * as es from 'estree'
 
-import { make, simple } from '../../utils/walkers'
+import { make, simple } from '../../utils/ast/walkers'
 
 /*
  * GPU Body verifier helps to ensure the body is parallelizable

src/index.ts

@@ -21,16 +21,16 @@ import {
   SVMProgram,
   Variant
 } from './types'
-import { findNodeAt } from './utils/walkers'
+import { findNodeAt } from './utils/ast/walkers'
 import { assemble } from './vm/svml-assembler'
 import { compileToIns } from './vm/svml-compiler'
 export { SourceDocumentation } from './editors/ace/docTooltip'
 import * as es from 'estree'
 
 import { ECEResultPromise, resumeEvaluate } from './ec-evaluator/interpreter'
 import { CannotFindModuleError } from './errors/localImportErrors'
-import { validateFilePath } from './localImports/filePaths'
-import preprocessFileImports from './localImports/preprocessor'
+import { validateFilePath } from './modules/preprocessor/filePaths'
+import preprocessFileImports from './modules/preprocessor/preprocessor'
 import { getKeywords, getProgramNames, NameDeclaration } from './name-extractor'
 import { parse } from './parser/parser'
 import { parseWithComments } from './parser/utils'

src/infiniteLoops/detect.ts

@@ -1,7 +1,7 @@
 import { generate } from 'astring'
 import * as es from 'estree'
 
-import { simple } from '../utils/walkers'
+import { simple } from '../utils/ast/walkers'
 import { InfiniteLoopError, InfiniteLoopErrorType } from './errors'
 import { getOriginalName } from './instrument'
 import * as st from './state'

src/infiniteLoops/instrument.ts

@@ -2,8 +2,8 @@ import { generate } from 'astring'
 import * as es from 'estree'
 
 import { transformImportDeclarations } from '../transpiler/transpiler'
-import * as create from '../utils/astCreator'
-import { recursive, simple, WalkerCallback } from '../utils/walkers'
+import * as create from '../utils/ast/astCreator'
+import { recursive, simple, WalkerCallback } from '../utils/ast/walkers'
 // transforms AST of program
 
 const globalIds = {

src/infiniteLoops/runtime.ts

@@ -6,7 +6,7 @@ import { getRequireProvider } from '../modules/requireProvider'
 import { parse } from '../parser/parser'
 import * as stdList from '../stdlib/list'
 import { Chapter, Variant } from '../types'
-import * as create from '../utils/astCreator'
+import * as create from '../utils/ast/astCreator'
 import { checkForInfiniteLoop } from './detect'
 import { InfiniteLoopError } from './errors'
 import {

src/infiniteLoops/state.ts

@@ -1,7 +1,7 @@
 import { generate } from 'astring'
 import * as es from 'estree'
 
-import { identifier } from '../utils/astCreator'
+import { identifier } from '../utils/ast/astCreator'
 import * as sym from './symbolic'
 
 // Object + functions called during runtime to check for infinite loops

src/infiniteLoops/symbolic.ts

@@ -1,6 +1,6 @@
 import * as es from 'estree'
 
-import * as create from '../utils/astCreator'
+import * as create from '../utils/ast/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 
 // data structure for symbolic + hybrid values

src/interpreter/closure.ts

@@ -8,7 +8,7 @@ import {
   callExpression,
   identifier,
   returnStatement
-} from '../utils/astCreator'
+} from '../utils/ast/astCreator'
 import { dummyLocation } from '../utils/dummyAstCreator'
 import { apply } from './interpreter'
 

src/interpreter/interpreter-non-det.ts

@@ -6,7 +6,7 @@ import { CUT, UNKNOWN_LOCATION } from '../constants'
 import * as errors from '../errors/errors'
 import { RuntimeSourceError } from '../errors/runtimeSourceError'
 import { Context, Environment, Frame, Value } from '../types'
-import { conditionalExpression, literal, primitive } from '../utils/astCreator'
+import { conditionalExpression, literal, primitive } from '../utils/ast/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
 import Closure from './closure'

src/interpreter/interpreter.ts

@@ -11,8 +11,8 @@ import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader'
 import { ModuleFunctions } from '../modules/moduleTypes'
 import { checkEditorBreakpoints } from '../stdlib/inspector'
 import { Context, ContiguousArrayElements, Environment, Frame, Value, Variant } from '../types'
-import * as create from '../utils/astCreator'
-import { conditionalExpression, literal, primitive } from '../utils/astCreator'
+import * as create from '../utils/ast/astCreator'
+import { conditionalExpression, literal, primitive } from '../utils/ast/astCreator'
 import { evaluateBinaryExpression, evaluateUnaryExpression } from '../utils/operators'
 import * as rttc from '../utils/rttc'
 import Closure from './closure'

src/lazy/lazy.ts

@@ -1,8 +1,8 @@
 import * as es from 'estree'
 
-import * as create from '../utils/astCreator'
+import * as create from '../utils/ast/astCreator'
 import { getIdentifiersInProgram } from '../utils/uniqueIds'
-import { simple } from '../utils/walkers'
+import { simple } from '../utils/ast/walkers'
 
 const lazyPrimitives = new Set(['makeLazyFunction', 'wrapLazyCallee', 'forceIt', 'delayIt'])
 

src/modules/__tests__/moduleLoader.ts

@@ -1,11 +1,9 @@
 import { createEmptyContext } from '../../createContext'
-import { ModuleConnectionError, ModuleInternalError } from '../../errors/moduleErrors'
 import { Variant } from '../../types'
 import { stripIndent } from '../../utils/formatters'
-import * as moduleLoader from '../moduleLoader'
+import { ModuleConnectionError, ModuleInternalError } from '../errors'
 
-// Mock memoize function from lodash
-jest.mock('lodash', () => ({ memoize: jest.fn(func => func) }))
+const moduleLoader = jest.requireActual('../moduleLoader')
 
 /**
  * Mock XMLHttpRequest from jsdom environment
@@ -80,7 +78,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => {
     const sampleResponse = `(function () {'use strict'; function index(_params) { return { }; } return index; })();`
     const correctUrl = moduleLoader.MODULES_STATIC_URL + `/bundles/${validModuleBundle}.js`
     const mockedXMLHttpRequest = mockXMLHttpRequest({ responseText: sampleResponse })
-    const response = moduleLoader.memoizedGetModuleFile(validModuleBundle, 'bundle')
+    const response = moduleLoader.memoizedGetBundle(validModuleBundle)
     expect(mockedXMLHttpRequest.open).toHaveBeenCalledTimes(1)
     expect(mockedXMLHttpRequest.open).toHaveBeenCalledWith('GET', correctUrl, false)
     expect(mockedXMLHttpRequest.send).toHaveBeenCalledTimes(1)
@@ -93,7 +91,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => {
     const sampleResponse = `(function (React) {});`
     const correctUrl = moduleLoader.MODULES_STATIC_URL + `/tabs/${validModuleTab}.js`
     const mockedXMLHttpRequest = mockXMLHttpRequest({ responseText: sampleResponse })
-    const response = moduleLoader.memoizedGetModuleFile(validModuleTab, 'tab')
+    const response = moduleLoader.memoizedGetTab(validModuleTab)
     expect(mockedXMLHttpRequest.open).toHaveBeenCalledTimes(1)
     expect(mockedXMLHttpRequest.open).toHaveBeenCalledWith('GET', correctUrl, false)
     expect(mockedXMLHttpRequest.send).toHaveBeenCalledTimes(1)
@@ -112,7 +110,8 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => {
     mockXMLHttpRequest({ responseText: sampleResponse })
     const loadedBundle = moduleLoader.loadModuleBundle(
       'module',
-      createEmptyContext(1, Variant.DEFAULT, [])
+      createEmptyContext(1, Variant.DEFAULT, []),
+      false
     )
     expect(loadedBundle.make_empty_array()).toEqual([])
   })
@@ -123,7 +122,7 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => {
     const wrongModuleText = `export function es6_function(params) {};`
     mockXMLHttpRequest({ responseText: wrongModuleText })
     expect(() =>
-      moduleLoader.loadModuleBundle('module', createEmptyContext(1, Variant.DEFAULT, []))
+      moduleLoader.loadModuleBundle('module', createEmptyContext(1, Variant.DEFAULT, []), false)
     ).toThrow(ModuleInternalError)
   })
 
@@ -183,4 +182,4 @@ describe('Testing modules/moduleLoader.ts in a jsdom environment', () => {
     expect(mockedXMLHttpRequest3.send).toHaveBeenCalledTimes(1)
     expect(mockedXMLHttpRequest3.send).toHaveBeenCalledWith(null)
   })
-})
+})