From 3506ee1aa6351f5b23b15c61d77c8dd1e37a962a Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Thu, 28 Mar 2019 10:56:39 +0200 Subject: [PATCH] Run imports once per VU and support import cycles (#975) Previously scripts would've been ran for each time they were imported this is both slower and not how it works in most(all?) interpreters so we are changing the behavious to be more compliant. Just like in c3d3fa5cff4b60e4839298643052bb3e77f1f15e , but now we clear the cached exports when we copy the programs when making new VUs. This is needed because otherwise: 1. We share exports between VUs so there are race conditions 2. The exports from the initial compilation without context are used (when not running from archive) and this leads to not being able to call functions that need context inside imported scripts. Closes #659 and fixes #969 Additionally import cycles are now supported.This works by pre-populating what a given file(module) will export before actually evaluating the code whatsoever. This way if it requires something that requires the original module it will get the exports object as it is at that time. And because of some magic (probably in babel) we also get that the exports are bindings[1]. So even if something is not exported at the time of the import, it will still be populated later when the original file finish loading. [1]: http://2ality.com/2015/07/es6-module-exports.html#why-export-bindings Fixes #502 --- js/initcontext.go | 90 ++++++----- js/module_loading_test.go | 320 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 373 insertions(+), 37 deletions(-) create mode 100644 js/module_loading_test.go diff --git a/js/initcontext.go b/js/initcontext.go index 1dcf6d402f1..a10bc88c814 100644 --- a/js/initcontext.go +++ b/js/initcontext.go @@ -35,11 +35,12 @@ import ( ) type programWithSource struct { - pgm *goja.Program - src string + pgm *goja.Program + src string + exports goja.Value } -// Provides APIs for use in the init context. +// InitContext provides APIs for use in the init context. type InitContext struct { // Bound runtime; used to instantiate objects. runtime *goja.Runtime @@ -57,7 +58,10 @@ type InitContext struct { files map[string][]byte } -func NewInitContext(rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string) *InitContext { +// NewInitContext creates a new initcontext with the provided arguments +func NewInitContext( + rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string, +) *InitContext { return &InitContext{ runtime: rt, compiler: compiler, @@ -71,6 +75,16 @@ func NewInitContext(rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *conte } func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Runtime) *InitContext { + // we don't copy the exports as otherwise they will be shared and we don't want this. + // this means that all the files will be executed again but once again only once per compilation + // of the main file. + var programs = make(map[string]programWithSource, len(base.programs)) + for key, program := range base.programs { + programs[key] = programWithSource{ + src: program.src, + pgm: program.pgm, + } + } return &InitContext{ runtime: rt, ctxPtr: ctxPtr, @@ -79,11 +93,12 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru pwd: base.pwd, compiler: base.compiler, - programs: base.programs, + programs: programs, files: base.files, } } +// Require is called when a module/file needs to be loaded by a script func (i *InitContext) Require(arg string) goja.Value { switch { case arg == "k6", strings.HasPrefix(arg, "k6/"): @@ -116,47 +131,48 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) { // Resolve the file path, push the target directory as pwd to make relative imports work. pwd := i.pwd filename := loader.Resolve(pwd, name) - i.pwd = loader.Dir(filename) - defer func() { i.pwd = pwd }() - - // Swap the importing scope's exports out, then put it back again. - oldExports := i.runtime.Get("exports") - defer i.runtime.Set("exports", oldExports) - oldModule := i.runtime.Get("module") - defer i.runtime.Set("module", oldModule) - exports := i.runtime.NewObject() - i.runtime.Set("exports", exports) - module := i.runtime.NewObject() - _ = module.Set("exports", exports) - i.runtime.Set("module", module) // First, check if we have a cached program already. pgm, ok := i.programs[filename] - if !ok { - // Load the sources; the loader takes care of remote loading, etc. - data, err := loader.Load(i.fs, pwd, name) - if err != nil { - return goja.Undefined(), err - } - - // Compile the sources; this handles ES5 vs ES6 automatically. - src := string(data.Data) - pgm_, err := i.compileImport(src, data.Filename) - if err != nil { - return goja.Undefined(), err + if !ok || pgm.exports == nil { + i.pwd = loader.Dir(filename) + defer func() { i.pwd = pwd }() + + // Swap the importing scope's exports out, then put it back again. + oldExports := i.runtime.Get("exports") + defer i.runtime.Set("exports", oldExports) + oldModule := i.runtime.Get("module") + defer i.runtime.Set("module", oldModule) + exports := i.runtime.NewObject() + i.runtime.Set("exports", exports) + module := i.runtime.NewObject() + _ = module.Set("exports", exports) + i.runtime.Set("module", module) + if pgm.pgm == nil { + // Load the sources; the loader takes care of remote loading, etc. + data, err := loader.Load(i.fs, pwd, name) + if err != nil { + return goja.Undefined(), err + } + pgm.src = string(data.Data) + + // Compile the sources; this handles ES5 vs ES6 automatically. + pgm.pgm, err = i.compileImport(pgm.src, data.Filename) + if err != nil { + return goja.Undefined(), err + } } - // Cache the compiled program. - pgm = programWithSource{pgm_, src} + pgm.exports = module.Get("exports") i.programs[filename] = pgm - } - // Run the program. - if _, err := i.runtime.RunProgram(pgm.pgm); err != nil { - return goja.Undefined(), err + // Run the program. + if _, err := i.runtime.RunProgram(pgm.pgm); err != nil { + return goja.Undefined(), err + } } - return module.Get("exports"), nil + return pgm.exports, nil } func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) { diff --git a/js/module_loading_test.go b/js/module_loading_test.go new file mode 100644 index 00000000000..e5e51fbaaee --- /dev/null +++ b/js/module_loading_test.go @@ -0,0 +1,320 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2019 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package js + +import ( + "context" + "os" + "testing" + + "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/testutils" + "github.com/loadimpact/k6/stats" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" +) + +func newDevNullSampleChannel() chan stats.SampleContainer { + var ch = make(chan stats.SampleContainer, 100) + go func() { + for range ch { + } + }() + return ch +} + +func TestLoadOnceGlobalVars(t *testing.T) { + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/C.js", []byte(` + var globalVar; + if (!globalVar) { + globalVar = Math.random(); + } + export function C() { + return globalVar; + } + `), os.ModePerm)) + + require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(` + import { C } from "./C.js"; + export function A() { + return C(); + } + `), os.ModePerm)) + require.NoError(t, afero.WriteFile(fs, "/B.js", []byte(` + import { C } from "./C.js"; + export function B() { + return C(); + } + `), os.ModePerm)) + r1, err := New(&lib.SourceData{ + Filename: "/script.js", + Data: []byte(` + import { A } from "./A.js"; + import { B } from "./B.js"; + + export default function(data) { + if (A() === undefined) { + throw new Error("A() is undefined"); + } + if (A() != B()) { + throw new Error("A() != B() (" + A() + ") != (" + B() + ")"); + } + } + `), + }, fs, lib.RuntimeOptions{}) + require.NoError(t, err) + + arc := r1.MakeArchive() + arc.Files = make(map[string][]byte) + r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) + require.NoError(t, err) + + runners := map[string]*Runner{"Source": r1, "Archive": r2} + for name, r := range runners { + r := r + t.Run(name, func(t *testing.T) { + ch := newDevNullSampleChannel() + defer close(ch) + vu, err := r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + }) + } +} + +func TestLoadDoesntBreakHTTPGet(t *testing.T) { + // This test that functions such as http.get which require context still work if they are called + // inside script that is imported + + tb := testutils.NewHTTPMultiBin(t) + defer tb.Cleanup() + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(tb.Replacer.Replace(` + import http from "k6/http"; + export function A() { + return http.get("HTTPBIN_URL/get"); + } + `)), os.ModePerm)) + r1, err := New(&lib.SourceData{ + Filename: "/script.js", + Data: []byte(` + import { A } from "./A.js"; + + export default function(data) { + let resp = A(); + if (resp.status != 200) { + throw new Error("wrong status "+ resp.status); + } + } + `), + }, fs, lib.RuntimeOptions{}) + require.NoError(t, err) + + require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts})) + arc := r1.MakeArchive() + arc.Files = make(map[string][]byte) + r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) + require.NoError(t, err) + + runners := map[string]*Runner{"Source": r1, "Archive": r2} + for name, r := range runners { + r := r + t.Run(name, func(t *testing.T) { + ch := newDevNullSampleChannel() + defer close(ch) + vu, err := r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + }) + } +} + +func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) { + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(` + var globalVar = 0; + export function A() { + globalVar += 1 + return globalVar; + } + `), os.ModePerm)) + r1, err := New(&lib.SourceData{ + Filename: "/script.js", + Data: []byte(` + import { A } from "./A.js"; + + export default function(data) { + var a = A(); + if (a == 1) { + a = 2; + } else { + throw new Error("wrong value of a " + a); + } + } + `), + }, fs, lib.RuntimeOptions{}) + require.NoError(t, err) + + arc := r1.MakeArchive() + arc.Files = make(map[string][]byte) + r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) + require.NoError(t, err) + + runners := map[string]*Runner{"Source": r1, "Archive": r2} + for name, r := range runners { + r := r + t.Run(name, func(t *testing.T) { + ch := newDevNullSampleChannel() + defer close(ch) + vu, err := r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + + // run a second VU + vu, err = r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + }) + } +} + +func TestLoadCycle(t *testing.T) { + // This is mostly the example from https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/ + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/counter.js", []byte(` + let message = require("./main.js").message; + exports.count = 5; + export function a() { + return message; + } + `), os.ModePerm)) + + require.NoError(t, afero.WriteFile(fs, "/main.js", []byte(` + let counter = require("./counter.js"); + let count = counter.count; + let a = counter.a; + let message= "Eval complete"; + exports.message = message; + + export default function() { + if (count != 5) { + throw new Error("Wrong value of count "+ count); + } + let aMessage = a(); + if (aMessage != message) { + throw new Error("Wrong value of a() "+ aMessage); + } + } + `), os.ModePerm)) + data, err := afero.ReadFile(fs, "/main.js") + require.NoError(t, err) + r1, err := New(&lib.SourceData{ + Filename: "/main.js", + Data: data, + }, fs, lib.RuntimeOptions{}) + require.NoError(t, err) + + arc := r1.MakeArchive() + arc.Files = make(map[string][]byte) + r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) + require.NoError(t, err) + + runners := map[string]*Runner{"Source": r1, "Archive": r2} + for name, r := range runners { + r := r + t.Run(name, func(t *testing.T) { + ch := newDevNullSampleChannel() + defer close(ch) + vu, err := r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + }) + } + +} + +func TestLoadCycleBinding(t *testing.T) { + // This is mostly the example from + // http://2ality.com/2015/07/es6-module-exports.html#why-export-bindings + fs := afero.NewMemMapFs() + require.NoError(t, afero.WriteFile(fs, "/a.js", []byte(` + import {bar} from './b.js'; + export function foo(a) { + if (a !== undefined) { + return "foo" + a; + } + return "foo" + bar(3); + } + `), os.ModePerm)) + + require.NoError(t, afero.WriteFile(fs, "/b.js", []byte(` + import {foo} from './a.js'; + export function bar(a) { + if (a !== undefined) { + return "bar" + a; + } + return "bar" + foo(5); + } + `), os.ModePerm)) + + r1, err := New(&lib.SourceData{ + Filename: "/main.js", + Data: []byte(` + import {foo} from './a.js'; + import {bar} from './b.js'; + export default function() { + let fooMessage = foo(); + if (fooMessage != "foobar3") { + throw new Error("Wrong value of foo() "+ fooMessage); + } + let barMessage = bar(); + if (barMessage != "barfoo5") { + throw new Error("Wrong value of bar() "+ barMessage); + } + } + `), + }, fs, lib.RuntimeOptions{}) + require.NoError(t, err) + + arc := r1.MakeArchive() + arc.Files = make(map[string][]byte) + r2, err := NewFromArchive(arc, lib.RuntimeOptions{}) + require.NoError(t, err) + + runners := map[string]*Runner{"Source": r1, "Archive": r2} + for name, r := range runners { + r := r + t.Run(name, func(t *testing.T) { + ch := newDevNullSampleChannel() + defer close(ch) + vu, err := r.NewVU(ch) + require.NoError(t, err) + err = vu.RunOnce(context.Background()) + require.NoError(t, err) + }) + } +}