Skip to content

Commit

Permalink
Run imports once per VU
Browse files Browse the repository at this point in the history
Previously scripts would've been ran for each time they were imported
this is both slower and not how it works in most(all?) interpreters
so we are changing the behavious to be more compliant.

Just like in c3d3fa5 , but now we clear
the cached exports when we copy the programs when making new VUs.

This is needed because otherwise:
1. We share exports between VUs so there are race conditions
2. The exports from the initial compilation without context are used
(when not running from archive) and this leads to not being able to call
functions that need context inside imported scripts.

Closes #659 and fixes #969
  • Loading branch information
mstoykov committed Mar 22, 2019
1 parent 672769c commit 0d7c20f
Show file tree
Hide file tree
Showing 2 changed files with 236 additions and 36 deletions.
87 changes: 51 additions & 36 deletions js/initcontext.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,12 @@ import (
)

type programWithSource struct {
pgm *goja.Program
src string
pgm *goja.Program
src string
exports goja.Value
}

// Provides APIs for use in the init context.
// InitContext provides APIs for use in the init context.
type InitContext struct {
// Bound runtime; used to instantiate objects.
runtime *goja.Runtime
Expand All @@ -57,7 +58,10 @@ type InitContext struct {
files map[string][]byte
}

func NewInitContext(rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string) *InitContext {
// NewInitContext creates a new initcontext with the provided arguments
func NewInitContext(
rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *context.Context, fs afero.Fs, pwd string,
) *InitContext {
return &InitContext{
runtime: rt,
compiler: compiler,
Expand All @@ -71,6 +75,16 @@ func NewInitContext(rt *goja.Runtime, compiler *compiler.Compiler, ctxPtr *conte
}

func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Runtime) *InitContext {
// we don't copy the exports as otherwise they will be shared and we don't want this.
// this means that all the files will be executed again but once again only once per compilation
// of the main file.
var programs = make(map[string]programWithSource, len(base.programs))
for key, program := range base.programs {
programs[key] = programWithSource{
src: program.src,
pgm: program.pgm,
}
}
return &InitContext{
runtime: rt,
ctxPtr: ctxPtr,
Expand All @@ -79,11 +93,12 @@ func newBoundInitContext(base *InitContext, ctxPtr *context.Context, rt *goja.Ru
pwd: base.pwd,
compiler: base.compiler,

programs: base.programs,
programs: programs,
files: base.files,
}
}

// Require is called when a module/file needs to be loaded by a script
func (i *InitContext) Require(arg string) goja.Value {
switch {
case arg == "k6", strings.HasPrefix(arg, "k6/"):
Expand Down Expand Up @@ -116,47 +131,47 @@ func (i *InitContext) requireFile(name string) (goja.Value, error) {
// Resolve the file path, push the target directory as pwd to make relative imports work.
pwd := i.pwd
filename := loader.Resolve(pwd, name)
i.pwd = loader.Dir(filename)
defer func() { i.pwd = pwd }()

// Swap the importing scope's exports out, then put it back again.
oldExports := i.runtime.Get("exports")
defer i.runtime.Set("exports", oldExports)
oldModule := i.runtime.Get("module")
defer i.runtime.Set("module", oldModule)
exports := i.runtime.NewObject()
i.runtime.Set("exports", exports)
module := i.runtime.NewObject()
_ = module.Set("exports", exports)
i.runtime.Set("module", module)

// First, check if we have a cached program already.
pgm, ok := i.programs[filename]
if !ok {
// Load the sources; the loader takes care of remote loading, etc.
data, err := loader.Load(i.fs, pwd, name)
if err != nil {
return goja.Undefined(), err
if !ok || pgm.exports == nil {
i.pwd = loader.Dir(filename)
defer func() { i.pwd = pwd }()

// Swap the importing scope's exports out, then put it back again.
oldExports := i.runtime.Get("exports")
defer i.runtime.Set("exports", oldExports)
oldModule := i.runtime.Get("module")
defer i.runtime.Set("module", oldModule)
exports := i.runtime.NewObject()
i.runtime.Set("exports", exports)
module := i.runtime.NewObject()
_ = module.Set("exports", exports)
i.runtime.Set("module", module)
if pgm.pgm == nil {
// Load the sources; the loader takes care of remote loading, etc.
data, err := loader.Load(i.fs, pwd, name)
if err != nil {
return goja.Undefined(), err
}
pgm.src = string(data.Data)

// Compile the sources; this handles ES5 vs ES6 automatically.
pgm.pgm, err = i.compileImport(pgm.src, data.Filename)
if err != nil {
return goja.Undefined(), err
}
}

// Compile the sources; this handles ES5 vs ES6 automatically.
src := string(data.Data)
pgm_, err := i.compileImport(src, data.Filename)
if err != nil {
// Run the program.
if _, err := i.runtime.RunProgram(pgm.pgm); err != nil {
return goja.Undefined(), err
}

// Cache the compiled program.
pgm = programWithSource{pgm_, src}
pgm.exports = module.Get("exports")
i.programs[filename] = pgm
}

// Run the program.
if _, err := i.runtime.RunProgram(pgm.pgm); err != nil {
return goja.Undefined(), err
}

return module.Get("exports"), nil
return pgm.exports, nil
}

func (i *InitContext) compileImport(src, filename string) (*goja.Program, error) {
Expand Down
185 changes: 185 additions & 0 deletions js/module_loading_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package js

import (
"context"
"os"
"testing"

"github.com/loadimpact/k6/lib"
"github.com/loadimpact/k6/lib/testutils"
"github.com/loadimpact/k6/stats"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)

func newDevNullSampleChannel() chan stats.SampleContainer {
var ch = make(chan stats.SampleContainer, 100)
go func() {
for range ch {
}
}()
return ch
}

func TestLoadOnceGlobalVars(t *testing.T) {
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/C.js", []byte(`
var globalVar;
if (!globalVar) {
globalVar = Math.random();
}
export function C() {
return globalVar;
}
`), os.ModePerm))

require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(`
import { C } from "./C.js";
export function A() {
return C();
}
`), os.ModePerm))
require.NoError(t, afero.WriteFile(fs, "/B.js", []byte(`
import { C } from "./C.js";
export function B() {
return C();
}
`), os.ModePerm))
r1, err := New(&lib.SourceData{
Filename: "/script.js",
Data: []byte(`
import { A } from "./A.js";
import { B } from "./B.js";
export default function(data) {
if (A() === undefined) {
throw new Error("A() is undefined");
}
if (A() != B()) {
throw new Error("A() != B() (" + A() + ") != (" + B() + ")");
}
}
`),
}, fs, lib.RuntimeOptions{})
require.NoError(t, err)

arc := r1.MakeArchive()
arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)

runners := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range runners {
r := r
t.Run(name, func(t *testing.T) {
ch := newDevNullSampleChannel()
defer close(ch)
vu, err := r.NewVU(ch)
require.NoError(t, err)
err = vu.RunOnce(context.Background())
require.NoError(t, err)
})
}
}

func TestLoadDoesntBreakHTTPGet(t *testing.T) {
// This test that functions such as http.get which require context still work if they are called
// inside script that is imported

tb := testutils.NewHTTPMultiBin(t)
defer tb.Cleanup()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(tb.Replacer.Replace(`
import http from "k6/http";
export function A() {
return http.get("HTTPBIN_URL/get");
}
`)), os.ModePerm))
r1, err := New(&lib.SourceData{
Filename: "/script.js",
Data: []byte(`
import { A } from "./A.js";
export default function(data) {
let resp = A();
if (resp.status != 200) {
throw new Error(JSON.stringify(resp));
throw new Error("wrong status "+ resp.status);
}
}
`),
}, fs, lib.RuntimeOptions{})
require.NoError(t, err)

require.NoError(t, r1.SetOptions(lib.Options{Hosts: tb.Dialer.Hosts}))
arc := r1.MakeArchive()
arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)

runners := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range runners {
r := r
t.Run(name, func(t *testing.T) {
ch := newDevNullSampleChannel()
defer close(ch)
vu, err := r.NewVU(ch)
require.NoError(t, err)
err = vu.RunOnce(context.Background())
require.NoError(t, err)
})
}
}

func TestLoadGlobalVarsAreNotSharedBetweenVUs(t *testing.T) {
// This test that functions such as http.get which require context still work if they are called
// inside script that is imported
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/A.js", []byte(`
var globalVar = 0;
export function A() {
globalVar += 1
return globalVar;
}
`), os.ModePerm))
r1, err := New(&lib.SourceData{
Filename: "/script.js",
Data: []byte(`
import { A } from "./A.js";
export default function(data) {
var a = A();
if (a == 1) {
a = 2;
} else {
throw new Error("wrong value of a " + a);
}
}
`),
}, fs, lib.RuntimeOptions{})
require.NoError(t, err)

arc := r1.MakeArchive()
arc.Files = make(map[string][]byte)
r2, err := NewFromArchive(arc, lib.RuntimeOptions{})
require.NoError(t, err)

runners := map[string]*Runner{"Source": r1, "Archive": r2}
for name, r := range runners {
r := r
t.Run(name, func(t *testing.T) {
ch := newDevNullSampleChannel()
defer close(ch)
vu, err := r.NewVU(ch)
require.NoError(t, err)
err = vu.RunOnce(context.Background())
require.NoError(t, err)

// run a second VU
vu, err = r.NewVU(ch)
require.NoError(t, err)
err = vu.RunOnce(context.Background())
require.NoError(t, err)
})
}
}

0 comments on commit 0d7c20f

Please sign in to comment.