Skip to content

Commit

Permalink
feat(go): require go 1.16, use native embed (#2603)
Browse files Browse the repository at this point in the history
Instead of generating code to build byte slices with the embedded
assets, upgraded the baseline go requirement to go 1.16, and use the new
native embed feature released as part of this go release.

This includes the update to `jsii/superhain` to the correct go release,
updates to the `.github/workflows/main.yml` workflow to ensure the
correct go version is installed in PR validation builds, and the changes
in `jsii-pacmak` and `@jsii/go-runtime` to leverage the feature.

This makes the code easier to deal with by IDEs (the byte slices were
otherwise very large and caused IDEs to occasionally choke on them),
and removes the risk we make a mistake in generating the slices. It
also makes the generated code easier to troubleshoot, as the tarball
is present as-is, and can be inspected without additional hoop jumping.

See: https://pkg.go.dev/embed
  • Loading branch information
RomainMuller authored Feb 25, 2021
1 parent ffb0032 commit 67cd3ce
Show file tree
Hide file tree
Showing 16 changed files with 221 additions and 260 deletions.
24 changes: 24 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: '5.0.x'
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Set up Java 8
uses: actions/setup-java@v1
with:
Expand Down Expand Up @@ -113,6 +117,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: '5.0.x'
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Set up Java 8
uses: actions/setup-java@v1
with:
Expand Down Expand Up @@ -197,6 +205,7 @@ jobs:
matrix:
# All currently supported node versions (Maintenance LTS, Active LTS, Current)
dotnet: ['3.1.x']
go: ['1.16']
java: ['8']
node: ['10', '12', '14']
os: [ubuntu-latest]
Expand All @@ -206,40 +215,47 @@ jobs:
# Test using Windows
- os: windows-latest
dotnet: '3.1.x'
go: '1.16'
java: '8'
node: '10'
python: '3.6'
# Test using macOS
- os: macos-latest
dotnet: '3.1.x'
go: '1.16'
java: '8'
node: '10'
python: '3.6'
# Test alternate .NETs
- java: '8'
dotnet: '5.0.x'
go: '1.16'
node: '10'
os: ubuntu-latest
python: '3.6'
# Test alternate Javas
- java: '11'
dotnet: '3.1.x'
go: '1.16'
node: '10'
os: ubuntu-latest
python: '3.6'
# Test alternate Pythons
- python: '3.7'
dotnet: '3.1.x'
go: '1.16'
java: '8'
node: '10'
os: ubuntu-latest
- python: '3.8'
dotnet: '3.1.x'
go: '1.16'
java: '8'
node: '10'
os: ubuntu-latest
- python: '3.9'
dotnet: '3.1.x'
go: '1.16'
java: '8'
node: '10'
os: ubuntu-latest
Expand All @@ -252,6 +268,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: ${{ matrix.dotnet }}
- name: Set up Go ${{ matrix.go }}
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go }}
- name: Set up Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
Expand Down Expand Up @@ -336,6 +356,10 @@ jobs:
uses: actions/setup-dotnet@v1
with:
dotnet-version: '5.0.x'
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Set up Java 8
uses: actions/setup-java@v1
with:
Expand Down
4 changes: 2 additions & 2 deletions packages/@jsii/go-runtime/.gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/jsii-calc/
*.generated.go
*.generated_test.go
/jsii-runtime-go/embedded/resources/
*.generated.*

*.js
*.d.ts
184 changes: 17 additions & 167 deletions packages/@jsii/go-runtime/build-tools/gen.ts
Original file line number Diff line number Diff line change
@@ -1,129 +1,34 @@
#!/usr/bin/env npx ts-node

import { CodeMaker } from 'codemaker';
import { createHash } from 'crypto';
import { readdirSync, readFileSync, statSync } from 'fs';
import { resolve } from 'path';
import { copySync, mkdirpSync } from 'fs-extra';
import { join, resolve } from 'path';

const EMBEDDED_RUNTIME_ROOT = resolve(
const EMBEDDED_RUNTIME_SOURCE_ROOT = resolve(
__dirname,
'..',
'..',
'runtime',
'webpack',
);

const OUTPUT_DIR = resolve(__dirname, '..', 'jsii-runtime-go', 'kernel');
const RUNTIME_ROOT = resolve(__dirname, '..', 'jsii-runtime-go');

const RUNTIME_FILE = 'embeddedruntime.generated.go';
const RUNTIME_TEST_FILE = 'embeddedruntime.generated_test.go';
const VERSION_FILE = 'version.generated.go';

const code = new CodeMaker({ indentationLevel: 1, indentCharacter: '\t' });

code.openFile(RUNTIME_FILE);
code.line('package kernel');
code.line();
code.open('var embeddedruntime = map[string][]byte{');
const fileInfo: Record<
string,
{ readonly size: number; readonly hash: readonly string[] }
> = {};

(function emitFiles(directory: string, prefix?: string) {
for (const file of readdirSync(directory)) {
// Ignore dot-files
if (file.startsWith('.')) {
continue;
}
const fullPath = resolve(directory, file);

if (statSync(fullPath).isDirectory()) {
// Not using path.join because we don't want Windows delimiters here!
emitFiles(fullPath, prefix ? `${prefix}/${file}` : file);
continue;
}

const key = prefix ? `${prefix}/${file}` : file;

const { byteSlice, hash } = getByteSlice(fullPath);
fileInfo[key] = {
size: byteSlice.length,
hash,
};
code.open(`${JSON.stringify(key)}: []byte{`);
formatBytes(code, byteSlice);
code.close('},');
}
})(EMBEDDED_RUNTIME_ROOT);

code.close('}');
code.line();
const mainKey = JSON.stringify(
Object.keys(fileInfo).find((f) => f.endsWith('jsii-runtime.js')),
)!;
code.line(`const embeddedruntimeMain = ${mainKey}`);
code.closeFile(RUNTIME_FILE);

// This allows us to sanity-check we've generated correct data
code.openFile(RUNTIME_TEST_FILE);
code.line('package kernel');
code.line();
code.open('import (');
code.line('"crypto/sha512"');
code.line('"testing"');
code.close(')');
code.line();
code.openBlock('func TestEmbeddedruntime(t *testing.T)');
const EMBEDDED_RESOURCE_DIR = join(RUNTIME_ROOT, 'embedded', 'resources');

code.open(
't.Run("embeddedruntime[embeddedruntimeMain] exists", func(t *testing.T) {',
);
code.openBlock('if _, exists := embeddedruntime[embeddedruntimeMain]; !exists');
code.line(
't.Errorf("embeddedruntimeMain refers to non-existent file %s", embeddedruntimeMain)',
);
code.closeBlock();
code.close('})');

for (const [file, { size, hash }] of Object.entries(fileInfo)) {
code.line();
code.open(`t.Run("embeddedruntime[\\"${file}\\"]", func(t *testing.T) {`);

code.open('checkEmbeddedFile(');
code.line('t,');
code.line(`"${file}",`);
code.line(`${readableNumber(size)},`);
code.open('[sha512.Size]byte{');
formatBytes(code, hash);
code.close('},');
code.close(')');
mkdirpSync(EMBEDDED_RESOURCE_DIR);
copySync(EMBEDDED_RUNTIME_SOURCE_ROOT, EMBEDDED_RESOURCE_DIR, {
dereference: true,
errorOnExist: false,
overwrite: true,
preserveTimestamps: true,
recursive: true,
});

code.close('})');
}
code.closeBlock();
code.line();
code.openBlock(
'func checkEmbeddedFile(t *testing.T, name string, expectedSize int, expectedHash [sha512.Size]byte)',
);
code.line('data := embeddedruntime[name]');
code.line();
code.line('size := len(data)');
code.openBlock('if size != expectedSize');
code.line(
't.Errorf("Size mismatch: expected %d bytes, got %d", expectedSize, size)',
);
code.closeBlock();
code.line();
code.line('hash := sha512.Sum512(data)');
code.openBlock('if hash != expectedHash');
code.line(
't.Errorf("SHA512 do not match:\\nExpected: %x\\nActual: %x", expectedHash, hash)',
);
code.closeBlock();
code.closeBlock();
code.closeFile(RUNTIME_TEST_FILE);
const KERNEL_LIB_DIR = resolve(RUNTIME_ROOT, 'kernel');
const code = new CodeMaker({ indentationLevel: 1, indentCharacter: '\t' });

const VERSION_FILE = 'version.generated.go';
code.openFile(VERSION_FILE);
code.line('package kernel');
code.line();
Expand All @@ -133,59 +38,4 @@ const thisVersion = require('../package.json').version;
code.line(`const version = ${JSON.stringify(thisVersion)}`);
code.closeFile(VERSION_FILE);

code.save(OUTPUT_DIR).catch(console.error);

function getByteSlice(path: string): { byteSlice: string[]; hash: string[] } {
const rawData = readFileSync(path);
return {
byteSlice: toHexBytes(rawData),
hash: toHexBytes(createHash('SHA512').update(rawData).digest()),
};
}

function toHexBytes(rawData: Buffer): string[] {
const hexString = rawData.toString('hex');
const result = [];
for (let i = 0; i < hexString.length; i += 2) {
result.push(`0x${hexString[i]}${hexString[i + 1]}`);
}
return result;
}

function formatBytes(
code: CodeMaker,
byteSlice: readonly string[],
bytesPerLine = 16,
) {
for (let i = 0; i < byteSlice.length; i += bytesPerLine) {
const line = byteSlice.slice(i, i + bytesPerLine);
code.line(`${line.join(', ')},`);
}
}

/**
* Turns a integer into a "human-readable" format, adding an `_` thousand
* separator.
*
* @param val an integer to be formatted.
*
* @returns the formatted number with thousand separators.
*/
function readableNumber(val: number): string {
return val.toFixed(0).replace(
// This regex can be a little jarring, so it is annotated below with the
// corresponding explanation. It can also be explained in plain english:
// matches the position before any sequence of N consecutive digits (0-9)
// where N is a multiple of 3.
/**/ /\B(?=(\d{3})+(?!\d))/g,
// \B -- not a word boundary (i.e: not start of input)
// (?= ) -- positive lookahead (does not consume input)
// ( )+ -- repeated one or more times
// \d -- any digit (0-9)
// {3} -- repeated exactly 3 times
// (?! ) -- negative lookahead (does not consume input)
// \d -- any digit (0-9), negated by surrounding group
//
'_',
);
}
code.save(KERNEL_LIB_DIR).catch(console.error);
56 changes: 56 additions & 0 deletions packages/@jsii/go-runtime/jsii-runtime-go/embedded/embedded.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package embedded

import (
"embed"
"os"
"path"
)

// embeddedRootDir is the name of the root directory for the embeddedFS variable.
const embeddedRootDir string = "resources"

//go:embed resources/*
var embeddedFS embed.FS

// entrypointName is the path to the entry point relative to the embeddedRootDir.
var entrypointName string = path.Join("bin", "jsii-runtime.js")

// ExtractRuntime extracts a copy of the embedded runtime library into
// the designated directory.
func ExtractRuntime(into string) (entrypoint string, err error) {
err = extractRuntime(into, embeddedRootDir)
if err == nil {
entrypoint = path.Join(into, entrypointName)
}
return
}

// extractRuntime copies the contents of embeddedFS at "from" to the provided "into"
// directory, recursively.
func extractRuntime(into string, from string) error {
files, err := embeddedFS.ReadDir(from)
if err != nil {
return err
}
for _, file := range files {
src := path.Join(from, file.Name())
dest := path.Join(into, file.Name())
if file.IsDir() {
if err = os.Mkdir(dest, 0o700); err != nil {
return err
}
if err = extractRuntime(dest, src); err != nil {
return err
}
} else {
data, err := embeddedFS.ReadFile(src)
if err != nil {
return err
}
if err = os.WriteFile(dest, data, 0o600); err != nil {
return err
}
}
}
return nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package embedded

import (
"path"
"testing"
)

func TestEntryPointExists(t *testing.T) {
if bytes, err := embeddedFS.ReadFile(path.Join(embeddedRootDir, entrypointName)); err != nil {
t.Errorf("unable to read entry point data: %v", err)
} else if len(bytes) == 0 {
t.Error("entry point file is empty")
}

}
2 changes: 1 addition & 1 deletion packages/@jsii/go-runtime/jsii-runtime-go/go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
module github.com/aws/jsii-runtime-go

go 1.15
go 1.16
Loading

0 comments on commit 67cd3ce

Please sign in to comment.