Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JQ Support #9

Merged
merged 43 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
a15b51c
Added jq wasm binary
evg4b Jul 14, 2024
66bf860
Updated jq
evg4b Jul 14, 2024
d1a41a4
Reorganize structure
evg4b Jul 14, 2024
299788c
Created new structure of packages
evg4b Jul 14, 2024
0e15feb
Refactor
evg4b Jul 14, 2024
a1a6fad
Fixed linting
evg4b Jul 14, 2024
9fb2561
Update workflow
evg4b Jul 14, 2024
a221bf6
Updates
evg4b Jul 14, 2024
9ef1e2b
Rebuild
evg4b Jul 14, 2024
c32275c
comment tests
evg4b Jul 14, 2024
addc686
comment tests
evg4b Jul 14, 2024
aee9442
Test
evg4b Jul 14, 2024
eb870a8
Updated git ignore
evg4b Jul 14, 2024
e0d9f25
Added jq wrapper
evg4b Jul 17, 2024
7309020
Synchronize wasm outputs
evg4b Jul 17, 2024
0ea545b
Added buttons
evg4b Jul 19, 2024
cec1427
Added elemtns
evg4b Jul 19, 2024
1f1dae0
Updated jq query pkg
evg4b Jul 19, 2024
b4a3e5c
Rebuild wasms
evg4b Jul 19, 2024
0f35295
Added transforming
evg4b Jul 20, 2024
e8b7e89
Added correct object and array structure
evg4b Jul 20, 2024
29c12f0
Updaated multiple row responce
evg4b Jul 21, 2024
5a4d04e
Updated readme
evg4b Jul 21, 2024
c7832ef
split tsconfigs
evg4b Jul 21, 2024
8a898e2
Added global make file
evg4b Jul 21, 2024
eb73b6e
Updated tsconfigs
evg4b Jul 23, 2024
fcd97b6
Updated error casting
evg4b Jul 23, 2024
8794db1
Updated error nodes
evg4b Jul 28, 2024
e756177
Updated
evg4b Jul 30, 2024
77ade1d
Added tuple support
evg4b Jul 30, 2024
b563b2e
Added tests for packages
evg4b Jul 30, 2024
b9a1c5c
Fixed linter issues
evg4b Jul 30, 2024
4a97de8
Path test ouputs
evg4b Jul 30, 2024
5f7de3a
Reorganize package
evg4b Jul 31, 2024
aaa1a7c
Update
evg4b Jul 31, 2024
9243ffb
Update
evg4b Jul 31, 2024
fa939f1
Reorganize ts porject
evg4b Aug 4, 2024
1bc9cdd
Updated wasm
evg4b Aug 5, 2024
1f095ce
Added tests
evg4b Aug 5, 2024
5fee9b4
x
evg4b Aug 5, 2024
3229f85
Increase test coverage
evg4b Aug 5, 2024
3aab587
Updated screens
evg4b Aug 5, 2024
8dcf202
Updated media data
evg4b Aug 5, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,16 @@ jobs:
go-version: '1.22.2'

- name: Build JSON Tokenizer
working-directory: ./tokenizer
run: GOOS=js GOARCH=wasm go build -o tokenizer.wasm -v .
working-directory: ./packages
run: GOOS=js GOARCH=wasm go build -o tokenizer.wasm -v ./tokenizer

- name: Test JSON Tokenizer
working-directory: ./tokenizer
run: go test -v -timeout 1m -race -coverprofile=coverage.out -json ./tokenizer/... > test-report.out
- name: Build JQ
working-directory: ./packages
run: GOOS=js GOARCH=wasm go build -o tokenizer.wasm -v ./jq

- name: Test GO code
working-directory: ./packages
run: go test -v -timeout 1m -coverprofile=coverage.out -json ./pkg/... > test-report.out

- name: Use Node.js
uses: actions/setup-node@v3
Expand Down
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -130,5 +130,6 @@ dist
.pnp.*
.idea
.DS_Store
tokenizer/test-report.out
tokenizer/coverage.out
extention.zip
packages/coverage.out
packages/test-report.out
11 changes: 11 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
default: build-packages build-extension pack-extension

build-extension:
yarn build:production

build-packages:
@echo "Building wasms..."
cd packages && $(MAKE)

pack-extension:
cd ./dist && zip -r -X ../extention.zip *
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
- Fast parse and format JSON
- Supports big numbers (includes x64 integer and float)
- Guaranteed order of keys
- JQ expression support
- Formatted and Raw view switcher
- Array and Object expand/collapse
- Safe for selecting and copying JSON
Expand Down
2 changes: 1 addition & 1 deletion eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ export default tseslint.config(
'**/node_modules/',
'.git/',
'dist/',
'tokenizer/',
'jest.config.js',
'media_data/',
'tsup.config.js',
'packages/wasm_exec.js'
],
},
eslint.configs.recommended,
Expand Down
7 changes: 6 additions & 1 deletion jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,13 @@ module.exports = {
'!tokenizer/**/*.spec.ts',
'!tokenizer/tests/**/*',
],
setupFiles: ['<rootDir>/src/jest.setup.ts'],
moduleDirectories: ['node_modules', 'src'],
setupFiles: ['<rootDir>/jest.setup.ts'],
verbose: true,
moduleNameMapper: {
'^@packages/jq/(.*)$': '<rootDir>packages/jq/$1',
'^@packages/tokenizer/(.*)$': '<rootDir>packages/tokenizer/$1',
},
transform: {
'^.+\\.[tj]sx?$': [
'ts-jest',
Expand Down
2 changes: 2 additions & 0 deletions src/tokenizer/polifills.ts → jest.setup.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { TextDecoder, TextEncoder } from 'text-encoding';
global.TextDecoder = TextDecoder;
global.TextEncoder = TextEncoder;
import './packages/wasm_exec.js';

Binary file modified media_data/.yarn/install-state.gz
Binary file not shown.
Original file line number Diff line number Diff line change
Expand Up @@ -1199,4 +1199,4 @@
"title": "ipsam aperiam voluptates qui",
"completed": false
}
]
]
1 change: 1 addition & 0 deletions media_data/queries.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.[] | { "id": .id, name: .title }
Binary file modified media_data/screens/screen2.psd
Binary file not shown.
Binary file modified media_data/screens/screen3.psd
Binary file not shown.
Binary file modified media_data/screens/screen4.psd
Binary file not shown.
Binary file modified media_data/screens/screen5.psd
Binary file not shown.
1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
"devDependencies": {
"@eslint/js": "^9.8.0",
"@jest/globals": "^29.7.0",
"@jgoz/esbuild-plugin-livereload": "^2.1.1",
"@types/chrome": "^0.0.269",
"@types/golang-wasm-exec": "^1.15.2",
"@types/jest": "^29.5.12",
Expand Down
52 changes: 52 additions & 0 deletions packages/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
default: tokenizer jq exec-script

test:
@echo "Testing..."
go test -v ./pkg/...

exec-script:
@echo "Copying exec scripts..."
rm -f ./wasm_exec.js
cp "$(shell go env GOROOT)/misc/wasm/wasm_exec.js" .

# Tokenizer

tokenizer: tokenizer-clean tokenizer-build tokenizer-optimize

TOKENIZER_FILE=tokenizer/tokenizer.wasm
TOKENIZER_FILE_BAK=${TOKENIZER_FILE}.bak

tokenizer-build:
@echo "Building wasm..."
go build -trimpath -ldflags="-s -w -buildid=" -o ${TOKENIZER_FILE} ./tokenizer

tokenizer-optimize:
@echo "Optimizing wasm..."
cp ${TOKENIZER_FILE} ${TOKENIZER_FILE_BAK}
wasm-opt --enable-bulk-memory -Oz -o ${TOKENIZER_FILE} ${TOKENIZER_FILE_BAK}
rm -f ${TOKENIZER_FILE_BAK}

tokenizer-clean:
@echo "Cleaning..."
rm -f ${TOKENIZER_FILE} ${TOKENIZER_FILE_BAK}

# JQ

jq: jq-clean jq-build jq-optimize

JQ_FILE=jq/jq.wasm
JQ_FILE_BAK=${TOKENIZER_FILE}.bak

jq-build:
@echo "Building wasm..."
go build -trimpath -ldflags="-s -w -buildid=" -o ${JQ_FILE} ./jq

jq-optimize:
@echo "Optimizing wasm..."
cp ${JQ_FILE} ${JQ_FILE_BAK}
wasm-opt --enable-bulk-memory -Oz -o ${JQ_FILE} ${JQ_FILE_BAK}
rm -f ${JQ_FILE_BAK}

jq-clean:
@echo "Cleaning..."
rm -f ${JQ_FILE} ${JQ_FILE_BAK}
15 changes: 14 additions & 1 deletion tokenizer/README.md → packages/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# WASM json tokenizer
# WASMs for modern-json-formatter

Custom json tokenizer written in GO and compiled to WASM.

Expand Down Expand Up @@ -64,3 +64,16 @@ Array schema:
]
}
```

Tuple schema:

```js
{
type: "tuple",
items: [
// inner value...
// inner value...
// inner value...
]
}
```
13 changes: 13 additions & 0 deletions packages/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
module packages

go 1.22.4

require github.com/itchyny/gojq v0.12.16

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/itchyny/timefmt-go v0.1.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.9.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
5 changes: 4 additions & 1 deletion tokenizer/go.sum → packages/go.sum
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/itchyny/gojq v0.12.16 h1:yLfgLxhIr/6sJNVmYfQjTIv0jGctu6/DgDoivmxTr7g=
github.com/itchyny/gojq v0.12.16/go.mod h1:6abHbdC2uB9ogMS38XsErnfqJ94UlngIJGlRAIj4jTM=
github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
13 changes: 13 additions & 0 deletions packages/jq/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { TokenizerResponse } from '@packages/tokenizer';
import { importWasm } from '../shared';

let go = new Go();

export const jq = async (data: string, query: string): Promise<TokenizerResponse> => {
if (!('___jq' in window) || go.exited) {
go = new Go();
await importWasm(go, 'jq.wasm');
}

return window.___jq(data, query);
};
22 changes: 22 additions & 0 deletions packages/jq/jq.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { readFileSync } from 'fs';
import { chromeMockAfter, chromeMockBefore, tNumber } from '../../testing';
import { jq } from './index';

jest.mock('../shared/wasm_helpers.ts', () => ({
loadWasm: (_: string, imports: WebAssembly.Imports) => {
const data = readFileSync('packages/jq/jq.wasm');
return WebAssembly.instantiate(data, imports);
},
}));

describe('jq', () => {
beforeAll(chromeMockBefore);

afterAll(chromeMockAfter);

test('should return a TokenizerResponse', async () => {
const data = await jq('{ "data": 123 }', '.data');

expect(data).toEqual(tNumber(`123`));
});
});
Binary file added packages/jq/jq.wasm
Binary file not shown.
34 changes: 34 additions & 0 deletions packages/jq/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package main

import (
"errors"
"packages/pkg/jq"
"packages/pkg/tokens"
"syscall/js"
)

func main() {
window := js.Global()
window.Set("___jq", wrapper(jq.Query))
<-make(chan struct{})
}

func wrapper(query func(input string, query string) (any, error)) js.Func {
return js.FuncOf(func(_ js.Value, args []js.Value) any {
if len(args) != 2 {
return js.ValueOf(tokens.ErrorNode(
"jq",
errors.New("invalid arguments passed"),
))
}

if jsonTree, err := query(args[0].String(), args[1].String()); err != nil {
return js.ValueOf(tokens.ErrorNode(
"jq",
err,
))
} else {
return js.ValueOf(jsonTree)
}
})
}
1 change: 1 addition & 0 deletions packages/jq/types.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
declare function ___jq(data: string, query: string): TokenizerResponse;
34 changes: 34 additions & 0 deletions packages/pkg/jq/helpres.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package jq

import (
"packages/pkg/tokens"
"strconv"
)

func formatFloat(value any) string {
return strconv.FormatFloat(value.(float64), 'f', -1, 64)
}

func formatInt(value any) string {
return strconv.Itoa(value.(int))
}

func arrayItems(value any) []any {
items := value.([]any)
mapped := make([]any, len(items))
for i, item := range items {
mapped[i] = normalise(item)
}

return mapped
}

func objectProperties(value any) []any {
properties := value.(map[string]any)
mapped := make([]any, 0, len(properties))
for key, value := range properties {
mapped = append(mapped, tokens.PropertyNode(key, normalise(value)))
}

return mapped
}
29 changes: 29 additions & 0 deletions packages/pkg/jq/normalise.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package jq

import (
"math/big"
"packages/pkg/tokens"
)

func normalise(value any) any {
switch value.(type) {
case string:
return tokens.StringNode(value.(string))
case float64:
return tokens.NumberNode(formatFloat(value))
case bool:
return tokens.BoolNode(value.(bool))
case int:
return tokens.NumberNode(formatInt(value))
case nil:
return tokens.NullNode()
case *big.Int:
return tokens.NumberNode(value.(*big.Int).String())
case []any:
return tokens.ArrayNode(arrayItems(value))
case map[string]any:
return tokens.ObjectNode(objectProperties(value))
default:
panic("unexpected type")
}
}
50 changes: 50 additions & 0 deletions packages/pkg/jq/query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package jq

import (
"encoding/json"
"errors"
"github.com/itchyny/gojq"
"packages/pkg/tokens"
"strings"
)

func Query(jsonString string, queryString string) (any, error) {
decoder := json.NewDecoder(strings.NewReader(jsonString))
decoder.UseNumber()

var data any
if err := decoder.Decode(&data); err != nil {
return nil, err
}

query, err := gojq.Parse(queryString)
if err != nil {
return nil, err
}

results := make([]any, 0, 10)

iterator := query.Run(data)
for {
v, ok := iterator.Next()
if !ok {
break
}
if err, ok := v.(error); ok {
var haltError *gojq.HaltError
if errors.As(err, &haltError) && haltError.Value() == nil {
break
}

return nil, err
}

results = append(results, normalise(v))
}

if len(results) == 1 {
return results[0], nil
}

return tokens.TupleNode(results), nil
}
Loading
Loading