Skip to content

Commit 4acfad3

Browse files
x1unixx1unix
authored
feat: local completion index (#419)
* feat: refactor symbols db * feat: provide project id * feat: build import context * feat: provide monitoring range * feat: use cache * feat: implement GOROOT indexer * fix: fix comment validator * fix: fix package filter * feat: update response model * feat: debug parser * fix: fix doc generation * fix: fix doc gen for types * feat: add filter for builtins * feat: keep only func signature * feat: finish docutil * fix: fix IsGoSourceFile * feat: use predefined buff sizes * feat: restructure pkgindex * chore: require protobuf for future developments * feat: use structure of arrays to save space * feat: add tests * chore: goimports * chore: add pprof * chore: fix docs * chore: update js types * fix: respect build constraints during scan * feat: flatten symbol source struct * fix: fix indexing and fallbacks * chore: make monaco-editor work in vitest * fix: fix imports traversal * fix: fix single import parse * fix: fix start line number * fix: add additional index * chore: use built-in cache in setup-go * feat: show completion after dot * fix: fix PR job * fix: linter * fix: linter * fix: linter * chore: bump golangci-lint * chore: bump golangci-lint * chore: use node cache * fix: tests
1 parent a03da77 commit 4acfad3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+4710
-734
lines changed

‎.github/workflows/pull_request.yml

+9-16
Original file line numberDiff line numberDiff line change
@@ -12,21 +12,7 @@ jobs:
1212
test:
1313
runs-on: ubuntu-latest
1414
steps:
15-
- id: go-cache-paths
16-
run: |
17-
echo "::set-output name=go-build::$(go env GOCACHE)"
18-
echo "::set-output name=go-mod::$(go env GOMODCACHE)"
1915
- uses: actions/checkout@v4
20-
- name: Go Build Cache
21-
uses: actions/cache@v4
22-
with:
23-
path: ${{ steps.go-cache-paths.outputs.go-build }}
24-
key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
25-
- name: Go Mod Cache
26-
uses: actions/cache@v4
27-
with:
28-
path: ${{ steps.go-cache-paths.outputs.go-mod }}
29-
key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
3016
- uses: actions/cache@v4
3117
env:
3218
cache-name: npm-cache
@@ -35,6 +21,12 @@ jobs:
3521
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
3622
restore-keys: |
3723
${{ runner.os }}-build-${{ env.cache-name }}-
24+
- name: Setup node
25+
uses: actions/setup-node@v4
26+
with:
27+
cache: 'yarn'
28+
node-version-file: '.nvmrc'
29+
cache-dependency-path: 'web/yarn.lock'
3830
- name: Install modules
3931
run: yarn
4032
working-directory: ./web
@@ -47,10 +39,11 @@ jobs:
4739
- name: Setup Go
4840
uses: actions/setup-go@v5
4941
with:
50-
go-version: "^1.21.0"
42+
go-version: "^1.23.0"
43+
cache-dependency-path: go.sum
5144
- name: golangci-lint
5245
uses: golangci/golangci-lint-action@v6.1.0
5346
with:
54-
version: v1.54.2
47+
version: v1.61.0
5548
- run: go version
5649
- run: go test ./...

‎HACKING.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ Please ensure that you have installed:
7474
* [Node Version Manager](https://github.com/nvm-sh/nvm) or Node.js 20 (`lts/iron`)
7575
* [Yarn](https://yarnpkg.com/) package manager.
7676
* Go 1.22+
77+
* Protobuf:
78+
* [protoc](https://developers.google.com/protocol-buffers)
79+
* [Protobuf Go plugins](https://grpc.io/docs/languages/go/quickstart/)
7780

7881
### First-time setup
7982

@@ -83,7 +86,7 @@ Run following commands to configure a just cloned project:
8386
|-------------------|----------------------------------------------------------|
8487
| `make preinstall` | Installs NPM packages for a web app. |
8588
| `make wasm` | Builds WebAssembly binaries used by the web app. |
86-
| `make pkg-index` | Generates Go packages index for autocomplete in web app. |
89+
| `make go-index` | Generates Go packages index for autocomplete in web app. |
8790

8891
### Running Project
8992

‎build.mk

+11-6
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,15 @@ check-go:
3232
exit 1; \
3333
fi
3434

35-
.PHONY: pkg-index
36-
pkg-index:
37-
@echo ":: Generating Go packages index..." && \
38-
$(GO) run ./tools/pkgindexer -o $(UI)/public/data/imports.json
35+
.PHONY: imports-index
36+
imports-index:
37+
@echo ":: Generating Go imports index..." && \
38+
$(GO) run ./tools/pkgindexer imports -o $(UI)/public/data/imports.json $(OPTS)
39+
40+
.PHONY: go-index
41+
go-index:
42+
@echo ":: Generating Go symbols index..." && \
43+
$(GO) run ./tools/pkgindexer index -o $(UI)/public/data/go-index.json $(OPTS)
3944

4045
.PHONY:check-yarn
4146
check-yarn:
@@ -68,12 +73,12 @@ analyzer.wasm:
6873
wasm: wasm_exec.js analyzer.wasm
6974

7075
.PHONY: build
71-
build: check-go check-yarn clean preinstall gen build-server wasm pkg-index build-ui
76+
build: check-go check-yarn clean preinstall gen build-server wasm go-index imports-index build-ui
7277
@echo ":: Copying assets..." && \
7378
cp -rfv ./data $(TARGET)/data && \
7479
mv -v $(UI)/build $(TARGET)/public && \
7580
echo ":: Build done - $(TARGET)"
7681

7782
.PHONY:gen
7883
gen:
79-
@find . -name '*_gen.go' -exec go generate -v {} \;
84+
@find . -name 'generate.go' -exec go generate -v {} \;

‎go.mod

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
module github.com/x1unix/go-playground
22

3-
go 1.21
3+
go 1.23
4+
5+
toolchain go1.23.1
46

57
require (
68
github.com/TheZeroSlave/zapsentry v1.10.0
@@ -25,12 +27,15 @@ require (
2527
github.com/benbjohnson/clock v1.1.0 // indirect
2628
github.com/davecgh/go-spew v1.1.1 // indirect
2729
github.com/getsentry/sentry-go v0.13.0 // indirect
30+
github.com/hashicorp/go-set/v3 v3.0.0 // indirect
2831
github.com/inconshreveable/mousetrap v1.1.0 // indirect
2932
github.com/pmezard/go-difflib v1.0.0 // indirect
3033
github.com/spf13/cobra v1.8.1 // indirect
3134
github.com/spf13/pflag v1.0.5 // indirect
3235
go.uber.org/atomic v1.9.0 // indirect
3336
go.uber.org/multierr v1.8.0 // indirect
34-
golang.org/x/sys v0.3.0 // indirect
37+
golang.org/x/sys v0.21.0 // indirect
38+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect
39+
google.golang.org/protobuf v1.34.2 // indirect
3540
gopkg.in/yaml.v3 v3.0.1 // indirect
3641
)

‎go.sum

+10
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6
1414
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
1515
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
1616
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
17+
github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA=
18+
github.com/hashicorp/go-set/v3 v3.0.0/go.mod h1:IEghM2MpE5IaNvL+D7X480dfNtxjRXZ6VMpK3C8s2ok=
1719
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
1820
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
1921
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
@@ -88,6 +90,8 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
8890
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
8991
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
9092
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
93+
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
94+
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
9195
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
9296
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
9397
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -100,6 +104,12 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
100104
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
101105
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
102106
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
107+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 h1:F29+wU6Ee6qgu9TddPgooOdaqsxTMunOoj8KA5yuS5A=
108+
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1/go.mod h1:5KF+wpkbTSbGcR9zteSqZV6fqFOWBl4Yde8En8MryZA=
109+
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
110+
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
111+
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
112+
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
103113
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
104114
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
105115
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎internal/analyzer/docfmt.go

+26-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package analyzer
22

3-
import "strings"
3+
import (
4+
"bytes"
5+
"go/ast"
6+
"go/doc/comment"
7+
"strings"
8+
)
49

510
const (
611
newLineChar = '\n'
@@ -27,7 +32,27 @@ func isDocLine(line string) bool {
2732
return false
2833
}
2934

35+
// ParseCommentGroup parses comments from AST and returns them in Markdown format.
36+
func ParseCommentGroup(group *ast.CommentGroup) []byte {
37+
if group == nil || len(group.List) == 0 {
38+
return nil
39+
}
40+
41+
var (
42+
parser comment.Parser
43+
printer comment.Printer
44+
)
45+
46+
str := group.Text()
47+
parsedDoc := parser.Parse(str)
48+
mdDoc := printer.Markdown(parsedDoc)
49+
mdDoc = bytes.TrimSuffix(mdDoc, []byte("\n"))
50+
return mdDoc
51+
}
52+
3053
// FormatDocString parses Go comment and returns a markdown-formatted string.
54+
//
55+
// Deprecated: use ParseCommentGroup instead.
3156
func FormatDocString(str string) MarkdownString {
3257
if str == "" {
3358
return MarkdownString{Value: str}

‎internal/pkgindex/cmd/common.go

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package cmd
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io"
7+
"os"
8+
"path/filepath"
9+
)
10+
11+
func writeOutput(flags importsFlags, data any) error {
12+
if flags.outFile == "" {
13+
return getEncoder(os.Stdout, true).Encode(data)
14+
}
15+
16+
if err := os.MkdirAll(filepath.Dir(flags.outFile), 0755); err != nil {
17+
return fmt.Errorf("failed to pre-create parent directories: %w", err)
18+
}
19+
20+
f, err := os.OpenFile(flags.outFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
21+
defer silentClose(f)
22+
23+
if err != nil {
24+
return fmt.Errorf("can't create output file: %w", err)
25+
}
26+
27+
if err := getEncoder(f, flags.prettyPrint).Encode(data); err != nil {
28+
return fmt.Errorf("can't write JSON to file %q: %w", flags.outFile, err)
29+
}
30+
31+
return nil
32+
}
33+
34+
func getEncoder(dst io.Writer, pretty bool) *json.Encoder {
35+
enc := json.NewEncoder(dst)
36+
if pretty {
37+
enc.SetIndent("", " ")
38+
}
39+
40+
return enc
41+
}
42+
43+
func silentClose(c io.Closer) {
44+
// I don't care
45+
_ = c.Close()
46+
}

‎internal/pkgindex/cmd/flags.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/x1unix/go-playground/internal/pkgindex/imports"
7+
)
8+
9+
type globalFlags struct {
10+
goRoot string
11+
heapProfFile string
12+
}
13+
14+
func (f globalFlags) withDefaults() (globalFlags, error) {
15+
if f.goRoot != "" {
16+
return f, nil
17+
}
18+
19+
goRoot, err := imports.ResolveGoRoot()
20+
if err != nil {
21+
return f, fmt.Errorf(
22+
"cannot find GOROOT, please set GOROOT path or check if Go is installed.\nError: %w",
23+
err,
24+
)
25+
}
26+
f.goRoot = goRoot
27+
return f, err
28+
}
29+
30+
type importsFlags struct {
31+
*globalFlags
32+
verbose bool
33+
prettyPrint bool
34+
stdout bool
35+
outFile string
36+
}
37+
38+
func (f importsFlags) validate() error {
39+
if f.outFile == "" && !f.stdout {
40+
return fmt.Errorf("missing output file flag. Use --stdout flag to print into stdout")
41+
}
42+
43+
if f.stdout && f.outFile != "" {
44+
return fmt.Errorf("ambiguous output flag: --stdout and output file flag can't be together")
45+
}
46+
47+
return nil
48+
}

‎internal/pkgindex/cmd/imports.go

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package cmd
2+
3+
import (
4+
"log"
5+
6+
"github.com/spf13/cobra"
7+
"github.com/x1unix/go-playground/internal/pkgindex/imports"
8+
)
9+
10+
func newCmdImports(g *globalFlags) *cobra.Command {
11+
flags := importsFlags{
12+
globalFlags: g,
13+
}
14+
cmd := &cobra.Command{
15+
Use: "imports [-r goroot] [-o output]",
16+
Short: "Generate imports.json file for old Playground version",
17+
Long: "Generate imports file which contains list of all importable packages. Used in legacy app versions",
18+
PreRunE: func(_ *cobra.Command, _ []string) error {
19+
return flags.validate()
20+
},
21+
RunE: func(_ *cobra.Command, _ []string) error {
22+
return runGenImports(flags)
23+
},
24+
}
25+
26+
cmd.Flags().StringVarP(&flags.outFile, "output", "o", "", "Path to output file. When enpty, prints to stdout")
27+
cmd.Flags().BoolVarP(&flags.prettyPrint, "pretty", "P", false, "Add indents to JSON output")
28+
cmd.Flags().BoolVar(&flags.stdout, "stdout", false, "Dump result into stdout")
29+
return cmd
30+
}
31+
32+
func runGenImports(flags importsFlags) error {
33+
scanner := imports.NewGoRootScanner(flags.goRoot)
34+
results, err := scanner.Scan()
35+
if err != nil {
36+
return err
37+
}
38+
39+
if err := writeOutput(flags, results); err != nil {
40+
return err
41+
}
42+
43+
log.Printf("Scanned %d packages", len(results.Packages))
44+
return nil
45+
}

‎internal/pkgindex/cmd/index.go

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package cmd
2+
3+
import (
4+
"log"
5+
6+
"github.com/spf13/cobra"
7+
"github.com/x1unix/go-playground/internal/pkgindex/index"
8+
)
9+
10+
func newCmdIndex(g *globalFlags) *cobra.Command {
11+
flags := importsFlags{
12+
globalFlags: g,
13+
}
14+
15+
cmd := &cobra.Command{
16+
Use: "index [-r goroot] [-o output]",
17+
Short: "Generate index file with standard Go packages and symbols",
18+
Long: "Generate a JSON file that contains list of all standard Go packages and its symbols. Used in new version of app",
19+
PreRunE: func(_ *cobra.Command, _ []string) error {
20+
index.Debug = flags.verbose
21+
return flags.validate()
22+
},
23+
RunE: func(_ *cobra.Command, _ []string) error {
24+
return runGenIndex(flags)
25+
},
26+
}
27+
28+
cmd.Flags().StringVarP(&flags.outFile, "output", "o", "", "Path to output file. When enpty, prints to stdout")
29+
cmd.Flags().BoolVarP(&flags.prettyPrint, "pretty", "P", false, "Add indents to JSON output")
30+
cmd.Flags().BoolVar(&flags.stdout, "stdout", false, "Dump result into stdout")
31+
cmd.Flags().BoolVarP(&flags.verbose, "verbose", "v", false, "Enable verbose logging")
32+
return cmd
33+
}
34+
35+
func runGenIndex(flags importsFlags) error {
36+
entries, err := index.ScanRoot(flags.goRoot)
37+
if err != nil {
38+
return err
39+
}
40+
41+
if err := writeOutput(flags, entries); err != nil {
42+
return err
43+
}
44+
45+
log.Printf("Scanned %d packages and %d symbols", len(entries.Packages.Names), len(entries.Symbols.Names))
46+
return nil
47+
}

0 commit comments

Comments
 (0)