✨ Refactor generic-external-provider (#512)

Clone of #387

We needed this fix
b2df101

Follow up issue created in
#513

---------

Signed-off-by: JonahSussman <sussmanjonah@gmail.com>
Signed-off-by: Pranav Gaikwad <pgaikwad@redhat.com>
Co-authored-by: JonahSussman <sussmanjonah@gmail.com>

.gitignore

@@ -20,3 +20,4 @@ go.work.sum
 
 external-providers/generic-external-provider/generic-external-provider
 external-providers/golang-dependency-provider/golang-dependency-provider
+external-providers/generic-external-provider/debug/

Dockerfile

@@ -3,8 +3,10 @@ WORKDIR /analyzer-lsp
 
 COPY cmd /analyzer-lsp/cmd
 COPY engine /analyzer-lsp/engine
+COPY  event /analyzer-lsp/event
 COPY output /analyzer-lsp/output
 COPY jsonrpc2 /analyzer-lsp/jsonrpc2
+COPY  jsonrpc2_v2 /analyzer-lsp/jsonrpc2_v2
 COPY lsp /analyzer-lsp/lsp
 COPY parser /analyzer-lsp/parser
 COPY provider /analyzer-lsp/provider
@@ -31,7 +33,12 @@ FROM quay.io/konveyor/jdtls-server-base
 
 RUN microdnf install gcc-c++ python-devel python3-devel -y
 RUN python3 -m ensurepip --upgrade
-RUN python3 -m pip install python-lsp-server
+RUN python3 -m pip install 'python-lsp-server>=1.8.2'
+
+ENV NODEJS_VERSION=18
+RUN echo -e "[nodejs]\nname=nodejs\nstream=${NODEJS_VERSION}\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module
+RUN microdnf install nodejs -y
+RUN npm install -g typescript-language-server typescript
 
 COPY --from=jaeger-builder /go/bin/all-in-one-linux /usr/local/bin/all-in-one-linux
 COPY --from=yq-builder /usr/bin/yq /usr/bin/yq

cmd/dep/main.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"sort"
+	"time"
 
 	"github.com/bombsimon/logrusr/v3"
 	"github.com/konveyor/analyzer-lsp/engine/labels"
@@ -85,10 +87,16 @@ func main() {
 				os.Exit(1)
 			}
 		}
+
+		time.Sleep(5 * time.Second)
+
 		err = prov.ProviderInit(ctx)
+		b, _ := json.Marshal(config)
 		if err != nil {
-			log.Error(err, "unable to init the providers", "provider", config.Name)
+			log.Error(err, "unable to init the providers", "provider", config.Name, "the-error-is", err, "config", string(b))
 			os.Exit(1)
+		} else {
+			log.Error(err, "init'd provider", "provider", config.Name, "config", string(b))
 		}
 		providers[config.Name] = prov
 

demo-output.yaml

@@ -438,11 +438,6 @@
         lineNumber: 2
         variables:
           file: file:///analyzer-lsp/examples/python/file_a.py
-      - uri: file:///analyzer-lsp/examples/python/file_b.py
-        message: python sample rule 001
-        lineNumber: 0
-        variables:
-          file: file:///analyzer-lsp/examples/python/file_b.py
     python-sample-rule-002:
       description: ""
       category: potential
@@ -452,11 +447,6 @@
         lineNumber: 5
         variables:
           file: file:///analyzer-lsp/examples/python/file_a.py
-      - uri: file:///analyzer-lsp/examples/python/file_b.py
-        message: python sample rule 002
-        lineNumber: 7
-        variables:
-          file: file:///analyzer-lsp/examples/python/file_b.py
     python-sample-rule-003:
       description: ""
       category: potential

docs/experimental/lsp-notes.md

@@ -0,0 +1,36 @@
+Using gopls as the reference implementation to ensure feature parity across lsp servers.
+
+[settings.md](https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/doc/settings.md) is very useful
+
+The `workspace/symbol` method returns all the top-level declarations that match the given query.
+
+References (each one builds off the next):
+
+- `workspace/symbol` calls this [WorkspaceSymbol function](https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/internal/lsp/source/workspace_symbol.go#L54)
+- Which calls [collectSymbols](https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/internal/lsp/source/workspace_symbol.go#L299)
+- Which calls [snapshot.Symbols](https://github.com/golang/tools/blob/master/gopls/internal/lsp/cache/snapshot.go#L1109) which calls [symbolize](https://github.com/golang/tools/blob/master/gopls/internal/lsp/cache/symbols.go#L21)
+- Which calls [symbolizeImpl](https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/internal/lsp/cache/symbols.go#L72), which after parsing the file, only takes the Decls property of the ast.File, [which are the top-level declarations](https://pkg.go.dev/go/ast#File)
+  - https://github.com/golang/tools/blob/master/gopls/internal/lsp/cache/parse.go#L59
+- https://github.com/golang/tools/blob/master/gopls/internal/lsp/cache/symbols.go#L115
+- https://pkg.go.dev/go/ast#File
+
+symbolScope controls which packages are searched for workspace/symbol requests.
+Default: `"all"` matches symbols in any loaded package, including dependencies.
+
+symbolMatcher sets the algorithm that is used when finding workspace symbols.
+Default: `"FastFuzzy"`.
+- https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/internal/lsp/source/workspace_symbol.go#L160
+-
+
+symbolStyle controls how symbols are qualified in symbol responses.
+Default: `"Dynamic"` uses whichever qualifier results in the highest scoring
+match for the given symbol query. Here a "qualifier" is any "/" or "."
+delimited suffix of the fully qualified symbol. i.e. "to/pkg.Foo.Field" or
+just "Foo.Field".
+- https://github.com/golang/tools/blob/ecbfa885b278478686e8b8efb52535e934c53ec5/gopls/internal/lsp/source/workspace_symbol.go#L99
+
+
+`workspace/didChangeConfiguration` to change settings
+
+
+https://jedi.readthedocs.io/en/latest/docs/api.html#jedi.Project.search

event/bench_test.go

@@ -0,0 +1,158 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package event_test
+
+import (
+	"context"
+	"io"
+	"log"
+	"testing"
+
+	"github.com/konveyor/analyzer-lsp/event"
+	"github.com/konveyor/analyzer-lsp/event/core"
+	"github.com/konveyor/analyzer-lsp/event/export"
+	"github.com/konveyor/analyzer-lsp/event/keys"
+	"github.com/konveyor/analyzer-lsp/event/label"
+)
+
+type Hooks struct {
+	A func(ctx context.Context, a int) (context.Context, func())
+	B func(ctx context.Context, b string) (context.Context, func())
+}
+
+var (
+	aValue  = keys.NewInt("a", "")
+	bValue  = keys.NewString("b", "")
+	aCount  = keys.NewInt64("aCount", "Count of time A is called.")
+	aStat   = keys.NewInt("aValue", "A value.")
+	bCount  = keys.NewInt64("B", "Count of time B is called.")
+	bLength = keys.NewInt("BLen", "B length.")
+
+	Baseline = Hooks{
+		A: func(ctx context.Context, a int) (context.Context, func()) {
+			return ctx, func() {}
+		},
+		B: func(ctx context.Context, b string) (context.Context, func()) {
+			return ctx, func() {}
+		},
+	}
+
+	StdLog = Hooks{
+		A: func(ctx context.Context, a int) (context.Context, func()) {
+			log.Printf("A where a=%d", a)
+			return ctx, func() {}
+		},
+		B: func(ctx context.Context, b string) (context.Context, func()) {
+			log.Printf("B where b=%q", b)
+			return ctx, func() {}
+		},
+	}
+
+	Log = Hooks{
+		A: func(ctx context.Context, a int) (context.Context, func()) {
+			core.Log1(ctx, "A", aValue.Of(a))
+			return ctx, func() {}
+		},
+		B: func(ctx context.Context, b string) (context.Context, func()) {
+			core.Log1(ctx, "B", bValue.Of(b))
+			return ctx, func() {}
+		},
+	}
+
+	Trace = Hooks{
+		A: func(ctx context.Context, a int) (context.Context, func()) {
+			return core.Start1(ctx, "A", aValue.Of(a))
+		},
+		B: func(ctx context.Context, b string) (context.Context, func()) {
+			return core.Start1(ctx, "B", bValue.Of(b))
+		},
+	}
+
+	Stats = Hooks{
+		A: func(ctx context.Context, a int) (context.Context, func()) {
+			core.Metric1(ctx, aStat.Of(a))
+			core.Metric1(ctx, aCount.Of(1))
+			return ctx, func() {}
+		},
+		B: func(ctx context.Context, b string) (context.Context, func()) {
+			core.Metric1(ctx, bLength.Of(len(b)))
+			core.Metric1(ctx, bCount.Of(1))
+			return ctx, func() {}
+		},
+	}
+
+	initialList = []int{0, 1, 22, 333, 4444, 55555, 666666, 7777777}
+	stringList  = []string{
+		"A value",
+		"Some other value",
+		"A nice longer value but not too long",
+		"V",
+		"",
+		"ı",
+		"prime count of values",
+	}
+)
+
+type namedBenchmark struct {
+	name string
+	test func(*testing.B)
+}
+
+func Benchmark(b *testing.B) {
+	b.Run("Baseline", Baseline.runBenchmark)
+	b.Run("StdLog", StdLog.runBenchmark)
+	benchmarks := []namedBenchmark{
+		{"Log", Log.runBenchmark},
+		{"Trace", Trace.runBenchmark},
+		{"Stats", Stats.runBenchmark},
+	}
+
+	event.SetExporter(nil)
+	for _, t := range benchmarks {
+		b.Run(t.name+"NoExporter", t.test)
+	}
+
+	event.SetExporter(noopExporter)
+	for _, t := range benchmarks {
+		b.Run(t.name+"Noop", t.test)
+	}
+
+	event.SetExporter(export.Spans(export.LogWriter(io.Discard, false)))
+	for _, t := range benchmarks {
+		b.Run(t.name, t.test)
+	}
+}
+
+func A(ctx context.Context, hooks Hooks, a int) int {
+	ctx, done := hooks.A(ctx, a)
+	defer done()
+	return B(ctx, hooks, a, stringList[a%len(stringList)])
+}
+
+func B(ctx context.Context, hooks Hooks, a int, b string) int {
+	_, done := hooks.B(ctx, b)
+	defer done()
+	return a + len(b)
+}
+
+func (hooks Hooks) runBenchmark(b *testing.B) {
+	ctx := context.Background()
+	b.ReportAllocs()
+	b.ResetTimer()
+	var acc int
+	for i := 0; i < b.N; i++ {
+		for _, value := range initialList {
+			acc += A(ctx, hooks, value)
+		}
+	}
+}
+
+func init() {
+	log.SetOutput(io.Discard)
+}
+
+func noopExporter(ctx context.Context, ev core.Event, lm label.Map) context.Context {
+	return ctx
+}

event/core/event.go

@@ -0,0 +1,85 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package core provides support for event based telemetry.
+package core
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/konveyor/analyzer-lsp/event/label"
+)
+
+// Event holds the information about an event of note that occurred.
+type Event struct {
+	at time.Time
+
+	// As events are often on the stack, storing the first few labels directly
+	// in the event can avoid an allocation at all for the very common cases of
+	// simple events.
+	// The length needs to be large enough to cope with the majority of events
+	// but no so large as to cause undue stack pressure.
+	// A log message with two values will use 3 labels (one for each value and
+	// one for the message itself).
+
+	static  [3]label.Label // inline storage for the first few labels
+	dynamic []label.Label  // dynamically sized storage for remaining labels
+}
+
+// eventLabelMap implements label.Map for a the labels of an Event.
+type eventLabelMap struct {
+	event Event
+}
+
+func (ev Event) At() time.Time { return ev.at }
+
+func (ev Event) Format(f fmt.State, r rune) {
+	if !ev.at.IsZero() {
+		fmt.Fprint(f, ev.at.Format("2006/01/02 15:04:05 "))
+	}
+	for index := 0; ev.Valid(index); index++ {
+		if l := ev.Label(index); l.Valid() {
+			fmt.Fprintf(f, "\n\t%v", l)
+		}
+	}
+}
+
+func (ev Event) Valid(index int) bool {
+	return index >= 0 && index < len(ev.static)+len(ev.dynamic)
+}
+
+func (ev Event) Label(index int) label.Label {
+	if index < len(ev.static) {
+		return ev.static[index]
+	}
+	return ev.dynamic[index-len(ev.static)]
+}
+
+func (ev Event) Find(key label.Key) label.Label {
+	for _, l := range ev.static {
+		if l.Key() == key {
+			return l
+		}
+	}
+	for _, l := range ev.dynamic {
+		if l.Key() == key {
+			return l
+		}
+	}
+	return label.Label{}
+}
+
+func MakeEvent(static [3]label.Label, labels []label.Label) Event {
+	return Event{
+		static:  static,
+		dynamic: labels,
+	}
+}
+
+// CloneEvent event returns a copy of the event with the time adjusted to at.
+func CloneEvent(ev Event, at time.Time) Event {
+	ev.at = at
+	return ev
+}