Skip to content

Commit

Permalink
feat!: using Gazelle's lifecycle manager to manage external processes (
Browse files Browse the repository at this point in the history
…bazelbuild#1284)

Gazelle v0.30.0 introduced a lifecycle manager. We can use that to start
and shutdown parser and stdmodule processes. So we don't need to use
`init` function or creating `context.WithTimeout`.

BREAKING CHANGES:
This requires the users of this Gazelle extension to upgrade to Gazelle
v0.30.0 or above.
  • Loading branch information
linzhp authored Jun 21, 2023
1 parent 0d59fcf commit fe2c325
Show file tree
Hide file tree
Showing 7 changed files with 73 additions and 25 deletions.
9 changes: 6 additions & 3 deletions gazelle/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ module github.com/bazelbuild/rules_python/gazelle
go 1.19

require (
github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b
github.com/bazelbuild/bazel-gazelle v0.31.1
github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d
github.com/bazelbuild/rules_go v0.39.1
github.com/bmatcuk/doublestar v1.3.4
github.com/emirpasic/gods v1.18.1
github.com/ghodss/yaml v1.0.0
Expand All @@ -12,6 +14,7 @@ require (

require (
github.com/google/go-cmp v0.5.9 // indirect
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/tools v0.9.1 // indirect
)
22 changes: 12 additions & 10 deletions gazelle/go.sum
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/bazelbuild/bazel-gazelle v0.27.0 h1:+/ZhUxlDy4XnyMIGeKkbRZoIGssy1eO51GijwIvvuwE=
github.com/bazelbuild/bazel-gazelle v0.27.0/go.mod h1:2K6B42/loq8ext4JObmam4gTYx4En1MUSzHFKQF8hPM=
github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b h1:jhiMzJ+8unnLRtV8rpbWBFE9pFNzIqgUTyZU5aA++w8=
github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
github.com/bazelbuild/rules_go v0.35.0 h1:ViPR65vOrg74JKntAUFY6qZkheBKGB6to7wFd8gCRU4=
github.com/bazelbuild/rules_go v0.35.0/go.mod h1:ahciH68Viyxtm/gvCQplaAiu8buhf/b+gWswcPjFixI=
github.com/bazelbuild/bazel-gazelle v0.31.1 h1:ROyUyUHzoEdvoOs1e0haxJx1l5EjZX6AOqiKdVlaBbg=
github.com/bazelbuild/bazel-gazelle v0.31.1/go.mod h1:Ul0pqz50f5wxz0QNzsZ+mrEu4AVAVJZEB5xLnHgIG9c=
github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d h1:Fl1FfItZp34QIQmmDTbZXHB5XA6JfbNNfH7tRRGWvQo=
github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo=
github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=
github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down Expand Up @@ -45,6 +45,8 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand All @@ -56,15 +58,15 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
Expand Down
1 change: 1 addition & 0 deletions gazelle/python/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ go_library(
"generate.go",
"kinds.go",
"language.go",
"lifecycle.go",
"parser.go",
"resolve.go",
"std_modules.go",
Expand Down
1 change: 1 addition & 0 deletions gazelle/python/language.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
type Python struct {
Configurer
Resolver
LifeCycleManager
}

// NewLanguage initializes a new Python that satisfies the language.Language
Expand Down
37 changes: 37 additions & 0 deletions gazelle/python/lifecycle.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2023 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package python

import (
"context"
"github.com/bazelbuild/bazel-gazelle/language"
)

type LifeCycleManager struct {
language.BaseLifecycleManager
}

func (l *LifeCycleManager) Before(ctx context.Context) {
startParserProcess(ctx)
startStdModuleProcess(ctx)
}

func (l *LifeCycleManager) DoneGeneratingRules() {
shutdownParserProcess()
}

func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) {
shutdownStdModuleProcess()
}
14 changes: 8 additions & 6 deletions gazelle/python/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,25 @@ import (
"os/exec"
"strings"
"sync"
"time"

"github.com/bazelbuild/rules_go/go/tools/bazel"
"github.com/emirpasic/gods/sets/treeset"
godsutils "github.com/emirpasic/gods/utils"
)

var (
parserStdin io.Writer
parserStdin io.WriteCloser
parserStdout io.Reader
parserMutex sync.Mutex
)

func init() {
func startParserProcess(ctx context.Context) {
parseScriptRunfile, err := bazel.Runfile("python/parse")
if err != nil {
log.Printf("failed to initialize parser: %v\n", err)
os.Exit(1)
}

ctx := context.Background()
ctx, parserCancel := context.WithTimeout(ctx, time.Minute*10)
cmd := exec.CommandContext(ctx, parseScriptRunfile)

cmd.Stderr = os.Stderr
Expand All @@ -71,14 +68,19 @@ func init() {
}

go func() {
defer parserCancel()
if err := cmd.Wait(); err != nil {
log.Printf("failed to wait for parser: %v\n", err)
os.Exit(1)
}
}()
}

func shutdownParserProcess() {
if err := parserStdin.Close(); err != nil {
fmt.Fprintf(os.Stderr, "error closing parser: %v", err)
}
}

// python3Parser implements a parser for Python files that extracts the modules
// as seen in the import statements.
type python3Parser struct {
Expand Down
14 changes: 8 additions & 6 deletions gazelle/python/std_modules.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,18 @@ import (
"strconv"
"strings"
"sync"
"time"

"github.com/bazelbuild/rules_go/go/tools/bazel"
)

var (
stdModulesStdin io.Writer
stdModulesStdin io.WriteCloser
stdModulesStdout io.Reader
stdModulesMutex sync.Mutex
stdModulesSeen map[string]struct{}
)

func init() {
func startStdModuleProcess(ctx context.Context) {
stdModulesSeen = make(map[string]struct{})

stdModulesScriptRunfile, err := bazel.Runfile("python/std_modules")
Expand All @@ -46,8 +45,6 @@ func init() {
os.Exit(1)
}

ctx := context.Background()
ctx, stdModulesCancel := context.WithTimeout(ctx, time.Minute*10)
cmd := exec.CommandContext(ctx, stdModulesScriptRunfile)

cmd.Stderr = os.Stderr
Expand All @@ -73,14 +70,19 @@ func init() {
}

go func() {
defer stdModulesCancel()
if err := cmd.Wait(); err != nil {
log.Printf("failed to wait for std_modules: %v\n", err)
os.Exit(1)
}
}()
}

func shutdownStdModuleProcess() {
if err := stdModulesStdin.Close(); err != nil {
fmt.Fprintf(os.Stderr, "error closing std module: %v", err)
}
}

func isStdModule(m module) (bool, error) {
if _, seen := stdModulesSeen[m.Name]; seen {
return true, nil
Expand Down

0 comments on commit fe2c325

Please sign in to comment.