diff --git a/cmds/houndd/main.go b/cmds/houndd/main.go index eb8038a2..30db78f6 100644 --- a/cmds/houndd/main.go +++ b/cmds/houndd/main.go @@ -1,11 +1,14 @@ package main import ( - "encoding/json" "flag" "fmt" + "github.com/blang/semver/v4" + "github.com/fsnotify/fsnotify" + "github.com/hound-search/hound/config" + "github.com/hound-search/hound/searcher" + "github.com/hound-search/hound/web" "log" - "net/http" "os" "os/exec" "os/signal" @@ -13,13 +16,6 @@ import ( "runtime" "strings" "syscall" - - "github.com/blang/semver/v4" - "github.com/hound-search/hound/api" - "github.com/hound-search/hound/config" - "github.com/hound-search/hound/searcher" - "github.com/hound-search/hound/ui" - "github.com/hound-search/hound/web" ) const gracefulShutdownSignal = syscall.SIGTERM @@ -31,30 +27,30 @@ var ( basepath = filepath.Dir(b) ) -func makeSearchers(cfg *config.Config) (map[string]*searcher.Searcher, bool, error) { +func makeSearchers(cfg *config.Config, searchers map[string]*searcher.Searcher) (bool, error) { // Ensure we have a dbpath if _, err := os.Stat(cfg.DbPath); err != nil { if err := os.MkdirAll(cfg.DbPath, os.ModePerm); err != nil { - return nil, false, err + return false, err } } - searchers, errs, err := searcher.MakeAll(cfg) + errs, err := searcher.MakeAll(cfg, searchers) if err != nil { - return nil, false, err + return false, err } if len(errs) > 0 { // NOTE: This mutates the original config so the repos // are not even seen by other code paths. - for name, _ := range errs { //nolint + for name := range errs { //nolint delete(cfg.Repos, name) } - return searchers, false, nil + return false, nil } - return searchers, true, nil + return true, nil } func handleShutdown(shutdownCh <-chan os.Signal, searchers map[string]*searcher.Searcher) { @@ -79,42 +75,6 @@ func registerShutdownSignal() <-chan os.Signal { return shutdownCh } -func makeTemplateData(cfg *config.Config) (interface{}, error) { //nolint - var data struct { - ReposAsJson string - } - - res := map[string]*config.Repo{} - for name, repo := range cfg.Repos { - res[name] = repo - } - - b, err := json.Marshal(res) - if err != nil { - return nil, err - } - - data.ReposAsJson = string(b) - return &data, nil -} - -func runHttp( //nolint - addr string, - dev bool, - cfg *config.Config, - idx map[string]*searcher.Searcher) error { - m := http.DefaultServeMux - - h, err := ui.Content(dev, cfg) - if err != nil { - return err - } - - m.Handle("/", h) - api.Setup(m, idx, cfg.ResultLimit) - return http.ListenAndServe(addr, m) -} - // TODO: Automatically increment this when building a release func getVersion() semver.Version { return semver.Version{ @@ -141,29 +101,38 @@ func main() { os.Exit(0) } + idx := make(map[string]*searcher.Searcher) var cfg config.Config - if err := cfg.LoadFromFile(*flagConf); err != nil { - panic(err) + + loadConfig := func() { + if err := cfg.LoadFromFile(*flagConf); err != nil { + panic(err) + } + // It's not safe to be killed during makeSearchers, so register the + // shutdown signal here and defer processing it until we are ready. + shutdownCh := registerShutdownSignal() + ok, err := makeSearchers(&cfg, idx) + if err != nil { + log.Panic(err) + } + if !ok { + info_log.Println("Some repos failed to index, see output above") + } else { + info_log.Println("All indexes built!") + } + handleShutdown(shutdownCh, idx) } + loadConfig() + + // watch for config file changes + configWatcher := config.NewWatcher(*flagConf) + configWatcher.OnChange(func(fsnotify.Event) { + loadConfig() + }) // Start the web server on a background routine. ws := web.Start(&cfg, *flagAddr, *flagDev) - // It's not safe to be killed during makeSearchers, so register the - // shutdown signal here and defer processing it until we are ready. - shutdownCh := registerShutdownSignal() - idx, ok, err := makeSearchers(&cfg) - if err != nil { - log.Panic(err) - } - if !ok { - info_log.Println("Some repos failed to index, see output above") - } else { - info_log.Println("All indexes built!") - } - - handleShutdown(shutdownCh, idx) - host := *flagAddr if strings.HasPrefix(host, ":") { //nolint host = "localhost" + host @@ -175,8 +144,7 @@ func main() { webpack.Dir = basepath + "/../../" webpack.Stdout = os.Stdout webpack.Stderr = os.Stderr - err = webpack.Start() - if err != nil { + if err := webpack.Start(); err != nil { error_log.Println(err) } } diff --git a/config/watcher.go b/config/watcher.go new file mode 100644 index 00000000..bd3a0486 --- /dev/null +++ b/config/watcher.go @@ -0,0 +1,79 @@ +package config + +import ( + "log" + "sync" + + "github.com/fsnotify/fsnotify" +) + +// WatcherListenerFunc defines the signature for listner functions +type WatcherListenerFunc func(fsnotify.Event) + +// Watcher watches for configuration updates and provides hooks for +// triggering post events +type Watcher struct { + listeners []WatcherListenerFunc +} + +// NewWatcher returns a new file watcher +func NewWatcher(cfgPath string) *Watcher { + log.Printf("setting up watcher for %s", cfgPath) + w := Watcher{} + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Panic(err) + } + defer watcher.Close() + // Event listener setup + eventWG := sync.WaitGroup{} + eventWG.Add(1) + go func() { + defer eventWG.Done() + for { + select { + case event, ok := <-watcher.Events: + if !ok { + // events channel is closed + log.Printf("error: events channel is closed\n") + return + } + // only trigger on creates and writes of the watched config file + if event.Name == cfgPath && event.Op&fsnotify.Write == fsnotify.Write { + log.Printf("change in config file (%s) detected\n", cfgPath) + for _, listener := range w.listeners { + listener(event) + } + } + case err, ok := <-watcher.Errors: + if !ok { + // errors channel is closed + log.Printf("error: errors channel is closed\n") + return + } + log.Println("error:", err) + return + } + } + }() + // add config file + if err := watcher.Add(cfgPath); err != nil { + log.Fatalf("failed to watch %s", cfgPath) + } + // setup is complete + wg.Done() + // wait for the event listener to complete before exiting + eventWG.Wait() + }() + // wait for watcher setup to complete + wg.Wait() + return &w +} + +// OnChange registers a listener function to be called if a file changes +func (w *Watcher) OnChange(listener WatcherListenerFunc) { + w.listeners = append(w.listeners, listener) +} diff --git a/go.mod b/go.mod index 3d5151e6..67d773d2 100644 --- a/go.mod +++ b/go.mod @@ -4,5 +4,6 @@ go 1.16 require ( github.com/blang/semver/v4 v4.0.0 + github.com/fsnotify/fsnotify v1.6.0 golang.org/x/mod v0.10.0 ) diff --git a/go.sum b/go.sum index ab72a812..16c3c8bd 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -16,6 +18,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/index/index.go b/index/index.go index f84d933b..8838212a 100644 --- a/index/index.go +++ b/index/index.go @@ -4,6 +4,7 @@ import ( "compress/gzip" "encoding/gob" "encoding/json" + "errors" "io" "os" "path/filepath" @@ -80,16 +81,17 @@ type IndexRef struct { Url string Rev string Time time.Time - dir string + IdxDir string + VcsDir string AutoGeneratedFiles []string } func (r *IndexRef) Dir() string { - return r.dir + return r.IdxDir } func (r *IndexRef) writeManifest() error { - w, err := os.Create(filepath.Join(r.dir, manifestFilename)) + w, err := os.Create(filepath.Join(r.IdxDir, manifestFilename)) if err != nil { return err } @@ -101,12 +103,16 @@ func (r *IndexRef) writeManifest() error { func (r *IndexRef) Open() (*Index, error) { return &Index{ Ref: r, - idx: index.Open(filepath.Join(r.dir, "tri")), + idx: index.Open(filepath.Join(r.IdxDir, "tri")), }, nil } func (r *IndexRef) Remove() error { - return os.RemoveAll(r.dir) + if err := os.RemoveAll(r.IdxDir); err != nil { + return err + } + return os.RemoveAll(r.VcsDir) + return nil } func (n *Index) Close() error { @@ -125,7 +131,7 @@ func (n *Index) Destroy() error { } func (n *Index) GetDir() string { - return n.Ref.dir + return n.Ref.IdxDir } func toStrings(lines [][]byte) []string { @@ -206,7 +212,7 @@ func (n *Index) Search(pat string, opt *SearchOptions) (*SearchResponse, error) } filesOpened++ - if err := g.grep2File(filepath.Join(n.Ref.dir, "raw", name), re, int(opt.LinesOfContext), + if err := g.grep2File(filepath.Join(n.Ref.IdxDir, "raw", name), re, int(opt.LinesOfContext), func(line []byte, lineno int, before [][]byte, after [][]byte) (bool, error) { hasMatch = true @@ -469,7 +475,7 @@ func indexAllFiles(opt *IndexOptions, dst, src string) error { // include only the path) func Read(dir string) (*IndexRef, error) { m := &IndexRef{ - dir: dir, + IdxDir: dir, } r, err := os.Open(filepath.Join(dir, manifestFilename)) @@ -482,6 +488,10 @@ func Read(dir string) (*IndexRef, error) { return m, err } + if m.VcsDir == "" { + return m, errors.New("Metadata missing!\n Did you just upgrade to thos version? If so, please remove all data") + } + return m, nil } @@ -504,7 +514,8 @@ func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) { Url: url, Rev: rev, Time: time.Now(), - dir: dst, + VcsDir: src, + IdxDir: dst, AutoGeneratedFiles: opt.AutoGeneratedFiles, } @@ -515,7 +526,7 @@ func Build(opt *IndexOptions, dst, src, url, rev string) (*IndexRef, error) { return r, nil } -// Open the index in dir for searching. +// Open the index in IdxDir for searching. func Open(dir string) (*Index, error) { r, err := Read(dir) if err != nil { diff --git a/searcher/searcher.go b/searcher/searcher.go index 791ce810..88ff36d8 100644 --- a/searcher/searcher.go +++ b/searcher/searcher.go @@ -52,7 +52,7 @@ type limiter chan bool */ type foundRefs struct { refs []*index.IndexRef - claimed map[*index.IndexRef]bool + claimed map[string]bool lock sync.Mutex } @@ -89,7 +89,7 @@ func (r *foundRefs) claim(ref *index.IndexRef) { r.lock.Lock() defer r.lock.Unlock() - r.claimed[ref] = true + r.claimed[ref.Dir()] = true } /** @@ -101,7 +101,7 @@ func (r *foundRefs) removeUnclaimed() error { defer r.lock.Unlock() for _, ref := range r.refs { - if r.claimed[ref] { + if r.claimed[ref.Dir()] { continue } @@ -217,13 +217,16 @@ func findExistingRefs(dbpath string) (*foundRefs, error) { var refs []*index.IndexRef for _, dir := range dirs { - r, _ := index.Read(dir) + r, err := index.Read(dir) + if err != nil { + return nil, err + } refs = append(refs, r) } return &foundRefs{ refs: refs, - claimed: map[*index.IndexRef]bool{}, + claimed: map[string]bool{}, }, nil } @@ -264,7 +267,7 @@ func reportOnMemory() { // Utility function for producing a hex encoded sha1 hash for a string. func hashFor(name string) string { h := sha1.New() - h.Write([]byte(name)) //nolint + h.Write([]byte(name)) //nolint return hex.EncodeToString(h.Sum(nil)) } @@ -282,24 +285,34 @@ func init() { // occurred and no other return values are valid. If an error occurs that is specific // to a particular searcher, that searcher will not be present in the searcher map and // will have an error entry in the error map. -func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) { +func MakeAll(cfg *config.Config, searchers map[string]*Searcher) (map[string]error, error) { errs := map[string]error{} - searchers := map[string]*Searcher{} refs, err := findExistingRefs(cfg.DbPath) if err != nil { - return nil, nil, err + return nil, err } lim := makeLimiter(cfg.MaxConcurrentIndexers) - n := len(cfg.Repos) + n := 0 + for name := range cfg.Repos { + if s, ok := searchers[name]; ok { + // claim any already running searcher refs so that they don't get removed + refs.claim(s.idx.Ref) + continue + } + n++ + } // Channel to receive the results from newSearcherConcurrent function. resultCh := make(chan searcherResult, n) // Start new searchers for all repos in different go routines while // respecting cfg.MaxConcurrentIndexers. for name, repo := range cfg.Repos { + if _, ok := searchers[name]; ok { + continue + } go newSearcherConcurrent(cfg.DbPath, name, repo, refs, lim, resultCh) } @@ -315,7 +328,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) } if err := refs.removeUnclaimed(); err != nil { - return nil, nil, err + return nil, err } // after all the repos are in good shape, we start their polling @@ -323,7 +336,7 @@ func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) s.begin() } - return searchers, errs, nil + return errs, nil } // Creates a new Searcher that is available for searches as soon as this returns. @@ -407,7 +420,6 @@ func newSearcher( return nil, err } - rev, err := wd.PullOrClone(vcsDir, repo.Url) if err != nil { return nil, err diff --git a/ui/ui.go b/ui/ui.go index 30d558e3..1db88240 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -128,7 +128,7 @@ func (h *prdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ct := h.content[p] if ct != nil { // if so, render it - if err := renderForPrd(w, ct, h.cfg, h.cfgJson, r); err != nil { + if err := renderForPrd(w, ct, h.cfg, r); err != nil { log.Panic(err) } return @@ -142,7 +142,7 @@ func (h *prdHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Renders a templated asset in prd-mode. This strategy will embed // the sources directly in a script tag on the templated page. -func renderForPrd(w io.Writer, c *content, cfg *config.Config, cfgJson string, r *http.Request) error { +func renderForPrd(w io.Writer, c *content, cfg *config.Config, r *http.Request) error { var buf bytes.Buffer buf.WriteString("") + json, err := cfg.ToJsonString() + if err != nil { + return err + } + return c.tpl.Execute(w, map[string]interface{}{ "ReactVersion": ReactVersion, - "ReposAsJson": cfgJson, + "ReposAsJson": json, "Title": cfg.Title, "Source": html_template.HTML(buf.String()), "Host": r.Host,