diff --git a/README.md b/README.md index 845e7062..1b5c7255 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,16 @@ publisher: url: "gcppubsub://foobar.com" http_port: 8080 + +cutoff_delta: 5m ``` -To specify this configuration file, define its path in your environment under the `PACKAGE_FEEDS_CONFIG_PATH` variable. +`cutoff_delta` string formatted for [duration parser](https://golang.org/pkg/time/#ParseDuration). To specify this configuration file, define its path in your environment under the `PACKAGE_FEEDS_CONFIG_PATH` variable. ## Legacy Configuration Legacy configuration methods are still supported. By default, without a configuration file all feeds will be enabled. The environment variable `OSSMALWARE_TOPIC_URL` can be used to select the GCP pubsub publisher and `PORT` will configure the port for the HTTP server. +The default `cutoff_delta` is 5 minutes, this is used to generate a cutoff point for feed events relative to the given time at execution. # Contributing diff --git a/cmd/scheduled-feed/main.go b/cmd/scheduled-feed/main.go index d9c9f9d1..7f985a4e 100644 --- a/cmd/scheduled-feed/main.go +++ b/cmd/scheduled-feed/main.go @@ -6,8 +6,8 @@ import ( "fmt" "net/http" "os" - "time" "strings" + "time" "github.com/ossf/package-feeds/config" "github.com/ossf/package-feeds/feeds/scheduler" @@ -16,16 +16,15 @@ import ( log "github.com/sirupsen/logrus" ) -const delta = 5 * time.Minute - // FeedHandler is a handler that fetches new packages from various feeds type FeedHandler struct { scheduler *scheduler.Scheduler pub publisher.Publisher + delta time.Duration } func (handler *FeedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - cutoff := time.Now().UTC().Add(-delta) + cutoff := time.Now().UTC().Add(-handler.delta) pkgs, errs := handler.scheduler.Poll(cutoff) if len(errs) > 0 { for _, err := range errs { @@ -87,9 +86,14 @@ func main() { sched := scheduler.New(feeds) log.Printf("listening on port %v", appConfig.HttpPort) + delta, err := time.ParseDuration(appConfig.CutoffDelta) + if err != nil { + log.Fatal(err) + } handler := &FeedHandler{ scheduler: sched, pub: pub, + delta: delta, } http.Handle("/", handler) if err := http.ListenAndServe(fmt.Sprintf(":%v", appConfig.HttpPort), nil); err != nil { diff --git a/config/config_test.go b/config/config_test.go index 3f880bb1..847df227 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -22,6 +22,7 @@ publisher: endpoint: "https://foobaz.com" http_port: 8080 +cutoff_delta: 5m ` TestConfigStrUnknownFeedType = ` enabled_feeds: diff --git a/config/scheduledfeed.go b/config/scheduledfeed.go index 1cfde4c8..520a2fb7 100644 --- a/config/scheduledfeed.go +++ b/config/scheduledfeed.go @@ -152,7 +152,8 @@ func Default() *ScheduledFeedConfig { PubConfig: PublisherConfig{ Type: stdout.PublisherType, }, - HttpPort: 8080, + HttpPort: 8080, + CutoffDelta: "5m", } config.applyEnvVars() return config diff --git a/config/structs.go b/config/structs.go index a8bb655b..81411bf3 100644 --- a/config/structs.go +++ b/config/structs.go @@ -4,6 +4,7 @@ type ScheduledFeedConfig struct { PubConfig PublisherConfig `yaml:"publisher"` EnabledFeeds []string `yaml:"enabled_feeds"` HttpPort int `yaml:"http_port,omitempty"` + CutoffDelta string `yaml:"cutoff_delta"` } type PublisherConfig struct {