Add a threshold so small batches don't have jitter. Bump jitter window.

- 10 seconds should space out requests more to reduce congestion.
- Early requests don't have jitter appled to avoid slowing down small
  batch sizes.

See #337

Signed-off-by: Caleb Brown <calebbrown@google.com>

pkg/feeds/npm/npm.go

@@ -29,7 +29,13 @@ const (
 	// maxJitterMillis is the upper bound of random jitter introcude while
 	// issuing requests to NPM. Random jitter will be generated between 0 and
 	// maxJitterMillis.
-	maxJitterMillis = 1000
+	maxJitterMillis = 10000
+
+	// minJitterThreshold is the minimum number of packages that need to be fetched
+	// before the jitter is applied. This allows a number of packages to
+	// fetched without a delay. The number is chosen somewhat arbitrarily, but
+	// is 10% of the rssLimit above.
+	minJitterThreshold = 20
 )
 
 var (
@@ -179,11 +185,14 @@ func fetchAllPackages(registryURL string) ([]*feeds.Package, []error) {
 		uniquePackages[pkg.Title]++
 	}
 
+	n := 0
 	for pkgTitle, count := range uniquePackages {
-		go func(pkgTitle string, count int) {
-			// Before requesting, wait, so all the requests don't arrive at once.
-			jitter := time.Duration(rand.Intn(maxJitterMillis)) * time.Millisecond //nolint:gosec
-			time.Sleep(jitter)
+		go func(id int, pkgTitle string, count int) {
+			if id > minJitterThreshold {
+				// Before requesting, wait, so all the requests don't arrive at once.
+				jitter := time.Duration(rand.Intn(maxJitterMillis)) * time.Millisecond //nolint:gosec
+				time.Sleep(jitter)
+			}
 
 			pkgs, err := fetchPackage(registryURL, pkgTitle)
 			if err != nil {
@@ -203,7 +212,8 @@ func fetchAllPackages(registryURL string) ([]*feeds.Package, []error) {
 			} else {
 				packageChannel <- pkgs
 			}
-		}(pkgTitle, count)
+		}(n, pkgTitle, count)
+		n++
 	}
 
 	// Collect all the worker.