Set soft Ulimit to hard Ulimit value, fixes Steampipe failing to run …

…when too many benchmarks use the same controls. v0.5.3. Closes #528. Closes #530

CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.5.3 [2021-06-14]
+_Bug fixes_
+* Fixes Steampipe failing to run when too many benchmarks use the same controls. ([#528](https://github.com/turbot/steampipe/issues/528))
+
 ## v0.5.2 [2021-06-10]
 _Bug fixes_
 * Ensure consistent ordering of query result cache key when more than one qual is used. ([#53](https://github.com/turbot/steampipe-postgres-fdw/issues/53))

README.md

@@ -31,7 +31,7 @@ Check the verison
 
 ```
 $ steampipe -v
-steampipe version 0.5.2
+steampipe version 0.5.3
 ```
 
 Install a plugin

db/interactive_client.go

@@ -3,7 +3,6 @@ package db
 import (
 	"context"
 	"fmt"
-
 	"os"
 	"os/signal"
 	"sort"

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.7.1
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	github.com/turbot/go-kit v0.2.1
+	github.com/turbot/go-kit v0.2.2-0.20210614162002-b56bc76ce374
 	github.com/turbot/steampipe-plugin-sdk v0.2.8
 	github.com/ulikunitz/xz v0.5.8
 	github.com/zclconf/go-cty v1.8.2
@@ -54,5 +54,3 @@ require (
 )
 
 replace github.com/c-bata/go-prompt => github.com/turbot/go-prompt v0.2.6-steampipe
-
-replace github.com/turbot/go-kit => github.com/turbot/go-kit v0.2.2-0.20210517131416-052d9629cbd5

main.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/hashicorp/go-hclog"
 	_ "github.com/lib/pq"
+	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/steampipe-plugin-sdk/logging"
 	"github.com/turbot/steampipe/cmd"
@@ -25,6 +26,11 @@ func main() {
 
 	// ensure steampipe is not being run as root
 	checkRoot()
+
+	// increase the soft ULIMIT to match the hard limit
+	err := setULimit()
+	utils.FailOnErrorWithMessage(err, "failed to increase the file limit")
+
 	cmd.InitCmd()
 
 	// execute the command
@@ -34,6 +40,23 @@ func main() {
 	utils.DisplayProfileData()
 }
 
+// set the current to the max to avoid any file handle shortages
+func setULimit() error {
+	ulimit, err := filehelpers.GetULimit()
+	if err != nil {
+		return err
+	}
+
+	// set the current ulimit to 8192 (or the max, if less)
+	// this is to ensure we do not run out of file handler when watching files
+	var newULimit uint64 = 8192
+	if newULimit > ulimit.Max {
+		newULimit = ulimit.Max
+	}
+	err = filehelpers.SetULimit(newULimit)
+	return err
+}
+
 // this is to replicate the user security mechanism of out underlying
 // postgresql engine.
 func checkRoot() {

version/version.go

@@ -19,7 +19,7 @@ Also https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-versi
 **/
 
 // The main version number that is being run at the moment.
-var steampipeVersion = "0.5.2"
+var steampipeVersion = "0.5.3"
 
 // A pre-release marker for the version. If this is "" (empty string)
 // then it means that it is a final release. Otherwise, this is a pre-release