feat: flush inactive/expired login and consent requests

.github/workflows/oidc-conformity.yml

@@ -16,7 +16,7 @@ jobs:
           fetch-depth: 2
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.16.1
+          go-version: '^1.16.1'
       - name: Start service
         run: ./test/conformance/start.sh
       - name: Run tests

Makefile

@@ -70,7 +70,7 @@ test-resetdb: node_modules
 		docker rm -f hydra_test_database_cockroach || true
 		docker run --rm --name hydra_test_database_mysql -p 3444:3306 -e MYSQL_ROOT_PASSWORD=secret -d mysql:5.7
 		docker run --rm --name hydra_test_database_postgres -p 3445:5432 -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=postgres -d postgres:9.6
-		docker run --rm --name hydra_test_database_cockroach -p 3446:26257 -d cockroachdb/cockroach:v2.1.6 start --insecure
+		docker run --rm --name hydra_test_database_cockroach -p 3446:26257 -d cockroachdb/cockroach:v20.2.5 start-single-node --insecure
 
 # Runs tests in short mode, without database adapters
 .PHONY: docker

cmd/cli/handler.go

@@ -37,6 +37,7 @@ type Handler struct {
 	Introspection *IntrospectionHandler
 	Token         *TokenHandler
 	Migration     *MigrateHandler
+	Janitor       *JanitorHandler
 }
 
 func Remote(cmd *cobra.Command) string {
@@ -63,5 +64,6 @@ func NewHandler() *Handler {
 		Introspection: newIntrospectionHandler(),
 		Token:         newTokenHandler(),
 		Migration:     newMigrateHandler(),
+		Janitor:       newJanitorHandler(),
 	}
 }

cmd/cli/handler_janitor.go

@@ -0,0 +1,114 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/ory/x/flagx"
+
+	"github.com/spf13/cobra"
+
+	"github.com/ory/hydra/driver"
+	"github.com/ory/hydra/driver/config"
+	"github.com/ory/x/configx"
+	"github.com/ory/x/errorsx"
+)
+
+type JanitorHandler struct{}
+
+func newJanitorHandler() *JanitorHandler {
+	return &JanitorHandler{}
+}
+
+func (j *JanitorHandler) Purge(cmd *cobra.Command, args []string) error {
+	var d driver.Registry
+
+	co := []configx.OptionModifier{
+		configx.WithFlags(cmd.Flags()),
+		configx.SkipValidation(),
+	}
+
+	keys := map[string]string{
+		"access-lifespan":          config.KeyAccessTokenLifespan,
+		"refresh-lifespan":         config.KeyRefreshTokenLifespan,
+		"consent-request-lifespan": config.KeyConsentRequestMaxAge,
+	}
+
+	for k, v := range keys {
+		if x := flagx.MustGetString(cmd, k); x != "" {
+			if xp, err := time.ParseDuration(x); err == nil {
+				co = append(co, configx.WithValue(v, xp))
+			}
+		}
+	}
+
+	notAfter := time.Now()
+
+	if keepYounger := flagx.MustGetString(cmd, "keep-if-younger"); keepYounger != "" {
+		if keepYoungerDuration, err := time.ParseDuration(keepYounger); err == nil {
+			notAfter = notAfter.Add(-keepYoungerDuration)
+		}
+	}
+
+	if !flagx.MustGetBool(cmd, "read-from-env") && len(flagx.MustGetStringSlice(cmd, "config")) == 0 {
+		co = append(co, configx.WithValue(config.KeyDSN, args[0]))
+	}
+
+	do := []driver.OptionsModifier{
+		driver.DisableValidation(),
+		driver.DisablePreloading(),
+		driver.WithOptions(co...),
+	}
+
+	d = driver.New(cmd.Context(), do...)
+
+	if len(d.Config().DSN()) == 0 {
+		return fmt.Errorf("%s\n%s\n%s\n", cmd.UsageString(),
+			"When using flag -e, environment variable DSN must be set.",
+			"When using flag -c, the dsn property should be set.")
+	}
+
+	if err := d.Init(cmd.Context()); err != nil {
+		return fmt.Errorf("%s\n%s\n", cmd.UsageString(),
+			"Janitor can only be executed against a SQL-compatible driver but DSN is not a SQL source.")
+	}
+
+	p := d.Persister()
+
+	var routines []cleanupRoutine
+
+	if flagx.MustGetBool(cmd, "tokens") {
+		routines = append(routines, cleanup(p.FlushInactiveAccessTokens, "access tokens"))
+		routines = append(routines, cleanup(p.FlushInactiveRefreshTokens, "refresh tokens"))
+	}
+
+	if flagx.MustGetBool(cmd, "requests") {
+		routines = append(routines, cleanup(p.FlushInactiveLoginConsentRequests, "login-consent requests"))
+	}
+
+	return cleanupRun(cmd.Context(), notAfter, routines...)
+}
+
+type cleanupRoutine func(ctx context.Context, notAfter time.Time) error
+
+func cleanup(cr cleanupRoutine, routineName string) cleanupRoutine {
+	return func(ctx context.Context, notAfter time.Time) error {
+		if err := cr(ctx, notAfter); err != nil {
+			return errors.Wrap(errorsx.WithStack(err), fmt.Sprintf("Could not cleanup inactive %s", routineName))
+		}
+		fmt.Printf("Successfully completed Janitor run on %s\n", routineName)
+		return nil
+	}
+}
+
+func cleanupRun(ctx context.Context, notAfter time.Time, routines ...cleanupRoutine) error {
+	for _, r := range routines {
+		if err := r(ctx, notAfter); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/cli/handler_janitor_test.go

@@ -0,0 +1,247 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"github.com/ory/hydra/internal/testhelpers"
+	"github.com/ory/x/cmdx"
+	"github.com/ory/x/configx"
+	"github.com/ory/x/flagx"
+
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	keepIfYounger          = "keep-if-younger"
+	accessLifespan         = "access-lifespan"
+	refreshLifespan        = "refresh-lifespan"
+	consentRequestLifespan = "consent-request-lifespan"
+	onlyTokens             = "tokens"
+	onlyRequests           = "requests"
+)
+
+func newJanitorCmd() *cobra.Command {
+	janitor := newJanitorHandler()
+	JanitorCmd := &cobra.Command{
+		Use:  "janitor",
+		RunE: janitor.Purge,
+		Args: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 &&
+				!flagx.MustGetBool(cmd, "read-from-env") &&
+				len(flagx.MustGetStringSlice(cmd, "config")) == 0 {
+
+				fmt.Printf("%s\n", cmd.UsageString())
+				return fmt.Errorf("%s\n%s\n%s\n",
+					"A DSN is required as a positional argument when not passing any of the following flags:",
+					"- Using the environment variable with flag -e, --read-from-env",
+					"- Using the config file with flag -c, --config")
+			}
+
+			if (!flagx.MustGetBool(cmd, onlyTokens) && !flagx.MustGetBool(cmd, onlyRequests)) || (flagx.MustGetBool(cmd, onlyTokens) && flagx.MustGetBool(cmd, onlyRequests)) {
+				return fmt.Errorf("%s\n%s\n", cmd.UsageString(),
+					"Janitor requires either --tokens or --requests to be set")
+			}
+
+			return nil
+		},
+	}
+	JanitorCmd.Flags().String(keepIfYounger, "", "Keep database records that are younger than a specified duration e.g. 1s, 1m, 1h.")
+	JanitorCmd.Flags().String(accessLifespan, "", "Set the access token lifespan e.g. 1s, 1m, 1h.")
+	JanitorCmd.Flags().String(refreshLifespan, "", "Set the refresh token lifespan e.g. 1s, 1m, 1h.")
+	JanitorCmd.Flags().String(consentRequestLifespan, "", "Set the login/consent request lifespan e.g. 1s, 1m, 1h")
+	JanitorCmd.Flags().Bool(onlyRequests, false, "This will only run the cleanup on requests and will skip token cleanup.")
+	JanitorCmd.Flags().Bool(onlyTokens, false, "This will only run the cleanup on tokens and will skip requests cleanup.")
+
+	JanitorCmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
+	configx.RegisterFlags(JanitorCmd.PersistentFlags())
+	return JanitorCmd
+}
+
+func TestJanitorHandler_PurgeTokenNotAfter(t *testing.T) {
+	ctx := context.Background()
+	testCycles := testhelpers.NewConsentJanitorTestHelper("").GetNotAfterTestCycles()
+
+	for k, v := range testCycles {
+		t.Run(fmt.Sprintf("case=%s", k), func(t *testing.T) {
+			jt := testhelpers.NewConsentJanitorTestHelper(t.Name())
+			reg, err := jt.GetRegistry(ctx, k)
+			require.NoError(t, err)
+
+			// setup test
+			t.Run("step=setup-access", jt.AccessTokenNotAfterSetup(ctx, reg.ClientManager(), reg.OAuth2Storage()))
+			t.Run("step=setup-refresh", jt.RefreshTokenNotAfterSetup(ctx, reg.ClientManager(), reg.OAuth2Storage()))
+
+			// run the cleanup routine
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s=%s", keepIfYounger, v.String()),
+					fmt.Sprintf("--%s=%s", accessLifespan, jt.GetAccessTokenLifespan().String()),
+					fmt.Sprintf("--%s=%s", refreshLifespan, jt.GetRefreshTokenLifespan().String()),
+					fmt.Sprintf("--%s", onlyTokens),
+					jt.GetDSN(),
+				)
+			})
+
+			// validate test
+			notAfter := time.Now().Round(time.Second).Add(-v)
+			t.Run("step=validate-access", jt.AccessTokenNotAfterValidate(ctx, notAfter, reg.OAuth2Storage()))
+			t.Run("step=validate-refresh", jt.RefreshTokenNotAfterValidate(ctx, notAfter, reg.OAuth2Storage()))
+		})
+	}
+}
+
+func TestJanitorHandler_PurgeLoginConsentNotAfter(t *testing.T) {
+	ctx := context.Background()
+
+	testCycles := testhelpers.NewConsentJanitorTestHelper("").GetNotAfterTestCycles()
+
+	for k, v := range testCycles {
+		jt := testhelpers.NewConsentJanitorTestHelper(k)
+		reg, err := jt.GetRegistry(ctx, k)
+		require.NoError(t, err)
+
+		t.Run(fmt.Sprintf("case=%s", k), func(t *testing.T) {
+			// Setup the test
+			t.Run("step=setup", jt.LoginConsentNotAfterSetup(ctx, reg.ConsentManager(), reg.ClientManager()))
+			// Run the cleanup routine
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s=%s", keepIfYounger, v.String()),
+					fmt.Sprintf("--%s=%s", consentRequestLifespan, jt.GetConsentRequestLifespan().String()),
+					fmt.Sprintf("--%s", onlyRequests),
+					jt.GetDSN(),
+				)
+			})
+
+			notAfter := time.Now().Round(time.Second).Add(-v)
+			consentLifespan := time.Now().Round(time.Second).Add(-jt.GetConsentRequestLifespan())
+			t.Run("step=validate", jt.LoginConsentNotAfterValidate(ctx, notAfter, consentLifespan, reg.ConsentManager()))
+		})
+	}
+
+}
+
+func TestJanitorHandler_PurgeLoginConsent(t *testing.T) {
+	/*
+		Login and Consent also needs to be purged on two conditions besides the KeyConsentRequestMaxAge and notAfter time
+		- when a login/consent request was never completed (timed out)
+		- when a login/consent request was rejected
+	*/
+
+	t.Run("case=login-consent-timeout", func(t *testing.T) {
+		t.Run("case=login-timeout", func(t *testing.T) {
+			ctx := context.Background()
+			jt := testhelpers.NewConsentJanitorTestHelper(t.Name())
+			reg, err := jt.GetRegistry(ctx, t.Name())
+			require.NoError(t, err)
+
+			// setup
+			t.Run("step=setup", jt.LoginTimeoutSetup(ctx, reg.ConsentManager(), reg.ClientManager()))
+
+			// cleanup
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s", onlyRequests),
+					jt.GetDSN(),
+				)
+			})
+
+			t.Run("step=validate", jt.LoginTimeoutValidate(ctx, reg.ConsentManager()))
+
+		})
+
+		t.Run("case=consent-timeout", func(t *testing.T) {
+			ctx := context.Background()
+			jt := testhelpers.NewConsentJanitorTestHelper(t.Name())
+			reg, err := jt.GetRegistry(ctx, t.Name())
+			require.NoError(t, err)
+
+			// setup
+			t.Run("step=setup", jt.ConsentTimeoutSetup(ctx, reg.ConsentManager(), reg.ClientManager()))
+
+			// run cleanup
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s", onlyRequests),
+					jt.GetDSN(),
+				)
+			})
+
+			// validate
+			t.Run("step=validate", jt.ConsentTimeoutValidate(ctx, reg.ConsentManager()))
+		})
+
+	})
+
+	t.Run("case=login-consent-rejection", func(t *testing.T) {
+		ctx := context.Background()
+
+		t.Run("case=login-rejection", func(t *testing.T) {
+			jt := testhelpers.NewConsentJanitorTestHelper(t.Name())
+			reg, err := jt.GetRegistry(ctx, t.Name())
+			require.NoError(t, err)
+
+			// setup
+			t.Run("step=setup", jt.LoginRejectionSetup(ctx, reg.ConsentManager(), reg.ClientManager()))
+
+			// cleanup
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s", onlyRequests),
+					jt.GetDSN(),
+				)
+			})
+
+			// validate
+			t.Run("step=validate", jt.LoginRejectionValidate(ctx, reg.ConsentManager()))
+		})
+
+		t.Run("case=consent-rejection", func(t *testing.T) {
+			jt := testhelpers.NewConsentJanitorTestHelper(t.Name())
+			reg, err := jt.GetRegistry(ctx, t.Name())
+			require.NoError(t, err)
+
+			// setup
+			t.Run("step=setup", jt.ConsentRejectionSetup(ctx, reg.ConsentManager(), reg.ClientManager()))
+
+			// cleanup
+			t.Run("step=cleanup", func(t *testing.T) {
+				cmdx.ExecNoErr(t, newJanitorCmd(),
+					fmt.Sprintf("--%s", onlyRequests),
+					jt.GetDSN(),
+				)
+			})
+
+			// validate
+			t.Run("step=validate", jt.ConsentRejectionValidate(ctx, reg.ConsentManager()))
+		})
+
+	})
+
+}
+
+/*
+// TODO: this throws a panic like error instead of a pass on an expected error
+func TestJanitorHandler_Arguments(t *testing.T) {
+	cmdx.ExecNoErr(t, newJanitorCmd(),
+		fmt.Sprintf("--%s", onlyRequests),
+		"memory",
+	)
+	cmdx.ExecNoErr(t, newJanitorCmd(),
+		fmt.Sprintf("--%s", onlyTokens),
+		"memory",
+	)
+	cmdx.ExecExpectedErr(t, newJanitorCmd(),
+		fmt.Sprintf("--%s", onlyRequests),
+		fmt.Sprintf("--%s", onlyTokens),
+		"memory",
+	)
+	cmdx.ExecExpectedErr(t, newJanitorCmd(),
+		"memory",
+	)
+}*/