diff --git a/CHANGELOG.md b/CHANGELOG.md index abcf91055b9..0db48e3a206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,9 +24,8 @@ We use *breaking* word for marking changes that are not backward compatible (rel ### Changed -- [#2136](https://github.com/thanos-io/thanos/pull/2136) store, compact, bucket: schedule block deletion by adding deletion-mark.json. This adds a consistent way for multiple readers and writers to access object storage. -Since there are no consistency guarantees provided by some Object Storage providers, this PR adds a consistent lock-free way of dealing with Object Storage irrespective of the choice of object storage. In order to achieve this co-ordination, blocks are not deleted directly. Instead, blocks are marked for deletion by uploading `deletion-mark.json` file for the block that was chosen to be deleted. This file contains unix time of when the block was marked for deletion. - +- [#2136](https://github.com/thanos-io/thanos/pull/2136) *breaking* store, compact, bucket: schedule block deletion by adding deletion-mark.json. This adds a consistent way for multiple readers and writers to access object storage. +Since there are no consistency guarantees provided by some Object Storage providers, this PR adds a consistent lock-free way of dealing with Object Storage irrespective of the choice of object storage. In order to achieve this co-ordination, blocks are not deleted directly. Instead, blocks are marked for deletion by uploading `deletion-mark.json` file for the block that was chosen to be deleted. This file contains unix time of when the block was marked for deletion. If you want to keep existing behavior, you should add `--delete-delay=0s` as a flag. - [#2090](https://github.com/thanos-io/thanos/issues/2090) *breaking* Downsample command: the `downsample` command has moved as the `thanos bucket` sub-command, and cannot be called via `thanos downsample` any more. ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0) - 2020.03.02 diff --git a/Makefile b/Makefile index ca5193f4546..b274cf55da1 100644 --- a/Makefile +++ b/Makefile @@ -245,8 +245,12 @@ test-local: .PHONY: test-e2e test-e2e: ## Runs all Thanos e2e docker-based e2e tests from test/e2e. Required access to docker daemon. test-e2e: docker + @echo ">> cleaning docker environment." + @docker system prune -f --volumes + @echo ">> cleaning e2e test garbage." + @rm -rf ./test/e2e/e2e_integration_test* @echo ">> running /test/e2e tests." - @go test -v ./test/e2e/... + @go test -failfast -parallel 1 -timeout 5m -v ./test/e2e/... .PHONY: install-deps install-deps: ## Installs dependencies for integration tests. It installs supported versions of Prometheus and alertmanager to test against in integration tests. diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 4f88b0dba03..4b95702ca4a 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -41,6 +41,34 @@ type syncMetrics struct { modified *extprom.TxGaugeVec } +func (s *syncMetrics) submit() { + if s == nil { + return + } + + if s.synced != nil { + s.synced.Submit() + } + + if s.modified != nil { + s.modified.Submit() + } +} + +func (s *syncMetrics) resetTx() { + if s == nil { + return + } + + if s.synced != nil { + s.synced.ResetTx() + } + + if s.modified != nil { + s.modified.ResetTx() + } +} + const ( syncMetricSubSys = "blocks_meta" @@ -260,8 +288,7 @@ func (s *MetaFetcher) Fetch(ctx context.Context) (metas map[ulid.ULID]*metadata. metaErrs tsdberrors.MultiError ) - s.metrics.synced.ResetTx() - s.metrics.modified.ResetTx() + s.metrics.resetTx() for i := 0; i < s.concurrency; i++ { wg.Add(1) @@ -364,8 +391,7 @@ func (s *MetaFetcher) Fetch(ctx context.Context) (metas map[ulid.ULID]*metadata. } s.metrics.synced.WithLabelValues(loadedMeta).Set(float64(len(metas))) - s.metrics.synced.Submit() - s.metrics.modified.Submit() + s.metrics.submit() if incompleteView { return metas, partial, errors.Wrap(metaErrs, "incomplete view") diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 720328caccc..f45688d7287 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -30,7 +30,6 @@ import ( ) func TestCompact(t *testing.T) { - t.Parallel() l := log.NewLogfmtLogger(os.Stdout) // blockDesc describes a recipe to generate blocks from the given series and external labels. @@ -109,25 +108,25 @@ func TestCompact(t *testing.T) { expectOfChunks: 2, }, { - name: "(full) vertically overlapping blocks with replica labels, downsampling disabled", + name: "(full) vertically overlapping blocks with replica labels downsampling disabled", blocks: []blockDesc{ { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "1"), + extLset: labels.FromStrings("ext1", "value1", "ext2", "value2", "replica", "1"), mint: timestamp.FromTime(now), maxt: timestamp.FromTime(now.Add(2 * time.Hour)), samplesPerSeries: 120, }, { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "2"), + extLset: labels.FromStrings("ext2", "value2", "ext1", "value1", "replica", "2"), mint: timestamp.FromTime(now), maxt: timestamp.FromTime(now.Add(2 * time.Hour)), samplesPerSeries: 120, }, { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "rule_replica", "1"), + extLset: labels.FromStrings("ext1", "value1", "rule_replica", "1", "ext2", "value2"), mint: timestamp.FromTime(now), maxt: timestamp.FromTime(now.Add(2 * time.Hour)), samplesPerSeries: 120, @@ -142,6 +141,7 @@ func TestCompact(t *testing.T) { "a": "1", "b": "2", "ext1": "value1", + "ext2": "value2", }, }, expectOfModBlocks: 3, @@ -151,25 +151,46 @@ func TestCompact(t *testing.T) { expectOfChunks: 2, }, { - name: "(partial) vertically overlapping blocks with replica labels", + name: "(full) vertically overlapping blocks with replica labels, downsampling disabled and extra blocks", blocks: []blockDesc{ { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "1"), + extLset: labels.FromStrings("ext1", "value1", "ext2", "value2", "replica", "1"), mint: timestamp.FromTime(now), maxt: timestamp.FromTime(now.Add(2 * time.Hour)), samplesPerSeries: 120, }, { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "2"), + extLset: labels.FromStrings("ext2", "value2", "ext1", "value1", "replica", "2"), mint: timestamp.FromTime(now), - maxt: timestamp.FromTime(now.Add(1 * time.Hour)), - samplesPerSeries: 60, + maxt: timestamp.FromTime(now.Add(2 * time.Hour)), + samplesPerSeries: 120, + }, + { + series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, + extLset: labels.FromStrings("ext1", "value1", "rule_replica", "1", "ext2", "value2"), + mint: timestamp.FromTime(now), + maxt: timestamp.FromTime(now.Add(2 * time.Hour)), + samplesPerSeries: 120, + }, + { + series: []labels.Labels{labels.FromStrings("c", "1", "d", "2")}, + extLset: labels.FromStrings("ext1", "value1", "ext2", "value2"), + mint: timestamp.FromTime(now), + maxt: timestamp.FromTime(now.Add(2 * time.Hour)), + samplesPerSeries: 120, + }, + { + series: []labels.Labels{labels.FromStrings("c", "1", "d", "2")}, + extLset: labels.FromStrings("ext3", "value3"), + mint: timestamp.FromTime(now), + maxt: timestamp.FromTime(now.Add(2 * time.Hour)), + samplesPerSeries: 120, }, }, - replicaLabels: []string{"replica"}, - downsamplingEnabled: true, + replicaLabels: []string{"replica", "rule_replica"}, + downsamplingEnabled: false, query: "{a=\"1\"}", expected: []model.Metric{ @@ -177,30 +198,31 @@ func TestCompact(t *testing.T) { "a": "1", "b": "2", "ext1": "value1", + "ext2": "value2", }, }, - expectOfModBlocks: 2, - expectOfBlocks: 1, - expectOfSamples: 179, // TODO(kakkoyun): ? - expectOfSeries: 1, - expectOfChunks: 2, + expectOfModBlocks: 3, + expectOfBlocks: 2, + expectOfSamples: 360, + expectOfSeries: 3, + expectOfChunks: 6, }, { - name: "(contains) vertically overlapping blocks with replica labels", + name: "(partial) vertically overlapping blocks with replica labels", blocks: []blockDesc{ { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "1"), - mint: timestamp.FromTime(now.Add(30 * time.Minute)), - maxt: timestamp.FromTime(now.Add(1 * time.Hour)), - samplesPerSeries: 90, + extLset: labels.FromStrings("ext1", "value1", "ext2", "value2", "replica", "1"), + mint: timestamp.FromTime(now), + maxt: timestamp.FromTime(now.Add(2 * time.Hour)), + samplesPerSeries: 119, }, { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, - extLset: labels.FromStrings("ext1", "value1", "replica", "2"), + extLset: labels.FromStrings("ext2", "value2", "ext1", "value1", "replica", "2"), mint: timestamp.FromTime(now), - maxt: timestamp.FromTime(now.Add(2 * time.Hour)), - samplesPerSeries: 120, + maxt: timestamp.FromTime(now.Add(1 * time.Hour)), + samplesPerSeries: 59, }, }, replicaLabels: []string{"replica"}, @@ -212,11 +234,12 @@ func TestCompact(t *testing.T) { "a": "1", "b": "2", "ext1": "value1", + "ext2": "value2", }, }, expectOfModBlocks: 2, expectOfBlocks: 1, - expectOfSamples: 210, // TODO(kakkoyun): ? + expectOfSamples: 119, expectOfSeries: 1, expectOfChunks: 2, }, @@ -228,14 +251,14 @@ func TestCompact(t *testing.T) { extLset: labels.FromStrings("ext1", "value1", "replica", "1"), mint: timestamp.FromTime(now.Add(30 * time.Minute)), maxt: timestamp.FromTime(now.Add(150 * time.Minute)), - samplesPerSeries: 120, + samplesPerSeries: 119, }, { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, extLset: labels.FromStrings("ext1", "value1", "replica", "2"), - mint: timestamp.FromTime(now.Add(-30 * time.Minute)), - maxt: timestamp.FromTime(now.Add(90 * time.Minute)), - samplesPerSeries: 120, + mint: timestamp.FromTime(now), + maxt: timestamp.FromTime(now.Add(120 * time.Minute)), + samplesPerSeries: 119, }, }, replicaLabels: []string{"replica"}, @@ -251,12 +274,12 @@ func TestCompact(t *testing.T) { }, expectOfModBlocks: 2, expectOfBlocks: 1, - expectOfSamples: 240, // TODO(kakkoyun): ? + expectOfSamples: 149, expectOfSeries: 1, expectOfChunks: 2, }, { - name: "(full) vertically overlapping blocks with replica labels, retention specified", + name: "(full) vertically overlapping blocks with replica labels retention specified", blocks: []blockDesc{ { series: []labels.Labels{labels.FromStrings("a", "1", "b", "2")}, @@ -345,11 +368,9 @@ func TestCompact(t *testing.T) { i := i tcase := tcase t.Run(tcase.name, func(t *testing.T) { - t.Parallel() - s, err := e2e.NewScenario("e2e_test_compact_" + strconv.Itoa(i)) testutil.Ok(t, err) - defer s.Close() + defer s.Close() // TODO(kakkoyun): Change with t.CleanUp after go 1.14 update. dir := filepath.Join(s.SharedDir(), "tmp_"+strconv.Itoa(i)) testutil.Ok(t, os.MkdirAll(filepath.Join(s.SharedDir(), dir), os.ModePerm)) @@ -368,8 +389,8 @@ func TestCompact(t *testing.T) { }, "test-feed") testutil.Ok(t, err) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() // TODO(kakkoyun): Change with t.CleanUp after go 1.14 update. var rawBlockIds []ulid.ULID for _, b := range tcase.blocks { @@ -405,7 +426,6 @@ func TestCompact(t *testing.T) { tcase.downsamplingEnabled, append(dedupFlags, retenFlags...)..., ) - testutil.Ok(t, err) testutil.Ok(t, s.StartAndWaitReady(cmpt)) testutil.Ok(t, cmpt.WaitSumMetrics(e2e.Equals(float64(len(rawBlockIds))), "thanos_blocks_meta_synced")) diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 201b2f959bc..6bd9a9fed18 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -10,8 +10,10 @@ import ( "os" "path/filepath" "strconv" + "time" "github.com/cortexproject/cortex/integration/e2e" + "github.com/cortexproject/cortex/pkg/util" "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" @@ -25,6 +27,11 @@ import ( const logLevel = "info" +var defaultBackoffConfig = util.BackoffConfig{ + MinBackoff: 300 * time.Millisecond, + MaxBackoff: 30 * time.Second, +} + // TODO(bwplotka): Run against multiple? func DefaultPrometheusImage() string { return "quay.io/prometheus/prometheus:v2.16.0" @@ -69,6 +76,7 @@ func NewPrometheus(sharedDir string, name string, config, promImage string) (*e2 9090, ) prom.SetUser("root") + prom.SetBackoff(defaultBackoffConfig) return prom, container, nil } @@ -78,6 +86,7 @@ func NewPrometheusWithSidecar(sharedDir string, netName string, name string, con if err != nil { return nil, nil, err } + prom.SetBackoff(defaultBackoffConfig) sidecar := NewService( fmt.Sprintf("sidecar-%s", name), @@ -95,6 +104,8 @@ func NewPrometheusWithSidecar(sharedDir string, netName string, name string, con 80, 9091, ) + sidecar.SetBackoff(defaultBackoffConfig) + return prom, sidecar, nil } @@ -109,6 +120,7 @@ func NewQuerier(sharedDir string, name string, storeAddresses []string, fileSDSt "--query.replica-label": replicaLabel, "--store.sd-dns-interval": "5s", "--log.level": logLevel, + "--query.max-concurrent": "1", "--store.sd-interval": "5s", }) for _, addr := range storeAddresses { @@ -139,14 +151,17 @@ func NewQuerier(sharedDir string, name string, storeAddresses []string, fileSDSt args = append(args, "--store.sd-files="+filepath.Join(container, "filesd.yaml")) } - return NewService( + querier := NewService( fmt.Sprintf("querier-%v", name), DefaultImage(), e2e.NewCommand("query", args...), e2e.NewReadinessProbe(80, "/-/ready", 200), 80, 9091, - ), nil + ) + querier.SetBackoff(defaultBackoffConfig) + + return querier, nil } func RemoteWriteEndpoint(addr string) string { return fmt.Sprintf("http://%s/api/v1/receive", addr) } @@ -171,7 +186,7 @@ func NewReceiver(sharedDir string, networkName string, name string, replicationF return nil, errors.Wrap(err, "creating receive config") } - return NewService( + receiver := NewService( fmt.Sprintf("receive-%v", name), DefaultImage(), // TODO(bwplotka): BuildArgs should be interface. @@ -193,7 +208,10 @@ func NewReceiver(sharedDir string, networkName string, name string, replicationF 80, 9091, 81, - ), nil + ) + receiver.SetBackoff(defaultBackoffConfig) + + return receiver, nil } func NewRuler(sharedDir string, name string, ruleSubDir string, amCfg []alert.AlertmanagerConfig, queryCfg []query.Config) (*Service, error) { @@ -215,7 +233,7 @@ func NewRuler(sharedDir string, name string, ruleSubDir string, amCfg []alert.Al return nil, errors.Wrapf(err, "generate query file: %v", queryCfg) } - return NewService( + ruler := NewService( fmt.Sprintf("rule-%v", name), DefaultImage(), e2e.NewCommand("rule", e2e.BuildArgs(map[string]string{ @@ -226,7 +244,7 @@ func NewRuler(sharedDir string, name string, ruleSubDir string, amCfg []alert.Al "--label": fmt.Sprintf(`replica="%s"`, name), "--data-dir": container, "--rule-file": filepath.Join(e2e.ContainerSharedDir, ruleSubDir, "*.yaml"), - "--eval-interval": "1s", + "--eval-interval": "3s", "--alertmanagers.config": string(amCfgBytes), "--alertmanagers.sd-dns-interval": "1s", "--log.level": logLevel, @@ -237,7 +255,10 @@ func NewRuler(sharedDir string, name string, ruleSubDir string, amCfg []alert.Al e2e.NewReadinessProbe(80, "/-/ready", 200), 80, 9091, - ), nil + ) + ruler.SetBackoff(defaultBackoffConfig) + + return ruler, nil } func NewAlertmanager(sharedDir string, name string) (*e2e.HTTPService, error) { @@ -263,15 +284,19 @@ receivers: fmt.Sprintf("alertmanager-%v", name), DefaultAlertmanagerImage(), e2e.NewCommandWithoutEntrypoint("/bin/alertmanager", e2e.BuildArgs(map[string]string{ - "--config.file": filepath.Join(container, "config.yaml"), - "--web.listen-address": "0.0.0.0:80", - "--log.level": logLevel, - "--storage.path": container, + "--config.file": filepath.Join(container, "config.yaml"), + "--web.listen-address": "0.0.0.0:80", + "--log.level": logLevel, + "--storage.path": container, + "--web.get-concurrency": "1", + "--web.timeout": "2m", })...), e2e.NewReadinessProbe(80, "/-/ready", 200), 80, ) s.SetUser("root") + s.SetBackoff(defaultBackoffConfig) + return s, nil } @@ -292,7 +317,7 @@ func NewStoreGW(sharedDir string, name string, bucketConfig client.BucketConfig, return nil, errors.Wrapf(err, "generate store relabel file: %v", relabelConfig) } - return NewService( + store := NewService( fmt.Sprintf("store-gw-%v", name), DefaultImage(), e2e.NewCommand("store", append(e2e.BuildArgs(map[string]string{ @@ -304,17 +329,22 @@ func NewStoreGW(sharedDir string, name string, bucketConfig client.BucketConfig, "--data-dir": container, "--objstore.config": string(bktConfigBytes), // Accelerated sync time for quicker test (3m by default). - "--sync-block-duration": "1s", - "--selector.relabel-config": string(relabelConfigBytes), - "--consistency-delay": "30m", + "--sync-block-duration": "3s", + "--block-sync-concurrency": "1", + "--store.grpc.series-max-concurrency": "1", + "--selector.relabel-config": string(relabelConfigBytes), + "--consistency-delay": "30m", }), "--experimental.enable-index-header")...), e2e.NewReadinessProbe(80, "/-/ready", 200), 80, 9091, - ), nil + ) + store.SetBackoff(defaultBackoffConfig) + + return store, nil } -func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfig, relabelConfig []relabel.Config, downsamplingEnabled bool, extArgs ...string) (*Service, error) { +func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfig, relabelConfig []relabel.Config, downsamplingEnabled bool, extArgs ...string) (*e2e.HTTPService, error) { dir := filepath.Join(sharedDir, "data", "compact", name) container := filepath.Join(e2e.ContainerSharedDir, "data", "compact", name) @@ -336,7 +366,7 @@ func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfi extArgs = append(extArgs, "--downsampling.disable") } - return NewService( + compactor := e2e.NewHTTPService( fmt.Sprintf("compact-%s", name), DefaultImage(), e2e.NewCommand("compact", append(e2e.BuildArgs(map[string]string{ @@ -345,11 +375,15 @@ func NewCompactor(sharedDir string, name string, bucketConfig client.BucketConfi "--data-dir": container, "--objstore.config": string(bktConfigBytes), "--http-address": ":80", + "--delete-delay": "0s", + "--block-sync-concurrency": "1", "--selector.relabel-config": string(relabelConfigBytes), "--wait": "", }), extArgs...)...), e2e.NewReadinessProbe(80, "/-/ready", 200), 80, - 9091, - ), nil + ) + compactor.SetBackoff(defaultBackoffConfig) + + return compactor, nil } diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 914d1c6a929..5e11841bc66 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -63,8 +63,6 @@ func sortResults(res model.Vector) { } func TestQuery(t *testing.T) { - t.Parallel() - s, err := e2e.NewScenario("e2e_test_query") testutil.Ok(t, err) defer s.Close() diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index fda1f595912..b772e5eef40 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -17,11 +17,7 @@ import ( ) func TestReceive(t *testing.T) { - t.Parallel() - t.Run("hashring", func(t *testing.T) { - t.Parallel() - s, err := e2e.NewScenario("e2e_test_receive_hashring") testutil.Ok(t, err) defer s.Close() @@ -98,7 +94,6 @@ func TestReceive(t *testing.T) { }) t.Run("replication", func(t *testing.T) { - t.Parallel() s, err := e2e.NewScenario("e2e_test_receive_replication") testutil.Ok(t, err) @@ -170,7 +165,6 @@ func TestReceive(t *testing.T) { }) t.Run("replication_with_outage", func(t *testing.T) { - t.Parallel() s, err := e2e.NewScenario("e2e_test_receive_replication_with_outage") testutil.Ok(t, err) diff --git a/test/e2e/rule_test.go b/test/e2e/rule_test.go index 5e41c5f941d..62f773efa73 100644 --- a/test/e2e/rule_test.go +++ b/test/e2e/rule_test.go @@ -179,8 +179,6 @@ func (m *mockAlertmanager) ServeHTTP(resp http.ResponseWriter, req *http.Request func TestRule_AlertmanagerHTTPClient(t *testing.T) { t.Skip("TODO: Allow HTTP ports from binaries running on host to be accessible.") - t.Parallel() - s, err := e2e.NewScenario("e2e_test_rule_am_http_client") testutil.Ok(t, err) defer s.Close() @@ -265,8 +263,6 @@ func TestRule_AlertmanagerHTTPClient(t *testing.T) { } func TestRule(t *testing.T) { - t.Parallel() - s, err := e2e.NewScenario("e2e_test_rule") testutil.Ok(t, err) defer s.Close() @@ -435,7 +431,7 @@ func TestRule(t *testing.T) { testutil.Ok(t, r.WaitSumMetrics(e2e.Equals(1), "thanos_ruler_alertmanagers_dns_provider_results")) }) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() queryAndAssert(t, ctx, q.HTTPEndpoint(), "ALERTS", promclient.QueryOptions{ diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index d663c71e710..0abe8fdbbe6 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -31,8 +31,6 @@ import ( // TODO(bwplotka): Extend this test to have multiple stores and memcached. // TODO(bwplotka): Extend this test for downsampling. func TestStoreGateway(t *testing.T) { - t.Parallel() - s, err := e2e.NewScenario("e2e_test_store_gateway") testutil.Ok(t, err) defer s.Close()