From a652a250002fb05eda76c91f9e7067694f0e9ba4 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Wed, 13 May 2020 14:37:30 -0700 Subject: [PATCH 01/12] wip --- go/tasks/plugins/hive/config/config.go | 35 ++++++++---- go/tasks/plugins/hive/executor.go | 1 + go/tasks/plugins/hive/plugin.go | 78 ++++++++++++++++++++++++++ go/tasks/plugins/hive/transformer.go | 1 + 4 files changed, 105 insertions(+), 10 deletions(-) create mode 100644 go/tasks/plugins/hive/plugin.go create mode 100644 go/tasks/plugins/hive/transformer.go diff --git a/go/tasks/plugins/hive/config/config.go b/go/tasks/plugins/hive/config/config.go index 25228525c..56ae8c70c 100644 --- a/go/tasks/plugins/hive/config/config.go +++ b/go/tasks/plugins/hive/config/config.go @@ -5,6 +5,9 @@ package config import ( "context" "net/url" + "time" + + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" "github.com/lyft/flytestdlib/config" "github.com/lyft/flytestdlib/logger" @@ -45,10 +48,21 @@ var ( CommandAPIPath: MustParse("/api/v1.2/commands/"), AnalyzeLinkPath: MustParse("/v2/analyze"), TokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN", - LruCacheSize: 2000, - Workers: 15, ClusterConfigs: []ClusterConfig{{PrimaryLabel: "default", Labels: []string{"default"}, Limit: 100, ProjectScopeQuotaProportionCap: 0.7, NamespaceScopeQuotaProportionCap: 0.7}}, DestinationClusterConfigs: []DestinationClusterConfig{}, + Caching: remote.CachingProperties{ + Size: 2000, + ResyncInterval: config.Duration{Duration: 20 * time.Second}, + Workers: 10, + }, + WriteRateLimiter: remote.RateLimiterProperties{ + QPS: 100, + Burst: 200, + }, + ReadRateLimiter: remote.RateLimiterProperties{ + QPS: 100, + Burst: 200, + }, } quboleConfigSection = pluginsConfig.MustRegisterSubSection(quboleConfigSectionKey, &defaultConfig) @@ -56,14 +70,15 @@ var ( // Qubole plugin configs type Config struct { - Endpoint config.URL `json:"endpoint" pflag:",Endpoint for qubole to use"` - CommandAPIPath config.URL `json:"commandApiPath" pflag:",API Path where commands can be launched on Qubole. Should be a valid url."` - AnalyzeLinkPath config.URL `json:"analyzeLinkPath" pflag:",URL path where queries can be visualized on qubole website. Should be a valid url."` - TokenKey string `json:"quboleTokenKey" pflag:",Name of the key where to find Qubole token in the secret manager."` - LruCacheSize int `json:"lruCacheSize" pflag:",Size of the AutoRefreshCache"` - Workers int `json:"workers" pflag:",Number of parallel workers to refresh the cache"` - ClusterConfigs []ClusterConfig `json:"clusterConfigs" pflag:"-,A list of cluster configs. Each of the configs corresponds to a service cluster"` - DestinationClusterConfigs []DestinationClusterConfig `json:"destinationClusterConfigs" pflag:"-,A list configs specifying the destination service cluster for (project, domain)"` + Endpoint config.URL `json:"endpoint" pflag:",Endpoint for qubole to use"` + CommandAPIPath config.URL `json:"commandApiPath" pflag:",API Path where commands can be launched on Qubole. Should be a valid url."` + AnalyzeLinkPath config.URL `json:"analyzeLinkPath" pflag:",URL path where queries can be visualized on qubole website. Should be a valid url."` + TokenKey string `json:"quboleTokenKey" pflag:",Name of the key where to find Qubole token in the secret manager."` + ClusterConfigs []ClusterConfig `json:"clusterConfigs" pflag:"-,A list of cluster configs. Each of the configs corresponds to a service cluster"` + DestinationClusterConfigs []DestinationClusterConfig `json:"destinationClusterConfigs" pflag:"-,A list configs specifying the destination service cluster for (project, domain)"` + ReadRateLimiter remote.RateLimiterProperties `json:"readRateLimiter" pflag:",Defines rate limiter properties for read actions (e.g. retrieve status)."` + WriteRateLimiter remote.RateLimiterProperties `json:"writeRateLimiter" pflag:",Defines rate limiter properties for write actions."` + Caching remote.CachingProperties `json:"caching" pflag:",Defines caching characteristics."` } // Retrieves the current config value or default. diff --git a/go/tasks/plugins/hive/executor.go b/go/tasks/plugins/hive/executor.go index f9e8fbdd3..65698ed92 100644 --- a/go/tasks/plugins/hive/executor.go +++ b/go/tasks/plugins/hive/executor.go @@ -111,6 +111,7 @@ func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace] for _, clusterCfg := range cfg { resourceConfig[core.ResourceNamespace(clusterCfg.PrimaryLabel)] = clusterCfg.Limit } + return resourceConfig } diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go new file mode 100644 index 000000000..1331803f5 --- /dev/null +++ b/go/tasks/plugins/hive/plugin.go @@ -0,0 +1,78 @@ +package hive + +import ( + "context" + + "github.com/lyft/flyteplugins/go/tasks/errors" + + "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" + "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" + + pluginMachinery "github.com/lyft/flyteplugins/go/tasks/pluginmachinery" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" +) + +type QuboleHivePlugin struct { + client client.QuboleClient + resourceQuotas map[core.ResourceNamespace]int + properties remote.PluginProperties +} + +func (q QuboleHivePlugin) GetPluginProperties() remote.PluginProperties { + return q.properties +} + +func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote.PluginContext) ( + namespace core.ResourceNamespace, constraints core.ResourceConstraintsSpec, err error) { + uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() + + clusterPrimaryLabel, err := composeResourceNamespaceWithClusterPrimaryLabel(ctx, tCtx) + if err != nil { + return "", core.ResourceConstraintsSpec{}, errors.Wrapf(errors.ResourceManagerFailure, err, "Error getting query info when requesting allocation token %s", uniqueID) + } + + resourceConstraintsSpec := createResourceConstraintsSpec(ctx, tCtx, clusterPrimaryLabel) + return clusterPrimaryLabel, resourceConstraintsSpec, nil +} + +func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.PluginContext) (createdResources remote.ResourceKey, err error) { + panic("implement me") +} + +func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceKey) (resource remote.Resource, err error) { + panic("implement me") +} + +func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceKey) error { + panic("implement me") +} + +func (q QuboleHivePlugin) Status(ctx context.Context, resource remote.Resource) (phase core.PhaseInfo, err error) { + panic("implement me") +} + +func QuboleHivePluginLoader(ctx context.Context, iCtx remote.PluginSetupContext) ( + remote.Plugin, error) { + + cfg := config.GetQuboleConfig() + + return QuboleHivePlugin{ + client: client.NewQuboleClient(cfg), + properties: remote.PluginProperties{ + ResourceQuotas: BuildResourceConfig(cfg.ClusterConfigs), + ReadRateLimiter: cfg.ReadRateLimiter, + WriteRateLimiter: cfg.WriteRateLimiter, + Caching: cfg.Caching, + }, + }, nil +} + +func init() { + pluginMachinery.PluginRegistry().RegisterRemotePlugin( + remote.PluginEntry{ + ID: quboleHiveExecutorID, + SupportedTaskTypes: []core.TaskType{hiveTaskType}, + PluginLoader: QuboleHivePluginLoader, + }) +} diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go new file mode 100644 index 000000000..17e668609 --- /dev/null +++ b/go/tasks/plugins/hive/transformer.go @@ -0,0 +1 @@ +package hive From be93b103fa351d5aee730fa845a6b231f8881cda Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Wed, 13 May 2020 15:43:34 -0700 Subject: [PATCH 02/12] wip --- go/tasks/plugins/hive/execution_state.go | 12 ++-- go/tasks/plugins/hive/plugin.go | 80 +++++++++++++++++++++--- go/tasks/plugins/hive/resource.go | 53 ++++++++++++++++ 3 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 go/tasks/plugins/hive/resource.go diff --git a/go/tasks/plugins/hive/execution_state.go b/go/tasks/plugins/hive/execution_state.go index c0caa5c15..8b0ff9388 100644 --- a/go/tasks/plugins/hive/execution_state.go +++ b/go/tasks/plugins/hive/execution_state.go @@ -6,6 +6,8 @@ import ( "strconv" "time" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" + "github.com/lyft/flytestdlib/cache" idlCore "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" @@ -144,7 +146,7 @@ func ConstructTaskInfo(e ExecutionState) *core.TaskInfo { return nil } -func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx core.TaskExecutionContext) (core.ResourceNamespace, error) { +func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext) (core.ResourceNamespace, error) { _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) if err != nil { return "", err @@ -153,7 +155,7 @@ func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx c return core.ResourceNamespace(clusterPrimaryLabel), nil } -func createResourceConstraintsSpec(ctx context.Context, _ core.TaskExecutionContext, targetClusterPrimaryLabel core.ResourceNamespace) core.ResourceConstraintsSpec { +func createResourceConstraintsSpec(ctx context.Context, _ remote.TaskExecutionContext, targetClusterPrimaryLabel core.ResourceNamespace) core.ResourceConstraintsSpec { cfg := config.GetQuboleConfig() constraintsSpec := core.ResourceConstraintsSpec{ ProjectScopeResourceConstraint: nil, @@ -229,7 +231,7 @@ func validateQuboleHiveJob(hiveJob plugins.QuboleHiveJob) error { // This function is the link between the output written by the SDK, and the execution side. It extracts the query // out of the task template. -func GetQueryInfo(ctx context.Context, tCtx core.TaskExecutionContext) ( +func GetQueryInfo(ctx context.Context, tCtx remote.TaskExecutionContext) ( query string, cluster string, tags []string, timeoutSec uint32, err error) { taskTemplate, err := tCtx.TaskReader().Read(ctx) @@ -288,7 +290,7 @@ func mapLabelToPrimaryLabel(ctx context.Context, quboleCfg *config.Config, label return primaryLabel, found } -func mapProjectDomainToDestinationClusterLabel(ctx context.Context, tCtx core.TaskExecutionContext, quboleCfg *config.Config) (string, bool) { +func mapProjectDomainToDestinationClusterLabel(ctx context.Context, tCtx remote.TaskExecutionContext, quboleCfg *config.Config) (string, bool) { tExecID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID() project := tExecID.NodeExecutionId.GetExecutionId().GetProject() domain := tExecID.NodeExecutionId.GetExecutionId().GetDomain() @@ -305,7 +307,7 @@ func mapProjectDomainToDestinationClusterLabel(ctx context.Context, tCtx core.Ta return "", false } -func getClusterPrimaryLabel(ctx context.Context, tCtx core.TaskExecutionContext, clusterLabelOverride string) string { +func getClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext, clusterLabelOverride string) string { cfg := config.GetQuboleConfig() // If override is not empty and if it has a mapping, we return the mapped primary label diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 1331803f5..38fea29ce 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -2,6 +2,11 @@ package hive import ( "context" + "fmt" + "reflect" + "strconv" + + "github.com/lyft/flytestdlib/logger" "github.com/lyft/flyteplugins/go/tasks/errors" @@ -15,6 +20,7 @@ import ( type QuboleHivePlugin struct { client client.QuboleClient + apiKey string resourceQuotas map[core.ResourceNamespace]int properties remote.PluginProperties } @@ -23,7 +29,7 @@ func (q QuboleHivePlugin) GetPluginProperties() remote.PluginProperties { return q.properties } -func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote.PluginContext) ( +func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote.TaskExecutionContext) ( namespace core.ResourceNamespace, constraints core.ResourceConstraintsSpec, err error) { uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() @@ -36,29 +42,87 @@ func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote. return clusterPrimaryLabel, resourceConstraintsSpec, nil } -func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.PluginContext) (createdResources remote.ResourceKey, err error) { - panic("implement me") +func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionContext) ( + createdResources remote.ResourceKey, err error) { + query, clusterLabelOverride, tags, timeoutSec, err := GetQueryInfo(ctx, tCtx) + if err != nil { + return remote.ResourceKey{}, err + } + + clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) + + cmdDetails, err := q.client.ExecuteHiveCommand(ctx, query, timeoutSec, + clusterPrimaryLabel, q.apiKey, tags) + if err != nil { + return remote.ResourceKey{}, err + } + + // If we succeed, then store the command id returned from Qubole, and update our state. Also, add to the + // AutoRefreshCache so we start getting updates. + commandID := strconv.FormatInt(cmdDetails.ID, 10) + logger.Infof(ctx, "Created Qubole ID [%s]", commandID) + + return remote.ResourceKey{ + Name: commandID, + }, nil } -func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceKey) (resource remote.Resource, err error) { - panic("implement me") +func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceKey) ( + resource remote.Resource, err error) { + + logger.Debugf(ctx, "Retrieving Hive job [%s]", key.Name) + + // Get an updated status from Qubole + commandStatus, err := q.client.GetCommandStatus(ctx, key.Name, q.apiKey) + if err != nil { + logger.Errorf(ctx, "Error from Qubole command %s. Error: %v", key.Name, err) + return nil, err + } + + newExecutionPhase, err := QuboleStatusToExecutionPhase(commandStatus) + if err != nil { + return nil, err + } + + return Resource{ + Phase: newExecutionPhase, + }, nil } func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceKey) error { - panic("implement me") + err := q.client.KillCommand(ctx, key.Name, q.apiKey) + if err != nil { + logger.Errorf(ctx, "Error terminating Qubole command [%s]. Error: %v", key.Name, err) + return err + } + + return nil } -func (q QuboleHivePlugin) Status(ctx context.Context, resource remote.Resource) (phase core.PhaseInfo, err error) { - panic("implement me") +func (q QuboleHivePlugin) Status(ctx context.Context, resource remote.Resource) ( + phase core.PhaseInfo, err error) { + + r, casted := resource.(Resource) + if !casted { + return core.PhaseInfo{}, fmt.Errorf("failed to cast resource to the expected type. Input type: %v", + reflect.TypeOf(resource)) + } + + return r.GetPhaseInfo(), nil } func QuboleHivePluginLoader(ctx context.Context, iCtx remote.PluginSetupContext) ( remote.Plugin, error) { cfg := config.GetQuboleConfig() + apiKey, err := iCtx.SecretManager().Get(ctx, cfg.TokenKey) + if err != nil { + return nil, errors.Wrapf(errors.RuntimeFailure, err, "Failed to read token from secrets manager") + } return QuboleHivePlugin{ client: client.NewQuboleClient(cfg), + apiKey: apiKey, properties: remote.PluginProperties{ ResourceQuotas: BuildResourceConfig(cfg.ClusterConfigs), ReadRateLimiter: cfg.ReadRateLimiter, diff --git a/go/tasks/plugins/hive/resource.go b/go/tasks/plugins/hive/resource.go new file mode 100644 index 000000000..394e44f21 --- /dev/null +++ b/go/tasks/plugins/hive/resource.go @@ -0,0 +1,53 @@ +package hive + +import ( + "fmt" + "time" + + idlCore "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + + "github.com/lyft/flyteplugins/go/tasks/errors" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" +) + +type Resource struct { + CommandID string + Phase ExecutionPhase + URI string +} + +func (r Resource) ConstructTaskInfo() *core.TaskInfo { + logs := make([]*idlCore.TaskLog, 0, 1) + t := time.Now() + logs = append(logs) + return &core.TaskInfo{ + Logs: []*idlCore.TaskLog{ + { + Name: fmt.Sprintf("Status: %s [%s]", r.Phase, r.CommandID), + MessageFormat: idlCore.TaskLog_UNKNOWN, + Uri: r.URI, + }, + }, + OccurredAt: &t, + } +} + +func (r Resource) GetPhaseInfo() core.PhaseInfo { + var phaseInfo core.PhaseInfo + t := time.Now() + + switch r.Phase { + case PhaseNotStarted: + phaseInfo = core.PhaseInfoNotReady(t, core.DefaultPhaseVersion, "Haven't received allocation token") + case PhaseQueued: + phaseInfo = core.PhaseInfoQueued(t, core.DefaultPhaseVersion, "Waiting for Qubole launch") + case PhaseSubmitted: + phaseInfo = core.PhaseInfoRunning(core.DefaultPhaseVersion, r.ConstructTaskInfo()) + case PhaseQuerySucceeded: + phaseInfo = core.PhaseInfoSuccess(r.ConstructTaskInfo()) + case PhaseQueryFailed: + phaseInfo = core.PhaseInfoFailure(errors.DownstreamSystemError, "Query failed", r.ConstructTaskInfo()) + } + + return phaseInfo +} From a3731694df827146ef2a61e120f6d54059373198 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 13:49:01 -0700 Subject: [PATCH 03/12] adapt --- go/tasks/plugins/hive/plugin.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 38fea29ce..5d3962b6b 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -43,10 +43,10 @@ func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote. } func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionContext) ( - createdResources remote.ResourceKey, err error) { + createdResources remote.ResourceMeta, err error) { query, clusterLabelOverride, tags, timeoutSec, err := GetQueryInfo(ctx, tCtx) if err != nil { - return remote.ResourceKey{}, err + return remote.ResourceMeta{}, err } clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) @@ -54,7 +54,7 @@ func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionC cmdDetails, err := q.client.ExecuteHiveCommand(ctx, query, timeoutSec, clusterPrimaryLabel, q.apiKey, tags) if err != nil { - return remote.ResourceKey{}, err + return remote.ResourceMeta{}, err } // If we succeed, then store the command id returned from Qubole, and update our state. Also, add to the @@ -62,13 +62,13 @@ func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionC commandID := strconv.FormatInt(cmdDetails.ID, 10) logger.Infof(ctx, "Created Qubole ID [%s]", commandID) - return remote.ResourceKey{ + return remote.ResourceMeta{ Name: commandID, }, nil } -func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceKey) ( - resource remote.Resource, err error) { +func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceMeta) ( + resource remote.ResourceMeta, err error) { logger.Debugf(ctx, "Retrieving Hive job [%s]", key.Name) @@ -76,20 +76,22 @@ func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceKey) ( commandStatus, err := q.client.GetCommandStatus(ctx, key.Name, q.apiKey) if err != nil { logger.Errorf(ctx, "Error from Qubole command %s. Error: %v", key.Name, err) - return nil, err + return remote.ResourceMeta{}, err } newExecutionPhase, err := QuboleStatusToExecutionPhase(commandStatus) if err != nil { - return nil, err + return remote.ResourceMeta{}, err } - return Resource{ - Phase: newExecutionPhase, + return remote.ResourceMeta{ + Custom: Resource{ + Phase: newExecutionPhase, + }, }, nil } -func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceKey) error { +func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceMeta) error { err := q.client.KillCommand(ctx, key.Name, q.apiKey) if err != nil { logger.Errorf(ctx, "Error terminating Qubole command [%s]. Error: %v", key.Name, err) @@ -99,10 +101,10 @@ func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceKey) er return nil } -func (q QuboleHivePlugin) Status(ctx context.Context, resource remote.Resource) ( +func (q QuboleHivePlugin) Status(_ context.Context, resource remote.ResourceMeta) ( phase core.PhaseInfo, err error) { - r, casted := resource.(Resource) + r, casted := resource.Custom.(Resource) if !casted { return core.PhaseInfo{}, fmt.Errorf("failed to cast resource to the expected type. Input type: %v", reflect.TypeOf(resource)) @@ -128,6 +130,7 @@ func QuboleHivePluginLoader(ctx context.Context, iCtx remote.PluginSetupContext) ReadRateLimiter: cfg.ReadRateLimiter, WriteRateLimiter: cfg.WriteRateLimiter, Caching: cfg.Caching, + CustomState: Resource{}, }, }, nil } From 4f9121ca99124d1fcb7abfc74ed10a7e05938ebd Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:09:04 -0700 Subject: [PATCH 04/12] more simplification --- .../internal/remote/launcher.go | 2 + .../remote/missingresourcepolicy_enumer.go | 68 --------------- .../pluginmachinery/remote/mocks/plugin.go | 69 +++++++++++---- .../remote/mocks/remote_resource.go | 10 --- .../pluginmachinery/remote/mocks/resource.go | 84 ------------------- go/tasks/plugins/hive/plugin.go | 48 ++++++----- 6 files changed, 82 insertions(+), 199 deletions(-) delete mode 100644 go/tasks/pluginmachinery/remote/missingresourcepolicy_enumer.go delete mode 100644 go/tasks/pluginmachinery/remote/mocks/remote_resource.go delete mode 100644 go/tasks/pluginmachinery/remote/mocks/resource.go diff --git a/go/tasks/pluginmachinery/internal/remote/launcher.go b/go/tasks/pluginmachinery/internal/remote/launcher.go index d675ea068..d9628fe0b 100644 --- a/go/tasks/pluginmachinery/internal/remote/launcher.go +++ b/go/tasks/pluginmachinery/internal/remote/launcher.go @@ -33,5 +33,7 @@ func launch(ctx context.Context, p remote.Plugin, tCtx core.TaskExecutionContext return nil, core.PhaseInfo{}, err } + state.ResourceMeta = r + return state, core.PhaseInfoQueued(time.Now(), 1, "launched"), nil } diff --git a/go/tasks/pluginmachinery/remote/missingresourcepolicy_enumer.go b/go/tasks/pluginmachinery/remote/missingresourcepolicy_enumer.go deleted file mode 100644 index d09b0dd1e..000000000 --- a/go/tasks/pluginmachinery/remote/missingresourcepolicy_enumer.go +++ /dev/null @@ -1,68 +0,0 @@ -// Code generated by "enumer -type=MissingResourcePolicy -trimprefix=MissingResourcePolicy -json"; DO NOT EDIT. - -// -package remote - -import ( - "encoding/json" - "fmt" -) - -const _MissingResourcePolicyName = "MissingResourceFailMissingResourceRetry" - -var _MissingResourcePolicyIndex = [...]uint8{0, 19, 39} - -func (i MissingResourcePolicy) String() string { - if i >= MissingResourcePolicy(len(_MissingResourcePolicyIndex)-1) { - return fmt.Sprintf("MissingResourcePolicy(%d)", i) - } - return _MissingResourcePolicyName[_MissingResourcePolicyIndex[i]:_MissingResourcePolicyIndex[i+1]] -} - -var _MissingResourcePolicyValues = []MissingResourcePolicy{0, 1} - -var _MissingResourcePolicyNameToValueMap = map[string]MissingResourcePolicy{ - _MissingResourcePolicyName[0:19]: 0, - _MissingResourcePolicyName[19:39]: 1, -} - -// MissingResourcePolicyString retrieves an enum value from the enum constants string name. -// Throws an error if the param is not part of the enum. -func MissingResourcePolicyString(s string) (MissingResourcePolicy, error) { - if val, ok := _MissingResourcePolicyNameToValueMap[s]; ok { - return val, nil - } - return 0, fmt.Errorf("%s does not belong to MissingResourcePolicy values", s) -} - -// MissingResourcePolicyValues returns all values of the enum -func MissingResourcePolicyValues() []MissingResourcePolicy { - return _MissingResourcePolicyValues -} - -// IsAMissingResourcePolicy returns "true" if the value is listed in the enum definition. "false" otherwise -func (i MissingResourcePolicy) IsAMissingResourcePolicy() bool { - for _, v := range _MissingResourcePolicyValues { - if i == v { - return true - } - } - return false -} - -// MarshalJSON implements the json.Marshaler interface for MissingResourcePolicy -func (i MissingResourcePolicy) MarshalJSON() ([]byte, error) { - return json.Marshal(i.String()) -} - -// UnmarshalJSON implements the json.Unmarshaler interface for MissingResourcePolicy -func (i *MissingResourcePolicy) UnmarshalJSON(data []byte) error { - var s string - if err := json.Unmarshal(data, &s); err != nil { - return fmt.Errorf("MissingResourcePolicy should be a string, got %s", data) - } - - var err error - *i, err = MissingResourcePolicyString(s) - return err -} diff --git a/go/tasks/pluginmachinery/remote/mocks/plugin.go b/go/tasks/pluginmachinery/remote/mocks/plugin.go index bab0cbf64..1cd8dbf60 100644 --- a/go/tasks/pluginmachinery/remote/mocks/plugin.go +++ b/go/tasks/pluginmachinery/remote/mocks/plugin.go @@ -20,7 +20,7 @@ type Plugin_Create struct { *mock.Call } -func (_m Plugin_Create) Return(resource remote.Resource, err error) *Plugin_Create { +func (_m Plugin_Create) Return(resource interface{}, err error) *Plugin_Create { return &Plugin_Create{Call: _m.Call.Return(resource, err)} } @@ -35,15 +35,15 @@ func (_m *Plugin) OnCreateMatch(matchers ...interface{}) *Plugin_Create { } // Create provides a mock function with given fields: ctx, tCtx -func (_m *Plugin) Create(ctx context.Context, tCtx remote.TaskExecutionContext) (remote.Resource, error) { +func (_m *Plugin) Create(ctx context.Context, tCtx remote.TaskExecutionContext) (interface{}, error) { ret := _m.Called(ctx, tCtx) - var r0 remote.Resource - if rf, ok := ret.Get(0).(func(context.Context, remote.TaskExecutionContext) remote.Resource); ok { + var r0 interface{} + if rf, ok := ret.Get(0).(func(context.Context, remote.TaskExecutionContext) interface{}); ok { r0 = rf(ctx, tCtx) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(remote.Resource) + r0 = ret.Get(0).(interface{}) } } @@ -65,7 +65,7 @@ func (_m Plugin_Delete) Return(_a0 error) *Plugin_Delete { return &Plugin_Delete{Call: _m.Call.Return(_a0)} } -func (_m *Plugin) OnDelete(ctx context.Context, cached remote.Resource) *Plugin_Delete { +func (_m *Plugin) OnDelete(ctx context.Context, cached interface{}) *Plugin_Delete { c := _m.On("Delete", ctx, cached) return &Plugin_Delete{Call: c} } @@ -76,11 +76,11 @@ func (_m *Plugin) OnDeleteMatch(matchers ...interface{}) *Plugin_Delete { } // Delete provides a mock function with given fields: ctx, cached -func (_m *Plugin) Delete(ctx context.Context, cached remote.Resource) error { +func (_m *Plugin) Delete(ctx context.Context, cached interface{}) error { ret := _m.Called(ctx, cached) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, remote.Resource) error); ok { + if rf, ok := ret.Get(0).(func(context.Context, interface{}) error); ok { r0 = rf(ctx, cached) } else { r0 = ret.Error(0) @@ -93,11 +93,11 @@ type Plugin_Get struct { *mock.Call } -func (_m Plugin_Get) Return(latest remote.Resource, err error) *Plugin_Get { +func (_m Plugin_Get) Return(latest interface{}, err error) *Plugin_Get { return &Plugin_Get{Call: _m.Call.Return(latest, err)} } -func (_m *Plugin) OnGet(ctx context.Context, cached remote.Resource) *Plugin_Get { +func (_m *Plugin) OnGet(ctx context.Context, cached interface{}) *Plugin_Get { c := _m.On("Get", ctx, cached) return &Plugin_Get{Call: c} } @@ -108,20 +108,20 @@ func (_m *Plugin) OnGetMatch(matchers ...interface{}) *Plugin_Get { } // Get provides a mock function with given fields: ctx, cached -func (_m *Plugin) Get(ctx context.Context, cached remote.Resource) (remote.Resource, error) { +func (_m *Plugin) Get(ctx context.Context, cached interface{}) (interface{}, error) { ret := _m.Called(ctx, cached) - var r0 remote.Resource - if rf, ok := ret.Get(0).(func(context.Context, remote.Resource) remote.Resource); ok { + var r0 interface{} + if rf, ok := ret.Get(0).(func(context.Context, interface{}) interface{}); ok { r0 = rf(ctx, cached) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(remote.Resource) + r0 = ret.Get(0).(interface{}) } } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, remote.Resource) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, interface{}) error); ok { r1 = rf(ctx, cached) } else { r1 = ret.Error(1) @@ -207,3 +207,42 @@ func (_m *Plugin) ResourceRequirements(ctx context.Context, tCtx remote.TaskExec return r0, r1, r2 } + +type Plugin_Status struct { + *mock.Call +} + +func (_m Plugin_Status) Return(phase core.PhaseInfo, err error) *Plugin_Status { + return &Plugin_Status{Call: _m.Call.Return(phase, err)} +} + +func (_m *Plugin) OnStatus(ctx context.Context, resource interface{}) *Plugin_Status { + c := _m.On("Status", ctx, resource) + return &Plugin_Status{Call: c} +} + +func (_m *Plugin) OnStatusMatch(matchers ...interface{}) *Plugin_Status { + c := _m.On("Status", matchers...) + return &Plugin_Status{Call: c} +} + +// Status provides a mock function with given fields: ctx, resource +func (_m *Plugin) Status(ctx context.Context, resource interface{}) (core.PhaseInfo, error) { + ret := _m.Called(ctx, resource) + + var r0 core.PhaseInfo + if rf, ok := ret.Get(0).(func(context.Context, interface{}) core.PhaseInfo); ok { + r0 = rf(ctx, resource) + } else { + r0 = ret.Get(0).(core.PhaseInfo) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, interface{}) error); ok { + r1 = rf(ctx, resource) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/go/tasks/pluginmachinery/remote/mocks/remote_resource.go b/go/tasks/pluginmachinery/remote/mocks/remote_resource.go deleted file mode 100644 index 5770fab12..000000000 --- a/go/tasks/pluginmachinery/remote/mocks/remote_resource.go +++ /dev/null @@ -1,10 +0,0 @@ -// Code generated by mockery v1.0.1. DO NOT EDIT. - -package mocks - -import mock "github.com/stretchr/testify/mock" - -// RemoteResource is an autogenerated mock type for the RemoteResource type -type RemoteResource struct { - mock.Mock -} diff --git a/go/tasks/pluginmachinery/remote/mocks/resource.go b/go/tasks/pluginmachinery/remote/mocks/resource.go deleted file mode 100644 index 9d8d150f2..000000000 --- a/go/tasks/pluginmachinery/remote/mocks/resource.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by mockery v1.0.1. DO NOT EDIT. - -package mocks - -import ( - core "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" - mock "github.com/stretchr/testify/mock" -) - -// Resource is an autogenerated mock type for the Resource type -type Resource struct { - mock.Mock -} - -type Resource_ID struct { - *mock.Call -} - -func (_m Resource_ID) Return(_a0 string) *Resource_ID { - return &Resource_ID{Call: _m.Call.Return(_a0)} -} - -func (_m *Resource) OnID() *Resource_ID { - c := _m.On("ID") - return &Resource_ID{Call: c} -} - -func (_m *Resource) OnIDMatch(matchers ...interface{}) *Resource_ID { - c := _m.On("ID", matchers...) - return &Resource_ID{Call: c} -} - -// ID provides a mock function with given fields: -func (_m *Resource) ID() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -type Resource_Status struct { - *mock.Call -} - -func (_m Resource_Status) Return(phase core.PhaseInfo, err error) *Resource_Status { - return &Resource_Status{Call: _m.Call.Return(phase, err)} -} - -func (_m *Resource) OnStatus() *Resource_Status { - c := _m.On("Status") - return &Resource_Status{Call: c} -} - -func (_m *Resource) OnStatusMatch(matchers ...interface{}) *Resource_Status { - c := _m.On("Status", matchers...) - return &Resource_Status{Call: c} -} - -// Status provides a mock function with given fields: -func (_m *Resource) Status() (core.PhaseInfo, error) { - ret := _m.Called() - - var r0 core.PhaseInfo - if rf, ok := ret.Get(0).(func() core.PhaseInfo); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(core.PhaseInfo) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 5d3962b6b..3f63d50a3 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -46,7 +46,7 @@ func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionC createdResources remote.ResourceMeta, err error) { query, clusterLabelOverride, tags, timeoutSec, err := GetQueryInfo(ctx, tCtx) if err != nil { - return remote.ResourceMeta{}, err + return nil, err } clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) @@ -54,7 +54,7 @@ func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionC cmdDetails, err := q.client.ExecuteHiveCommand(ctx, query, timeoutSec, clusterPrimaryLabel, q.apiKey, tags) if err != nil { - return remote.ResourceMeta{}, err + return nil, err } // If we succeed, then store the command id returned from Qubole, and update our state. Also, add to the @@ -62,39 +62,44 @@ func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionC commandID := strconv.FormatInt(cmdDetails.ID, 10) logger.Infof(ctx, "Created Qubole ID [%s]", commandID) - return remote.ResourceMeta{ - Name: commandID, + return Resource{ + CommandID: commandID, + URI: cmdDetails.URI.String(), }, nil } -func (q QuboleHivePlugin) Get(ctx context.Context, key remote.ResourceMeta) ( - resource remote.ResourceMeta, err error) { - - logger.Debugf(ctx, "Retrieving Hive job [%s]", key.Name) +func (q QuboleHivePlugin) Get(ctx context.Context, meta remote.ResourceMeta) ( + newMeta remote.ResourceMeta, err error) { + r := meta.(Resource) + logger.Debugf(ctx, "Retrieving Hive job [%s]", r.CommandID) // Get an updated status from Qubole - commandStatus, err := q.client.GetCommandStatus(ctx, key.Name, q.apiKey) + commandStatus, err := q.client.GetCommandStatus(ctx, r.CommandID, q.apiKey) if err != nil { - logger.Errorf(ctx, "Error from Qubole command %s. Error: %v", key.Name, err) - return remote.ResourceMeta{}, err + logger.Errorf(ctx, "Error from Qubole command %s. Error: %v", r.CommandID, err) + return nil, err } newExecutionPhase, err := QuboleStatusToExecutionPhase(commandStatus) if err != nil { - return remote.ResourceMeta{}, err + return nil, err } - return remote.ResourceMeta{ - Custom: Resource{ - Phase: newExecutionPhase, - }, + return Resource{ + Phase: newExecutionPhase, + CommandID: r.CommandID, + URI: r.URI, }, nil } -func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceMeta) error { - err := q.client.KillCommand(ctx, key.Name, q.apiKey) +func (q QuboleHivePlugin) Delete(ctx context.Context, meta remote.ResourceMeta) error { + r := meta.(Resource) + logger.Debugf(ctx, "Killing Hive job [%s]", r.CommandID) + + err := q.client.KillCommand(ctx, r.CommandID, q.apiKey) if err != nil { - logger.Errorf(ctx, "Error terminating Qubole command [%s]. Error: %v", key.Name, err) + logger.Errorf(ctx, "Error terminating Qubole command [%s]. Error: %v", + r.CommandID, err) return err } @@ -103,8 +108,7 @@ func (q QuboleHivePlugin) Delete(ctx context.Context, key remote.ResourceMeta) e func (q QuboleHivePlugin) Status(_ context.Context, resource remote.ResourceMeta) ( phase core.PhaseInfo, err error) { - - r, casted := resource.Custom.(Resource) + r, casted := resource.(Resource) if !casted { return core.PhaseInfo{}, fmt.Errorf("failed to cast resource to the expected type. Input type: %v", reflect.TypeOf(resource)) @@ -130,7 +134,7 @@ func QuboleHivePluginLoader(ctx context.Context, iCtx remote.PluginSetupContext) ReadRateLimiter: cfg.ReadRateLimiter, WriteRateLimiter: cfg.WriteRateLimiter, Caching: cfg.Caching, - CustomState: Resource{}, + ResourceMeta: Resource{}, }, }, nil } From 3c0190cc6b32bd5746bf6c5a015a73ff8b3f6db1 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:13:56 -0700 Subject: [PATCH 05/12] delete old code --- go/tasks/plugins/hive/config/config_flags.go | 9 +- .../plugins/hive/config/config_flags_test.go | 146 +++++++-- go/tasks/plugins/hive/execution_state.go | 297 ------------------ go/tasks/plugins/hive/execution_state_test.go | 294 +---------------- go/tasks/plugins/hive/executions_cache.go | 172 ---------- .../plugins/hive/executions_cache_test.go | 91 ------ go/tasks/plugins/hive/executor.go | 168 ---------- 7 files changed, 139 insertions(+), 1038 deletions(-) delete mode 100644 go/tasks/plugins/hive/executions_cache.go delete mode 100644 go/tasks/plugins/hive/executions_cache_test.go delete mode 100644 go/tasks/plugins/hive/executor.go diff --git a/go/tasks/plugins/hive/config/config_flags.go b/go/tasks/plugins/hive/config/config_flags.go index 5b82f436d..f469db49a 100755 --- a/go/tasks/plugins/hive/config/config_flags.go +++ b/go/tasks/plugins/hive/config/config_flags.go @@ -45,8 +45,13 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.String(fmt.Sprintf("%v%v", prefix, "commandApiPath"), defaultConfig.CommandAPIPath.String(), "API Path where commands can be launched on Qubole. Should be a valid url.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "analyzeLinkPath"), defaultConfig.AnalyzeLinkPath.String(), "URL path where queries can be visualized on qubole website. Should be a valid url.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "quboleTokenKey"), defaultConfig.TokenKey, "Name of the key where to find Qubole token in the secret manager.") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "lruCacheSize"), defaultConfig.LruCacheSize, "Size of the AutoRefreshCache") - cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "workers"), defaultConfig.Workers, "Number of parallel workers to refresh the cache") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "defaultClusterLabel"), defaultConfig.DefaultClusterLabel, "The default cluster label. This will be used if label is not specified on the hive job.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "readRateLimiter.qps"), defaultConfig.ReadRateLimiter.QPS, "Defines the max rate of calls per second.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "readRateLimiter.burst"), defaultConfig.ReadRateLimiter.Burst, "Defines the maximum burst size.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "writeRateLimiter.qps"), defaultConfig.WriteRateLimiter.QPS, "Defines the max rate of calls per second.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "writeRateLimiter.burst"), defaultConfig.WriteRateLimiter.Burst, "Defines the maximum burst size.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "caching.size"), defaultConfig.Caching.Size, "Defines the maximum number of items to cache.") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "caching.resyncInterval"), defaultConfig.Caching.ResyncInterval.String(), "Defines the sync interval.") + cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "caching.workers"), defaultConfig.Caching.Workers, "Defines the number of workers to start up to process items.") return cmdFlags } diff --git a/go/tasks/plugins/hive/config/config_flags_test.go b/go/tasks/plugins/hive/config/config_flags_test.go index a7fdb6063..67be1f288 100755 --- a/go/tasks/plugins/hive/config/config_flags_test.go +++ b/go/tasks/plugins/hive/config/config_flags_test.go @@ -187,11 +187,11 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) - t.Run("Test_lruCacheSize", func(t *testing.T) { + t.Run("Test_defaultClusterLabel", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("lruCacheSize"); err == nil { - assert.Equal(t, int(defaultConfig.LruCacheSize), vInt) + if vString, err := cmdFlags.GetString("defaultClusterLabel"); err == nil { + assert.Equal(t, string(defaultConfig.DefaultClusterLabel), vString) } else { assert.FailNow(t, err.Error()) } @@ -200,20 +200,20 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("lruCacheSize", testValue) - if vInt, err := cmdFlags.GetInt("lruCacheSize"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.LruCacheSize) + cmdFlags.Set("defaultClusterLabel", testValue) + if vString, err := cmdFlags.GetString("defaultClusterLabel"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.DefaultClusterLabel) } else { assert.FailNow(t, err.Error()) } }) }) - t.Run("Test_workers", func(t *testing.T) { + t.Run("Test_readRateLimiter.qps", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vInt, err := cmdFlags.GetInt("workers"); err == nil { - assert.Equal(t, int(defaultConfig.Workers), vInt) + if vInt, err := cmdFlags.GetInt("readRateLimiter.qps"); err == nil { + assert.Equal(t, int(defaultConfig.ReadRateLimiter.QPS), vInt) } else { assert.FailNow(t, err.Error()) } @@ -222,20 +222,20 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("workers", testValue) - if vInt, err := cmdFlags.GetInt("workers"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.Workers) + cmdFlags.Set("readRateLimiter.qps", testValue) + if vInt, err := cmdFlags.GetInt("readRateLimiter.qps"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.ReadRateLimiter.QPS) } else { assert.FailNow(t, err.Error()) } }) }) - t.Run("Test_defaultClusterLabel", func(t *testing.T) { + t.Run("Test_readRateLimiter.burst", func(t *testing.T) { t.Run("DefaultValue", func(t *testing.T) { // Test that default value is set properly - if vString, err := cmdFlags.GetString("defaultClusterLabel"); err == nil { - assert.Equal(t, string(defaultConfig.DefaultClusterLabel), vString) + if vInt, err := cmdFlags.GetInt("readRateLimiter.burst"); err == nil { + assert.Equal(t, int(defaultConfig.ReadRateLimiter.Burst), vInt) } else { assert.FailNow(t, err.Error()) } @@ -244,9 +244,119 @@ func TestConfig_SetFlags(t *testing.T) { t.Run("Override", func(t *testing.T) { testValue := "1" - cmdFlags.Set("defaultClusterLabel", testValue) - if vString, err := cmdFlags.GetString("defaultClusterLabel"); err == nil { - testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.DefaultClusterLabel) + cmdFlags.Set("readRateLimiter.burst", testValue) + if vInt, err := cmdFlags.GetInt("readRateLimiter.burst"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.ReadRateLimiter.Burst) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_writeRateLimiter.qps", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vInt, err := cmdFlags.GetInt("writeRateLimiter.qps"); err == nil { + assert.Equal(t, int(defaultConfig.WriteRateLimiter.QPS), vInt) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("writeRateLimiter.qps", testValue) + if vInt, err := cmdFlags.GetInt("writeRateLimiter.qps"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.WriteRateLimiter.QPS) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_writeRateLimiter.burst", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vInt, err := cmdFlags.GetInt("writeRateLimiter.burst"); err == nil { + assert.Equal(t, int(defaultConfig.WriteRateLimiter.Burst), vInt) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("writeRateLimiter.burst", testValue) + if vInt, err := cmdFlags.GetInt("writeRateLimiter.burst"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.WriteRateLimiter.Burst) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_caching.size", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vInt, err := cmdFlags.GetInt("caching.size"); err == nil { + assert.Equal(t, int(defaultConfig.Caching.Size), vInt) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("caching.size", testValue) + if vInt, err := cmdFlags.GetInt("caching.size"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.Caching.Size) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_caching.resyncInterval", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("caching.resyncInterval"); err == nil { + assert.Equal(t, string(defaultConfig.Caching.ResyncInterval.String()), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := defaultConfig.Caching.ResyncInterval.String() + + cmdFlags.Set("caching.resyncInterval", testValue) + if vString, err := cmdFlags.GetString("caching.resyncInterval"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.Caching.ResyncInterval) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_caching.workers", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vInt, err := cmdFlags.GetInt("caching.workers"); err == nil { + assert.Equal(t, int(defaultConfig.Caching.Workers), vInt) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("caching.workers", testValue) + if vInt, err := cmdFlags.GetInt("caching.workers"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vInt), &actual.Caching.Workers) } else { assert.FailNow(t, err.Error()) diff --git a/go/tasks/plugins/hive/execution_state.go b/go/tasks/plugins/hive/execution_state.go index 8b0ff9388..a06290875 100644 --- a/go/tasks/plugins/hive/execution_state.go +++ b/go/tasks/plugins/hive/execution_state.go @@ -3,14 +3,9 @@ package hive import ( "context" "fmt" - "strconv" - "time" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" - "github.com/lyft/flytestdlib/cache" - - idlCore "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/plugins" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" @@ -18,134 +13,9 @@ import ( "github.com/lyft/flyteplugins/go/tasks/errors" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" "github.com/lyft/flytestdlib/logger" ) -type ExecutionPhase int - -const ( - PhaseNotStarted ExecutionPhase = iota - PhaseQueued // resource manager token gotten - PhaseSubmitted // Sent off to Qubole - - PhaseQuerySucceeded - PhaseQueryFailed -) - -func (p ExecutionPhase) String() string { - switch p { - case PhaseNotStarted: - return "PhaseNotStarted" - case PhaseQueued: - return "PhaseQueued" - case PhaseSubmitted: - return "PhaseSubmitted" - case PhaseQuerySucceeded: - return "PhaseQuerySucceeded" - case PhaseQueryFailed: - return "PhaseQueryFailed" - } - return "Bad Qubole execution phase" -} - -type ExecutionState struct { - Phase ExecutionPhase - - // This will store the command ID from Qubole - CommandID string `json:"command_id,omitempty"` - URI string `json:"uri,omitempty"` - - // This number keeps track of the number of failures within the sync function. Without this, what happens in - // the sync function is entirely opaque. Note that this field is completely orthogonal to Flyte system/node/task - // level retries, just errors from hitting the Qubole API, inside the sync loop - SyncFailureCount int `json:"sync_failure_count,omitempty"` - - // In kicking off the Qubole command, this is the number of failures - CreationFailureCount int `json:"creation_failure_count,omitempty"` - - // The time the execution first requests for an allocation token - AllocationTokenRequestStartTime time.Time `json:"allocation_token_request_start_time,omitempty"` -} - -// This is the main state iteration -func HandleExecutionState(ctx context.Context, tCtx core.TaskExecutionContext, currentState ExecutionState, quboleClient client.QuboleClient, - executionsCache cache.AutoRefresh, cfg *config.Config, metrics QuboleHiveExecutorMetrics) (ExecutionState, error) { - - var transformError error - var newState ExecutionState - - switch currentState.Phase { - case PhaseNotStarted: - newState, transformError = GetAllocationToken(ctx, tCtx, currentState, metrics) - - case PhaseQueued: - newState, transformError = KickOffQuery(ctx, tCtx, currentState, quboleClient, executionsCache, cfg) - - case PhaseSubmitted: - newState, transformError = MonitorQuery(ctx, tCtx, currentState, executionsCache) - - case PhaseQuerySucceeded: - newState = currentState - transformError = nil - - case PhaseQueryFailed: - newState = currentState - transformError = nil - } - - return newState, transformError -} - -func MapExecutionStateToPhaseInfo(state ExecutionState, quboleClient client.QuboleClient) core.PhaseInfo { - var phaseInfo core.PhaseInfo - t := time.Now() - - switch state.Phase { - case PhaseNotStarted: - phaseInfo = core.PhaseInfoNotReady(t, core.DefaultPhaseVersion, "Haven't received allocation token") - case PhaseQueued: - // TODO: Turn into config - if state.CreationFailureCount > 5 { - phaseInfo = core.PhaseInfoSystemRetryableFailure("QuboleFailure", "Too many creation attempts", nil) - } else { - phaseInfo = core.PhaseInfoQueued(t, uint32(state.CreationFailureCount), "Waiting for Qubole launch") - } - case PhaseSubmitted: - phaseInfo = core.PhaseInfoRunning(core.DefaultPhaseVersion, ConstructTaskInfo(state)) - - case PhaseQuerySucceeded: - phaseInfo = core.PhaseInfoSuccess(ConstructTaskInfo(state)) - - case PhaseQueryFailed: - phaseInfo = core.PhaseInfoFailure(errors.DownstreamSystemError, "Query failed", ConstructTaskInfo(state)) - } - - return phaseInfo -} - -func ConstructTaskLog(e ExecutionState) *idlCore.TaskLog { - return &idlCore.TaskLog{ - Name: fmt.Sprintf("Status: %s [%s]", e.Phase, e.CommandID), - MessageFormat: idlCore.TaskLog_UNKNOWN, - Uri: e.URI, - } -} - -func ConstructTaskInfo(e ExecutionState) *core.TaskInfo { - logs := make([]*idlCore.TaskLog, 0, 1) - t := time.Now() - if e.CommandID != "" { - logs = append(logs, ConstructTaskLog(e)) - return &core.TaskInfo{ - Logs: logs, - OccurredAt: &t, - } - } - - return nil -} - func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext) (core.ResourceNamespace, error) { _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) if err != nil { @@ -176,51 +46,6 @@ func createResourceConstraintsSpec(ctx context.Context, _ remote.TaskExecutionCo return constraintsSpec } -func GetAllocationToken(ctx context.Context, tCtx core.TaskExecutionContext, currentState ExecutionState, metric QuboleHiveExecutorMetrics) (ExecutionState, error) { - newState := ExecutionState{} - uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() - - clusterPrimaryLabel, err := composeResourceNamespaceWithClusterPrimaryLabel(ctx, tCtx) - if err != nil { - return newState, errors.Wrapf(errors.ResourceManagerFailure, err, "Error getting query info when requesting allocation token %s", uniqueID) - } - - resourceConstraintsSpec := createResourceConstraintsSpec(ctx, tCtx, clusterPrimaryLabel) - - allocationStatus, err := tCtx.ResourceManager().AllocateResource(ctx, clusterPrimaryLabel, uniqueID, resourceConstraintsSpec) - if err != nil { - logger.Errorf(ctx, "Resource manager failed for TaskExecId [%s] token [%s]. error %s", - tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID(), uniqueID, err) - return newState, errors.Wrapf(errors.ResourceManagerFailure, err, "Error requesting allocation token %s", uniqueID) - } - logger.Infof(ctx, "Allocation result for [%s] is [%s]", uniqueID, allocationStatus) - - // Emitting the duration this execution has been waiting for a token allocation - if currentState.AllocationTokenRequestStartTime.IsZero() { - newState.AllocationTokenRequestStartTime = time.Now() - } else { - newState.AllocationTokenRequestStartTime = currentState.AllocationTokenRequestStartTime - } - waitTime := time.Since(newState.AllocationTokenRequestStartTime) - metric.ResourceWaitTime.Observe(waitTime.Seconds()) - - if allocationStatus == core.AllocationStatusGranted { - metric.AllocationGranted.Inc(ctx) - newState.Phase = PhaseQueued - } else if allocationStatus == core.AllocationStatusExhausted { - metric.AllocationNotGranted.Inc(ctx) - newState.Phase = PhaseNotStarted - } else if allocationStatus == core.AllocationStatusNamespaceQuotaExceeded { - metric.AllocationNotGranted.Inc(ctx) - newState.Phase = PhaseNotStarted - } else { - return newState, errors.Errorf(errors.ResourceManagerFailure, "Got bad allocation result [%s] for token [%s]", - allocationStatus, uniqueID) - } - - return newState, nil -} - func validateQuboleHiveJob(hiveJob plugins.QuboleHiveJob) error { if hiveJob.Query == nil { return errors.Errorf(errors.BadTaskSpecification, @@ -326,125 +151,3 @@ func getClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContex // Else we return the default primary label return cfg.DefaultClusterLabel } - -func KickOffQuery(ctx context.Context, tCtx core.TaskExecutionContext, currentState ExecutionState, quboleClient client.QuboleClient, - cache cache.AutoRefresh, cfg *config.Config) (ExecutionState, error) { - - uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() - apiKey, err := tCtx.SecretManager().Get(ctx, cfg.TokenKey) - if err != nil { - return currentState, errors.Wrapf(errors.RuntimeFailure, err, "Failed to read token from secrets manager") - } - - query, clusterLabelOverride, tags, timeoutSec, err := GetQueryInfo(ctx, tCtx) - if err != nil { - return currentState, err - } - - clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) - - cmdDetails, err := quboleClient.ExecuteHiveCommand(ctx, query, timeoutSec, - clusterPrimaryLabel, apiKey, tags) - if err != nil { - // If we failed, we'll keep the NotStarted state - currentState.CreationFailureCount = currentState.CreationFailureCount + 1 - logger.Warnf(ctx, "Error creating Qubole query for %s, failure counts %d. Error: %s", uniqueID, currentState.CreationFailureCount, err) - } else { - // If we succeed, then store the command id returned from Qubole, and update our state. Also, add to the - // AutoRefreshCache so we start getting updates. - commandID := strconv.FormatInt(cmdDetails.ID, 10) - logger.Infof(ctx, "Created Qubole ID [%s] for token %s", commandID, uniqueID) - currentState.CommandID = commandID - currentState.Phase = PhaseSubmitted - currentState.URI = cmdDetails.URI.String() - - executionStateCacheItem := ExecutionStateCacheItem{ - ExecutionState: currentState, - Identifier: uniqueID, - } - - // The first time we put it in the cache, we know it won't have succeeded so we don't need to look at it - _, err := cache.GetOrCreate(uniqueID, executionStateCacheItem) - if err != nil { - // This means that our cache has fundamentally broken... return a system error - logger.Errorf(ctx, "Cache failed to GetOrCreate for execution [%s] cache key [%s], owner [%s]. Error %s", - tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID(), uniqueID, - tCtx.TaskExecutionMetadata().GetOwnerReference(), err) - return currentState, err - } - } - - return currentState, nil -} - -func MonitorQuery(ctx context.Context, tCtx core.TaskExecutionContext, currentState ExecutionState, cache cache.AutoRefresh) ( - ExecutionState, error) { - - uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() - executionStateCacheItem := ExecutionStateCacheItem{ - ExecutionState: currentState, - Identifier: uniqueID, - } - - cachedItem, err := cache.GetOrCreate(uniqueID, executionStateCacheItem) - if err != nil { - // This means that our cache has fundamentally broken... return a system error - logger.Errorf(ctx, "Cache is broken on execution [%s] cache key [%s], owner [%s]. Error %s", - tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID(), uniqueID, - tCtx.TaskExecutionMetadata().GetOwnerReference(), err) - return currentState, errors.Wrapf(errors.CacheFailed, err, "Error when GetOrCreate while monitoring") - } - - cachedExecutionState, ok := cachedItem.(ExecutionStateCacheItem) - if !ok { - logger.Errorf(ctx, "Error casting cache object into ExecutionState") - return currentState, errors.Errorf(errors.CacheFailed, "Failed to cast [%v]", cachedItem) - } - - // TODO: Add a couple of debug lines here - did it change or did it not? - - // If there were updates made to the state, we'll have picked them up automatically. Nothing more to do. - return cachedExecutionState.ExecutionState, nil -} - -func Abort(ctx context.Context, tCtx core.TaskExecutionContext, currentState ExecutionState, qubole client.QuboleClient, apiKey string) error { - // Cancel Qubole query if non-terminal state - if !InTerminalState(currentState) && currentState.CommandID != "" { - err := qubole.KillCommand(ctx, currentState.CommandID, apiKey) - if err != nil { - logger.Errorf(ctx, "Error terminating Qubole command in Finalize [%s]", err) - return err - } - } - return nil -} - -func Finalize(ctx context.Context, tCtx core.TaskExecutionContext, _ ExecutionState, metrics QuboleHiveExecutorMetrics) error { - // Release allocation token - uniqueID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName() - clusterPrimaryLabel, err := composeResourceNamespaceWithClusterPrimaryLabel(ctx, tCtx) - if err != nil { - return errors.Wrapf(errors.ResourceManagerFailure, err, "Error getting query info when releasing allocation token %s", uniqueID) - } - - err = tCtx.ResourceManager().ReleaseResource(ctx, clusterPrimaryLabel, uniqueID) - - if err != nil { - metrics.ResourceReleaseFailed.Inc(ctx) - logger.Errorf(ctx, "Error releasing allocation token [%s] in Finalize [%s]", uniqueID, err) - return err - } - metrics.ResourceReleased.Inc(ctx) - return nil -} - -func InTerminalState(e ExecutionState) bool { - return e.Phase == PhaseQuerySucceeded || e.Phase == PhaseQueryFailed -} - -func IsNotYetSubmitted(e ExecutionState) bool { - if e.Phase == PhaseNotStarted || e.Phase == PhaseQueued { - return true - } - return false -} diff --git a/go/tasks/plugins/hive/execution_state_test.go b/go/tasks/plugins/hive/execution_state_test.go index 9ebd6a9b3..fcbf81f13 100644 --- a/go/tasks/plugins/hive/execution_state_test.go +++ b/go/tasks/plugins/hive/execution_state_test.go @@ -2,77 +2,31 @@ package hive import ( "context" - "net/url" "testing" - "time" "github.com/lyft/flytestdlib/contextutils" "github.com/lyft/flytestdlib/promutils/labeled" - "github.com/lyft/flytestdlib/promutils" - idlCore "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" "github.com/lyft/flyteidl/gen/pb-go/flyteidl/plugins" - mocks2 "github.com/lyft/flytestdlib/cache/mocks" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core/mocks" pluginsCoreMocks "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core/mocks" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" - quboleMocks "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client/mocks" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" ) +const ( + DefaultClusterPrimaryLabel = "default" +) + func init() { labeled.SetMetricKeys(contextutils.NamespaceKey) } -func TestInTerminalState(t *testing.T) { - var stateTests = []struct { - phase ExecutionPhase - isTerminal bool - }{ - {phase: PhaseNotStarted, isTerminal: false}, - {phase: PhaseQueued, isTerminal: false}, - {phase: PhaseSubmitted, isTerminal: false}, - {phase: PhaseQuerySucceeded, isTerminal: true}, - {phase: PhaseQueryFailed, isTerminal: true}, - } - - for _, tt := range stateTests { - t.Run(tt.phase.String(), func(t *testing.T) { - e := ExecutionState{Phase: tt.phase} - res := InTerminalState(e) - assert.Equal(t, tt.isTerminal, res) - }) - } -} - -func TestIsNotYetSubmitted(t *testing.T) { - var stateTests = []struct { - phase ExecutionPhase - isNotYetSubmitted bool - }{ - {phase: PhaseNotStarted, isNotYetSubmitted: true}, - {phase: PhaseQueued, isNotYetSubmitted: true}, - {phase: PhaseSubmitted, isNotYetSubmitted: false}, - {phase: PhaseQuerySucceeded, isNotYetSubmitted: false}, - {phase: PhaseQueryFailed, isNotYetSubmitted: false}, - } - - for _, tt := range stateTests { - t.Run(tt.phase.String(), func(t *testing.T) { - e := ExecutionState{Phase: tt.phase} - res := IsNotYetSubmitted(e) - assert.Equal(t, tt.isNotYetSubmitted, res) - }) - } -} - func TestGetQueryInfo(t *testing.T) { ctx := context.Background() @@ -106,246 +60,6 @@ func TestValidateQuboleHiveJob(t *testing.T) { assert.Error(t, err) } -func TestConstructTaskLog(t *testing.T) { - expected := "https://wellness.qubole.com/v2/analyze?command_id=123" - u, err := url.Parse(expected) - assert.NoError(t, err) - taskLog := ConstructTaskLog(ExecutionState{CommandID: "123", URI: u.String()}) - assert.Equal(t, expected, taskLog.Uri) -} - -func TestConstructTaskInfo(t *testing.T) { - empty := ConstructTaskInfo(ExecutionState{}) - assert.Nil(t, empty) - - expected := "https://wellness.qubole.com/v2/analyze?command_id=123" - u, err := url.Parse(expected) - assert.NoError(t, err) - - e := ExecutionState{ - Phase: PhaseQuerySucceeded, - CommandID: "123", - SyncFailureCount: 0, - URI: u.String(), - } - - taskInfo := ConstructTaskInfo(e) - assert.Equal(t, "https://wellness.qubole.com/v2/analyze?command_id=123", taskInfo.Logs[0].Uri) -} - -func TestMapExecutionStateToPhaseInfo(t *testing.T) { - c := client.NewQuboleClient(config.GetQuboleConfig()) - t.Run("NotStarted", func(t *testing.T) { - e := ExecutionState{ - Phase: PhaseNotStarted, - } - phaseInfo := MapExecutionStateToPhaseInfo(e, c) - assert.Equal(t, core.PhaseNotReady, phaseInfo.Phase()) - }) - - t.Run("Queued", func(t *testing.T) { - e := ExecutionState{ - Phase: PhaseQueued, - CreationFailureCount: 0, - } - phaseInfo := MapExecutionStateToPhaseInfo(e, c) - assert.Equal(t, core.PhaseQueued, phaseInfo.Phase()) - - e = ExecutionState{ - Phase: PhaseQueued, - CreationFailureCount: 100, - } - phaseInfo = MapExecutionStateToPhaseInfo(e, c) - assert.Equal(t, core.PhaseRetryableFailure, phaseInfo.Phase()) - - }) - - t.Run("Submitted", func(t *testing.T) { - e := ExecutionState{ - Phase: PhaseSubmitted, - } - phaseInfo := MapExecutionStateToPhaseInfo(e, c) - assert.Equal(t, core.PhaseRunning, phaseInfo.Phase()) - }) -} - -func TestGetAllocationToken(t *testing.T) { - ctx := context.Background() - - t.Run("allocation granted", func(t *testing.T) { - tCtx := GetMockTaskExecutionContext() - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("AllocateResource", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(core.AllocationStatusGranted, nil) - - mockCurrentState := ExecutionState{AllocationTokenRequestStartTime: time.Now()} - mockMetrics := getQuboleHiveExecutorMetrics(promutils.NewTestScope()) - state, err := GetAllocationToken(ctx, tCtx, mockCurrentState, mockMetrics) - assert.NoError(t, err) - assert.Equal(t, PhaseQueued, state.Phase) - }) - - t.Run("exhausted", func(t *testing.T) { - tCtx := GetMockTaskExecutionContext() - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("AllocateResource", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(core.AllocationStatusExhausted, nil) - - mockCurrentState := ExecutionState{AllocationTokenRequestStartTime: time.Now()} - mockMetrics := getQuboleHiveExecutorMetrics(promutils.NewTestScope()) - state, err := GetAllocationToken(ctx, tCtx, mockCurrentState, mockMetrics) - assert.NoError(t, err) - assert.Equal(t, PhaseNotStarted, state.Phase) - }) - - t.Run("namespace exhausted", func(t *testing.T) { - tCtx := GetMockTaskExecutionContext() - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("AllocateResource", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(core.AllocationStatusNamespaceQuotaExceeded, nil) - - mockCurrentState := ExecutionState{AllocationTokenRequestStartTime: time.Now()} - mockMetrics := getQuboleHiveExecutorMetrics(promutils.NewTestScope()) - state, err := GetAllocationToken(ctx, tCtx, mockCurrentState, mockMetrics) - assert.NoError(t, err) - assert.Equal(t, PhaseNotStarted, state.Phase) - }) - - t.Run("Request start time, if empty in current state, should be set", func(t *testing.T) { - tCtx := GetMockTaskExecutionContext() - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("AllocateResource", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(core.AllocationStatusNamespaceQuotaExceeded, nil) - - mockCurrentState := ExecutionState{} - mockMetrics := getQuboleHiveExecutorMetrics(promutils.NewTestScope()) - state, err := GetAllocationToken(ctx, tCtx, mockCurrentState, mockMetrics) - assert.NoError(t, err) - assert.Equal(t, state.AllocationTokenRequestStartTime.IsZero(), false) - }) - - t.Run("Request start time, if already set in current state, should be maintained", func(t *testing.T) { - tCtx := GetMockTaskExecutionContext() - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("AllocateResource", mock.Anything, mock.Anything, mock.Anything, mock.Anything). - Return(core.AllocationStatusGranted, nil) - - startTime := time.Now() - mockCurrentState := ExecutionState{AllocationTokenRequestStartTime: startTime} - mockMetrics := getQuboleHiveExecutorMetrics(promutils.NewTestScope()) - state, err := GetAllocationToken(ctx, tCtx, mockCurrentState, mockMetrics) - assert.NoError(t, err) - assert.Equal(t, state.AllocationTokenRequestStartTime.IsZero(), false) - assert.Equal(t, state.AllocationTokenRequestStartTime, startTime) - }) -} - -func TestAbort(t *testing.T) { - ctx := context.Background() - - t.Run("Terminate called when not in terminal state", func(t *testing.T) { - var x = false - mockQubole := &quboleMocks.QuboleClient{} - mockQubole.On("KillCommand", mock.Anything, mock.MatchedBy(func(commandId string) bool { - return commandId == "123456" - }), mock.Anything).Run(func(_ mock.Arguments) { - x = true - }).Return(nil) - - err := Abort(ctx, GetMockTaskExecutionContext(), ExecutionState{Phase: PhaseSubmitted, CommandID: "123456"}, mockQubole, "fake-key") - assert.NoError(t, err) - assert.True(t, x) - }) - - t.Run("Terminate not called when in terminal state", func(t *testing.T) { - var x = false - mockQubole := &quboleMocks.QuboleClient{} - mockQubole.On("KillCommand", mock.Anything, mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - x = true - }).Return(nil) - - err := Abort(ctx, GetMockTaskExecutionContext(), ExecutionState{ - Phase: PhaseQuerySucceeded, - CommandID: "123456", - }, mockQubole, "fake-key") - assert.NoError(t, err) - assert.False(t, x) - }) -} - -func TestFinalize(t *testing.T) { - // Test that Finalize releases resources - ctx := context.Background() - tCtx := GetMockTaskExecutionContext() - state := ExecutionState{} - var called = false - mockResourceManager := tCtx.ResourceManager() - x := mockResourceManager.(*mocks.ResourceManager) - x.On("ReleaseResource", mock.Anything, mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - called = true - }).Return(nil) - - err := Finalize(ctx, tCtx, state, getQuboleHiveExecutorMetrics(promutils.NewTestScope())) - assert.NoError(t, err) - assert.True(t, called) -} - -func TestMonitorQuery(t *testing.T) { - ctx := context.Background() - tCtx := GetMockTaskExecutionContext() - state := ExecutionState{ - Phase: PhaseSubmitted, - } - var getOrCreateCalled = false - mockCache := &mocks2.AutoRefresh{} - mockCache.OnGetOrCreateMatch("my_wf_exec_project:my_wf_exec_domain:my_wf_exec_name", mock.Anything).Return(ExecutionStateCacheItem{ - ExecutionState: ExecutionState{Phase: PhaseQuerySucceeded}, - Identifier: "my_wf_exec_project:my_wf_exec_domain:my_wf_exec_name", - }, nil).Run(func(_ mock.Arguments) { - getOrCreateCalled = true - }) - - newState, err := MonitorQuery(ctx, tCtx, state, mockCache) - assert.NoError(t, err) - assert.True(t, getOrCreateCalled) - assert.Equal(t, PhaseQuerySucceeded, newState.Phase) -} - -func TestKickOffQuery(t *testing.T) { - ctx := context.Background() - tCtx := GetMockTaskExecutionContext() - - var quboleCalled = false - quboleCommandDetails := &client.QuboleCommandDetails{ - ID: int64(453298043), - Status: client.QuboleStatusWaiting, - } - mockQubole := &quboleMocks.QuboleClient{} - mockQubole.OnExecuteHiveCommandMatch(mock.Anything, mock.Anything, mock.Anything, mock.Anything, - mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - quboleCalled = true - }).Return(quboleCommandDetails, nil) - - var getOrCreateCalled = false - mockCache := &mocks2.AutoRefresh{} - mockCache.OnGetOrCreate(mock.Anything, mock.Anything).Run(func(_ mock.Arguments) { - getOrCreateCalled = true - }).Return(ExecutionStateCacheItem{}, nil) - - state := ExecutionState{} - newState, err := KickOffQuery(ctx, tCtx, state, mockQubole, mockCache, config.GetQuboleConfig()) - assert.NoError(t, err) - assert.Equal(t, PhaseSubmitted, newState.Phase) - assert.Equal(t, "453298043", newState.CommandID) - assert.True(t, getOrCreateCalled) - assert.True(t, quboleCalled) -} - func createMockQuboleCfg() *config.Config { return &config.Config{ DefaultClusterLabel: "default", diff --git a/go/tasks/plugins/hive/executions_cache.go b/go/tasks/plugins/hive/executions_cache.go deleted file mode 100644 index 3e7347ffa..000000000 --- a/go/tasks/plugins/hive/executions_cache.go +++ /dev/null @@ -1,172 +0,0 @@ -package hive - -import ( - "context" - "time" - - "k8s.io/client-go/util/workqueue" - - "github.com/lyft/flytestdlib/cache" - - "github.com/lyft/flyteplugins/go/tasks/errors" - stdErrors "github.com/lyft/flytestdlib/errors" - - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" - - "github.com/lyft/flytestdlib/logger" - "github.com/lyft/flytestdlib/promutils" -) - -const ResyncDuration = 30 * time.Second - -const ( - BadQuboleReturnCodeError stdErrors.ErrorCode = "QUBOLE_RETURNED_UNKNOWN" -) - -type QuboleHiveExecutionsCache struct { - cache.AutoRefresh - quboleClient client.QuboleClient - secretManager core.SecretManager - scope promutils.Scope - cfg *config.Config -} - -func NewQuboleHiveExecutionsCache(ctx context.Context, quboleClient client.QuboleClient, - secretManager core.SecretManager, cfg *config.Config, scope promutils.Scope) (QuboleHiveExecutionsCache, error) { - - q := QuboleHiveExecutionsCache{ - quboleClient: quboleClient, - secretManager: secretManager, - scope: scope, - cfg: cfg, - } - autoRefreshCache, err := cache.NewAutoRefreshCache("qubole", q.SyncQuboleQuery, workqueue.DefaultControllerRateLimiter(), ResyncDuration, cfg.Workers, cfg.LruCacheSize, scope) - if err != nil { - logger.Errorf(ctx, "Could not create AutoRefreshCache in QuboleHiveExecutor. [%s]", err) - return q, errors.Wrapf(errors.CacheFailed, err, "Error creating AutoRefreshCache") - } - q.AutoRefresh = autoRefreshCache - return q, nil -} - -type ExecutionStateCacheItem struct { - ExecutionState - - // This ID is the cache key and so will need to be unique across all objects in the cache (it will probably be - // unique across all of Flyte) and needs to be deterministic. - // This will also be used as the allocation token for now. - Identifier string `json:"id"` -} - -func (e ExecutionStateCacheItem) ID() string { - return e.Identifier -} - -// This basically grab an updated status from the Qubole API and store it in the cache -// All other handling should be in the synchronous loop. -func (q *QuboleHiveExecutionsCache) SyncQuboleQuery(ctx context.Context, batch cache.Batch) ( - updatedBatch []cache.ItemSyncResponse, err error) { - - resp := make([]cache.ItemSyncResponse, 0, len(batch)) - for _, query := range batch { - // Cast the item back to the thing we want to work with. - executionStateCacheItem, ok := query.GetItem().(ExecutionStateCacheItem) - if !ok { - logger.Errorf(ctx, "Sync loop - Error casting cache object into ExecutionState") - return nil, errors.Errorf(errors.CacheFailed, "Failed to cast [%v]", batch[0].GetID()) - } - - if executionStateCacheItem.CommandID == "" { - logger.Warnf(ctx, "Sync loop - CommandID is blank for [%s] skipping", executionStateCacheItem.Identifier) - resp = append(resp, cache.ItemSyncResponse{ - ID: query.GetID(), - Item: query.GetItem(), - Action: cache.Unchanged, - }) - - continue - } - - logger.Debugf(ctx, "Sync loop - processing Hive job [%s] - cache key [%s]", - executionStateCacheItem.CommandID, executionStateCacheItem.Identifier) - - quboleAPIKey, err := q.secretManager.Get(ctx, q.cfg.TokenKey) - if err != nil { - return nil, err - } - - if InTerminalState(executionStateCacheItem.ExecutionState) { - logger.Debugf(ctx, "Sync loop - Qubole id [%s] in terminal state [%s]", - executionStateCacheItem.CommandID, executionStateCacheItem.Identifier) - - resp = append(resp, cache.ItemSyncResponse{ - ID: query.GetID(), - Item: query.GetItem(), - Action: cache.Unchanged, - }) - - continue - } - - // Get an updated status from Qubole - logger.Debugf(ctx, "Querying Qubole for %s - %s", executionStateCacheItem.CommandID, executionStateCacheItem.Identifier) - commandStatus, err := q.quboleClient.GetCommandStatus(ctx, executionStateCacheItem.CommandID, quboleAPIKey) - if err != nil { - logger.Errorf(ctx, "Error from Qubole command %s", executionStateCacheItem.CommandID) - executionStateCacheItem.SyncFailureCount++ - // Make sure we don't return nil for the first argument, because that deletes it from the cache. - resp = append(resp, cache.ItemSyncResponse{ - ID: query.GetID(), - Item: executionStateCacheItem, - Action: cache.Update, - }) - - continue - } - - newExecutionPhase, err := QuboleStatusToExecutionPhase(commandStatus) - if err != nil { - return nil, err - } - - if newExecutionPhase > executionStateCacheItem.Phase { - logger.Infof(ctx, "Moving ExecutionPhase for %s %s from %s to %s", executionStateCacheItem.CommandID, - executionStateCacheItem.Identifier, executionStateCacheItem.Phase, newExecutionPhase) - - executionStateCacheItem.Phase = newExecutionPhase - - resp = append(resp, cache.ItemSyncResponse{ - ID: query.GetID(), - Item: executionStateCacheItem, - Action: cache.Update, - }) - } - } - - return resp, nil -} - -// We need some way to translate results we get from Qubole, into a plugin phase -// NB: This function should only return plugin phases that are greater than (">") phases that represent states before -// the query was kicked off. That is, it will never make sense to go back to PhaseNotStarted, after we've -// submitted the query to Qubole. -func QuboleStatusToExecutionPhase(s client.QuboleStatus) (ExecutionPhase, error) { - switch s { - case client.QuboleStatusDone: - return PhaseQuerySucceeded, nil - case client.QuboleStatusCancelled: - return PhaseQueryFailed, nil - case client.QuboleStatusError: - return PhaseQueryFailed, nil - case client.QuboleStatusWaiting: - return PhaseSubmitted, nil - case client.QuboleStatusRunning: - return PhaseSubmitted, nil - case client.QuboleStatusUnknown: - return PhaseQueryFailed, errors.Errorf(BadQuboleReturnCodeError, "Qubole returned status Unknown") - default: - return PhaseQueryFailed, errors.Errorf(BadQuboleReturnCodeError, "default fallthrough case") - } -} diff --git a/go/tasks/plugins/hive/executions_cache_test.go b/go/tasks/plugins/hive/executions_cache_test.go deleted file mode 100644 index cc33365b3..000000000 --- a/go/tasks/plugins/hive/executions_cache_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package hive - -import ( - "context" - "testing" - - "github.com/lyft/flytestdlib/cache" - cacheMocks "github.com/lyft/flytestdlib/cache/mocks" - - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core/mocks" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" - quboleMocks "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client/mocks" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" - - "github.com/lyft/flytestdlib/promutils" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestQuboleHiveExecutionsCache_SyncQuboleQuery(t *testing.T) { - ctx := context.Background() - - t.Run("terminal state return unchanged", func(t *testing.T) { - mockCache := &cacheMocks.AutoRefresh{} - mockQubole := &quboleMocks.QuboleClient{} - testScope := promutils.NewTestScope() - - q := QuboleHiveExecutionsCache{ - AutoRefresh: mockCache, - quboleClient: mockQubole, - scope: testScope, - cfg: config.GetQuboleConfig(), - } - - state := ExecutionState{ - Phase: PhaseQuerySucceeded, - } - cacheItem := ExecutionStateCacheItem{ - ExecutionState: state, - Identifier: "some-id", - } - - iw := &cacheMocks.ItemWrapper{} - iw.OnGetItem().Return(cacheItem) - iw.OnGetID().Return("some-id") - - newCacheItem, err := q.SyncQuboleQuery(ctx, []cache.ItemWrapper{iw}) - assert.NoError(t, err) - assert.Equal(t, cache.Unchanged, newCacheItem[0].Action) - assert.Equal(t, cacheItem, newCacheItem[0].Item) - }) - - t.Run("move to success", func(t *testing.T) { - mockCache := &cacheMocks.AutoRefresh{} - mockQubole := &quboleMocks.QuboleClient{} - mockSecretManager := &mocks.SecretManager{} - mockSecretManager.OnGetMatch(mock.Anything, mock.Anything).Return("fake key", nil) - - testScope := promutils.NewTestScope() - - q := QuboleHiveExecutionsCache{ - AutoRefresh: mockCache, - quboleClient: mockQubole, - scope: testScope, - secretManager: mockSecretManager, - cfg: config.GetQuboleConfig(), - } - - state := ExecutionState{ - CommandID: "123456", - Phase: PhaseSubmitted, - } - cacheItem := ExecutionStateCacheItem{ - ExecutionState: state, - Identifier: "some-id", - } - mockQubole.OnGetCommandStatusMatch(mock.Anything, mock.MatchedBy(func(commandId string) bool { - return commandId == state.CommandID - }), mock.Anything).Return(client.QuboleStatusDone, nil) - - iw := &cacheMocks.ItemWrapper{} - iw.OnGetItem().Return(cacheItem) - iw.OnGetID().Return("some-id") - - newCacheItem, err := q.SyncQuboleQuery(ctx, []cache.ItemWrapper{iw}) - newExecutionState := newCacheItem[0].Item.(ExecutionStateCacheItem) - assert.NoError(t, err) - assert.Equal(t, cache.Update, newCacheItem[0].Action) - assert.Equal(t, PhaseQuerySucceeded, newExecutionState.Phase) - }) -} diff --git a/go/tasks/plugins/hive/executor.go b/go/tasks/plugins/hive/executor.go deleted file mode 100644 index 9ae116c8f..000000000 --- a/go/tasks/plugins/hive/executor.go +++ /dev/null @@ -1,168 +0,0 @@ -package hive - -import ( - "context" - - "github.com/lyft/flytestdlib/cache" - - "github.com/lyft/flyteplugins/go/tasks/errors" - pluginMachinery "github.com/lyft/flyteplugins/go/tasks/pluginmachinery" - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" - "github.com/lyft/flytestdlib/logger" - "github.com/lyft/flytestdlib/promutils" -) - -// This is the name of this plugin effectively. In Flyte plugin configuration, use this string to enable this plugin. -const quboleHiveExecutorID = "qubole-hive-executor" - -// Version of the custom state this plugin stores. Useful for backwards compatibility if you one day need to update -// the structure of the stored state -const pluginStateVersion = 0 - -const hiveTaskType = "hive" // This needs to match the type defined in Flytekit constants.py - -const DefaultClusterPrimaryLabel = "default" - -type QuboleHiveExecutor struct { - id string - metrics QuboleHiveExecutorMetrics - quboleClient client.QuboleClient - executionsCache cache.AutoRefresh - cfg *config.Config -} - -func (q QuboleHiveExecutor) GetID() string { - return q.id -} - -func (q QuboleHiveExecutor) Handle(ctx context.Context, tCtx core.TaskExecutionContext) (core.Transition, error) { - incomingState := ExecutionState{} - - // We assume here that the first time this function is called, the custom state we get back is whatever we passed in, - // namely the zero-value of our struct. - if _, err := tCtx.PluginStateReader().Get(&incomingState); err != nil { - logger.Errorf(ctx, "Plugin %s failed to unmarshal custom state when handling [%s] [%s]", - q.id, tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName(), err) - return core.UnknownTransition, errors.Wrapf(errors.CorruptedPluginState, err, - "Failed to unmarshal custom state in Handle") - } - - // Do what needs to be done, and give this function everything it needs to do its job properly - // TODO: Play around with making this return a transition directly. How will that pattern affect the multi-Qubole plugin - outgoingState, transformError := HandleExecutionState(ctx, tCtx, incomingState, q.quboleClient, q.executionsCache, q.cfg, q.metrics) - - // Return if there was an error - if transformError != nil { - return core.UnknownTransition, transformError - } - - // If no error, then infer the new Phase from the various states - phaseInfo := MapExecutionStateToPhaseInfo(outgoingState, q.quboleClient) - - if err := tCtx.PluginStateWriter().Put(pluginStateVersion, outgoingState); err != nil { - return core.UnknownTransition, err - } - - return core.DoTransitionType(core.TransitionTypeBarrier, phaseInfo), nil -} - -func (q QuboleHiveExecutor) Abort(ctx context.Context, tCtx core.TaskExecutionContext) error { - incomingState := ExecutionState{} - if _, err := tCtx.PluginStateReader().Get(&incomingState); err != nil { - logger.Errorf(ctx, "Plugin %s failed to unmarshal custom state in Finalize [%s] Err [%s]", - q.id, tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName(), err) - return errors.Wrapf(errors.CorruptedPluginState, err, "Failed to unmarshal custom state in Finalize") - } - - key, err := tCtx.SecretManager().Get(ctx, q.cfg.TokenKey) - if err != nil { - logger.Errorf(ctx, "Error reading token in Finalize [%s]", err) - return err - } - - return Abort(ctx, tCtx, incomingState, q.quboleClient, key) -} - -func (q QuboleHiveExecutor) Finalize(ctx context.Context, tCtx core.TaskExecutionContext) error { - incomingState := ExecutionState{} - if _, err := tCtx.PluginStateReader().Get(&incomingState); err != nil { - logger.Errorf(ctx, "Plugin %s failed to unmarshal custom state in Finalize [%s] Err [%s]", - q.id, tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName(), err) - return errors.Wrapf(errors.CorruptedPluginState, err, "Failed to unmarshal custom state in Finalize") - } - - return Finalize(ctx, tCtx, incomingState, q.metrics) -} - -func (q QuboleHiveExecutor) GetProperties() core.PluginProperties { - return core.PluginProperties{} -} - -func QuboleHiveExecutorLoader(ctx context.Context, iCtx core.SetupContext) (core.Plugin, error) { - cfg := config.GetQuboleConfig() - return InitializeHiveExecutor(ctx, iCtx, cfg, BuildResourceConfig(cfg.ClusterConfigs), client.NewQuboleClient(cfg)) -} - -func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace]int { - resourceConfig := make(map[core.ResourceNamespace]int, len(cfg)) - - for _, clusterCfg := range cfg { - resourceConfig[core.ResourceNamespace(clusterCfg.PrimaryLabel)] = clusterCfg.Limit - } - - return resourceConfig -} - -func InitializeHiveExecutor(ctx context.Context, iCtx core.SetupContext, cfg *config.Config, resourceConfig map[core.ResourceNamespace]int, - quboleClient client.QuboleClient) (core.Plugin, error) { - logger.Infof(ctx, "Initializing a Hive executor with a resource config [%v]", resourceConfig) - q, err := NewQuboleHiveExecutor(ctx, cfg, quboleClient, iCtx.SecretManager(), iCtx.MetricsScope()) - if err != nil { - logger.Errorf(ctx, "Failed to create a new QuboleHiveExecutor due to error: [%v]", err) - return nil, err - } - - for clusterPrimaryLabel, clusterLimit := range resourceConfig { - logger.Infof(ctx, "Registering resource quota ([%v]) and namespace quota cap ([%v]) for cluster [%v]", clusterPrimaryLabel) - if err := iCtx.ResourceRegistrar().RegisterResourceQuota(ctx, clusterPrimaryLabel, clusterLimit); err != nil { - logger.Errorf(ctx, "Resource quota registration for [%v] failed due to error [%v]", clusterPrimaryLabel, err) - return nil, err - } - } - - return q, nil -} - -// type PluginLoader func(ctx context.Context, iCtx SetupContext) (Plugin, error) -func NewQuboleHiveExecutor(ctx context.Context, cfg *config.Config, quboleClient client.QuboleClient, secretManager core.SecretManager, scope promutils.Scope) (QuboleHiveExecutor, error) { - executionsAutoRefreshCache, err := NewQuboleHiveExecutionsCache(ctx, quboleClient, secretManager, cfg, scope.NewSubScope(hiveTaskType)) - if err != nil { - logger.Errorf(ctx, "Failed to create AutoRefreshCache in QuboleHiveExecutor Setup. Error: %v", err) - return QuboleHiveExecutor{}, err - } - - err = executionsAutoRefreshCache.Start(ctx) - if err != nil { - logger.Errorf(ctx, "Failed to start AutoRefreshCache. Error: %v", err) - } - - return QuboleHiveExecutor{ - id: quboleHiveExecutorID, - cfg: cfg, - metrics: getQuboleHiveExecutorMetrics(scope.NewSubScope("hive")), - quboleClient: quboleClient, - executionsCache: executionsAutoRefreshCache, - }, nil -} - -func init() { - pluginMachinery.PluginRegistry().RegisterCorePlugin( - core.PluginEntry{ - ID: quboleHiveExecutorID, - RegisteredTaskTypes: []core.TaskType{hiveTaskType}, - LoadPlugin: QuboleHiveExecutorLoader, - IsDefault: false, - }) -} From a0f02ed8c9a47e00969521360a94fa41028f937e Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:24:50 -0700 Subject: [PATCH 06/12] wip --- go/tasks/plugins/hive/plugin.go | 2 +- go/tasks/plugins/hive/transformer.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 3f63d50a3..5e1a19281 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -80,7 +80,7 @@ func (q QuboleHivePlugin) Get(ctx context.Context, meta remote.ResourceMeta) ( return nil, err } - newExecutionPhase, err := QuboleStatusToExecutionPhase(commandStatus) + newExecutionPhase, err := QuboleStatusToPhase(commandStatus) if err != nil { return nil, err } diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index 17e668609..a0e3f4aa1 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -1 +1,23 @@ package hive + +import ( + "github.com/lyft/flyteplugins/go/tasks/plugins/array/core" + "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" +) + +func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { + switch status { + case client.QuboleStatusUnknown: + fallthrough + case client.QuboleStatusWaiting: + fallthrough + case client.QuboleStatusCancelled: + fallthrough + case client.QuboleStatusDone: + fallthrough + case client.QuboleStatusError: + fallthrough + case client.QuboleStatusRunning: + fallthrough + } +} \ No newline at end of file From 8f3f3c86da7968155814a5708fcbf52ccdadcfc1 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:30:11 -0700 Subject: [PATCH 07/12] wip --- go/tasks/plugins/hive/plugin.go | 7 +------ go/tasks/plugins/hive/resource.go | 12 ++++++------ go/tasks/plugins/hive/transformer.go | 18 +++++++++--------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 5e1a19281..80f78d77f 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -80,13 +80,8 @@ func (q QuboleHivePlugin) Get(ctx context.Context, meta remote.ResourceMeta) ( return nil, err } - newExecutionPhase, err := QuboleStatusToPhase(commandStatus) - if err != nil { - return nil, err - } - return Resource{ - Phase: newExecutionPhase, + Phase: QuboleStatusToPhase(commandStatus), CommandID: r.CommandID, URI: r.URI, }, nil diff --git a/go/tasks/plugins/hive/resource.go b/go/tasks/plugins/hive/resource.go index 394e44f21..e1ca9c295 100644 --- a/go/tasks/plugins/hive/resource.go +++ b/go/tasks/plugins/hive/resource.go @@ -12,7 +12,7 @@ import ( type Resource struct { CommandID string - Phase ExecutionPhase + Phase core.Phase URI string } @@ -37,15 +37,15 @@ func (r Resource) GetPhaseInfo() core.PhaseInfo { t := time.Now() switch r.Phase { - case PhaseNotStarted: + case core.PhaseNotReady: phaseInfo = core.PhaseInfoNotReady(t, core.DefaultPhaseVersion, "Haven't received allocation token") - case PhaseQueued: + case core.PhaseQueued: phaseInfo = core.PhaseInfoQueued(t, core.DefaultPhaseVersion, "Waiting for Qubole launch") - case PhaseSubmitted: + case core.PhaseRunning: phaseInfo = core.PhaseInfoRunning(core.DefaultPhaseVersion, r.ConstructTaskInfo()) - case PhaseQuerySucceeded: + case core.PhaseSuccess: phaseInfo = core.PhaseInfoSuccess(r.ConstructTaskInfo()) - case PhaseQueryFailed: + case core.PhaseRetryableFailure: phaseInfo = core.PhaseInfoFailure(errors.DownstreamSystemError, "Query failed", r.ConstructTaskInfo()) } diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index a0e3f4aa1..39e1c42b6 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -1,23 +1,23 @@ package hive import ( - "github.com/lyft/flyteplugins/go/tasks/plugins/array/core" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" ) func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { switch status { - case client.QuboleStatusUnknown: - fallthrough case client.QuboleStatusWaiting: - fallthrough - case client.QuboleStatusCancelled: - fallthrough + return core.PhaseNotReady case client.QuboleStatusDone: + return core.PhaseSuccess + case client.QuboleStatusRunning: + return core.PhaseRunning + case client.QuboleStatusCancelled: fallthrough case client.QuboleStatusError: fallthrough - case client.QuboleStatusRunning: - fallthrough + default: + return core.PhaseRetryableFailure } -} \ No newline at end of file +} From d30081ed18696b4952f8ca66d1ed53a8eadf9817 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:31:46 -0700 Subject: [PATCH 08/12] wip --- go/tasks/plugins/hive/plugin.go | 5 +++++ go/tasks/plugins/hive/transformer.go | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index 80f78d77f..b7e4f6b31 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -18,6 +18,11 @@ import ( "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" ) +const ( + quboleHiveExecutorID = "" + hiveTaskType = "" +) + type QuboleHivePlugin struct { client client.QuboleClient apiKey string diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index 39e1c42b6..3acdd69bd 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -3,6 +3,7 @@ package hive import ( "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" + "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" ) func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { @@ -21,3 +22,13 @@ func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { return core.PhaseRetryableFailure } } + +func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace]int { + resourceConfig := make(map[core.ResourceNamespace]int, len(cfg)) + + for _, clusterCfg := range cfg { + resourceConfig[core.ResourceNamespace(clusterCfg.PrimaryLabel)] = clusterCfg.Limit + } + + return resourceConfig +} From 1ead74df183b88f7647e7cf217c1a65359f8667b Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:34:07 -0700 Subject: [PATCH 09/12] wip --- go/tasks/plugins/hive/execution_state.go | 153 ------------------ go/tasks/plugins/hive/plugin.go | 4 +- go/tasks/plugins/hive/transformer.go | 144 +++++++++++++++++ ...tion_state_test.go => transformer_test.go} | 0 4 files changed, 146 insertions(+), 155 deletions(-) delete mode 100644 go/tasks/plugins/hive/execution_state.go rename go/tasks/plugins/hive/{execution_state_test.go => transformer_test.go} (100%) diff --git a/go/tasks/plugins/hive/execution_state.go b/go/tasks/plugins/hive/execution_state.go deleted file mode 100644 index a06290875..000000000 --- a/go/tasks/plugins/hive/execution_state.go +++ /dev/null @@ -1,153 +0,0 @@ -package hive - -import ( - "context" - "fmt" - - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" - - "github.com/lyft/flyteidl/gen/pb-go/flyteidl/plugins" - - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" - - "github.com/lyft/flyteplugins/go/tasks/errors" - "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" - "github.com/lyft/flytestdlib/logger" -) - -func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext) (core.ResourceNamespace, error) { - _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) - if err != nil { - return "", err - } - clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) - return core.ResourceNamespace(clusterPrimaryLabel), nil -} - -func createResourceConstraintsSpec(ctx context.Context, _ remote.TaskExecutionContext, targetClusterPrimaryLabel core.ResourceNamespace) core.ResourceConstraintsSpec { - cfg := config.GetQuboleConfig() - constraintsSpec := core.ResourceConstraintsSpec{ - ProjectScopeResourceConstraint: nil, - NamespaceScopeResourceConstraint: nil, - } - if cfg.ClusterConfigs == nil { - logger.Infof(ctx, "No cluster config is found. Returning an empty resource constraints spec") - return constraintsSpec - } - for _, cluster := range cfg.ClusterConfigs { - if cluster.PrimaryLabel == string(targetClusterPrimaryLabel) { - constraintsSpec.ProjectScopeResourceConstraint = &core.ResourceConstraint{Value: int64(float64(cluster.Limit) * cluster.ProjectScopeQuotaProportionCap)} - constraintsSpec.NamespaceScopeResourceConstraint = &core.ResourceConstraint{Value: int64(float64(cluster.Limit) * cluster.NamespaceScopeQuotaProportionCap)} - break - } - } - logger.Infof(ctx, "Created a resource constraints spec: [%v]", constraintsSpec) - return constraintsSpec -} - -func validateQuboleHiveJob(hiveJob plugins.QuboleHiveJob) error { - if hiveJob.Query == nil { - return errors.Errorf(errors.BadTaskSpecification, - "Query could not be found. Please ensure that you are at least on Flytekit version 0.3.0 or later.") - } - return nil -} - -// This function is the link between the output written by the SDK, and the execution side. It extracts the query -// out of the task template. -func GetQueryInfo(ctx context.Context, tCtx remote.TaskExecutionContext) ( - query string, cluster string, tags []string, timeoutSec uint32, err error) { - - taskTemplate, err := tCtx.TaskReader().Read(ctx) - if err != nil { - return "", "", []string{}, 0, err - } - - hiveJob := plugins.QuboleHiveJob{} - err = utils.UnmarshalStruct(taskTemplate.GetCustom(), &hiveJob) - if err != nil { - return "", "", []string{}, 0, err - } - - if err := validateQuboleHiveJob(hiveJob); err != nil { - return "", "", []string{}, 0, err - } - - query = hiveJob.Query.GetQuery() - cluster = hiveJob.ClusterLabel - timeoutSec = hiveJob.Query.TimeoutSec - tags = hiveJob.Tags - tags = append(tags, fmt.Sprintf("ns:%s", tCtx.TaskExecutionMetadata().GetNamespace())) - for k, v := range tCtx.TaskExecutionMetadata().GetLabels() { - tags = append(tags, fmt.Sprintf("%s:%s", k, v)) - } - logger.Debugf(ctx, "QueryInfo: query: [%v], cluster: [%v], timeoutSec: [%v], tags: [%v]", query, cluster, timeoutSec, tags) - return -} - -func mapLabelToPrimaryLabel(ctx context.Context, quboleCfg *config.Config, label string) (primaryLabel string, found bool) { - primaryLabel = quboleCfg.DefaultClusterLabel - found = false - - if label == "" { - logger.Debugf(ctx, "Input cluster label is an empty string; falling back to using the default primary label [%v]", label, primaryLabel) - return - } - - // Using a linear search because N is small and because of ClusterConfig's struct definition - // which is determined specifically for the readability of the corresponding configmap yaml file - for _, clusterCfg := range quboleCfg.ClusterConfigs { - for _, l := range clusterCfg.Labels { - if label != "" && l == label { - logger.Debugf(ctx, "Found the primary label [%v] for label [%v]", clusterCfg.PrimaryLabel, label) - primaryLabel, found = clusterCfg.PrimaryLabel, true - break - } - } - } - - if !found { - logger.Debugf(ctx, "Cannot find the primary cluster label for label [%v] in configmap; "+ - "falling back to using the default primary label [%v]", label, primaryLabel) - } - - return primaryLabel, found -} - -func mapProjectDomainToDestinationClusterLabel(ctx context.Context, tCtx remote.TaskExecutionContext, quboleCfg *config.Config) (string, bool) { - tExecID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID() - project := tExecID.NodeExecutionId.GetExecutionId().GetProject() - domain := tExecID.NodeExecutionId.GetExecutionId().GetDomain() - logger.Debugf(ctx, "No clusterLabelOverride. Finding the pre-defined cluster label for (project: %v, domain: %v)", project, domain) - // Using a linear search because N is small - for _, m := range quboleCfg.DestinationClusterConfigs { - if project == m.Project && domain == m.Domain { - logger.Debugf(ctx, "Found the pre-defined cluster label [%v] for (project: %v, domain: %v)", m.ClusterLabel, project, domain) - return m.ClusterLabel, true - } - } - - // This function finds the label, not primary label, so in the case where no mapping is found, this function should return an empty string - return "", false -} - -func getClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext, clusterLabelOverride string) string { - cfg := config.GetQuboleConfig() - - // If override is not empty and if it has a mapping, we return the mapped primary label - if clusterLabelOverride != "" { - if primaryLabel, found := mapLabelToPrimaryLabel(ctx, cfg, clusterLabelOverride); found { - return primaryLabel - } - } - - // If override is empty or if the override does not have a mapping, we return the primary label mapped using (project, domain) - if clusterLabel, found := mapProjectDomainToDestinationClusterLabel(ctx, tCtx, cfg); found { - primaryLabel, _ := mapLabelToPrimaryLabel(ctx, cfg, clusterLabel) - return primaryLabel - } - - // Else we return the default primary label - return cfg.DefaultClusterLabel -} diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index b7e4f6b31..bf4e12280 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -19,8 +19,8 @@ import ( ) const ( - quboleHiveExecutorID = "" - hiveTaskType = "" + quboleHiveExecutorID = "qubole-hive-executor" + hiveTaskType = "hive" ) type QuboleHivePlugin struct { diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index 3acdd69bd..d4ff86426 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -1,9 +1,17 @@ package hive import ( + "context" + "fmt" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/plugins" + "github.com/lyft/flyteplugins/go/tasks/errors" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" + "github.com/lyft/flytestdlib/logger" ) func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { @@ -32,3 +40,139 @@ func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace] return resourceConfig } + +func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext) (core.ResourceNamespace, error) { + _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) + if err != nil { + return "", err + } + clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) + return core.ResourceNamespace(clusterPrimaryLabel), nil +} + +func createResourceConstraintsSpec(ctx context.Context, _ remote.TaskExecutionContext, targetClusterPrimaryLabel core.ResourceNamespace) core.ResourceConstraintsSpec { + cfg := config.GetQuboleConfig() + constraintsSpec := core.ResourceConstraintsSpec{ + ProjectScopeResourceConstraint: nil, + NamespaceScopeResourceConstraint: nil, + } + if cfg.ClusterConfigs == nil { + logger.Infof(ctx, "No cluster config is found. Returning an empty resource constraints spec") + return constraintsSpec + } + for _, cluster := range cfg.ClusterConfigs { + if cluster.PrimaryLabel == string(targetClusterPrimaryLabel) { + constraintsSpec.ProjectScopeResourceConstraint = &core.ResourceConstraint{Value: int64(float64(cluster.Limit) * cluster.ProjectScopeQuotaProportionCap)} + constraintsSpec.NamespaceScopeResourceConstraint = &core.ResourceConstraint{Value: int64(float64(cluster.Limit) * cluster.NamespaceScopeQuotaProportionCap)} + break + } + } + logger.Infof(ctx, "Created a resource constraints spec: [%v]", constraintsSpec) + return constraintsSpec +} + +func validateQuboleHiveJob(hiveJob plugins.QuboleHiveJob) error { + if hiveJob.Query == nil { + return errors.Errorf(errors.BadTaskSpecification, + "Query could not be found. Please ensure that you are at least on Flytekit version 0.3.0 or later.") + } + return nil +} + +// This function is the link between the output written by the SDK, and the execution side. It extracts the query +// out of the task template. +func GetQueryInfo(ctx context.Context, tCtx remote.TaskExecutionContext) ( + query string, cluster string, tags []string, timeoutSec uint32, err error) { + + taskTemplate, err := tCtx.TaskReader().Read(ctx) + if err != nil { + return "", "", []string{}, 0, err + } + + hiveJob := plugins.QuboleHiveJob{} + err = utils.UnmarshalStruct(taskTemplate.GetCustom(), &hiveJob) + if err != nil { + return "", "", []string{}, 0, err + } + + if err := validateQuboleHiveJob(hiveJob); err != nil { + return "", "", []string{}, 0, err + } + + query = hiveJob.Query.GetQuery() + cluster = hiveJob.ClusterLabel + timeoutSec = hiveJob.Query.TimeoutSec + tags = hiveJob.Tags + tags = append(tags, fmt.Sprintf("ns:%s", tCtx.TaskExecutionMetadata().GetNamespace())) + for k, v := range tCtx.TaskExecutionMetadata().GetLabels() { + tags = append(tags, fmt.Sprintf("%s:%s", k, v)) + } + logger.Debugf(ctx, "QueryInfo: query: [%v], cluster: [%v], timeoutSec: [%v], tags: [%v]", query, cluster, timeoutSec, tags) + return +} + +func mapLabelToPrimaryLabel(ctx context.Context, quboleCfg *config.Config, label string) (primaryLabel string, found bool) { + primaryLabel = quboleCfg.DefaultClusterLabel + found = false + + if label == "" { + logger.Debugf(ctx, "Input cluster label is an empty string; falling back to using the default primary label [%v]", label, primaryLabel) + return + } + + // Using a linear search because N is small and because of ClusterConfig's struct definition + // which is determined specifically for the readability of the corresponding configmap yaml file + for _, clusterCfg := range quboleCfg.ClusterConfigs { + for _, l := range clusterCfg.Labels { + if label != "" && l == label { + logger.Debugf(ctx, "Found the primary label [%v] for label [%v]", clusterCfg.PrimaryLabel, label) + primaryLabel, found = clusterCfg.PrimaryLabel, true + break + } + } + } + + if !found { + logger.Debugf(ctx, "Cannot find the primary cluster label for label [%v] in configmap; "+ + "falling back to using the default primary label [%v]", label, primaryLabel) + } + + return primaryLabel, found +} + +func mapProjectDomainToDestinationClusterLabel(ctx context.Context, tCtx remote.TaskExecutionContext, quboleCfg *config.Config) (string, bool) { + tExecID := tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID() + project := tExecID.NodeExecutionId.GetExecutionId().GetProject() + domain := tExecID.NodeExecutionId.GetExecutionId().GetDomain() + logger.Debugf(ctx, "No clusterLabelOverride. Finding the pre-defined cluster label for (project: %v, domain: %v)", project, domain) + // Using a linear search because N is small + for _, m := range quboleCfg.DestinationClusterConfigs { + if project == m.Project && domain == m.Domain { + logger.Debugf(ctx, "Found the pre-defined cluster label [%v] for (project: %v, domain: %v)", m.ClusterLabel, project, domain) + return m.ClusterLabel, true + } + } + + // This function finds the label, not primary label, so in the case where no mapping is found, this function should return an empty string + return "", false +} + +func getClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext, clusterLabelOverride string) string { + cfg := config.GetQuboleConfig() + + // If override is not empty and if it has a mapping, we return the mapped primary label + if clusterLabelOverride != "" { + if primaryLabel, found := mapLabelToPrimaryLabel(ctx, cfg, clusterLabelOverride); found { + return primaryLabel + } + } + + // If override is empty or if the override does not have a mapping, we return the primary label mapped using (project, domain) + if clusterLabel, found := mapProjectDomainToDestinationClusterLabel(ctx, tCtx, cfg); found { + primaryLabel, _ := mapLabelToPrimaryLabel(ctx, cfg, clusterLabel) + return primaryLabel + } + + // Else we return the default primary label + return cfg.DefaultClusterLabel +} diff --git a/go/tasks/plugins/hive/execution_state_test.go b/go/tasks/plugins/hive/transformer_test.go similarity index 100% rename from go/tasks/plugins/hive/execution_state_test.go rename to go/tasks/plugins/hive/transformer_test.go From 7c0828bbdfbb52f3d02b13db507bbd6b5ba656c2 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Fri, 15 May 2020 14:50:37 -0700 Subject: [PATCH 10/12] wip --- go/tasks/plugins/hive/executor_metrics.go | 36 ----------------------- go/tasks/plugins/hive/plugin.go | 6 ++-- go/tasks/plugins/hive/resource.go | 24 ++++++++------- go/tasks/plugins/hive/transformer.go | 18 ------------ 4 files changed, 17 insertions(+), 67 deletions(-) delete mode 100644 go/tasks/plugins/hive/executor_metrics.go diff --git a/go/tasks/plugins/hive/executor_metrics.go b/go/tasks/plugins/hive/executor_metrics.go deleted file mode 100644 index 519a13602..000000000 --- a/go/tasks/plugins/hive/executor_metrics.go +++ /dev/null @@ -1,36 +0,0 @@ -package hive - -import ( - "github.com/lyft/flytestdlib/promutils" - "github.com/lyft/flytestdlib/promutils/labeled" - "github.com/prometheus/client_golang/prometheus" -) - -type QuboleHiveExecutorMetrics struct { - Scope promutils.Scope - ResourceReleased labeled.Counter - ResourceReleaseFailed labeled.Counter - AllocationGranted labeled.Counter - AllocationNotGranted labeled.Counter - ResourceWaitTime prometheus.Summary -} - -var ( - tokenAgeObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001, 1.0: 0.0} -) - -func getQuboleHiveExecutorMetrics(scope promutils.Scope) QuboleHiveExecutorMetrics { - return QuboleHiveExecutorMetrics{ - Scope: scope, - ResourceReleased: labeled.NewCounter("resource_release_success", - "Resource allocation token released", scope, labeled.EmitUnlabeledMetric), - ResourceReleaseFailed: labeled.NewCounter("resource_release_failed", - "Error releasing allocation token", scope, labeled.EmitUnlabeledMetric), - AllocationGranted: labeled.NewCounter("allocation_grant_success", - "Allocation request granted", scope, labeled.EmitUnlabeledMetric), - AllocationNotGranted: labeled.NewCounter("allocation_grant_failed", - "Allocation request did not fail but not granted", scope, labeled.EmitUnlabeledMetric), - ResourceWaitTime: scope.MustNewSummaryWithOptions("resource_wait_time", "Duration the execution has been waiting for a resource allocation token", - promutils.SummaryOptions{Objectives: tokenAgeObjectives}), - } -} diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index bf4e12280..ef8341dc5 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -86,9 +86,9 @@ func (q QuboleHivePlugin) Get(ctx context.Context, meta remote.ResourceMeta) ( } return Resource{ - Phase: QuboleStatusToPhase(commandStatus), - CommandID: r.CommandID, - URI: r.URI, + CommandStatus: commandStatus, + CommandID: r.CommandID, + URI: r.URI, }, nil } diff --git a/go/tasks/plugins/hive/resource.go b/go/tasks/plugins/hive/resource.go index e1ca9c295..e079dbbdb 100644 --- a/go/tasks/plugins/hive/resource.go +++ b/go/tasks/plugins/hive/resource.go @@ -4,6 +4,8 @@ import ( "fmt" "time" + "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" + idlCore "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" "github.com/lyft/flyteplugins/go/tasks/errors" @@ -11,9 +13,9 @@ import ( ) type Resource struct { - CommandID string - Phase core.Phase - URI string + CommandID string + CommandStatus client.QuboleStatus + URI string } func (r Resource) ConstructTaskInfo() *core.TaskInfo { @@ -23,7 +25,7 @@ func (r Resource) ConstructTaskInfo() *core.TaskInfo { return &core.TaskInfo{ Logs: []*idlCore.TaskLog{ { - Name: fmt.Sprintf("Status: %s [%s]", r.Phase, r.CommandID), + Name: fmt.Sprintf("Status: %s [%s]", r.CommandStatus, r.CommandID), MessageFormat: idlCore.TaskLog_UNKNOWN, Uri: r.URI, }, @@ -36,16 +38,18 @@ func (r Resource) GetPhaseInfo() core.PhaseInfo { var phaseInfo core.PhaseInfo t := time.Now() - switch r.Phase { - case core.PhaseNotReady: + switch r.CommandStatus { + case client.QuboleStatusUnknown: phaseInfo = core.PhaseInfoNotReady(t, core.DefaultPhaseVersion, "Haven't received allocation token") - case core.PhaseQueued: + case client.QuboleStatusWaiting: phaseInfo = core.PhaseInfoQueued(t, core.DefaultPhaseVersion, "Waiting for Qubole launch") - case core.PhaseRunning: + case client.QuboleStatusRunning: phaseInfo = core.PhaseInfoRunning(core.DefaultPhaseVersion, r.ConstructTaskInfo()) - case core.PhaseSuccess: + case client.QuboleStatusDone: phaseInfo = core.PhaseInfoSuccess(r.ConstructTaskInfo()) - case core.PhaseRetryableFailure: + case client.QuboleStatusCancelled: + fallthrough + case client.QuboleStatusError: phaseInfo = core.PhaseInfoFailure(errors.DownstreamSystemError, "Query failed", r.ConstructTaskInfo()) } diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index d4ff86426..91cfe7d77 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -9,28 +9,10 @@ import ( "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote" "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" - "github.com/lyft/flyteplugins/go/tasks/plugins/hive/client" "github.com/lyft/flyteplugins/go/tasks/plugins/hive/config" "github.com/lyft/flytestdlib/logger" ) -func QuboleStatusToPhase(status client.QuboleStatus) core.Phase { - switch status { - case client.QuboleStatusWaiting: - return core.PhaseNotReady - case client.QuboleStatusDone: - return core.PhaseSuccess - case client.QuboleStatusRunning: - return core.PhaseRunning - case client.QuboleStatusCancelled: - fallthrough - case client.QuboleStatusError: - fallthrough - default: - return core.PhaseRetryableFailure - } -} - func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace]int { resourceConfig := make(map[core.ResourceNamespace]int, len(cfg)) From 41f9f35cb3607599a562454c368fa4080f54e14a Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Wed, 21 Oct 2020 14:00:45 -0700 Subject: [PATCH 11/12] Updates --- go.mod | 3 +- go.sum | 206 ++-------------------- go/tasks/pluginmachinery/remote/plugin.go | 5 + 3 files changed, 25 insertions(+), 189 deletions(-) diff --git a/go.mod b/go.mod index 205ae1ec9..89f221e08 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20200410212604-780c48ecb21a github.com/aws/aws-sdk-go v1.29.23 github.com/coocood/freecache v1.1.0 - github.com/ghodss/yaml v1.0.0 github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-test/deep v1.0.5 github.com/golang/protobuf v1.3.5 @@ -33,7 +32,7 @@ require ( k8s.io/api v0.17.3 k8s.io/apimachinery v0.17.3 k8s.io/client-go v11.0.1-0.20190918222721-c0e3722d5cf0+incompatible - k8s.io/utils v0.0.0-20200124190032-861946025e34 // indirect + k8s.io/utils v0.0.0-20200124190032-861946025e34 sigs.k8s.io/controller-runtime v0.5.1 ) diff --git a/go.sum b/go.sum index 925e6f6e7..1dadcac3e 100644 --- a/go.sum +++ b/go.sum @@ -1,66 +1,50 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0 h1:GGslhk/BU052LPlnI1vpp3fcbUs+hQ3E+Doti/3/vF8= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v38.2.0+incompatible h1:ZeCdp1E/V5lI8oLR/BjWQh0OW9aFBYlgXGKRVIWNPXY= github.com/Azure/azure-sdk-for-go v38.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v39.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v40.3.0+incompatible h1:NthZg3psrLxvQLN6rVm07pZ9mv2wvGNaBNGQ3fnPvLE= -github.com/Azure/azure-sdk-for-go v40.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.4 h1:1cM+NmKw91+8h5vfjgzK4ZGLuN72k87XVZBWyGwNjUM= github.com/Azure/go-autorest/autorest v0.9.4/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY= -github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295/go.mod h1:e0aH495YLkrsIe9fhedd6aSR6fgU/qhKvtroi6y7G/M= github.com/Jeffail/gabs/v2 v2.0.0 h1:HDDyYkQSgnYNVuQzVc2Vy3Ezl5wkZ+HoJPQcZrZU/xw= github.com/Jeffail/gabs/v2 v2.0.0/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -69,9 +53,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Selvatico/go-mocket v1.0.7/go.mod h1:4gO2v+uQmsL+jzQgLANy3tyEFzaEzHlymVbZ3GP2Oes= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/adammck/venv v0.0.0-20160819025605-8a9c907a37d3 h1:O+yARMhWahUt25JGj1yEdUdQX6JlKkSZ3eZARw2rp80= github.com/adammck/venv v0.0.0-20160819025605-8a9c907a37d3/go.mod h1:3zXR2a/VSQndtpShh783rUTaEA2mpqN2VqZclBARBc0= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= @@ -82,7 +63,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -90,7 +70,6 @@ github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20200410212604-780c48e github.com/aws/amazon-sagemaker-operator-for-k8s v1.0.1-0.20200410212604-780c48ecb21a/go.mod h1:kw+Gl0uvPAMADPoubX+kLx0P7e7zWOr6rc+R7D24pbc= github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.28.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.23 h1:wtiGLOzxAP755OfuVTDIy/NbUIYEDxbIbBEDfNhUpeU= github.com/aws/aws-sdk-go v1.29.23/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go-v2 v0.20.0 h1:/yefUjgMrda9PNFwWctBU63nL10CJMdBwkAmaQ4w4Hs= @@ -107,6 +86,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -116,13 +96,10 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -137,10 +114,10 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -149,9 +126,6 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= @@ -160,7 +134,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 h1:cTavhURetDkezJCvxFggiyLeP40Mrk/TtVg2+ycw1Es= github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -176,18 +149,19 @@ github.com/fsnotify/fsnotify v1.4.8-0.20191012010759-4bf2d1fec783/go.mod h1:znqG github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -236,14 +210,11 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.5 h1:AKODKU3pDH1RzZzm6YZu77YWtEAq6uh1rLIAQlay2qc= github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -259,18 +230,13 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -287,8 +253,6 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -300,27 +264,20 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/graymeta/stow v0.2.4 h1:qDGstknYXqcnmBQ5TRJtxD9Qv1MuRbYRhLoSMeUDs7U= github.com/graymeta/stow v0.2.4/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrAeV2Kw= -github.com/graymeta/stow v0.2.5 h1:YFSo4nsAU4Fbi4r/neLIgVYlrMzA1ReDUkdLYTQm/RM= -github.com/graymeta/stow v0.2.5/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrAeV2Kw= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= -github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= -github.com/grpc-ecosystem/grpc-gateway v1.14.3/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -336,14 +293,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/jinzhu/gorm v1.9.11/go.mod h1:bu/pK8szGZ2puuErfU0RwyeNdsf3e6nCX/noXaVxkfw= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= @@ -357,6 +309,7 @@ github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGn github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -370,29 +323,19 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubeflow/pytorch-operator v0.6.0 h1:y9Vzk7Jd5H/s610Y+ucURypCHgJugB25UL8GEz4DRL4= github.com/kubeflow/pytorch-operator v0.6.0/go.mod h1:zHblV+yTwVG4PCgKTU2wPfOmQ6TJdfT87lDfHrP1a1Y= github.com/kubeflow/tf-operator v0.5.3 h1:Ejn5vEAwHBKHU2sJTlUIRpezqIX3WeqXZ2dZx6zn6vY= github.com/kubeflow/tf-operator v0.5.3/go.mod h1:EBtz5LQoKaHUl/5fV5vD1qXVNVNyn3TrFaH6eVoQ8SY= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0 h1:NGL46+1RYcCXb3sShp0nQq4W38fcgnpCD4+X02eeLL0= github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0/go.mod h1:/L5qH+AD540e7Cetbui1tuJeXdmNhO8jM6VkXeDdDhQ= github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f h1:PGuAMDzAen0AulUfaEhNQMYmUpa41pAVo3zHI+GJsCM= github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= -github.com/lyft/datacatalog v0.2.1/go.mod h1:ktrPvzTDUwHO5Lv0hLH38zLHnOJ++rGoAO0iQ/sIPJ4= -github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteidl v0.18.7/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= github.com/lyft/flyteidl v0.18.9 h1:p9gLp92whTSSOeMGPtZ4tkgsVHNGuBuXXMQ447s0J9E= github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= -github.com/lyft/flyteplugins v0.5.1/go.mod h1:8zhqFG9BzbHNQGEXzGYltTJLD+KTmQZkanxXgeFI25c= github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= -github.com/lyft/flytestdlib v0.3.2 h1:bY6Y+Fg6Jdc7zY4GAYuR7t2hjWwynIdmRvtLcRNaGnw= -github.com/lyft/flytestdlib v0.3.2/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= -github.com/lyft/flytestdlib v0.3.3 h1:MkWXPkwQinh6MR3Yf5siZhmRSt9r4YmsF+5kvVVVedE= -github.com/lyft/flytestdlib v0.3.3/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= github.com/lyft/flytestdlib v0.3.9/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/spark-on-k8s-operator v0.1.3 h1:rmke8lR2Oy8mvKXRhloKuEu7fgGuXepDxiBNiorVUFI= @@ -410,19 +353,15 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -438,36 +377,29 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/ncw/swift v1.0.50 h1:E01b5bVIssNhx2KnzAjMWEXkKrb8ytTqCDWY7lqmWjA= -github.com/ncw/swift v1.0.50/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -481,16 +413,12 @@ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prY github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -498,8 +426,6 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= @@ -507,16 +433,11 @@ github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.10 h1:QJQN3jYQhkamO4mhfUWqdDH2asK7ONOI9MTWjyAxNKM= -github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -531,10 +452,12 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= @@ -579,7 +502,6 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -588,20 +510,17 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -609,7 +528,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -627,12 +545,7 @@ golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -641,28 +554,20 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -673,24 +578,18 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -700,13 +599,11 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -720,30 +617,21 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775 h1:TC0v2RSO1u2kn1ZugjrFXkRZAEaqMN/RW+OTZkBzmLE= golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -753,7 +641,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -777,47 +664,25 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200124170513-3f4d10fc73b4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.16.0 h1:hhRbpE9nkabqMxGCewz2sikMYxm8yYWov7h2Eo4j3Is= google.golang.org/api v0.16.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -826,7 +691,6 @@ google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpC google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -834,21 +698,10 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67 h1:MBO9fkVSrTpJ8vgHLPi5gb+ZWXEy7/auJN8yqyu9EiE= google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672 h1:jiDSspVssiikoRPFHT6pYrL+CL6/yIc3b9AuHO/4xik= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -859,13 +712,13 @@ google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRn google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4= google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -877,8 +730,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.54.0 h1:oM5ElzbIi7gwLnNbPX2M25ED1vSAK3B6dex50eS/6Fs= -gopkg.in/ini.v1 v1.54.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -887,7 +738,6 @@ gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -901,24 +751,18 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.17.2 h1:cP579D2hSZNuO/rZj9XFRzwJNYb41DbNANJb6Kolpss= k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= -k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= k8s.io/client-go v0.0.0-20191016111102-bec269661e48 h1:C2XVy2z0dV94q9hSSoCuTPp1KOG7IegvbdXuz9VGxoU= k8s.io/client-go v0.0.0-20191016111102-bec269661e48/go.mod h1:hrwktSwYGI4JK+TJA3dMaFyyvHVi/aLarVHpbs8bgCU= -k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -931,34 +775,22 @@ k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe h1:GOfbcWvX5wW2vcfNch83xYp9SDZjRgAJk+t373yaHKk= -k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200124190032-861946025e34 h1:HjlUD6M0K3P8nRXmr2B9o4F9dUy9TCj/aEpReeyi6+k= k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab h1:I3f2hcBrepGRXI1z4sukzAb8w1R4eqbsHrAsx06LGYM= -k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.2.0/go.mod h1:ZHqrRDZi3f6BzONcvlUxkqCKgwasGk5FZrnSv9TVZF4= -sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/controller-runtime v0.5.1 h1:TNidCfVoU/cs2i+9xoTcL/l7yhl0bDhYXU0NCG6wmiE= sigs.k8s.io/controller-runtime v0.5.1/go.mod h1:Uojny7gvg55YLQnEGnPzRE3dC4ik2tRlZJgOUCWXAV4= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/go/tasks/pluginmachinery/remote/plugin.go b/go/tasks/pluginmachinery/remote/plugin.go index 573609efc..4a59d4f35 100644 --- a/go/tasks/pluginmachinery/remote/plugin.go +++ b/go/tasks/pluginmachinery/remote/plugin.go @@ -30,6 +30,11 @@ type PluginEntry struct { // Boolean that indicates if this plugin can be used as the default for unknown task types. There can only be // one default in the system IsDefault bool + + // A list of all task types for which this plugin should be default handler when multiple registered plugins + // support the same task type. This must be a subset of RegisteredTaskTypes and at most one default per task type + // is supported. + DefaultForTaskTypes []pluginsCore.TaskType } type PluginSetupContext interface { From 0412dd305304dad41899bb3c0f757f82f2f645e4 Mon Sep 17 00:00:00 2001 From: Haytham AbuelFutuh Date: Thu, 22 Oct 2020 13:46:00 -0700 Subject: [PATCH 12/12] cleanup --- go/tasks/plugins/hive/plugin.go | 11 ++--------- go/tasks/plugins/hive/transformer.go | 12 +++++++----- go/tasks/plugins/hive/transformer_test.go | 3 ++- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/go/tasks/plugins/hive/plugin.go b/go/tasks/plugins/hive/plugin.go index c16ca0c95..e42f02720 100644 --- a/go/tasks/plugins/hive/plugin.go +++ b/go/tasks/plugins/hive/plugin.go @@ -2,8 +2,6 @@ package hive import ( "context" - "fmt" - "reflect" "strconv" "github.com/lyft/flytestdlib/logger" @@ -49,7 +47,7 @@ func (q QuboleHivePlugin) ResourceRequirements(ctx context.Context, tCtx remote. func (q QuboleHivePlugin) Create(ctx context.Context, tCtx remote.TaskExecutionContext) ( createdResources remote.ResourceMeta, err error) { - query, clusterLabelOverride, tags, timeoutSec, taskName, err := GetQueryInfo(ctx, tCtx) + taskName, query, clusterLabelOverride, tags, timeoutSec, err := GetQueryInfo(ctx, tCtx) if err != nil { return nil, err } @@ -118,12 +116,7 @@ func (q QuboleHivePlugin) Delete(ctx context.Context, meta remote.ResourceMeta) func (q QuboleHivePlugin) Status(_ context.Context, resource remote.ResourceMeta) ( phase core.PhaseInfo, err error) { - r, casted := resource.(Resource) - if !casted { - return core.PhaseInfo{}, fmt.Errorf("failed to cast resource to the expected type. Input type: %v", - reflect.TypeOf(resource)) - } - + r := resource.(Resource) return r.GetPhaseInfo(), nil } diff --git a/go/tasks/plugins/hive/transformer.go b/go/tasks/plugins/hive/transformer.go index 0d7e965c1..ea4eaf583 100644 --- a/go/tasks/plugins/hive/transformer.go +++ b/go/tasks/plugins/hive/transformer.go @@ -24,10 +24,11 @@ func BuildResourceConfig(cfg []config.ClusterConfig) map[core.ResourceNamespace] } func composeResourceNamespaceWithClusterPrimaryLabel(ctx context.Context, tCtx remote.TaskExecutionContext) (core.ResourceNamespace, error) { - _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) + _, _, clusterLabelOverride, _, _, err := GetQueryInfo(ctx, tCtx) if err != nil { return "", err } + clusterPrimaryLabel := getClusterPrimaryLabel(ctx, tCtx, clusterLabelOverride) return core.ResourceNamespace(clusterPrimaryLabel), nil } @@ -64,21 +65,21 @@ func validateQuboleHiveJob(hiveJob plugins.QuboleHiveJob) error { // This function is the link between the output written by the SDK, and the execution side. It extracts the query // out of the task template. func GetQueryInfo(ctx context.Context, tCtx remote.TaskExecutionContext) ( - query string, cluster string, tags []string, timeoutSec uint32, taskName string, err error) { + taskName, query, cluster string, tags []string, timeoutSec uint32, err error) { taskTemplate, err := tCtx.TaskReader().Read(ctx) if err != nil { - return "", "", []string{}, 0, "", err + return "", "", "", []string{}, 0, err } hiveJob := plugins.QuboleHiveJob{} err = utils.UnmarshalStruct(taskTemplate.GetCustom(), &hiveJob) if err != nil { - return "", "", []string{}, 0, "", err + return "", "", "", []string{}, 0, err } if err := validateQuboleHiveJob(hiveJob); err != nil { - return "", "", []string{}, 0, "", err + return "", "", "", []string{}, 0, err } query = hiveJob.Query.GetQuery() @@ -90,6 +91,7 @@ func GetQueryInfo(ctx context.Context, tCtx remote.TaskExecutionContext) ( for k, v := range tCtx.TaskExecutionMetadata().GetLabels() { tags = append(tags, fmt.Sprintf("%s:%s", k, v)) } + logger.Debugf(ctx, "QueryInfo: query: [%v], cluster: [%v], timeoutSec: [%v], tags: [%v]", query, cluster, timeoutSec, tags) return } diff --git a/go/tasks/plugins/hive/transformer_test.go b/go/tasks/plugins/hive/transformer_test.go index fcbf81f13..d6f9498aa 100644 --- a/go/tasks/plugins/hive/transformer_test.go +++ b/go/tasks/plugins/hive/transformer_test.go @@ -42,8 +42,9 @@ func TestGetQueryInfo(t *testing.T) { taskMetadata.On("GetLabels").Return(map[string]string{"sample": "label"}) mockTaskExecutionContext.On("TaskExecutionMetadata").Return(taskMetadata) - query, cluster, tags, timeout, err := GetQueryInfo(ctx, &mockTaskExecutionContext) + taskName, query, cluster, tags, timeout, err := GetQueryInfo(ctx, &mockTaskExecutionContext) assert.NoError(t, err) + assert.Equal(t, "sample_hive_task_test_name", taskName) assert.Equal(t, "select 'one'", query) assert.Equal(t, "default", cluster) assert.Equal(t, []string{"flyte_plugin_test", "ns:myproject-staging", "sample:label"}, tags)