diff --git a/odpf/optimus/core/v1beta1/backup.proto b/odpf/optimus/core/v1beta1/backup.proto new file mode 100644 index 00000000..981412b6 --- /dev/null +++ b/odpf/optimus/core/v1beta1/backup.proto @@ -0,0 +1,118 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "BackupServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Backup Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service BackupService { + rpc BackupDryRun(BackupDryRunRequest) returns (BackupDryRunResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backupDryrun" + body: "*" + }; + } + rpc CreateBackup(CreateBackupRequest) returns (CreateBackupResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup" + body: "*" + }; + } + rpc ListBackups(ListBackupsRequest) returns (ListBackupsResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup" + }; + } + rpc GetBackup(GetBackupRequest) returns (GetBackupResponse) { + option (google.api.http) = { + get: "/v1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup/{id}" + }; + } +} + +message BackupDryRunRequest { + string project_name = 1; + string datastore_name = 2; + string resource_name = 3; + string namespace_name = 4; + string description = 5; + bool ignore_downstream = 6 [deprecated=true]; + + // represents which downstream to be backed up. + // possible values are the namespace names, *, or empty. + // '*' means all namespaces are allowed, empty list means all downstream will be ignored. + repeated string allowed_downstream_namespaces = 7; +} + +message BackupDryRunResponse { + repeated string resource_name = 1; + repeated string ignored_resources = 2; +} + +message CreateBackupRequest { + string project_name = 1; + string datastore_name = 2; + string resource_name = 3; + string namespace_name = 4; + string description = 5; + bool ignore_downstream = 6 [deprecated=true]; + map config = 7; + + // represents which downstream to be backed up. + // possible values are the namespace names, *, or empty. + // '*' means all namespaces are allowed, empty list means all downstream will be ignored. + repeated string allowed_downstream_namespaces = 8; +} + +message CreateBackupResponse { + repeated string urn = 1; + repeated string ignored_resources = 2; +} + +message ListBackupsRequest { + string project_name = 1; + string datastore_name = 2; + string namespace_name = 3; +} + +message ListBackupsResponse { + repeated BackupSpec backups = 1; +} + +message BackupSpec { + string id = 1; + string resource_name = 2; + google.protobuf.Timestamp created_at = 3; + string description = 4; + map config = 5; +} + +message GetBackupRequest { + string project_name = 1; + string datastore_name = 2; + string namespace_name = 3; + string id = 4; +} + +message GetBackupResponse { + BackupSpec spec = 1; + repeated string urn = 2; +} \ No newline at end of file diff --git a/odpf/optimus/core/v1beta1/job_run.proto b/odpf/optimus/core/v1beta1/job_run.proto new file mode 100644 index 00000000..fb85cfb9 --- /dev/null +++ b/odpf/optimus/core/v1beta1/job_run.proto @@ -0,0 +1,234 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "odpf/optimus/core/v1beta1/project.proto"; +import "odpf/optimus/core/v1beta1/namespace.proto"; +import "odpf/optimus/core/v1beta1/job_spec.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "JobRunManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Job Run Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service JobRunService { + // GetJobTask provides task details specific to plugin used in a job + rpc GetJobTask(GetJobTaskRequest) returns (GetJobTaskResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/task" + }; + } + // RegisterInstance is an internal admin command used during task/hook execution + // to pull task/hook compiled configuration and assets. + rpc RegisterInstance(RegisterInstanceRequest) returns (RegisterInstanceResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/job/{job_name}/instance" + body: "*" + }; + } + // JobStatus returns the current and past run status of jobs + rpc JobStatus(JobStatusRequest) returns (JobStatusResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/job/{job_name}/status" + }; + } + // JobRun returns the current and past run status of jobs on a given range + rpc JobRun(JobRunRequest) returns (JobRunResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/job/{job_name}/run" + }; + } + // GetWindow provides the start and end dates provided a scheduled date + // of the execution window + rpc GetWindow(GetWindowRequest) returns (GetWindowResponse) { + option (google.api.http) = { + get: "/v1beta1/window" + }; + } + // RunJob creates a job run and executes all included tasks/hooks instantly + // this doesn't necessarily deploy the job in db first + rpc RunJob(RunJobRequest) returns (RunJobResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/run" + body: "*" + }; + } + +} + +message GetJobTaskRequest { + string project_name = 1; + string namespace_name = 2; + string job_name = 3; +} + +message GetJobTaskResponse { + JobTask task = 1; +} + +message RegisterInstanceRequest { + reserved 3; + + string project_name = 1; + string job_name = 2; + google.protobuf.Timestamp scheduled_at = 4; + + string instance_name = 5; + InstanceSpec.Type instance_type = 6; + + // either set job_name if this is a scheduled execution + // or set jonrun_id if this is a manual triggered execution + // and not really registered as a valid job + string jobrun_id = 7; +} + +message RegisterInstanceResponse { + ProjectSpecification project = 1; + NamespaceSpecification namespace = 4; + JobSpecification job = 2; + + InstanceSpec instance = 3; + InstanceContext context = 5; +} + +message JobStatusRequest { + reserved 3; + + string project_name = 1; + string job_name = 2; +} + +message JobStatusResponse { + repeated JobStatus statuses = 1; +} + +message JobRunRequest { + string project_name = 1; + string job_name = 2; + google.protobuf.Timestamp start_date = 3; + google.protobuf.Timestamp end_date = 4; + repeated string filter = 5; +} + +message JobRunResponse { + repeated JobRun job_runs = 1; +} + +message JobRun { + string state = 1; + google.protobuf.Timestamp scheduled_at = 2; +} + +message GetWindowRequest { + google.protobuf.Timestamp scheduled_at = 1; + string size = 2; + string offset = 3; + string truncate_to = 4; +} + +message GetWindowResponse { + google.protobuf.Timestamp start = 1; + google.protobuf.Timestamp end = 2; +} + +message RunJobRequest { + string project_name = 1; + string namespace_name = 2; + + // job specification order of execution is undefined + // attributes realted to schedule behaviour are ignored like interval, + // start_date, end_date, catchup, etc + repeated JobSpecification specifications = 3; +} + +message RunJobResponse {} + + +// JobTask is part of a job that dictates main transformation +// each job has exactly one task +message JobTask { + string name = 1; + string description = 2; + string image = 3; + + message Destination { + string destination = 1; + string type = 2; + } + Destination destination = 4; + + message Dependency { + string dependency = 1; + } + repeated Dependency dependencies = 5; +} + +message InstanceSpec { + reserved 2, 4; + + string state = 1; + + // deprecated + // google.protobuf.Timestamp scheduled_at = 2; + // deprecated + // string job_name = 4; + + repeated InstanceSpecData data = 3; + google.protobuf.Timestamp executed_at = 5; + + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_TASK = 1; + TYPE_HOOK = 2; + } + string name = 6; + Type type = 7; +} + +message InstanceSpecData { + reserved 3, 4; + + string name = 1; + string value = 2; + + // type of data, could be an env var or file + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_ENV = 1; + TYPE_FILE = 2; + } + Type type = 5; +} + +message InstanceContext { + map envs = 1; + map files = 2; + map secrets = 3; +} + +message JobStatus { + string state = 1; + google.protobuf.Timestamp scheduled_at = 2; +} + +message TaskWindow { + google.protobuf.Duration size = 1; + google.protobuf.Duration offset = 2; + string truncate_to = 3; +} diff --git a/odpf/optimus/core/v1beta1/job_spec.proto b/odpf/optimus/core/v1beta1/job_spec.proto new file mode 100644 index 00000000..5467aa41 --- /dev/null +++ b/odpf/optimus/core/v1beta1/job_spec.proto @@ -0,0 +1,257 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/duration.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "JobSpecificationServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Job Specification Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service JobSpecificationService { + // DeployJobSpecification schedules jobs for execution + // returns a stream of messages which can be used to track the progress + // of deployments. Message containing ack are status events other are progress + // events + // State of the world request + rpc DeployJobSpecification(stream DeployJobSpecificationRequest) returns (stream DeployJobSpecificationResponse) {} + + // CreateJobSpecification registers a new job for a namespace which belongs to a project + rpc CreateJobSpecification(CreateJobSpecificationRequest) returns (CreateJobSpecificationResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job" + body: "*" + }; + } + + // GetJobSpecification reads a provided job spec of a namespace + rpc GetJobSpecification(GetJobSpecificationRequest) returns (GetJobSpecificationResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}" + }; + } + + // DeleteJobSpecification deletes a job spec of a namespace + rpc DeleteJobSpecification(DeleteJobSpecificationRequest) returns (DeleteJobSpecificationResponse) { + option (google.api.http) = { + delete: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}" + }; + } + + // ListJobSpecification returns list of jobs created in a project + rpc ListJobSpecification(ListJobSpecificationRequest) returns (ListJobSpecificationResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job" + }; + } + + // CheckJobSpecification checks if a job specification is valid + rpc CheckJobSpecification(CheckJobSpecificationRequest) returns (CheckJobSpecificationResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/job/check" + }; + } + // CheckJobSpecifications checks if the job specifications are valid + rpc CheckJobSpecifications(CheckJobSpecificationsRequest) returns (stream CheckJobSpecificationsResponse) {} +} + +message DeployJobSpecificationRequest { + string project_name = 1; // unique project identifier + repeated JobSpecification jobs = 2; + string namespace_name = 4; +} + +message DeployJobSpecificationResponse { + bool success = 1; + + // non ack responses are more of a progress/info response + // and not really success or failure statuses + bool ack = 2; + + string message = 3; + string job_name = 4; +} + +message CreateJobSpecificationRequest { + string project_name = 1; + string namespace_name = 2; + JobSpecification spec = 3; +} + +message CreateJobSpecificationResponse { + bool success = 1; + string message = 2; +} + +message GetJobSpecificationRequest { + string project_name = 1; + string namespace_name = 2; + string job_name = 3; +} + +message GetJobSpecificationResponse { + JobSpecification spec = 1; +} + +message DeleteJobSpecificationRequest { + string project_name = 1; + string namespace_name = 2; + string job_name = 3; +} + +message DeleteJobSpecificationResponse { + bool success = 1; + string message = 2; +} + +message ListJobSpecificationRequest { + string project_name = 1; + string namespace_name = 2; +} + +message ListJobSpecificationResponse{ + repeated JobSpecification jobs = 1; +} + +message CheckJobSpecificationRequest { + string project_name = 1; + JobSpecification job = 2; + string namespace_name = 3; +} + +message CheckJobSpecificationResponse { + bool success = 1; +} + +message CheckJobSpecificationsRequest { + string project_name = 1; + repeated JobSpecification jobs = 2; + string namespace_name = 3; +} + +message CheckJobSpecificationsResponse { + bool success = 1; + + // non ack responses are more of a progress/info response + // and not really success or failure statuses + bool ack = 2; + + string message = 3; + string job_name = 4; +} + +message JobSpecification { + int32 version = 1; + string name = 2; + string owner = 3; + + string start_date = 4; + string end_date = 5; // optional + string interval = 6; + + bool depends_on_past = 7; // should only execute today if yesterday was completed with success? + bool catch_up = 8; // should backfill till today? + + string task_name = 9; + repeated JobConfigItem config = 10; + + string window_size = 11; + string window_offset = 12; + string window_truncate_to = 13; + + repeated JobDependency dependencies = 14; // static dependencies + map assets = 15; + + repeated JobSpecHook hooks = 16; // optional + + string description = 17; // optional + map labels = 18; + + message Behavior { + + // retry behaviour if job failed to execute for the first time + message Retry { + int32 count = 1; + google.protobuf.Duration delay = 2; + bool exponential_backoff = 3; + } + Retry retry = 1; + + // Notifiers are used to set custom alerting in case of job failure/sla_miss + message Notifiers { + JobEvent.Type on = 1; + repeated string channels = 2; + map config = 3; + } + repeated Notifiers notify = 2; + } + Behavior behavior = 19; + + JobMetadata metadata = 20; +} + +message JobDependency { + string name = 1; + string type = 2; // intra/inter/extra + HttpDependency http_dependency = 3; // http sensor dependency +} + +message HttpDependency { + string name = 1; + string url = 2; + map headers = 3; + map params = 4; +} + +message JobSpecHook { + string name = 1; + repeated JobConfigItem config = 2; +} + +message JobConfigItem { + string name = 1; + string value = 2; +} + +message JobEvent { + enum Type { + TYPE_UNSPECIFIED = 0; + TYPE_SLA_MISS = 1; + TYPE_FAILURE = 2; + TYPE_SUCCESS = 3; + } + Type type = 1; + google.protobuf.Struct value = 2; +} + +message JobMetadata { + JobSpecMetadataResource resource = 1; +} + +message JobSpecMetadataResource { + JobSpecMetadataResourceConfig request = 1; + JobSpecMetadataResourceConfig limit = 2; +} + +message JobSpecMetadataResourceConfig { + string cpu = 1; + string memory = 2; +} + + diff --git a/odpf/optimus/core/v1beta1/namespace.proto b/odpf/optimus/core/v1beta1/namespace.proto new file mode 100644 index 00000000..3c8614b7 --- /dev/null +++ b/odpf/optimus/core/v1beta1/namespace.proto @@ -0,0 +1,63 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "NamespaceServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Namespace Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service NamespaceService { + // RegisterProjectNamespace creates a new namespace for a project + rpc RegisterProjectNamespace(RegisterProjectNamespaceRequest) returns (RegisterProjectNamespaceResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace" + body: "*" + }; + } + + // ListProjectNamespaces returns list of namespaces of a project + rpc ListProjectNamespaces(ListProjectNamespacesRequest) returns (ListProjectNamespacesResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace" + }; + } +} + +message RegisterProjectNamespaceRequest { + string project_name = 1; + NamespaceSpecification namespace = 2; +} + +message RegisterProjectNamespaceResponse { + bool success = 1; + string message = 2; +} + +message ListProjectNamespacesRequest { + string project_name = 1; +} + +message ListProjectNamespacesResponse { + repeated NamespaceSpecification namespaces = 1; +} + +message NamespaceSpecification { + string name = 1; + map config = 2; +} diff --git a/odpf/optimus/core/v1beta1/project.proto b/odpf/optimus/core/v1beta1/project.proto new file mode 100644 index 00000000..58dc8748 --- /dev/null +++ b/odpf/optimus/core/v1beta1/project.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "odpf/optimus/core/v1beta1/namespace.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "ProjectServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Project Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service ProjectService { + // RegisterProject creates a new optimus project + rpc RegisterProject(RegisterProjectRequest) returns (RegisterProjectResponse) { + option (google.api.http) = { + post: "/v1beta1/project" + body: "*" + }; + } + // ListProjects returns list of registered projects and configurations + rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse) { + option (google.api.http) = { + get: "/v1beta1/project" + }; + } +} + +message RegisterProjectRequest { + ProjectSpecification project = 1; + NamespaceSpecification namespace = 2 [deprecated=true]; +} + +message RegisterProjectResponse { + bool success = 1; + string message = 2; +} + +message ListProjectsRequest {} + +message ListProjectsResponse { + repeated ProjectSpecification projects = 1; +} + +message ProjectSpecification { + string name = 1; + map config = 2; + + message ProjectSecret { + string name = 1; + string value = 2; + } + repeated ProjectSecret secrets = 3; +} + diff --git a/odpf/optimus/core/v1beta1/replay.proto b/odpf/optimus/core/v1beta1/replay.proto new file mode 100644 index 00000000..ffd5d01c --- /dev/null +++ b/odpf/optimus/core/v1beta1/replay.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "ReplayServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Replay Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service ReplayService { + rpc ReplayDryRun(ReplayDryRunRequest) returns (ReplayDryRunResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/replay/dryrun" + body: "*" + }; + } + rpc Replay(ReplayRequest) returns (ReplayResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/replay" + body: "*" + }; + } + rpc GetReplayStatus(GetReplayStatusRequest) returns (GetReplayStatusResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/replay/{id}" + }; + } + rpc ListReplays(ListReplaysRequest) returns (ListReplaysResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/replay" + }; + } +} + +message ReplayRequest { + string project_name = 1; + string job_name = 2; + string namespace_name = 3; + string start_date = 4; + string end_date = 5; + bool force = 6; + + // represents which downstream to be replayed. + // possible values are the namespace names, *, or empty. + // '*' means all namespaces are allowed, empty list means all downstream will be ignored. + repeated string allowed_downstream_namespaces = 7; +} + +message ReplayResponse { + string id = 1; + repeated string ignored_jobs = 2; +} + +message ReplayDryRunRequest { + string project_name = 1; + string job_name = 2; + string namespace_name = 3; + string start_date = 4; + string end_date = 5; + + // represents which downstream to be replayed. + // possible values are the namespace names, *, or empty. + // '*' means all namespaces are allowed, empty list means all downstream will be ignored. + repeated string allowed_downstream_namespaces = 6; +} + +message ReplayDryRunResponse { + bool success = 1; + ReplayExecutionTreeNode response = 2 [deprecated=true]; + ReplayExecutionTreeNode execution_tree = 3; + repeated string ignored_jobs = 4; +} + +message ReplayExecutionTreeNode { + string job_name = 1; + repeated ReplayExecutionTreeNode dependents = 2; + repeated google.protobuf.Timestamp runs = 3; +} + +message GetReplayStatusResponse { + string state = 1; + ReplayStatusTreeNode response = 2; +} + +message ReplayStatusTreeNode { + string job_name = 1; + repeated ReplayStatusTreeNode dependents = 2; + repeated ReplayStatusRun runs = 3; + string state = 4; +} + +message ReplayStatusRun { + google.protobuf.Timestamp run = 1; + string state = 2; +} + +message GetReplayStatusRequest { + string id = 1; + string job_name = 2; + string project_name = 3; +} + +message ListReplaysRequest { + string project_name = 1; +} + +message ListReplaysResponse { + repeated ReplaySpec replay_list = 1; +} + +message ReplaySpec { + string id = 1; + string job_name = 2; + google.protobuf.Timestamp start_date = 3; + google.protobuf.Timestamp end_date = 4; + string state = 5; + google.protobuf.Timestamp created_at = 6; + map config = 7; +} \ No newline at end of file diff --git a/odpf/optimus/core/v1beta1/resource.proto b/odpf/optimus/core/v1beta1/resource.proto new file mode 100644 index 00000000..17905a73 --- /dev/null +++ b/odpf/optimus/core/v1beta1/resource.proto @@ -0,0 +1,137 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/struct.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "ResourceServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Resource Management Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service ResourceService { + // DeployResourceSpecification migrate all resource specifications of a datastore in project + // State of the world request + rpc DeployResourceSpecification(stream DeployResourceSpecificationRequest) returns (stream DeployResourceSpecificationResponse) {} + // ListResourceSpecification lists all resource specifications of a datastore in project + rpc ListResourceSpecification(ListResourceSpecificationRequest) returns (ListResourceSpecificationResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" + }; + } + + // Database CRUD + // CreateResource registers a new resource of a namespace which belongs to a project + rpc CreateResource(CreateResourceRequest) returns (CreateResourceResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" + body: "*" + }; + } + // ReadResource reads a provided resource spec of a namespace + rpc ReadResource(ReadResourceRequest) returns (ReadResourceResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource/{resource_name}" + }; + } + // UpdateResource updates a resource specification of a datastore in project + rpc UpdateResource(UpdateResourceRequest) returns (UpdateResourceResponse) { + option (google.api.http) = { + put: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" + body: "*" + }; + } +} + +message DeployResourceSpecificationRequest { + string project_name = 1; + string datastore_name = 2; + repeated ResourceSpecification resources = 3; + string namespace_name = 4; +} + +message DeployResourceSpecificationResponse { + bool success = 1; + + // non ack responses are more of a progress/info response + // and not success or failure statuses + bool ack = 2; + + string message = 3; + string resource_name = 4; +} + +// ListResourceSpecificationRequest lists all resource specifications of a datastore in project +message ListResourceSpecificationRequest { + string project_name = 1; + string datastore_name = 2; + string namespace_name = 3; +} + +message ListResourceSpecificationResponse { + repeated ResourceSpecification resources = 1; +} + +message CreateResourceRequest { + string project_name = 1; + string datastore_name = 2; + ResourceSpecification resource = 3; + string namespace_name = 4; +} + +message CreateResourceResponse { + bool success = 1; + string message = 2; +} + +message ReadResourceRequest { + string project_name = 1; + string datastore_name = 2; + string resource_name = 3; + string namespace_name = 4; +} + +message ReadResourceResponse { + bool success = 1; + string message = 2; + ResourceSpecification resource = 3; +} + +message UpdateResourceRequest { + string project_name = 1; + string datastore_name = 2; + ResourceSpecification resource = 3; + string namespace_name = 4; +} + +message UpdateResourceResponse { + bool success = 1; + string message = 2; +} + +// ResourceSpecification are datastore specification representation of a resource +message ResourceSpecification { + reserved 3; // depricated "datastore" + + int32 version = 1; + string name = 2; + string type = 4; + + google.protobuf.Struct spec = 5; + map assets = 6; + map labels = 7; +} \ No newline at end of file diff --git a/odpf/optimus/core/v1beta1/runtime.proto b/odpf/optimus/core/v1beta1/runtime.proto index 26faeb8c..4a0b8fc3 100644 --- a/odpf/optimus/core/v1beta1/runtime.proto +++ b/odpf/optimus/core/v1beta1/runtime.proto @@ -3,9 +3,7 @@ package odpf.optimus.core.v1beta1; import "google/api/annotations.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; -import "google/protobuf/timestamp.proto"; -import "google/protobuf/struct.proto"; -import "google/protobuf/duration.proto"; +import "odpf/optimus/core/v1beta1/job_spec.proto"; option go_package = "github.com/odpf/proton/optimus"; option java_multiple_files = true; @@ -18,9 +16,9 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { version: "0.1"; }; external_docs: { - description: "Optimus server"; + description: "Optimus Runtime Service"; } - schemes: HTTP; + schemes: HTTP; host: "127.0.0.1:9100"; base_path: "/api"; }; @@ -34,139 +32,6 @@ service RuntimeService { body: "*" }; } - - // DeployJobSpecification schedules jobs for execution - // returns a stream of messages which can be used to track the progress - // of deployments. Message containing ack are status events other are progress - // events - // State of the world request - rpc DeployJobSpecification(stream DeployJobSpecificationRequest) returns (stream DeployJobSpecificationResponse) {} - - // CreateJobSpecification registers a new job for a namespace which belongs to a project - rpc CreateJobSpecification(CreateJobSpecificationRequest) returns (CreateJobSpecificationResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job" - body: "*" - }; - } - - // GetJobSpecification reads a provided job spec of a namespace - rpc GetJobSpecification(GetJobSpecificationRequest) returns (GetJobSpecificationResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}" - }; - } - - // DeleteJobSpecification deletes a job spec of a namespace - rpc DeleteJobSpecification(DeleteJobSpecificationRequest) returns (DeleteJobSpecificationResponse) { - option (google.api.http) = { - delete: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}" - }; - } - - // ListJobSpecification returns list of jobs created in a project - rpc ListJobSpecification(ListJobSpecificationRequest) returns (ListJobSpecificationResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job" - }; - } - - // GetJobTask provides task details specific to plugin used in a job - rpc GetJobTask(GetJobTaskRequest) returns (GetJobTaskResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/task" - }; - } - - // CheckJobSpecification checks if a job specification is valid - rpc CheckJobSpecification(CheckJobSpecificationRequest) returns (CheckJobSpecificationResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/job/check" - }; - } - // CheckJobSpecifications checks if the job specifications are valid - rpc CheckJobSpecifications(CheckJobSpecificationsRequest) returns (stream CheckJobSpecificationsResponse) {} - - // RegisterProject creates a new optimus project - rpc RegisterProject(RegisterProjectRequest) returns (RegisterProjectResponse) { - option (google.api.http) = { - post: "/v1beta1/project" - body: "*" - }; - } - // RegisterProjectNamespace creates a new namespace for a project - rpc RegisterProjectNamespace(RegisterProjectNamespaceRequest) returns (RegisterProjectNamespaceResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace" - body: "*" - }; - } - // RegisterSecret creates a new secret of a project - rpc RegisterSecret(RegisterSecretRequest) returns (RegisterSecretResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/secret/{secret_name}" - body: "*" - }; - } - - // UpdateSecret updates secret at project level - rpc UpdateSecret(UpdateSecretRequest) returns (UpdateSecretResponse) { - option (google.api.http) = { - put: "/v1beta1/project/{project_name}/secret/{secret_name}" - body: "*" - }; - } - - // ListSecrets shows the secrets registered for a project - rpc ListSecrets(ListSecretsRequest) returns (ListSecretsResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/secret" - }; - } - - // DeleteSecret deletes a secret for a project - rpc DeleteSecret(DeleteSecretRequest) returns (DeleteSecretResponse) { - option (google.api.http) = { - delete: "/v1beta1/project/{project_name}/secret/{secret_name}" - }; - } - - // ListProjects returns list of registered projects and configurations - rpc ListProjects(ListProjectsRequest) returns (ListProjectsResponse) { - option (google.api.http) = { - get: "/v1beta1/project" - }; - } - - // ListProjectNamespaces returns list of namespaces of a project - rpc ListProjectNamespaces(ListProjectNamespacesRequest) returns (ListProjectNamespacesResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace" - }; - } - - // RegisterInstance is an internal admin command used during task/hook execution - // to pull task/hook compiled configuration and assets. - rpc RegisterInstance(RegisterInstanceRequest) returns (RegisterInstanceResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/job/{job_name}/instance" - body: "*" - }; - } - // JobStatus returns the current and past run status of jobs - rpc JobStatus(JobStatusRequest) returns (JobStatusResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/job/{job_name}/status" - }; - } - - // JobRun returns the current and past run status of jobs on a given range - rpc JobRun(JobRunRequest) returns (JobRunResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/job/{job_name}/run" - }; - } - // RegisterJobEvent notifies optimus service about an event related to job rpc RegisterJobEvent(RegisterJobEventRequest) returns (RegisterJobEventResponse) { option (google.api.http) = { @@ -174,295 +39,7 @@ service RuntimeService { body: "*" }; } - // GetWindow provides the start and end dates provided a scheduled date - // of the execution window - rpc GetWindow(GetWindowRequest) returns (GetWindowResponse) { - option (google.api.http) = { - get: "/v1beta1/window" - }; - } - - // DeployResourceSpecification migrate all resource specifications of a datastore in project - // State of the world request - rpc DeployResourceSpecification(stream DeployResourceSpecificationRequest) returns (stream DeployResourceSpecificationResponse) {} - // ListResourceSpecification lists all resource specifications of a datastore in project - rpc ListResourceSpecification(ListResourceSpecificationRequest) returns (ListResourceSpecificationResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" - }; - } - - // Database CRUD - // CreateResource registers a new resource of a namespace which belongs to a project - rpc CreateResource(CreateResourceRequest) returns (CreateResourceResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" - body: "*" - }; - } - // ReadResource reads a provided resource spec of a namespace - rpc ReadResource(ReadResourceRequest) returns (ReadResourceResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource/{resource_name}" - }; - } - // UpdateResource updates a resource specification of a datastore in project - rpc UpdateResource(UpdateResourceRequest) returns (UpdateResourceResponse) { - option (google.api.http) = { - put: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/resource" - body: "*" - }; - } - rpc ReplayDryRun(ReplayDryRunRequest) returns (ReplayDryRunResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/replay/dryrun" - body: "*" - }; - } - rpc Replay(ReplayRequest) returns (ReplayResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/replay" - body: "*" - }; - } - rpc GetReplayStatus(GetReplayStatusRequest) returns (GetReplayStatusResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/replay/{id}" - }; - } - rpc ListReplays(ListReplaysRequest) returns (ListReplaysResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/replay" - }; - } - rpc BackupDryRun(BackupDryRunRequest) returns (BackupDryRunResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backupDryrun" - body: "*" - }; - } - rpc CreateBackup(CreateBackupRequest) returns (CreateBackupResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup" - body: "*" - }; - } - rpc ListBackups(ListBackupsRequest) returns (ListBackupsResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup" - }; - } - rpc GetBackup(GetBackupRequest) returns (GetBackupResponse) { - option (google.api.http) = { - get: "/v1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup/{id}" - }; - } - // TODO(kush.sharma): disabled ATM - // rpc DeleteResource(DeleteResourceRequest) returns (DeleteResourceResponse) {} - - // RunJob creates a job run and executes all included tasks/hooks instantly - // this doesn't necessarily deploy the job in db first - rpc RunJob(RunJobRequest) returns (RunJobResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/run" - body: "*" - }; - } -} - -message ProjectSpecification { - string name = 1; - map config = 2; - - message ProjectSecret { - string name = 1; - string value = 2; - } - repeated ProjectSecret secrets = 3; -} - -message NamespaceSpecification { - string name = 1; - map config = 2; -} - -message JobSpecHook { - string name = 1; - repeated JobConfigItem config = 2; -} - -message JobSpecMetadataResourceConfig { - string cpu = 1; - string memory = 2; } - -message JobSpecMetadataResource { - JobSpecMetadataResourceConfig request = 1; - JobSpecMetadataResourceConfig limit = 2; -} - -message JobMetadata { - JobSpecMetadataResource resource = 1; -} - -message JobSpecification { - int32 version = 1; - string name = 2; - string owner = 3; - - string start_date = 4; - string end_date = 5; // optional - string interval = 6; - - bool depends_on_past = 7; // should only execute today if yesterday was completed with success? - bool catch_up = 8; // should backfill till today? - - string task_name = 9; - repeated JobConfigItem config = 10; - - string window_size = 11; - string window_offset = 12; - string window_truncate_to = 13; - - repeated JobDependency dependencies = 14; // static dependencies - map assets = 15; - - repeated JobSpecHook hooks = 16; // optional - - string description = 17; // optional - map labels = 18; - - message Behavior { - - // retry behaviour if job failed to execute for the first time - message Retry { - int32 count = 1; - google.protobuf.Duration delay = 2; - bool exponential_backoff = 3; - } - Retry retry = 1; - - // Notifiers are used to set custom alerting in case of job failure/sla_miss - message Notifiers { - JobEvent.Type on = 1; - repeated string channels = 2; - map config = 3; - } - repeated Notifiers notify = 2; - } - Behavior behavior = 19; - - JobMetadata metadata = 20; -} - -message JobConfigItem { - string name = 1; - string value = 2; -} - -message JobDependency { - string name = 1; - string type = 2; // intra/inter/extra - HttpDependency http_dependency = 3; // http sensor dependency -} - -message InstanceSpec { - reserved 2, 4; - - string state = 1; - - // deprecated - // google.protobuf.Timestamp scheduled_at = 2; - // deprecated - // string job_name = 4; - - repeated InstanceSpecData data = 3; - google.protobuf.Timestamp executed_at = 5; - - enum Type { - TYPE_UNSPECIFIED = 0; - TYPE_TASK = 1; - TYPE_HOOK = 2; - } - string name = 6; - Type type = 7; -} - -message InstanceSpecData { - reserved 3, 4; - - string name = 1; - string value = 2; - - // type of data, could be an env var or file - enum Type { - TYPE_UNSPECIFIED = 0; - TYPE_ENV = 1; - TYPE_FILE = 2; - } - Type type = 5; -} - -message InstanceContext { - map envs = 1; - map files = 2; - map secrets = 3; -} - -message JobStatus { - string state = 1; - google.protobuf.Timestamp scheduled_at = 2; -} - -message JobEvent { - enum Type { - TYPE_UNSPECIFIED = 0; - TYPE_SLA_MISS = 1; - TYPE_FAILURE = 2; - TYPE_SUCCESS = 3; - } - Type type = 1; - google.protobuf.Struct value = 2; -} - -message TaskWindow { - google.protobuf.Duration size = 1; - google.protobuf.Duration offset = 2; - string truncate_to = 3; -} - -// ResourceSpecification are datastore specification representation of a resource -message ResourceSpecification { - reserved 3; // depricated "datastore" - - int32 version = 1; - string name = 2; - string type = 4; - - google.protobuf.Struct spec = 5; - map assets = 6; - map labels = 7; -} - -// JobTask is part of a job that dictates main transformation -// each job has exactly one task -message JobTask { - string name = 1; - string description = 2; - string image = 3; - - message Destination { - string destination = 1; - string type = 2; - } - Destination destination = 4; - - message Dependency { - string dependency = 1; - } - repeated Dependency dependencies = 5; -} - // rpc req/res message VersionRequest { @@ -473,374 +50,6 @@ message VersionResponse { string server = 1; } -message DeployJobSpecificationRequest { - string project_name = 1; // unique project identifier - repeated JobSpecification jobs = 2; - string namespace_name = 4; -} - -message DeployJobSpecificationResponse { - bool success = 1; - - // non ack responses are more of a progress/info response - // and not really success or failure statuses - bool ack = 2; - - string message = 3; - string job_name = 4; -} - -message ListJobSpecificationRequest { - string project_name = 1; - string namespace_name = 2; -} - -message ListJobSpecificationResponse{ - repeated JobSpecification jobs = 1; -} - -message GetJobTaskRequest { - string project_name = 1; - string namespace_name = 2; - string job_name = 3; -} - -message GetJobTaskResponse { - JobTask task = 1; -} - -message CheckJobSpecificationRequest { - string project_name = 1; - JobSpecification job = 2; - string namespace_name = 3; -} - -message CheckJobSpecificationResponse { - bool success = 1; -} - -message CheckJobSpecificationsRequest { - string project_name = 1; - repeated JobSpecification jobs = 2; - string namespace_name = 3; -} - -message CheckJobSpecificationsResponse { - bool success = 1; - - // non ack responses are more of a progress/info response - // and not really success or failure statuses - bool ack = 2; - - string message = 3; - string job_name = 4; -} - -message RegisterProjectRequest { - ProjectSpecification project = 1; - NamespaceSpecification namespace = 2 [deprecated=true]; -} - -message RegisterProjectResponse { - bool success = 1; - string message = 2; -} - -message RegisterProjectNamespaceRequest { - string project_name = 1; - NamespaceSpecification namespace = 2; -} - -message RegisterProjectNamespaceResponse { - bool success = 1; - string message = 2; -} - -message CreateJobSpecificationRequest { - string project_name = 1; - string namespace_name = 2; - JobSpecification spec = 3; -} - -message CreateJobSpecificationResponse { - bool success = 1; - string message = 2; -} - -message GetJobSpecificationRequest { - string project_name = 1; - string namespace_name = 2; - string job_name = 3; -} - -message GetJobSpecificationResponse { - JobSpecification spec = 1; -} - -message DeleteJobSpecificationRequest { - string project_name = 1; - string namespace_name = 2; - string job_name = 3; -} - -message DeleteJobSpecificationResponse { - bool success = 1; - string message = 2; -} - -message RegisterSecretRequest { - string project_name = 1; - string secret_name = 2; - string value = 3; // base64 encoded secret value - string namespace_name = 4; -} - -message RegisterSecretResponse {} - -message UpdateSecretRequest { - string project_name = 1; - string secret_name = 2; - string value = 3; // base64 encoded secret value - string namespace_name = 4; -} - -message UpdateSecretResponse {} - -message ListSecretsRequest { - string project_name = 1; -} - -message ListSecretsResponse { - message Secret { - string name = 1; - string digest = 2; - string namespace = 3; - google.protobuf.Timestamp updated_at = 4; - } - repeated Secret secrets = 1; -} - -message DeleteSecretRequest { - string project_name = 1; - string secret_name = 2; - string namespace_name = 3; -} - -message DeleteSecretResponse {} - -message ListProjectsRequest {} - -message ListProjectsResponse { - repeated ProjectSpecification projects = 1; -} - -message ListProjectNamespacesRequest { - string project_name = 1; -} - -message ListProjectNamespacesResponse { - repeated NamespaceSpecification namespaces = 1; -} - -message RegisterInstanceRequest { - reserved 3; - - string project_name = 1; - string job_name = 2; - google.protobuf.Timestamp scheduled_at = 4; - - string instance_name = 5; - InstanceSpec.Type instance_type = 6; - - // either set job_name if this is a scheduled execution - // or set jonrun_id if this is a manual triggered execution - // and not really registered as a valid job - string jobrun_id = 7; -} - -message RegisterInstanceResponse { - ProjectSpecification project = 1; - NamespaceSpecification namespace = 4; - JobSpecification job = 2; - - InstanceSpec instance = 3; - InstanceContext context = 5; -} - -message JobStatusRequest { - reserved 3; - - string project_name = 1; - string job_name = 2; -} - -message JobStatusResponse { - repeated JobStatus statuses = 1; -} - -message JobRunRequest { - string project_name = 1; - string job_name = 2; - google.protobuf.Timestamp start_date = 3; - google.protobuf.Timestamp end_date = 4; - repeated string filter = 5; -} - -message JobRunResponse { - repeated JobRun job_runs = 1; -} - -message JobRun { - string state = 1; - google.protobuf.Timestamp scheduled_at = 2; -} - -message GetWindowRequest { - google.protobuf.Timestamp scheduled_at = 1; - string size = 2; - string offset = 3; - string truncate_to = 4; -} - -message GetWindowResponse { - google.protobuf.Timestamp start = 1; - google.protobuf.Timestamp end = 2; -} - -message DeployResourceSpecificationRequest { - string project_name = 1; - string datastore_name = 2; - repeated ResourceSpecification resources = 3; - string namespace_name = 4; -} - -message DeployResourceSpecificationResponse { - bool success = 1; - - // non ack responses are more of a progress/info response - // and not success or failure statuses - bool ack = 2; - - string message = 3; - string resource_name = 4; -} - -// ListResourceSpecificationRequest lists all resource specifications of a datastore in project -message ListResourceSpecificationRequest { - string project_name = 1; - string datastore_name = 2; - string namespace_name = 3; -} - -message ListResourceSpecificationResponse { - repeated ResourceSpecification resources = 1; -} - -message CreateResourceRequest { - string project_name = 1; - string datastore_name = 2; - ResourceSpecification resource = 3; - string namespace_name = 4; -} - -message CreateResourceResponse { - bool success = 1; - string message = 2; -} - -message ReadResourceRequest { - string project_name = 1; - string datastore_name = 2; - string resource_name = 3; - string namespace_name = 4; -} - -message ReadResourceResponse { - bool success = 1; - string message = 2; - ResourceSpecification resource = 3; -} - -message UpdateResourceRequest { - string project_name = 1; - string datastore_name = 2; - ResourceSpecification resource = 3; - string namespace_name = 4; -} - -message UpdateResourceResponse { - bool success = 1; - string message = 2; -} - -message ReplayRequest { - string project_name = 1; - string job_name = 2; - string namespace_name = 3; - string start_date = 4; - string end_date = 5; - bool force = 6; - - // represents which downstream to be replayed. - // possible values are the namespace names, *, or empty. - // '*' means all namespaces are allowed, empty list means all downstream will be ignored. - repeated string allowed_downstream_namespaces = 7; -} - -message ReplayResponse { - string id = 1; - repeated string ignored_jobs = 2; -} - -message ReplayDryRunRequest { - string project_name = 1; - string job_name = 2; - string namespace_name = 3; - string start_date = 4; - string end_date = 5; - - // represents which downstream to be replayed. - // possible values are the namespace names, *, or empty. - // '*' means all namespaces are allowed, empty list means all downstream will be ignored. - repeated string allowed_downstream_namespaces = 6; -} - -message ReplayDryRunResponse { - bool success = 1; - ReplayExecutionTreeNode response = 2 [deprecated=true]; - ReplayExecutionTreeNode execution_tree = 3; - repeated string ignored_jobs = 4; -} - -message ReplayExecutionTreeNode { - string job_name = 1; - repeated ReplayExecutionTreeNode dependents = 2; - repeated google.protobuf.Timestamp runs = 3; -} - -message GetReplayStatusResponse { - string state = 1; - ReplayStatusTreeNode response = 2; -} - -message ReplayStatusTreeNode { - string job_name = 1; - repeated ReplayStatusTreeNode dependents = 2; - repeated ReplayStatusRun runs = 3; - string state = 4; -} - -message ReplayStatusRun { - google.protobuf.Timestamp run = 1; - string state = 2; -} - -message GetReplayStatusRequest { - string id = 1; - string job_name = 2; - string project_name = 3; -} - message RegisterJobEventRequest { string project_name = 1; string job_name = 2; @@ -850,109 +59,3 @@ message RegisterJobEventRequest { } message RegisterJobEventResponse {} - -message ListReplaysRequest { - string project_name = 1; -} - -message ListReplaysResponse { - repeated ReplaySpec replay_list = 1; -} - -message ReplaySpec { - string id = 1; - string job_name = 2; - google.protobuf.Timestamp start_date = 3; - google.protobuf.Timestamp end_date = 4; - string state = 5; - google.protobuf.Timestamp created_at = 6; - map config = 7; -} - -message RunJobRequest { - string project_name = 1; - string namespace_name = 2; - - // job specification order of execution is undefined - // attributes realted to schedule behaviour are ignored like interval, - // start_date, end_date, catchup, etc - repeated JobSpecification specifications = 3; -} - -message RunJobResponse {} - -message BackupDryRunRequest { - string project_name = 1; - string datastore_name = 2; - string resource_name = 3; - string namespace_name = 4; - string description = 5; - bool ignore_downstream = 6 [deprecated=true]; - - // represents which downstream to be backed up. - // possible values are the namespace names, *, or empty. - // '*' means all namespaces are allowed, empty list means all downstream will be ignored. - repeated string allowed_downstream_namespaces = 7; -} - -message BackupDryRunResponse { - repeated string resource_name = 1; - repeated string ignored_resources = 2; -} - -message CreateBackupRequest { - string project_name = 1; - string datastore_name = 2; - string resource_name = 3; - string namespace_name = 4; - string description = 5; - bool ignore_downstream = 6 [deprecated=true]; - map config = 7; - - // represents which downstream to be backed up. - // possible values are the namespace names, *, or empty. - // '*' means all namespaces are allowed, empty list means all downstream will be ignored. - repeated string allowed_downstream_namespaces = 8; -} - -message CreateBackupResponse { - repeated string urn = 1; - repeated string ignored_resources = 2; -} - -message ListBackupsRequest { - string project_name = 1; - string datastore_name = 2; - string namespace_name = 3; -} - -message ListBackupsResponse { - repeated BackupSpec backups = 1; -} - -message BackupSpec { - string id = 1; - string resource_name = 2; - google.protobuf.Timestamp created_at = 3; - string description = 4; - map config = 5; -} - -message GetBackupRequest { - string project_name = 1; - string datastore_name = 2; - string namespace_name = 3; - string id = 4; -} - -message GetBackupResponse { - BackupSpec spec = 1; - repeated string urn = 2; -} - -message HttpDependency { - string name = 1; - string url = 2; - map headers = 3; - map params = 4; -} \ No newline at end of file diff --git a/odpf/optimus/core/v1beta1/secret.proto b/odpf/optimus/core/v1beta1/secret.proto new file mode 100644 index 00000000..e73695c3 --- /dev/null +++ b/odpf/optimus/core/v1beta1/secret.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; +package odpf.optimus.core.v1beta1; + +import "google/api/annotations.proto"; +import "protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/odpf/proton/optimus"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.optimus"; +option java_outer_classname = "SecretServiceManager"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Optimus Secret Management Service"; + } + schemes: HTTP; + host: "127.0.0.1:9100"; + base_path: "/api"; +}; + +service SecretService{ + + // RegisterSecret creates a new secret of a project + rpc RegisterSecret(RegisterSecretRequest) returns (RegisterSecretResponse) { + option (google.api.http) = { + post: "/v1beta1/project/{project_name}/secret/{secret_name}" + body: "*" + }; + } + + // UpdateSecret updates secret at project level + rpc UpdateSecret(UpdateSecretRequest) returns (UpdateSecretResponse) { + option (google.api.http) = { + put: "/v1beta1/project/{project_name}/secret/{secret_name}" + body: "*" + }; + } + + // ListSecrets shows the secrets registered for a project + rpc ListSecrets(ListSecretsRequest) returns (ListSecretsResponse) { + option (google.api.http) = { + get: "/v1beta1/project/{project_name}/secret" + }; + } + + // DeleteSecret deletes a secret for a project + rpc DeleteSecret(DeleteSecretRequest) returns (DeleteSecretResponse) { + option (google.api.http) = { + delete: "/v1beta1/project/{project_name}/secret/{secret_name}" + }; + } + +} + +message RegisterSecretRequest { + string project_name = 1; + string secret_name = 2; + string value = 3; // base64 encoded secret value + string namespace_name = 4; +} + +message RegisterSecretResponse {} + +message UpdateSecretRequest { + string project_name = 1; + string secret_name = 2; + string value = 3; // base64 encoded secret value + string namespace_name = 4; +} + +message UpdateSecretResponse {} + +message ListSecretsRequest { + string project_name = 1; +} + +message ListSecretsResponse { + message Secret { + string name = 1; + string digest = 2; + string namespace = 3; + google.protobuf.Timestamp updated_at = 4; + } + repeated Secret secrets = 1; +} + +message DeleteSecretRequest { + string project_name = 1; + string secret_name = 2; + string namespace_name = 3; +} + +message DeleteSecretResponse {} diff --git a/odpf/optimus/plugins/v1beta1/cli.proto b/odpf/optimus/plugins/v1beta1/cli.proto index 3120a3b2..d1074118 100644 --- a/odpf/optimus/plugins/v1beta1/cli.proto +++ b/odpf/optimus/plugins/v1beta1/cli.proto @@ -8,7 +8,7 @@ option java_outer_classname = "CLIModProto"; import "google/protobuf/timestamp.proto"; import "odpf/optimus/plugins/v1beta1/base.proto"; -import "odpf/optimus/core/v1beta1/runtime.proto"; +import "odpf/optimus/core/v1beta1/job_run.proto"; // CLIModService must be implemented to interact with users via optimus cli service CLIModService { diff --git a/odpf/optimus/plugins/v1beta1/dependency_resolver.proto b/odpf/optimus/plugins/v1beta1/dependency_resolver.proto index 6bb8ff2a..e1ca2175 100644 --- a/odpf/optimus/plugins/v1beta1/dependency_resolver.proto +++ b/odpf/optimus/plugins/v1beta1/dependency_resolver.proto @@ -8,7 +8,7 @@ option java_outer_classname = "DependencyResolverModProto"; import "odpf/optimus/plugins/v1beta1/cli.proto"; import "odpf/optimus/plugins/v1beta1/base.proto"; -import "odpf/optimus/core/v1beta1/runtime.proto"; +import "odpf/optimus/core/v1beta1/project.proto"; // DependencyResolverModService must be implemented by all plugins want to support automatic // dependency resolution