refactor(optimus): Split optimus runtime proto

odpf/optimus/core/v1beta1/backup.proto

@@ -0,0 +1,118 @@
+syntax = "proto3";
+package odpf.optimus.core.v1beta1;
+
+import "google/api/annotations.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/odpf/proton/optimus";
+option java_multiple_files = true;
+option java_package = "io.odpf.proton.optimus";
+option java_outer_classname = "BackupServiceManager";
+
+// These annotations are used when generating the OpenAPI file.
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+    info: {
+      version: "0.1";
+    };
+    external_docs: {
+      description: "Optimus Backup Service";
+    }
+    schemes: HTTP;
+    host: "127.0.0.1:9100";
+    base_path: "/api";
+};
+
+service BackupService {
+  rpc BackupDryRun(BackupDryRunRequest) returns (BackupDryRunResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backupDryrun"
+      body: "*"
+    };
+  }
+  rpc CreateBackup(CreateBackupRequest) returns (CreateBackupResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup"
+      body: "*"
+    };
+  }
+  rpc ListBackups(ListBackupsRequest) returns (ListBackupsResponse) {
+    option (google.api.http) = {
+      get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup"
+    };
+  }
+  rpc GetBackup(GetBackupRequest) returns (GetBackupResponse) {
+    option (google.api.http) = {
+      get: "/v1/project/{project_name}/namespace/{namespace_name}/datastore/{datastore_name}/backup/{id}"
+    };
+  }
+}
+
+message BackupDryRunRequest {
+  string project_name = 1;
+  string datastore_name = 2;
+  string resource_name = 3;
+  string namespace_name = 4;
+  string description = 5;
+  bool ignore_downstream = 6 [deprecated=true];
+
+  // represents which downstream to be backed up.
+  // possible values are the namespace names, *, or empty.
+  // '*' means all namespaces are allowed, empty list means all downstream will be ignored.
+  repeated string allowed_downstream_namespaces = 7;
+}
+
+message BackupDryRunResponse {
+  repeated string resource_name = 1;
+  repeated string ignored_resources = 2;
+}
+
+message CreateBackupRequest {
+  string project_name = 1;
+  string datastore_name = 2;
+  string resource_name = 3;
+  string namespace_name = 4;
+  string description = 5;
+  bool ignore_downstream = 6 [deprecated=true];
+  map<string, string> config = 7;
+
+  // represents which downstream to be backed up.
+  // possible values are the namespace names, *, or empty.
+  // '*' means all namespaces are allowed, empty list means all downstream will be ignored.
+  repeated string allowed_downstream_namespaces = 8;
+}
+
+message CreateBackupResponse {
+  repeated string urn = 1;
+  repeated string ignored_resources = 2;
+}
+
+message ListBackupsRequest {
+  string project_name = 1;
+  string datastore_name = 2;
+  string namespace_name = 3;
+}
+
+message ListBackupsResponse {
+  repeated BackupSpec backups = 1;
+}
+
+message BackupSpec {
+  string id = 1;
+  string resource_name = 2;
+  google.protobuf.Timestamp created_at = 3;
+  string description = 4;
+  map<string, string> config = 5;
+}
+
+message GetBackupRequest {
+  string project_name = 1;
+  string datastore_name = 2;
+  string namespace_name = 3;
+  string id = 4;
+}
+
+message GetBackupResponse {
+  BackupSpec spec = 1;
+  repeated string urn = 2;
+}

odpf/optimus/core/v1beta1/job_run.proto

@@ -0,0 +1,234 @@
+syntax = "proto3";
+package odpf.optimus.core.v1beta1;
+
+import "google/api/annotations.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+import "odpf/optimus/core/v1beta1/project.proto";
+import "odpf/optimus/core/v1beta1/namespace.proto";
+import "odpf/optimus/core/v1beta1/job_spec.proto";
+
+option go_package = "github.com/odpf/proton/optimus";
+option java_multiple_files = true;
+option java_package = "io.odpf.proton.optimus";
+option java_outer_classname = "JobRunManager";
+
+// These annotations are used when generating the OpenAPI file.
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+    info: {
+      version: "0.1";
+    };
+    external_docs: {
+      description: "Optimus Job Run Service";
+    }
+    schemes: HTTP;
+    host: "127.0.0.1:9100";
+    base_path: "/api";
+};
+
+service JobRunService {
+  // GetJobTask provides task details specific to plugin used in a job
+  rpc GetJobTask(GetJobTaskRequest) returns (GetJobTaskResponse) {
+    option (google.api.http) = {
+      get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/task"
+    };
+  }
+  // RegisterInstance is an internal admin command used during task/hook execution
+  // to pull task/hook compiled configuration and assets.
+  rpc RegisterInstance(RegisterInstanceRequest) returns (RegisterInstanceResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/project/{project_name}/job/{job_name}/instance"
+      body: "*"
+    };
+  }
+  // JobStatus returns the current and past run status of jobs
+  rpc JobStatus(JobStatusRequest) returns (JobStatusResponse) {
+    option (google.api.http) = {
+      get: "/v1beta1/project/{project_name}/job/{job_name}/status"
+    };
+  }
+  // JobRun returns the current and past run status of jobs on a given range
+  rpc JobRun(JobRunRequest) returns (JobRunResponse) {
+    option (google.api.http) = {
+      get: "/v1beta1/project/{project_name}/job/{job_name}/run"
+    };
+  }
+  // GetWindow provides the start and end dates provided a scheduled date
+  // of the execution window
+  rpc GetWindow(GetWindowRequest) returns (GetWindowResponse) {
+    option (google.api.http) = {
+      get: "/v1beta1/window"
+    };
+  }
+  // RunJob creates a job run and executes all included tasks/hooks instantly
+  // this doesn't necessarily deploy the job in db first
+  rpc RunJob(RunJobRequest) returns (RunJobResponse) {
+    option (google.api.http) = {
+      post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/run"
+      body: "*"
+    };
+  }
+
+}
+
+message GetJobTaskRequest {
+  string project_name = 1;
+  string namespace_name = 2;
+  string job_name = 3;
+}
+
+message GetJobTaskResponse {
+  JobTask task = 1;
+}
+
+message RegisterInstanceRequest {
+  reserved 3;
+
+  string project_name = 1;
+  string job_name = 2;
+  google.protobuf.Timestamp scheduled_at = 4;
+
+  string instance_name = 5;
+  InstanceSpec.Type instance_type = 6;
+
+  // either set job_name if this is a scheduled execution
+  // or set jonrun_id if this is a manual triggered execution
+  // and not really registered as a valid job
+  string jobrun_id = 7;
+}
+
+message RegisterInstanceResponse {
+  ProjectSpecification project = 1;
+  NamespaceSpecification namespace = 4;
+  JobSpecification job = 2;
+
+  InstanceSpec instance = 3;
+  InstanceContext context = 5;
+}
+
+message JobStatusRequest {
+  reserved 3;
+
+  string project_name = 1;
+  string job_name = 2;
+}
+
+message JobStatusResponse {
+  repeated JobStatus statuses = 1;
+}
+
+message JobRunRequest {
+  string project_name = 1;
+  string job_name = 2;
+  google.protobuf.Timestamp start_date = 3;
+  google.protobuf.Timestamp end_date = 4;
+  repeated string filter = 5;
+}
+
+message JobRunResponse {
+  repeated JobRun job_runs = 1;
+}
+
+message JobRun {
+  string state = 1;
+  google.protobuf.Timestamp scheduled_at = 2;
+}
+
+message GetWindowRequest {
+  google.protobuf.Timestamp scheduled_at = 1;
+  string size = 2;
+  string offset = 3;
+  string truncate_to = 4;
+}
+
+message GetWindowResponse {
+  google.protobuf.Timestamp start = 1;
+  google.protobuf.Timestamp end = 2;
+}
+
+message RunJobRequest {
+  string project_name = 1;
+  string namespace_name = 2;
+
+  // job specification order of execution is undefined
+  // attributes realted to schedule behaviour are ignored like interval,
+  // start_date, end_date, catchup, etc
+  repeated JobSpecification specifications = 3;
+}
+
+message RunJobResponse {}
+
+
+// JobTask is part of a job that dictates main transformation
+// each job has exactly one task
+message JobTask {
+  string name = 1;
+  string description = 2;
+  string image = 3;
+
+  message Destination {
+    string destination = 1;
+    string type = 2;
+  }
+  Destination destination = 4;
+
+  message Dependency {
+    string dependency = 1;
+  }
+  repeated Dependency dependencies = 5;
+}
+
+message InstanceSpec {
+  reserved 2, 4;
+
+  string state = 1;
+
+  // deprecated
+  // google.protobuf.Timestamp scheduled_at = 2;
+  // deprecated
+  // string job_name = 4;
+
+  repeated InstanceSpecData data = 3;
+  google.protobuf.Timestamp executed_at = 5;
+
+  enum Type {
+    TYPE_UNSPECIFIED = 0;
+    TYPE_TASK = 1;
+    TYPE_HOOK = 2;
+  }
+  string name = 6;
+  Type type = 7;
+}
+
+message InstanceSpecData {
+  reserved 3, 4;
+
+  string name = 1;
+  string value = 2;
+
+  // type of data, could be an env var or file
+  enum Type {
+    TYPE_UNSPECIFIED = 0;
+    TYPE_ENV = 1;
+    TYPE_FILE = 2;
+  }
+  Type type = 5;
+}
+
+message InstanceContext {
+  map<string, string> envs = 1;
+  map<string, string> files = 2;
+  map<string, string> secrets = 3;
+}
+
+message JobStatus {
+  string state = 1;
+  google.protobuf.Timestamp scheduled_at = 2;
+}
+
+message TaskWindow {
+  google.protobuf.Duration size = 1;
+  google.protobuf.Duration offset = 2;
+  string truncate_to = 3;
+}