From 1c78cddd0cf7ee188f38aaa98ec1db9e5dd0e238 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Tue, 20 Jul 2021 14:02:38 +0700 Subject: [PATCH 01/18] feat: add metadata models --- odpf/meta/Chart.proto | 36 ++++++++++++++++++++++ odpf/meta/Dashboard.proto | 32 ++++++++++++++++++++ odpf/meta/Group.proto | 33 ++++++++++++++++++++ odpf/meta/Job.proto | 30 +++++++++++++++++++ odpf/meta/Table.proto | 40 +++++++++++++++++++++++++ odpf/meta/Topic.proto | 39 ++++++++++++++++++++++++ odpf/meta/User.proto | 48 ++++++++++++++++++++++++++++++ odpf/meta/common/Event.proto | 15 ++++++++++ odpf/meta/common/Resource.proto | 14 +++++++++ odpf/meta/common/Timestamp.proto | 14 +++++++++ odpf/meta/facets/Columns.proto | 19 ++++++++++++ odpf/meta/facets/Custom.proto | 11 +++++++ odpf/meta/facets/Lineage.proto | 14 +++++++++ odpf/meta/facets/Ownership.proto | 17 +++++++++++ odpf/meta/facets/Tags.proto | 11 +++++++ odpf/meta/facets/TopicSchema.proto | 12 ++++++++ 16 files changed, 385 insertions(+) create mode 100644 odpf/meta/Chart.proto create mode 100644 odpf/meta/Dashboard.proto create mode 100644 odpf/meta/Group.proto create mode 100644 odpf/meta/Job.proto create mode 100644 odpf/meta/Table.proto create mode 100644 odpf/meta/Topic.proto create mode 100644 odpf/meta/User.proto create mode 100644 odpf/meta/common/Event.proto create mode 100644 odpf/meta/common/Resource.proto create mode 100644 odpf/meta/common/Timestamp.proto create mode 100644 odpf/meta/facets/Columns.proto create mode 100644 odpf/meta/facets/Custom.proto create mode 100644 odpf/meta/facets/Lineage.proto create mode 100644 odpf/meta/facets/Ownership.proto create mode 100644 odpf/meta/facets/Tags.proto create mode 100644 odpf/meta/facets/TopicSchema.proto diff --git a/odpf/meta/Chart.proto b/odpf/meta/Chart.proto new file mode 100644 index 00000000..27c4d2c4 --- /dev/null +++ b/odpf/meta/Chart.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Chart"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/Ownership.proto"; +import "odpf/meta/facets/Lineage.proto"; +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message ChartKey { + string urn = 1; +} + +message Chart { + string urn = 1; + string name = 2; + string type = 3; + string source = 4; + string description = 5; + string url = 6; + string raw_query = 7; + string dashboard_urn = 8; + string dashboard_source = 9; + odpf.meta.facets.Ownership ownership = 21; + odpf.meta.facets.Lineage lineage = 22; + odpf.meta.facets.Tags tags = 23; + odpf.meta.facets.Custom custom = 24; + odpf.meta.common.Timestamp timestamps = 25; + odpf.meta.common.Event event = 100; +} diff --git a/odpf/meta/Dashboard.proto b/odpf/meta/Dashboard.proto new file mode 100644 index 00000000..a6dc539e --- /dev/null +++ b/odpf/meta/Dashboard.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Dashboard"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/Chart.proto"; +import "odpf/meta/facets/Ownership.proto"; +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message DashboardKey { + string urn = 1; +} + +message Dashboard { + string urn = 1; + string name = 2; + string source = 3; + string description = 4; + string url = 5; + repeated odpf.meta.Chart charts = 6; + odpf.meta.facets.Ownership ownership = 21; + odpf.meta.facets.Tags tags = 22; + odpf.meta.facets.Custom custom = 23; + odpf.meta.common.Timestamp timestamps = 24; + odpf.meta.common.Event event = 100; +} diff --git a/odpf/meta/Group.proto b/odpf/meta/Group.proto new file mode 100644 index 00000000..4dd24ba1 --- /dev/null +++ b/odpf/meta/Group.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Group"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message GroupKey { + string urn = 1; +} + +message Group { + string urn = 1; + string name = 2; + string email = 3; + string source = 4; + repeated Member members = 5; + odpf.meta.facets.Tags tags = 21; + odpf.meta.facets.Custom custom = 22; + odpf.meta.common.Timestamp timestamps = 23; + odpf.meta.common.Event event = 100; +} + +message Member { + string urn = 1; + string role = 2; +} \ No newline at end of file diff --git a/odpf/meta/Job.proto b/odpf/meta/Job.proto new file mode 100644 index 00000000..137f2081 --- /dev/null +++ b/odpf/meta/Job.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Job"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/Lineage.proto"; +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message JobKey { + string urn = 1; +} + +message Job { + string urn = 1; + string name = 2; + string source = 3; + string type = 4; + string description = 5; + repeated odpf.meta.facets.Lineage lineage = 6; + odpf.meta.facets.Tags tags = 21; + odpf.meta.facets.Custom custom = 22; + odpf.meta.common.Timestamp timestamps = 23; + odpf.meta.common.Event event = 100; +} diff --git a/odpf/meta/Table.proto b/odpf/meta/Table.proto new file mode 100644 index 00000000..2418b51a --- /dev/null +++ b/odpf/meta/Table.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Table"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/Columns.proto"; +import "odpf/meta/facets/Ownership.proto"; +import "odpf/meta/facets/Lineage.proto"; +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message TableKey { + string urn = 1; +} + +message Table { + string urn = 1; + string name = 2; + string source = 3; + string description = 4; + TableProfile profile = 5; + odpf.meta.facets.Columns schema = 6; + odpf.meta.facets.Ownership ownership = 7; + odpf.meta.facets.Lineage lineage = 8; + odpf.meta.facets.Tags tags = 21; + odpf.meta.facets.Custom custom = 22; + odpf.meta.common.Timestamp timestamps = 23; + odpf.meta.common.Event event = 100; +} + +message TableProfile { + int64 total_rows = 1; + string partition_key = 2; + string partition_value = 3; +} diff --git a/odpf/meta/Topic.proto b/odpf/meta/Topic.proto new file mode 100644 index 00000000..97a892c4 --- /dev/null +++ b/odpf/meta/Topic.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "Topic"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/TopicSchema.proto"; +import "odpf/meta/facets/Ownership.proto"; +import "odpf/meta/facets/Lineage.proto"; +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message TopicKey { + string urn = 1; +} + +message Topic { + string urn = 1; + string name = 2; + string source = 3; + string description = 4; + TopicProfile profile = 5; + odpf.meta.facets.TopicSchema schema = 6; + odpf.meta.facets.Ownership ownership = 7; + odpf.meta.facets.Lineage lineage = 8; + odpf.meta.facets.Tags tags = 21; + odpf.meta.facets.Custom custom = 22; + odpf.meta.common.Timestamp timestamps = 23; + odpf.meta.common.Event event = 100; +} + +message TopicProfile { + int64 throughput = 1; + int64 number_of_partitions = 2; +} diff --git a/odpf/meta/User.proto b/odpf/meta/User.proto new file mode 100644 index 00000000..e8d20e0b --- /dev/null +++ b/odpf/meta/User.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +package odpf.meta; + +option java_package = "io.odpf.proton.meta"; +option java_outer_classname = "User"; +option go_package = "github.com/odpf/proton/meta"; + +import "odpf/meta/facets/Tags.proto"; +import "odpf/meta/facets/Custom.proto"; +import "odpf/meta/common/Timestamp.proto"; +import "odpf/meta/common/Event.proto"; + +message UserKey { + string urn = 1; +} + +message User { + string urn = 1; + string source = 2; + string email = 3; + string username = 4; + string first_name = 5; + string last_name = 6; + string full_name = 7; + string display_name = 8; + string title = 9; + bool is_active = 10; + string manager_email = 11; + repeated Profile profiles = 12; + repeated Membership memberships = 20; + odpf.meta.facets.Tags tags = 21; + odpf.meta.facets.Custom custom = 22; + odpf.meta.common.Timestamp timestamps = 23; + odpf.meta.common.Event event = 100; +} + +message Membership { + string group_urn = 1; + repeated string role = 2; +} + +message Profile { + string id = 1; + string platform = 2; + string url = 3; +} + diff --git a/odpf/meta/common/Event.proto b/odpf/meta/common/Event.proto new file mode 100644 index 00000000..3cd37af6 --- /dev/null +++ b/odpf/meta/common/Event.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package odpf.meta.common; + +option java_package = "io.odpf.proton.meta.common"; +option java_outer_classname = "Event"; +option go_package = "github.com/odpf/proton/meta/common"; + +import "google/protobuf/timestamp.proto"; + +message Event { + string action = 1; + string description = 2; + google.protobuf.Timestamp timestamp = 100; +} diff --git a/odpf/meta/common/Resource.proto b/odpf/meta/common/Resource.proto new file mode 100644 index 00000000..fbdde09b --- /dev/null +++ b/odpf/meta/common/Resource.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package odpf.meta.common; + +option java_package = "io.odpf.proton.meta.common"; +option java_outer_classname = "Resource"; +option go_package = "github.com/odpf/proton/meta/common"; + +message Resource { + string urn = 1; + string name = 2; + string source = 3; + string type = 4; +} diff --git a/odpf/meta/common/Timestamp.proto b/odpf/meta/common/Timestamp.proto new file mode 100644 index 00000000..eec94f97 --- /dev/null +++ b/odpf/meta/common/Timestamp.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package odpf.meta.common; + +option java_package = "io.odpf.proton.meta.common"; +option java_outer_classname = "Timestamp"; +option go_package = "github.com/odpf/proton/meta/common"; + +import "google/protobuf/timestamp.proto"; + +message Timestamp { + google.protobuf.Timestamp created_at = 1; + google.protobuf.Timestamp updated_at = 2; +} diff --git a/odpf/meta/facets/Columns.proto b/odpf/meta/facets/Columns.proto new file mode 100644 index 00000000..c3756554 --- /dev/null +++ b/odpf/meta/facets/Columns.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "Columns"; +option go_package = "github.com/odpf/proton/meta/facets"; + +message Columns { + repeated Column columns = 1; +} + +message Column { + string name = 1; + string description = 2; + string data_type = 3; + bool is_nullable = 4; + int64 length = 5; +} diff --git a/odpf/meta/facets/Custom.proto b/odpf/meta/facets/Custom.proto new file mode 100644 index 00000000..5997697c --- /dev/null +++ b/odpf/meta/facets/Custom.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "Custom"; +option go_package = "github.com/odpf/proton/meta/facets"; + +message Custom { + map custom_properties = 1; +} diff --git a/odpf/meta/facets/Lineage.proto b/odpf/meta/facets/Lineage.proto new file mode 100644 index 00000000..0b54206b --- /dev/null +++ b/odpf/meta/facets/Lineage.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "Lineage"; +option go_package = "github.com/odpf/proton/meta/facets"; + +import "odpf/meta/common/Resource.proto"; + +message Lineage { + repeated odpf.meta.common.Resource upstreams = 1; + repeated odpf.meta.common.Resource downstreams = 2; +} diff --git a/odpf/meta/facets/Ownership.proto b/odpf/meta/facets/Ownership.proto new file mode 100644 index 00000000..da58c7f7 --- /dev/null +++ b/odpf/meta/facets/Ownership.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "Ownership"; +option go_package = "github.com/odpf/proton/meta/facets"; + +message Ownership { + repeated Owner owners = 7; +} + +message Owner { + string urn = 1; + string name = 2; + string role = 3; +} \ No newline at end of file diff --git a/odpf/meta/facets/Tags.proto b/odpf/meta/facets/Tags.proto new file mode 100644 index 00000000..22a6611a --- /dev/null +++ b/odpf/meta/facets/Tags.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "Tags"; +option go_package = "github.com/odpf/proton/meta/facets"; + +message Tags { + map tags = 1; +} diff --git a/odpf/meta/facets/TopicSchema.proto b/odpf/meta/facets/TopicSchema.proto new file mode 100644 index 00000000..4f7f9a55 --- /dev/null +++ b/odpf/meta/facets/TopicSchema.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package odpf.meta.facets; + +option java_package = "io.odpf.proton.meta.facets"; +option java_outer_classname = "TopicSchema"; +option go_package = "github.com/odpf/proton/meta/facets"; + +message TopicSchema { + string schema_url = 1; + string format = 2; +} From 8939ce229b1f419f0ac92a767473ccb77c445ce7 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Thu, 22 Jul 2021 14:53:11 +0700 Subject: [PATCH 02/18] feat: add ColumnProfile model --- odpf/meta/facets/Columns.proto | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/odpf/meta/facets/Columns.proto b/odpf/meta/facets/Columns.proto index c3756554..9dd01236 100644 --- a/odpf/meta/facets/Columns.proto +++ b/odpf/meta/facets/Columns.proto @@ -6,6 +6,8 @@ option java_package = "io.odpf.proton.meta.facets"; option java_outer_classname = "Columns"; option go_package = "github.com/odpf/proton/meta/facets"; +import "odpf/meta/facets/Custom.proto"; + message Columns { repeated Column columns = 1; } @@ -16,4 +18,16 @@ message Column { string data_type = 3; bool is_nullable = 4; int64 length = 5; + ColumnProfile profile = 6; + odpf.meta.facets.Custom custom = 22; +} + +message ColumnProfile { + string min = 1; + string max = 2; + float avg = 3; + float med = 4; + int64 unique = 5; + int64 count = 6; + string top = 7; } From cc6133f885e6c8978b78e8da9b47681f7c840a44 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Thu, 22 Jul 2021 17:41:00 +0700 Subject: [PATCH 03/18] feat(metadata): add data_source to Chart model --- odpf/meta/Chart.proto | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/odpf/meta/Chart.proto b/odpf/meta/Chart.proto index 27c4d2c4..a35b4ddc 100644 --- a/odpf/meta/Chart.proto +++ b/odpf/meta/Chart.proto @@ -24,9 +24,10 @@ message Chart { string source = 4; string description = 5; string url = 6; - string raw_query = 7; - string dashboard_urn = 8; - string dashboard_source = 9; + string data_source = 7; + string raw_query = 8; + string dashboard_urn = 9; + string dashboard_source = 10; odpf.meta.facets.Ownership ownership = 21; odpf.meta.facets.Lineage lineage = 22; odpf.meta.facets.Tags tags = 23; From e36f1c58ad3fbe8abb2e774f27389692a00e60c5 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Mon, 12 Jul 2021 10:22:16 +0700 Subject: [PATCH 04/18] add: add guardian proto --- odpf/guardian/guardian.proto | 370 +++++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 odpf/guardian/guardian.proto diff --git a/odpf/guardian/guardian.proto b/odpf/guardian/guardian.proto new file mode 100644 index 00000000..afdf5ebe --- /dev/null +++ b/odpf/guardian/guardian.proto @@ -0,0 +1,370 @@ +syntax = "proto3"; +package odpf.guardian; + +import "odpf/third_party/googleapis/google/api/annotations.proto"; +import "odpf/third_party/grpc-gateway/protoc-gen-openapiv2/options/annotations.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/odpf/proton/guardian/go_package"; +option java_multiple_files = true; +option java_package = "io.odpf.proton.guardian"; +option java_outer_classname = "Guardian"; + +// These annotations are used when generating the OpenAPI file. +option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { + info: { + version: "0.1"; + }; + external_docs: { + description: "Guardian server"; + } + schemes: HTTP; +}; + +// WARNING: This is still in active development and can have breaking changes +service Guardian { + rpc ListProviders(ListProvidersRequest) returns (ListProvidersResponse) { + option (google.api.http) = { + get: "/providers" + }; + } + + rpc CreateProvider(CreateProviderRequest) returns (CreateProviderResponse) { + option (google.api.http) = { + post: "/providers" + body: "*" + }; + } + + rpc UpdateProvider(UpdateProviderRequest) returns (UpdateProviderResponse) { + option (google.api.http) = { + put: "/providers/{id}" + body: "*" + }; + } + + rpc ListPolicies(ListPoliciesRequest) returns (ListPoliciesResponse) { + option (google.api.http) = { + get: "/policies" + }; + } + + rpc CreatePolicy(CreatePolicyRequest) returns (CreatePolicyResponse) { + option (google.api.http) = { + post: "/policies" + body: "*" + }; + } + + rpc UpdatePolicy(UpdatePolicyRequest) returns (UpdatePolicyResponse) { + option (google.api.http) = { + put: "/policies/{id}" + body: "*" + }; + } + + rpc ListResources(ListResourcesRequest) returns (ListResourcesResponse) { + option (google.api.http) = { + get: "/resources" + }; + } + + rpc UpdateResource(UpdateResourceRequest) returns (Resource) { + option (google.api.http) = { + put: "/resources/{id}" + body: "*" + }; + } + + rpc ListAppeals(ListAppealsRequest) returns (ListAppealsResponse) { + option (google.api.http) = { + get: "/appeals" + }; + } + + rpc GetAppeal(GetAppealRequest) returns (GetAppealResponse) { + option (google.api.http) = { + get: "/appeals/{id}" + }; + } + + rpc CancelAppeal(CancelAppealRequest) returns (CancelAppealResponse) { + option (google.api.http) = { + put: "/appeals/{id}/cancel" + }; + } + + rpc RevokeAppeal(RevokeAppealRequest) returns (RevokeAppealResponse) { + option (google.api.http) = { + put: "/appeals/{id}/revoke" + }; + } + + rpc CreateAppeal(CreateAppealRequest) returns (CreateAppealResponse) { + option (google.api.http) = { + post: "/appeals" + body: "*" + }; + } + + rpc ListApprovals(ListApprovalsRequest) returns (ListApprovalsResponse) { + option (google.api.http) = { + get: "/appeals/approvals" + }; + } + + rpc MakeAction(MakeActionRequest) returns (MakeActionResponse) { + option (google.api.http) = { + post: "/appeals/{id}/approvals/{approval_name}" + body: "*" + }; + } +} + +message ListProvidersRequest {} + +message ListProvidersResponse { + repeated Provider providers = 1; +} + +message CreateProviderRequest { + Provider provider = 1; +} + +message CreateProviderResponse { + Provider provider = 1; +} + +message UpdateProviderRequest { + uint32 id = 1; + Provider provider = 2; +} + +message UpdateProviderResponse { + Provider provider = 1; +} + +message ListPoliciesRequest {} + +message ListPoliciesResponse { + repeated Policy policies = 1; +} + +message CreatePolicyRequest { + Policy policy = 1; +} + +message CreatePolicyResponse { + Policy policy = 1; +} + +message UpdatePolicyRequest { + string id = 1; + Policy policy = 2; +} + +message UpdatePolicyResponse { + Policy policy = 1; +} + +message ListResourcesRequest {} + +message ListResourcesResponse { + repeated Resource resources = 1; +} + +message UpdateResourceRequest { + uint32 id = 1; + Resource resource = 2; +} + +message ListAppealsRequest { + string user = 1; +} + +message ListAppealsResponse { + repeated Appeal appeals = 1; +} + +message GetAppealRequest { + uint32 id = 1; +} + +message GetAppealResponse { + Appeal appeal = 1; +} + +message CancelAppealRequest { + uint32 id = 1; +} + +message CancelAppealResponse { + Appeal appeal = 1; +} + +message RevokeAppealRequest { + uint32 id = 1; +} + +message RevokeAppealResponse { + Appeal appeal = 1; +} + +message CreateAppealRequest { + string user = 1; + + message Resource { + uint32 id = 1; + string role = 2; + google.protobuf.Struct options = 3; + } + repeated Resource resources = 2; +} + +message CreateAppealResponse { + repeated Appeal appeals = 1; +} + +message ListApprovalsRequest { + string user = 1; +} + +message ListApprovalsResponse { + repeated Approval approvals = 1; +} + +message MakeActionRequest { + uint32 id = 1; + string approval_name = 2; + + message Action { + string action = 1; + } + Action action = 3; +} + +message MakeActionResponse { + Appeal appeal = 1; +} + +message Provider { + uint32 id = 1; + string type = 2; + string urn = 3; + + message ProviderConfig { + string type = 1; + string urn = 2; + google.protobuf.Struct labels = 3; + google.protobuf.Value credentials = 4; + + message AppealConfig { + bool allow_permanent_access = 1; + string allow_active_access_extension_in = 2; + } + AppealConfig appeal = 5; + + message ResourceConfig { + string type = 1; + + message PolicyConfig { + string id = 1; + int32 version = 2; + } + PolicyConfig policy = 2; + + message RoleConfig { + string id = 1; + string name = 2; + string description = 3; + repeated google.protobuf.Value permissions = 4; + } + repeated RoleConfig roles = 3; + } + ResourceConfig resources = 6; + } + ProviderConfig config = 4; + + google.protobuf.Timestamp created_at = 5; + google.protobuf.Timestamp updated_at = 6; +} + +message Policy { + string id = 1; + uint32 version = 2; + string description = 3; + + message ApprovalStep { + string name = 1; + string description = 2; // optional + + message Condition { + string field = 1; + + message MatchCondition { + google.protobuf.Value eq = 1; + } + MatchCondition match = 2; + } + repeated Condition conditions = 3; // optional + + bool allow_failed = 4; // optional + repeated string dependencies = 5; // optional + string approvers = 6; // optional + } + repeated ApprovalStep steps = 4; + + google.protobuf.Struct labels = 5; + google.protobuf.Timestamp created_at = 6; + google.protobuf.Timestamp updated_at = 7; +} + +message Appeal { + uint32 id = 1; + uint32 resource_id = 2; + string policy_id = 3; + uint32 policy_version = 4; + string status = 5; + string user = 6; + string role = 7; + + message AppealOptions { + google.protobuf.Timestamp expiration_date = 1; // optional + } + AppealOptions options = 8; // optional + + google.protobuf.Struct labels = 9; + Resource resource = 10; // optional + repeated Approval approvals = 11; + + google.protobuf.Timestamp created_at = 12; + google.protobuf.Timestamp updated_at = 13; +} + +message Approval { + uint32 id = 1; + string name = 2; + uint32 appeal_id = 3; + string status = 4; + string actor = 5; + string policy_id = 6; + uint32 policy_version = 7; + repeated string approvers = 8; + Appeal appeal = 9; + google.protobuf.Timestamp created_at = 10; + google.protobuf.Timestamp updated_at = 11; +} + +message Resource { + uint32 id = 1; + string provider_type = 2; + string provider_urn = 3; + string type = 4; + string urn = 5; + string name = 6; + google.protobuf.Struct details = 7; + google.protobuf.Struct labels = 8; + google.protobuf.Timestamp created_at = 9; + google.protobuf.Timestamp updated_at = 10; +} \ No newline at end of file From 83a05b7466ce96d9b6c07a93497a1e0de554cbf2 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Mon, 12 Jul 2021 10:24:44 +0700 Subject: [PATCH 05/18] fix: fix java outer classname --- odpf/guardian/guardian.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odpf/guardian/guardian.proto b/odpf/guardian/guardian.proto index afdf5ebe..54bc9a1e 100644 --- a/odpf/guardian/guardian.proto +++ b/odpf/guardian/guardian.proto @@ -9,7 +9,7 @@ import "google/protobuf/timestamp.proto"; option go_package = "github.com/odpf/proton/guardian/go_package"; option java_multiple_files = true; option java_package = "io.odpf.proton.guardian"; -option java_outer_classname = "Guardian"; +option java_outer_classname = "ServiceManager"; // These annotations are used when generating the OpenAPI file. option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { From 091bdcf079753f18be01b151339c01b5b97feb83 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Tue, 13 Jul 2021 13:30:06 +0700 Subject: [PATCH 06/18] fix: change go package name --- odpf/guardian/guardian.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odpf/guardian/guardian.proto b/odpf/guardian/guardian.proto index 54bc9a1e..5a55d1e3 100644 --- a/odpf/guardian/guardian.proto +++ b/odpf/guardian/guardian.proto @@ -6,7 +6,7 @@ import "odpf/third_party/grpc-gateway/protoc-gen-openapiv2/options/annotations.p import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/odpf/proton/guardian/go_package"; +option go_package = "github.com/odpf/proton/guardian"; option java_multiple_files = true; option java_package = "io.odpf.proton.guardian"; option java_outer_classname = "ServiceManager"; From 8dda6c0f5ef2ccecfcac833871d43afa4bba9355 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Thu, 15 Jul 2021 10:30:31 +0700 Subject: [PATCH 07/18] docs: add descriptive comments on each entity --- odpf/guardian/guardian.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/odpf/guardian/guardian.proto b/odpf/guardian/guardian.proto index 5a55d1e3..6ae5a2a7 100644 --- a/odpf/guardian/guardian.proto +++ b/odpf/guardian/guardian.proto @@ -248,6 +248,7 @@ message MakeActionResponse { Appeal appeal = 1; } +// Provider contains information about external data provider such as BigQuery, Metabase, etc., credentials, policy, and allowed roles message Provider { uint32 id = 1; string type = 2; @@ -290,6 +291,7 @@ message Provider { google.protobuf.Timestamp updated_at = 6; } +// Policy is a configurable steps for appeal's approval message Policy { string id = 1; uint32 version = 2; @@ -320,6 +322,7 @@ message Policy { google.protobuf.Timestamp updated_at = 7; } +// Appeal is created by user to get access to one or more resources message Appeal { uint32 id = 1; uint32 resource_id = 2; @@ -342,6 +345,7 @@ message Appeal { google.protobuf.Timestamp updated_at = 13; } +// Approval is an approval item that generated in an appeal based on the selected policy message Approval { uint32 id = 1; string name = 2; @@ -356,6 +360,7 @@ message Approval { google.protobuf.Timestamp updated_at = 11; } +// Resource contains information of resource from providers message Resource { uint32 id = 1; string provider_type = 2; From 48eedb3b379fdacd9fe4ffed879ac2b272030a88 Mon Sep 17 00:00:00 2001 From: Rahmat Hidayat Date: Thu, 15 Jul 2021 10:31:18 +0700 Subject: [PATCH 08/18] fix: rename MakeAction to UpdateApproval --- odpf/guardian/guardian.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/odpf/guardian/guardian.proto b/odpf/guardian/guardian.proto index 6ae5a2a7..b6aed6df 100644 --- a/odpf/guardian/guardian.proto +++ b/odpf/guardian/guardian.proto @@ -114,7 +114,7 @@ service Guardian { }; } - rpc MakeAction(MakeActionRequest) returns (MakeActionResponse) { + rpc UpdateApproval(UpdateApprovalRequest) returns (UpdateApprovalResponse) { option (google.api.http) = { post: "/appeals/{id}/approvals/{approval_name}" body: "*" @@ -234,7 +234,7 @@ message ListApprovalsResponse { repeated Approval approvals = 1; } -message MakeActionRequest { +message UpdateApprovalRequest { uint32 id = 1; string approval_name = 2; @@ -244,7 +244,7 @@ message MakeActionRequest { Action action = 3; } -message MakeActionResponse { +message UpdateApprovalResponse { Appeal appeal = 1; } From b90502d864b5c3cf72760a6dd1d600594726c8d7 Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Sat, 14 Aug 2021 22:44:56 -0700 Subject: [PATCH 09/18] feat: update entities schema --- .project | 28 +++++++ .settings/org.eclipse.buildship.core.prefs | 13 +++ odpf/entities/common/event.proto | 25 ++++++ odpf/entities/common/resource.proto | 25 ++++++ odpf/entities/common/timestamp.proto | 17 ++++ odpf/entities/facets/columns.proto | 34 ++++++++ odpf/entities/facets/lineage.proto | 20 +++++ odpf/entities/facets/ownership.proto | 29 +++++++ odpf/entities/facets/properties.proto | 22 +++++ odpf/entities/facets/schema.proto | 13 +++ odpf/entities/resources/dashboard.proto | 95 ++++++++++++++++++++++ odpf/entities/resources/group.proto | 53 ++++++++++++ odpf/entities/resources/job.proto | 51 ++++++++++++ odpf/entities/resources/table.proto | 66 +++++++++++++++ odpf/entities/resources/topic.proto | 64 +++++++++++++++ odpf/entities/resources/user.proto | 93 +++++++++++++++++++++ odpf/meta/Chart.proto | 36 -------- odpf/meta/Dashboard.proto | 32 -------- odpf/meta/Group.proto | 33 -------- odpf/meta/Job.proto | 30 ------- odpf/meta/Table.proto | 40 --------- odpf/meta/Topic.proto | 39 --------- odpf/meta/User.proto | 48 ----------- odpf/meta/common/Event.proto | 15 ---- odpf/meta/common/Resource.proto | 14 ---- odpf/meta/common/Timestamp.proto | 14 ---- odpf/meta/facets/Columns.proto | 19 ----- odpf/meta/facets/Custom.proto | 11 --- odpf/meta/facets/Lineage.proto | 14 ---- odpf/meta/facets/Ownership.proto | 17 ---- odpf/meta/facets/Tags.proto | 11 --- odpf/meta/facets/TopicSchema.proto | 12 --- odpf/metadata/dataset.proto | 5 ++ 33 files changed, 653 insertions(+), 385 deletions(-) create mode 100644 .project create mode 100644 .settings/org.eclipse.buildship.core.prefs create mode 100644 odpf/entities/common/event.proto create mode 100644 odpf/entities/common/resource.proto create mode 100644 odpf/entities/common/timestamp.proto create mode 100644 odpf/entities/facets/columns.proto create mode 100644 odpf/entities/facets/lineage.proto create mode 100644 odpf/entities/facets/ownership.proto create mode 100644 odpf/entities/facets/properties.proto create mode 100644 odpf/entities/facets/schema.proto create mode 100644 odpf/entities/resources/dashboard.proto create mode 100644 odpf/entities/resources/group.proto create mode 100644 odpf/entities/resources/job.proto create mode 100644 odpf/entities/resources/table.proto create mode 100644 odpf/entities/resources/topic.proto create mode 100644 odpf/entities/resources/user.proto delete mode 100644 odpf/meta/Chart.proto delete mode 100644 odpf/meta/Dashboard.proto delete mode 100644 odpf/meta/Group.proto delete mode 100644 odpf/meta/Job.proto delete mode 100644 odpf/meta/Table.proto delete mode 100644 odpf/meta/Topic.proto delete mode 100644 odpf/meta/User.proto delete mode 100644 odpf/meta/common/Event.proto delete mode 100644 odpf/meta/common/Resource.proto delete mode 100644 odpf/meta/common/Timestamp.proto delete mode 100644 odpf/meta/facets/Columns.proto delete mode 100644 odpf/meta/facets/Custom.proto delete mode 100644 odpf/meta/facets/Lineage.proto delete mode 100644 odpf/meta/facets/Ownership.proto delete mode 100644 odpf/meta/facets/Tags.proto delete mode 100644 odpf/meta/facets/TopicSchema.proto create mode 100644 odpf/metadata/dataset.proto diff --git a/.project b/.project new file mode 100644 index 00000000..cf127aba --- /dev/null +++ b/.project @@ -0,0 +1,28 @@ + + + proton + Project proton created by Buildship. + + + + + org.eclipse.buildship.core.gradleprojectbuilder + + + + + + org.eclipse.buildship.core.gradleprojectnature + + + + 1625977453235 + + 30 + + org.eclipse.core.resources.regexFilterMatcher + node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ + + + + diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs new file mode 100644 index 00000000..abac91ec --- /dev/null +++ b/.settings/org.eclipse.buildship.core.prefs @@ -0,0 +1,13 @@ +arguments= +auto.sync=false +build.scans.enabled=false +connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER) +connection.project.dir= +eclipse.preferences.version=1 +gradle.user.home= +java.home=/usr/local/Cellar/openjdk/16.0.1/libexec/openjdk.jdk/Contents/Home +jvm.arguments= +offline.mode=false +override.workspace.settings=true +show.console.view=true +show.executions.view=true diff --git a/odpf/entities/common/event.proto b/odpf/entities/common/event.proto new file mode 100644 index 00000000..b2bf5371 --- /dev/null +++ b/odpf/entities/common/event.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package odpf.entities.common; + +option java_package = "io.odpf.entities.common"; +option java_outer_classname = "Event"; +option go_package = "github.com/odpf/proton/entities/common"; + +import "google/protobuf/timestamp.proto"; + +// Event represents an event in the system. +// Event is majorly used to represent the state of the system in the form of events. +// It can be used in any schema which intend to produce events to message bus. +message Event { + // Required. The timestamp of the event. + // For example "2018-01-01T00:00:00Z". + google.protobuf.Timestamp timestamp = 1; + // Optional. The activity that created the event. + // For example "create", "update". + string action = 2; + // Optional. The description of the event. + // For example "user is created from signup form". + string description = 3; + +} diff --git a/odpf/entities/common/resource.proto b/odpf/entities/common/resource.proto new file mode 100644 index 00000000..bb84055f --- /dev/null +++ b/odpf/entities/common/resource.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package odpf.entities.common; + +option java_package = "io.odpf.entities.common"; +option java_outer_classname = "Resource"; +option go_package = "github.com/odpf/proton/entities/common"; + + +// Resource is a generic resource that represents a file or other resource. +// It can be a data asset, job, user or group. +message Resource { + // Required. The unique identifier of the resource. + // For example. "user:jdoe" or "group:accounting". + string urn = 1; + // Required. The name of the resource. + // For example. "John Doe" or "Accounting". + string name = 2; + // Required. The source of the resource. + // For example. "github" or "bigquery". + string source = 3; + // Required. The type of the resource. + // For example. "user" or "group". + string type = 4; +} diff --git a/odpf/entities/common/timestamp.proto b/odpf/entities/common/timestamp.proto new file mode 100644 index 00000000..1679891c --- /dev/null +++ b/odpf/entities/common/timestamp.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package odpf.entities.common; + +option java_package = "io.odpf.entities.common"; +option java_outer_classname = "Timestamp"; +option go_package = "github.com/odpf/proton/entities/common"; + +import "google/protobuf/timestamp.proto"; + +// Timestamp represents created and modified timestamps. +message Timestamp { + // Created is the timestamp when the object was created. + google.protobuf.Timestamp created_at = 1; + // Updated is the timestamp when the object was last modified. + google.protobuf.Timestamp updated_at = 2; +} diff --git a/odpf/entities/facets/columns.proto b/odpf/entities/facets/columns.proto new file mode 100644 index 00000000..e9c397d4 --- /dev/null +++ b/odpf/entities/facets/columns.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package odpf.entities.facets; + +option java_package = "io.odpf.entities.facets"; +option java_outer_classname = "Columns"; +option go_package = "github.com/odpf/proton/entities/facets"; + + +// Columns represents a list of columns. +// It is facet used to specify the schema of a table or a file. +message Columns { + // The list of columns. + repeated Column columns = 1; +} + +// Column represents a column in a table or a file. +message Column { + // Required. The name of the column. + // For example, "customer_id". + string name = 1; + // Optional. The description of the column. + // For example, "The unique id of the customer" + string description = 2; + // Required. The data type of the column. + // For example, "INT64". + string data_type = 3; + // Optional. The format of the column. + // For example, "true". + bool is_nullable = 4; + // Optional. The length of the column. + // For example, "10". + int64 length = 5; +} diff --git a/odpf/entities/facets/lineage.proto b/odpf/entities/facets/lineage.proto new file mode 100644 index 00000000..1318a9af --- /dev/null +++ b/odpf/entities/facets/lineage.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package odpf.entities.facets; + +option java_package = "io.odpf.entities.facets"; +option java_outer_classname = "Lineage"; +option go_package = "github.com/odpf/proton/entities/facets"; + +import "odpf/entities/common/resource.proto"; + +// Linage reprsents the relationship of resource to other resources. +// Relation is way of describing the relationship between two resources. +message Lineage { + // The resource that is the source of the relationship. + // For example, a resource that is the parent of another resource. + repeated odpf.entities.common.Resource upstreams = 1; + // The resource that is the destination of the relationship. + // For example, a resource that is the child of another resource. + repeated odpf.entities.common.Resource downstreams = 2; +} diff --git a/odpf/entities/facets/ownership.proto b/odpf/entities/facets/ownership.proto new file mode 100644 index 00000000..f54031c2 --- /dev/null +++ b/odpf/entities/facets/ownership.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package odpf.entities.facets; + +option java_package = "io.odpf.entities.facets"; +option java_outer_classname = "Ownership"; +option go_package = "github.com/odpf/proton/entities/facets"; + +import "odpf/entities/common/resource.proto"; + +// Ownership is a facet that describes the ownership of a resource. +message Ownership { + // Requird: The list of owners of the resource. + // For an example check the owner schema. + repeated Owner owners = 7; +} + +// Owner is a facet that describes the owner of a resource. +message Owner { + // Required: The name of the owner. + // For example: "user:johndoe". + string urn = 1; + // Optional: The name of the owner. + // For example: "John Doe". + string name = 2; + // Optional: The role of the owner. + // For example: "admin", "steward". + string role = 3; +} \ No newline at end of file diff --git a/odpf/entities/facets/properties.proto b/odpf/entities/facets/properties.proto new file mode 100644 index 00000000..d3fb09c9 --- /dev/null +++ b/odpf/entities/facets/properties.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package odpf.entities.facets; + +option java_package = "io.odpf.entities.facets"; +option java_outer_classname = "Properties"; +option go_package = "github.com/odpf/proton/entities/facets"; + +import "odpf/entities/common/resource.proto"; + +message Properties { + + // Optional: List of tags the user has. + repeated string tags = 1; + + //Optional. List of labels the user has. + map labels = 2; + + // Optional: List of properties the user has. + map fields = 3; + +} \ No newline at end of file diff --git a/odpf/entities/facets/schema.proto b/odpf/entities/facets/schema.proto new file mode 100644 index 00000000..dcea24a6 --- /dev/null +++ b/odpf/entities/facets/schema.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package odpf.entities.facets; + +option java_package = "io.odpf.entities.facets"; +option java_outer_classname = "Schema"; +option go_package = "github.com/odpf/proton/entities/facets"; + + +message TopicSchema { + string schema_url = 1; + string format = 2; +} diff --git a/odpf/entities/resources/dashboard.proto b/odpf/entities/resources/dashboard.proto new file mode 100644 index 00000000..09546f70 --- /dev/null +++ b/odpf/entities/resources/dashboard.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "Dashboard"; +option go_package = "github.com/odpf/proton/entities/resources"; + +import "odpf/entities/facets/schema.proto"; +import "odpf/entities/facets/ownership.proto"; +import "odpf/entities/facets/lineage.proto"; +import "odpf/entities/facets/properties.proto"; + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +// Dashboard is a resource that represents a dashboard. +message Dashboard { + // Required. The unique identifier for the dashboard. + // For example, "dashboard-1". + string urn = 1; + // Optional. The name of the dashboard. + // For example, "Dashboard 1". + string name = 2; + // Required. The source of the dashboard. + // For example, "metabase". + string source = 3; + // Optional. The description of the dashboard. + // For example, "This dashboard was created by the Metabase team." + string description = 4; + // Required. The url of the dashboard. + // For example, "https://metabase.com/dashboard/dashboard-1". + string url = 5; + // Optional. The list of the charts in the dashboard. + // For an example, check the schema of the chart. + Chart charts = 21; + // Optional. The ownership of the topic. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + odpf.entities.facets.Properties properties = 32; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 33; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +message Chart { + // Required. The URN of the chart. + // For example, chart:1. + string urn = 1; + // Required. The name of the chart. + // For example, "My Chart". + string name = 2; + // Optional. The type of the chart. + // For example, "line". + string type = 3; + // Optional. The source of the chart. + // For example, "metabase". + string source = 4; + // Optional. The description of the chart. + // For example, "This is a chart for my dashboard." + string description = 5; + // Optional. The url of the chart. + // For example, "http://metabase.com/charts/mychart". + string url = 6; + // Optional. The raw query of the chart. + // For example, "SELECT * FROM my_table". + string raw_query = 7; + // Optional. The dashboard ur of the chart. + // For example, "dashboard:1". + string dashboard_urn = 8; + // Optional. The source of the dashboard of the chart. + // For example, "metabase". + string dashboard_source = 9; + // Optional. The ownership of the topic. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + // Optional. The lineage of the topic. + // For an example check out lineage schema. + odpf.entities.facets.Lineage lineage = 32; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 33; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 34; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} diff --git a/odpf/entities/resources/group.proto b/odpf/entities/resources/group.proto new file mode 100644 index 00000000..1e600506 --- /dev/null +++ b/odpf/entities/resources/group.proto @@ -0,0 +1,53 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "Group"; +option go_package = "github.com/odpf/proton/entities/resources"; + + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +import "odpf/entities/facets/properties.proto"; + +// Group represents a group of users and resources. +message Group { + // Required. The unique identifier for the group. + // For example, "group:example". + string urn = 1; + // Optional. The name of the group. + // For example, "Example Group". + string name = 2; + // Optional. The email of the group. + // For example, "xyz@xyz.com" + string email = 3; + // Optional. Source of the group. + // For example, "example.com". + string source = 4; + // Required. The members of the group. + // For example look at schema of the member. + repeated Member members = 21; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 31; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 32; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +// Member represents a user. +message Member { + // Required. The unique identifier for the user. + // For example, "user:example". + string urn = 1; + // Optional. The role of the user. + // For example, "owner". + string role = 2; +} \ No newline at end of file diff --git a/odpf/entities/resources/job.proto b/odpf/entities/resources/job.proto new file mode 100644 index 00000000..a59fd1cd --- /dev/null +++ b/odpf/entities/resources/job.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "Job"; +option go_package = "github.com/odpf/proton/entities/resources"; + +import "odpf/entities/facets/columns.proto"; +import "odpf/entities/facets/ownership.proto"; +import "odpf/entities/facets/lineage.proto"; +import "odpf/entities/facets/properties.proto"; + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +// Job is a resource that represents a job. +message Job { + // Required. The unique identifier of the job. + // For example, "job_1". + string urn = 1; + // Optional. The name of the job. + // For example, "Job 1". + string name = 2; + // Optional. The source of the job. + // For example, "airflow". + string source = 3; + // Optional. The type of the job. + // For example, "firehose", "stream". + string type = 4; + // Optional. The description of the job. + // For example, "This job is used to process data from a stream." + string description = 5; + // Optional. The ownership of the topic. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + // Optional. The lineage of the topic. + // For an example check out lineage schema. + odpf.entities.facets.Lineage lineage = 32; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 33; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 34; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} \ No newline at end of file diff --git a/odpf/entities/resources/table.proto b/odpf/entities/resources/table.proto new file mode 100644 index 00000000..74e77a04 --- /dev/null +++ b/odpf/entities/resources/table.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "Table"; +option go_package = "github.com/odpf/proton/entities/resources"; + +import "odpf/entities/facets/columns.proto"; +import "odpf/entities/facets/ownership.proto"; +import "odpf/entities/facets/lineage.proto"; +import "odpf/entities/facets/properties.proto"; + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +// Table is a table in a database. +// It can be a file, a table, a view, a materialized view, a temporary table, or a virtual table. +message Table { + // Required. The URN of the table. + // For example, "my-database.my-schema.my-table". + string urn = 1; + // Required. The name of the table. + // For example, "my-table". + string name = 2; + // Optional. The source of the table. + // For example, "bigquery". + string source = 3; + // Optional. The description of the table. + // For example, "My User table". + string description = 4; + // Optional. The metrics about the table. + // For example check the profile schem. + TableProfile profile = 21; + // Optional. The columns of the table. + // For example, "id, name, age". + odpf.entities.facets.Columns schema = 22; + // Optional. The ownership of the topic. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + // Optional. The lineage of the topic. + // For an example check out lineage schema. + odpf.entities.facets.Lineage lineage = 32; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 33; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 34; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +// TableProfile is the metrics about the table. +message TableProfile { + // Optional. The number of rows in the table. + // For example, "100". + int64 total_rows = 1; + // Optional. The number of rows in the table that are not deleted. + // For example, "event_timestamp". + string partition_key = 2; + string partition_value = 3; +} diff --git a/odpf/entities/resources/topic.proto b/odpf/entities/resources/topic.proto new file mode 100644 index 00000000..8049bf2e --- /dev/null +++ b/odpf/entities/resources/topic.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "Topic"; +option go_package = "github.com/odpf/proton/entities/resources"; + +import "odpf/entities/facets/schema.proto"; +import "odpf/entities/facets/ownership.proto"; +import "odpf/entities/facets/lineage.proto"; +import "odpf/entities/facets/properties.proto"; + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +// Topic is resource that represents a logical group of messages. +message Topic { + // Required. The URN of the topic. + // For example, "urn:odpf:topic:my-topic". + string urn = 1; + // Required. The name of the topic. + // For example, "My Topic". + string name = 2; + // Optional. The source of the topic. + // For example, "kafka", "pubsub". + string source = 3; + // Optional. The description of the topic. + // For example, "This is a topic for my application." + string description = 4; + // Optional. The metrics of the topic. + // For an example check out topic profile schema. + TopicProfile profile = 21; + // Optional. The schama of the topic. + // For an example check out topic schema. + odpf.entities.facets.TopicSchema schema = 31; + // Optional. The ownership of the topic. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 32; + // Optional. The lineage of the topic. + // For an example check out lineage schema. + odpf.entities.facets.Lineage lineage = 33; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 34; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 35; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +// TopicProfile is the profile of the topic. +message TopicProfile { + // Optional. The thrroughput of the topic. + // For example: "1m/minute". + string throughput = 1; + // Optional. The number of partitions in the topic. + // For example: 12. + int64 number_of_partitions = 2; +} diff --git a/odpf/entities/resources/user.proto b/odpf/entities/resources/user.proto new file mode 100644 index 00000000..c46949ad --- /dev/null +++ b/odpf/entities/resources/user.proto @@ -0,0 +1,93 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "User"; +option go_package = "github.com/odpf/proton/entities/resources"; + + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +import "odpf/entities/facets/properties.proto"; + +// User is a person who uses or operates something. +// It can be a user of the system, or a user of a device. +// User is a resource that represents a user. +message User { + // Required. The unique identifier of the user + // For example "user:jdoe" + string urn = 1; + // Optional. The source of the user account. + // For example "github" + string source = 2; + // Required. The emai address of the user. + // For example "job.deo@gmail.com" + string email = 3; + // Required. The username of the user. + // For example "johndoe" + string username = 4; + // Optional. The first name of the user. + // For example "john" + string first_name = 5; + // Optional. The last name of the user. + // For example "doe" + string last_name = 6; + // Optional. The full name of the user. + // For example "john mayer doe" + string full_name = 7; + // Optional. The display name of the user. + // For example "John M. Doe" + string display_name = 8; + // Optional. The job title of the user, + // For example "data engineer" + string title = 9; + // Required. The status of the user. + // For example "true" + bool is_active = 10; + // Optional. The email of the manger of the user. + // For example "rambo.ryan@gmail.com" + string manager_email = 11; + // Optional. List of the user social media accounts. + // For an example check out the profile schema. + repeated Profile profiles = 21; + // Optional. List of the groups user belongs to. + // A user can be part of multiple groups and have a different role in every group. + // For an example check out the membership schema. + repeated Membership memberships = 22; + // Optional. List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 31; + // Optional. The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 32; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +// Membership is a relationship between a user and a group. +message Membership { + // Required. The unique identifier of the group. + // For example "group:mygroup" + string group_urn = 1; + // Required. The role user has in the group. + // For example "owner" + repeated string role = 2; +} + +// Profile is a social media account of the user. +message Profile { + // Required. The unique identifier of the profile. + // For example "profile:jdoe" + string id = 1; + // Required. The type of the profile. + // For example "github" + string platform = 2; + // Required. The url of the profile. + // For example "http://github.com/jdoe" + string url = 3; +} diff --git a/odpf/meta/Chart.proto b/odpf/meta/Chart.proto deleted file mode 100644 index 27c4d2c4..00000000 --- a/odpf/meta/Chart.proto +++ /dev/null @@ -1,36 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Chart"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/Ownership.proto"; -import "odpf/meta/facets/Lineage.proto"; -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message ChartKey { - string urn = 1; -} - -message Chart { - string urn = 1; - string name = 2; - string type = 3; - string source = 4; - string description = 5; - string url = 6; - string raw_query = 7; - string dashboard_urn = 8; - string dashboard_source = 9; - odpf.meta.facets.Ownership ownership = 21; - odpf.meta.facets.Lineage lineage = 22; - odpf.meta.facets.Tags tags = 23; - odpf.meta.facets.Custom custom = 24; - odpf.meta.common.Timestamp timestamps = 25; - odpf.meta.common.Event event = 100; -} diff --git a/odpf/meta/Dashboard.proto b/odpf/meta/Dashboard.proto deleted file mode 100644 index a6dc539e..00000000 --- a/odpf/meta/Dashboard.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Dashboard"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/Chart.proto"; -import "odpf/meta/facets/Ownership.proto"; -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message DashboardKey { - string urn = 1; -} - -message Dashboard { - string urn = 1; - string name = 2; - string source = 3; - string description = 4; - string url = 5; - repeated odpf.meta.Chart charts = 6; - odpf.meta.facets.Ownership ownership = 21; - odpf.meta.facets.Tags tags = 22; - odpf.meta.facets.Custom custom = 23; - odpf.meta.common.Timestamp timestamps = 24; - odpf.meta.common.Event event = 100; -} diff --git a/odpf/meta/Group.proto b/odpf/meta/Group.proto deleted file mode 100644 index 4dd24ba1..00000000 --- a/odpf/meta/Group.proto +++ /dev/null @@ -1,33 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Group"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message GroupKey { - string urn = 1; -} - -message Group { - string urn = 1; - string name = 2; - string email = 3; - string source = 4; - repeated Member members = 5; - odpf.meta.facets.Tags tags = 21; - odpf.meta.facets.Custom custom = 22; - odpf.meta.common.Timestamp timestamps = 23; - odpf.meta.common.Event event = 100; -} - -message Member { - string urn = 1; - string role = 2; -} \ No newline at end of file diff --git a/odpf/meta/Job.proto b/odpf/meta/Job.proto deleted file mode 100644 index 137f2081..00000000 --- a/odpf/meta/Job.proto +++ /dev/null @@ -1,30 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Job"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/Lineage.proto"; -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message JobKey { - string urn = 1; -} - -message Job { - string urn = 1; - string name = 2; - string source = 3; - string type = 4; - string description = 5; - repeated odpf.meta.facets.Lineage lineage = 6; - odpf.meta.facets.Tags tags = 21; - odpf.meta.facets.Custom custom = 22; - odpf.meta.common.Timestamp timestamps = 23; - odpf.meta.common.Event event = 100; -} diff --git a/odpf/meta/Table.proto b/odpf/meta/Table.proto deleted file mode 100644 index 2418b51a..00000000 --- a/odpf/meta/Table.proto +++ /dev/null @@ -1,40 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Table"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/Columns.proto"; -import "odpf/meta/facets/Ownership.proto"; -import "odpf/meta/facets/Lineage.proto"; -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message TableKey { - string urn = 1; -} - -message Table { - string urn = 1; - string name = 2; - string source = 3; - string description = 4; - TableProfile profile = 5; - odpf.meta.facets.Columns schema = 6; - odpf.meta.facets.Ownership ownership = 7; - odpf.meta.facets.Lineage lineage = 8; - odpf.meta.facets.Tags tags = 21; - odpf.meta.facets.Custom custom = 22; - odpf.meta.common.Timestamp timestamps = 23; - odpf.meta.common.Event event = 100; -} - -message TableProfile { - int64 total_rows = 1; - string partition_key = 2; - string partition_value = 3; -} diff --git a/odpf/meta/Topic.proto b/odpf/meta/Topic.proto deleted file mode 100644 index 97a892c4..00000000 --- a/odpf/meta/Topic.proto +++ /dev/null @@ -1,39 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "Topic"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/TopicSchema.proto"; -import "odpf/meta/facets/Ownership.proto"; -import "odpf/meta/facets/Lineage.proto"; -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message TopicKey { - string urn = 1; -} - -message Topic { - string urn = 1; - string name = 2; - string source = 3; - string description = 4; - TopicProfile profile = 5; - odpf.meta.facets.TopicSchema schema = 6; - odpf.meta.facets.Ownership ownership = 7; - odpf.meta.facets.Lineage lineage = 8; - odpf.meta.facets.Tags tags = 21; - odpf.meta.facets.Custom custom = 22; - odpf.meta.common.Timestamp timestamps = 23; - odpf.meta.common.Event event = 100; -} - -message TopicProfile { - int64 throughput = 1; - int64 number_of_partitions = 2; -} diff --git a/odpf/meta/User.proto b/odpf/meta/User.proto deleted file mode 100644 index e8d20e0b..00000000 --- a/odpf/meta/User.proto +++ /dev/null @@ -1,48 +0,0 @@ -syntax = "proto3"; - -package odpf.meta; - -option java_package = "io.odpf.proton.meta"; -option java_outer_classname = "User"; -option go_package = "github.com/odpf/proton/meta"; - -import "odpf/meta/facets/Tags.proto"; -import "odpf/meta/facets/Custom.proto"; -import "odpf/meta/common/Timestamp.proto"; -import "odpf/meta/common/Event.proto"; - -message UserKey { - string urn = 1; -} - -message User { - string urn = 1; - string source = 2; - string email = 3; - string username = 4; - string first_name = 5; - string last_name = 6; - string full_name = 7; - string display_name = 8; - string title = 9; - bool is_active = 10; - string manager_email = 11; - repeated Profile profiles = 12; - repeated Membership memberships = 20; - odpf.meta.facets.Tags tags = 21; - odpf.meta.facets.Custom custom = 22; - odpf.meta.common.Timestamp timestamps = 23; - odpf.meta.common.Event event = 100; -} - -message Membership { - string group_urn = 1; - repeated string role = 2; -} - -message Profile { - string id = 1; - string platform = 2; - string url = 3; -} - diff --git a/odpf/meta/common/Event.proto b/odpf/meta/common/Event.proto deleted file mode 100644 index 3cd37af6..00000000 --- a/odpf/meta/common/Event.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.common; - -option java_package = "io.odpf.proton.meta.common"; -option java_outer_classname = "Event"; -option go_package = "github.com/odpf/proton/meta/common"; - -import "google/protobuf/timestamp.proto"; - -message Event { - string action = 1; - string description = 2; - google.protobuf.Timestamp timestamp = 100; -} diff --git a/odpf/meta/common/Resource.proto b/odpf/meta/common/Resource.proto deleted file mode 100644 index fbdde09b..00000000 --- a/odpf/meta/common/Resource.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.common; - -option java_package = "io.odpf.proton.meta.common"; -option java_outer_classname = "Resource"; -option go_package = "github.com/odpf/proton/meta/common"; - -message Resource { - string urn = 1; - string name = 2; - string source = 3; - string type = 4; -} diff --git a/odpf/meta/common/Timestamp.proto b/odpf/meta/common/Timestamp.proto deleted file mode 100644 index eec94f97..00000000 --- a/odpf/meta/common/Timestamp.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.common; - -option java_package = "io.odpf.proton.meta.common"; -option java_outer_classname = "Timestamp"; -option go_package = "github.com/odpf/proton/meta/common"; - -import "google/protobuf/timestamp.proto"; - -message Timestamp { - google.protobuf.Timestamp created_at = 1; - google.protobuf.Timestamp updated_at = 2; -} diff --git a/odpf/meta/facets/Columns.proto b/odpf/meta/facets/Columns.proto deleted file mode 100644 index c3756554..00000000 --- a/odpf/meta/facets/Columns.proto +++ /dev/null @@ -1,19 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "Columns"; -option go_package = "github.com/odpf/proton/meta/facets"; - -message Columns { - repeated Column columns = 1; -} - -message Column { - string name = 1; - string description = 2; - string data_type = 3; - bool is_nullable = 4; - int64 length = 5; -} diff --git a/odpf/meta/facets/Custom.proto b/odpf/meta/facets/Custom.proto deleted file mode 100644 index 5997697c..00000000 --- a/odpf/meta/facets/Custom.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "Custom"; -option go_package = "github.com/odpf/proton/meta/facets"; - -message Custom { - map custom_properties = 1; -} diff --git a/odpf/meta/facets/Lineage.proto b/odpf/meta/facets/Lineage.proto deleted file mode 100644 index 0b54206b..00000000 --- a/odpf/meta/facets/Lineage.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "Lineage"; -option go_package = "github.com/odpf/proton/meta/facets"; - -import "odpf/meta/common/Resource.proto"; - -message Lineage { - repeated odpf.meta.common.Resource upstreams = 1; - repeated odpf.meta.common.Resource downstreams = 2; -} diff --git a/odpf/meta/facets/Ownership.proto b/odpf/meta/facets/Ownership.proto deleted file mode 100644 index da58c7f7..00000000 --- a/odpf/meta/facets/Ownership.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "Ownership"; -option go_package = "github.com/odpf/proton/meta/facets"; - -message Ownership { - repeated Owner owners = 7; -} - -message Owner { - string urn = 1; - string name = 2; - string role = 3; -} \ No newline at end of file diff --git a/odpf/meta/facets/Tags.proto b/odpf/meta/facets/Tags.proto deleted file mode 100644 index 22a6611a..00000000 --- a/odpf/meta/facets/Tags.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "Tags"; -option go_package = "github.com/odpf/proton/meta/facets"; - -message Tags { - map tags = 1; -} diff --git a/odpf/meta/facets/TopicSchema.proto b/odpf/meta/facets/TopicSchema.proto deleted file mode 100644 index 4f7f9a55..00000000 --- a/odpf/meta/facets/TopicSchema.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package odpf.meta.facets; - -option java_package = "io.odpf.proton.meta.facets"; -option java_outer_classname = "TopicSchema"; -option go_package = "github.com/odpf/proton/meta/facets"; - -message TopicSchema { - string schema_url = 1; - string format = 2; -} diff --git a/odpf/metadata/dataset.proto b/odpf/metadata/dataset.proto new file mode 100644 index 00000000..9cb3617b --- /dev/null +++ b/odpf/metadata/dataset.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +message Dataset { + +} \ No newline at end of file From f7acf6750ac5309aad39171e75f09f50abfca752 Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Sat, 14 Aug 2021 22:53:21 -0700 Subject: [PATCH 10/18] feat: update java outerclassnames --- odpf/entities/common/event.proto | 2 +- odpf/entities/common/resource.proto | 2 +- odpf/entities/common/timestamp.proto | 2 +- odpf/entities/facets/columns.proto | 2 +- odpf/entities/facets/lineage.proto | 2 +- odpf/entities/facets/ownership.proto | 2 +- odpf/entities/facets/properties.proto | 4 +--- odpf/entities/facets/schema.proto | 2 +- odpf/entities/resources/dashboard.proto | 3 +-- odpf/entities/resources/group.proto | 2 +- odpf/entities/resources/job.proto | 3 +-- odpf/entities/resources/table.proto | 2 +- odpf/entities/resources/topic.proto | 2 +- odpf/entities/resources/user.proto | 2 +- 14 files changed, 14 insertions(+), 18 deletions(-) diff --git a/odpf/entities/common/event.proto b/odpf/entities/common/event.proto index b2bf5371..f07715ed 100644 --- a/odpf/entities/common/event.proto +++ b/odpf/entities/common/event.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.common; option java_package = "io.odpf.entities.common"; -option java_outer_classname = "Event"; +option java_outer_classname = "EventProto"; option go_package = "github.com/odpf/proton/entities/common"; import "google/protobuf/timestamp.proto"; diff --git a/odpf/entities/common/resource.proto b/odpf/entities/common/resource.proto index bb84055f..de5b4c2f 100644 --- a/odpf/entities/common/resource.proto +++ b/odpf/entities/common/resource.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.common; option java_package = "io.odpf.entities.common"; -option java_outer_classname = "Resource"; +option java_outer_classname = "ResourceProto"; option go_package = "github.com/odpf/proton/entities/common"; diff --git a/odpf/entities/common/timestamp.proto b/odpf/entities/common/timestamp.proto index 1679891c..4908e419 100644 --- a/odpf/entities/common/timestamp.proto +++ b/odpf/entities/common/timestamp.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.common; option java_package = "io.odpf.entities.common"; -option java_outer_classname = "Timestamp"; +option java_outer_classname = "TimestampProto"; option go_package = "github.com/odpf/proton/entities/common"; import "google/protobuf/timestamp.proto"; diff --git a/odpf/entities/facets/columns.proto b/odpf/entities/facets/columns.proto index e9c397d4..754197e9 100644 --- a/odpf/entities/facets/columns.proto +++ b/odpf/entities/facets/columns.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.facets; option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "Columns"; +option java_outer_classname = "ColumnsProto"; option go_package = "github.com/odpf/proton/entities/facets"; diff --git a/odpf/entities/facets/lineage.proto b/odpf/entities/facets/lineage.proto index 1318a9af..944e582c 100644 --- a/odpf/entities/facets/lineage.proto +++ b/odpf/entities/facets/lineage.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.facets; option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "Lineage"; +option java_outer_classname = "LineageProto"; option go_package = "github.com/odpf/proton/entities/facets"; import "odpf/entities/common/resource.proto"; diff --git a/odpf/entities/facets/ownership.proto b/odpf/entities/facets/ownership.proto index f54031c2..a294e925 100644 --- a/odpf/entities/facets/ownership.proto +++ b/odpf/entities/facets/ownership.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.facets; option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "Ownership"; +option java_outer_classname = "OwnershipProto"; option go_package = "github.com/odpf/proton/entities/facets"; import "odpf/entities/common/resource.proto"; diff --git a/odpf/entities/facets/properties.proto b/odpf/entities/facets/properties.proto index d3fb09c9..e324af29 100644 --- a/odpf/entities/facets/properties.proto +++ b/odpf/entities/facets/properties.proto @@ -3,11 +3,9 @@ syntax = "proto3"; package odpf.entities.facets; option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "Properties"; +option java_outer_classname = "PropertiesProto"; option go_package = "github.com/odpf/proton/entities/facets"; -import "odpf/entities/common/resource.proto"; - message Properties { // Optional: List of tags the user has. diff --git a/odpf/entities/facets/schema.proto b/odpf/entities/facets/schema.proto index dcea24a6..431aa5c8 100644 --- a/odpf/entities/facets/schema.proto +++ b/odpf/entities/facets/schema.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.facets; option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "Schema"; +option java_outer_classname = "SchemaProto"; option go_package = "github.com/odpf/proton/entities/facets"; diff --git a/odpf/entities/resources/dashboard.proto b/odpf/entities/resources/dashboard.proto index 09546f70..62c1a3bd 100644 --- a/odpf/entities/resources/dashboard.proto +++ b/odpf/entities/resources/dashboard.proto @@ -3,10 +3,9 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "Dashboard"; +option java_outer_classname = "DashboardProto"; option go_package = "github.com/odpf/proton/entities/resources"; -import "odpf/entities/facets/schema.proto"; import "odpf/entities/facets/ownership.proto"; import "odpf/entities/facets/lineage.proto"; import "odpf/entities/facets/properties.proto"; diff --git a/odpf/entities/resources/group.proto b/odpf/entities/resources/group.proto index 1e600506..585b3951 100644 --- a/odpf/entities/resources/group.proto +++ b/odpf/entities/resources/group.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "Group"; +option java_outer_classname = "GroupProto"; option go_package = "github.com/odpf/proton/entities/resources"; diff --git a/odpf/entities/resources/job.proto b/odpf/entities/resources/job.proto index a59fd1cd..36a97d01 100644 --- a/odpf/entities/resources/job.proto +++ b/odpf/entities/resources/job.proto @@ -3,10 +3,9 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "Job"; +option java_outer_classname = "JobProto"; option go_package = "github.com/odpf/proton/entities/resources"; -import "odpf/entities/facets/columns.proto"; import "odpf/entities/facets/ownership.proto"; import "odpf/entities/facets/lineage.proto"; import "odpf/entities/facets/properties.proto"; diff --git a/odpf/entities/resources/table.proto b/odpf/entities/resources/table.proto index 74e77a04..bdda9bf1 100644 --- a/odpf/entities/resources/table.proto +++ b/odpf/entities/resources/table.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "Table"; +option java_outer_classname = "TableProto"; option go_package = "github.com/odpf/proton/entities/resources"; import "odpf/entities/facets/columns.proto"; diff --git a/odpf/entities/resources/topic.proto b/odpf/entities/resources/topic.proto index 8049bf2e..0a8c0cf8 100644 --- a/odpf/entities/resources/topic.proto +++ b/odpf/entities/resources/topic.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "Topic"; +option java_outer_classname = "TopicProto"; option go_package = "github.com/odpf/proton/entities/resources"; import "odpf/entities/facets/schema.proto"; diff --git a/odpf/entities/resources/user.proto b/odpf/entities/resources/user.proto index c46949ad..bd7ef729 100644 --- a/odpf/entities/resources/user.proto +++ b/odpf/entities/resources/user.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package odpf.entities.resources; option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "User"; +option java_outer_classname = "UserProto"; option go_package = "github.com/odpf/proton/entities/resources"; From cbb497afcc3b16c448f8973c1bf60f0ae2e0601e Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Tue, 24 Aug 2021 12:28:57 +0700 Subject: [PATCH 11/18] feat(metadata): add ColumnProfile --- odpf/entities/facets/columns.proto | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/odpf/entities/facets/columns.proto b/odpf/entities/facets/columns.proto index 754197e9..162328f8 100644 --- a/odpf/entities/facets/columns.proto +++ b/odpf/entities/facets/columns.proto @@ -6,6 +6,7 @@ option java_package = "io.odpf.entities.facets"; option java_outer_classname = "ColumnsProto"; option go_package = "github.com/odpf/proton/entities/facets"; +import "odpf/entities/facets/properties.proto"; // Columns represents a list of columns. // It is facet used to specify the schema of a table or a file. @@ -31,4 +32,20 @@ message Column { // Optional. The length of the column. // For example, "10". int64 length = 5; + // Optional. The profile of the column. + ColumnProfile profile = 6; + // Optional. List of the column's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a column. + // Properties are defined in the properties schema. + odpf.entities.facets.Properties properties = 31; +} + +message ColumnProfile { + string min = 1; + string max = 2; + double avg = 3; + double med = 4; + int64 unique = 5; + int64 count = 6; + string top = 7; } From 1785c28e7ef2bffbf2f6a1235aee25ed9f9874e6 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Tue, 24 Aug 2021 12:29:43 +0700 Subject: [PATCH 12/18] feat(metadata): update Properties facet to use dynamic custom fields --- odpf/entities/facets/properties.proto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/odpf/entities/facets/properties.proto b/odpf/entities/facets/properties.proto index e324af29..b9fbce1c 100644 --- a/odpf/entities/facets/properties.proto +++ b/odpf/entities/facets/properties.proto @@ -6,6 +6,8 @@ option java_package = "io.odpf.entities.facets"; option java_outer_classname = "PropertiesProto"; option go_package = "github.com/odpf/proton/entities/facets"; +import "google/protobuf/struct.proto"; + message Properties { // Optional: List of tags the user has. @@ -15,6 +17,6 @@ message Properties { map labels = 2; // Optional: List of properties the user has. - map fields = 3; + google.protobuf.Struct fields = 3; } \ No newline at end of file From 6d2478b2f3e0447886b5969cc5cb3279be040c0a Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Tue, 24 Aug 2021 12:30:01 +0700 Subject: [PATCH 13/18] feat(metadata): create bucket metadata model --- odpf/entities/resources/bucket.proto | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 odpf/entities/resources/bucket.proto diff --git a/odpf/entities/resources/bucket.proto b/odpf/entities/resources/bucket.proto new file mode 100644 index 00000000..69febf1e --- /dev/null +++ b/odpf/entities/resources/bucket.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package odpf.entities.resources; + +option java_package = "io.odpf.entities.resources"; +option java_outer_classname = "BucketProto"; +option go_package = "github.com/odpf/proton/entities/resources"; + +import "google/protobuf/timestamp.proto"; + +import "odpf/entities/facets/ownership.proto"; +import "odpf/entities/facets/lineage.proto"; +import "odpf/entities/facets/properties.proto"; + +import "odpf/entities/common/timestamp.proto"; +import "odpf/entities/common/event.proto"; + +message Bucket { + // Required. The unique identifier for the bucket. + // For example, "bucket-1". + string urn = 1; + // Optional. The name of the bucket. + // For example, "bucket 1". + string name = 2; + // Required. The source of the bucket. + // For example, "metabase". + string source = 3; + // Optional. The description of the bucket. + // For example, "This bucket was created by the product team." + string description = 4; + // Optional. The location of the bucket. Can differ based on cloud storage used. (e.g. GCS, S3, etc) + // For example, "ASIA" + string location = 5; + // Optional. The type of the storage. Can differ based on cloud storage used. (e.g. GCS, S3, etc) + // For example, "STANDARD" + string storage_type = 6; + repeated Blob blobs = 7; + // Optional. The ownership of the bucket. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + odpf.entities.facets.Properties properties = 32; + // Optional. The timestamp of the bucket's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a bucket. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 33; + // Optional. The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.entities.common.Event event = 100; +} + +message Blob { + // Required. The URN of the blob. + // For example, "location/bucket-name/file-name". + string urn = 1; + // Required. The name of the blob. + // For example, "file-name". + string name = 2; + // Optional. The source of the blob. + // For example, "gcs". + string source = 3; + // Optional. The description of the blob. + // For example, "This is a config file for x app" + string description = 4; + // Optional. The length of the object content. + // For example, 300 + int64 size = 5; + google.protobuf.Timestamp deleted_at = 6; + google.protobuf.Timestamp expired_at = 7; + // Optional. The ownership of the blob. + // For an example check out ownership. + odpf.entities.facets.Ownership ownership = 31; + odpf.entities.facets.Properties properties = 32; + // Optional. The timestamp of the blob's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a blob. + // Timestamps are defined in the timestamp schema. + odpf.entities.common.Timestamp timestamps = 33; +} \ No newline at end of file From 20251f9350537424a2e858e4952dbcd2a8ccd1c6 Mon Sep 17 00:00:00 2001 From: Stewart Jingga Date: Tue, 24 Aug 2021 12:30:52 +0700 Subject: [PATCH 14/18] feat(metadata): add data_source to Chart model --- odpf/entities/resources/dashboard.proto | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/odpf/entities/resources/dashboard.proto b/odpf/entities/resources/dashboard.proto index 62c1a3bd..7c87adc4 100644 --- a/odpf/entities/resources/dashboard.proto +++ b/odpf/entities/resources/dashboard.proto @@ -32,7 +32,7 @@ message Dashboard { string url = 5; // Optional. The list of the charts in the dashboard. // For an example, check the schema of the chart. - Chart charts = 21; + repeated Chart charts = 21; // Optional. The ownership of the topic. // For an example check out ownership. odpf.entities.facets.Ownership ownership = 31; @@ -68,24 +68,27 @@ message Chart { // Optional. The raw query of the chart. // For example, "SELECT * FROM my_table". string raw_query = 7; + // Optional. The source of the data. + // For example, "bigquery,graphite". + string data_source = 8; // Optional. The dashboard ur of the chart. // For example, "dashboard:1". - string dashboard_urn = 8; + string dashboard_urn = 9; // Optional. The source of the dashboard of the chart. // For example, "metabase". - string dashboard_source = 9; - // Optional. The ownership of the topic. + string dashboard_source = 10; + // Optional. The ownership of the dashboard. // For an example check out ownership. odpf.entities.facets.Ownership ownership = 31; - // Optional. The lineage of the topic. + // Optional. The lineage of the dashboard. // For an example check out lineage schema. odpf.entities.facets.Lineage lineage = 32; // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. + // Properties facet can be used to set custom properties, tags and labels for a dashboard. // Properties are defined in the properties schema. odpf.entities.facets.Properties properties = 33; // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. + // Timstamp facet can be used to set the creation and updation timestamp of a dashboard. // Timestamps are defined in the timestamp schema. odpf.entities.common.Timestamp timestamps = 34; // Optional. The timestamp of the generated event. From 3bf8d0dee8dcce0f4ed75f3c6c5ff341a0da7aa2 Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Thu, 26 Aug 2021 08:17:39 -0700 Subject: [PATCH 15/18] feat: update metadata models --- .settings/org.eclipse.buildship.core.prefs | 2 +- odpf/assets/bucket.proto | 102 +++++++++++++++ odpf/{entities => assets}/common/event.proto | 21 +-- odpf/assets/common/resource.proto | 32 +++++ odpf/assets/common/timestamp.proto | 28 ++++ odpf/assets/dashboard.proto | 121 ++++++++++++++++++ odpf/assets/facets/lineage.proto | 21 +++ .../facets/ownership.proto | 22 ++-- odpf/assets/facets/properties.proto | 22 ++++ odpf/assets/facets/schema.proto | 64 +++++++++ odpf/assets/group.proto | 58 +++++++++ odpf/assets/job.proto | 57 +++++++++ odpf/assets/table.proto | 74 +++++++++++ odpf/assets/topic.proto | 74 +++++++++++ odpf/assets/user.proto | 107 ++++++++++++++++ odpf/entities/common/resource.proto | 25 ---- odpf/entities/common/timestamp.proto | 17 --- odpf/entities/facets/columns.proto | 51 -------- odpf/entities/facets/lineage.proto | 20 --- odpf/entities/facets/properties.proto | 22 ---- odpf/entities/facets/schema.proto | 13 -- odpf/entities/resources/bucket.proto | 77 ----------- odpf/entities/resources/dashboard.proto | 97 -------------- odpf/entities/resources/group.proto | 53 -------- odpf/entities/resources/job.proto | 50 -------- odpf/entities/resources/table.proto | 66 ---------- odpf/entities/resources/topic.proto | 64 --------- odpf/entities/resources/user.proto | 93 -------------- 28 files changed, 784 insertions(+), 669 deletions(-) create mode 100644 odpf/assets/bucket.proto rename odpf/{entities => assets}/common/event.proto (50%) create mode 100644 odpf/assets/common/resource.proto create mode 100644 odpf/assets/common/timestamp.proto create mode 100644 odpf/assets/dashboard.proto create mode 100644 odpf/assets/facets/lineage.proto rename odpf/{entities => assets}/facets/ownership.proto (50%) create mode 100644 odpf/assets/facets/properties.proto create mode 100644 odpf/assets/facets/schema.proto create mode 100644 odpf/assets/group.proto create mode 100644 odpf/assets/job.proto create mode 100644 odpf/assets/table.proto create mode 100644 odpf/assets/topic.proto create mode 100644 odpf/assets/user.proto delete mode 100644 odpf/entities/common/resource.proto delete mode 100644 odpf/entities/common/timestamp.proto delete mode 100644 odpf/entities/facets/columns.proto delete mode 100644 odpf/entities/facets/lineage.proto delete mode 100644 odpf/entities/facets/properties.proto delete mode 100644 odpf/entities/facets/schema.proto delete mode 100644 odpf/entities/resources/bucket.proto delete mode 100644 odpf/entities/resources/dashboard.proto delete mode 100644 odpf/entities/resources/group.proto delete mode 100644 odpf/entities/resources/job.proto delete mode 100644 odpf/entities/resources/table.proto delete mode 100644 odpf/entities/resources/topic.proto delete mode 100644 odpf/entities/resources/user.proto diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs index abac91ec..656b35a5 100644 --- a/.settings/org.eclipse.buildship.core.prefs +++ b/.settings/org.eclipse.buildship.core.prefs @@ -5,7 +5,7 @@ connection.gradle.distribution=GRADLE_DISTRIBUTION(WRAPPER) connection.project.dir= eclipse.preferences.version=1 gradle.user.home= -java.home=/usr/local/Cellar/openjdk/16.0.1/libexec/openjdk.jdk/Contents/Home +java.home=/usr/local/Cellar/openjdk/16.0.2/libexec/openjdk.jdk/Contents/Home jvm.arguments= offline.mode=false override.workspace.settings=true diff --git a/odpf/assets/bucket.proto b/odpf/assets/bucket.proto new file mode 100644 index 00000000..d3b20071 --- /dev/null +++ b/odpf/assets/bucket.proto @@ -0,0 +1,102 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "BucketProto"; +option go_package = "github.com/odpf/proton/assets"; + +import "google/protobuf/timestamp.proto"; + +import "odpf/assets/facets/ownership.proto"; +import "odpf/assets/facets/lineage.proto"; +import "odpf/assets/facets/properties.proto"; + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +message Bucket { + // The unique identifier for the bucket. + // Example: `bucket-1`. + string urn = 1; + + // The name of the bucket. + // Example: `bucket 1`. + string name = 2; + + // The source of the bucket. + // Example: `metabase`. + string source = 3; + + // The description of the bucket. + // Example: `This bucket was created by the product team.` + string description = 4; + + // The location of the bucket. Can differ based on cloud storage used. (e.g. GCS, S3, etc) + // Example: `ASIA` + string location = 5; + + // The type of the storage. Can differ based on cloud storage used. (e.g. GCS, S3, etc) + // Example: `STANDARD` + string storage_type = 6; + + // List of blobs in the bucket. + repeated Blob blobs = 7; + + // The ownership of the bucket. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 32; + + // The timestamp of the bucket's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a bucket. + odpf.assets.common.Timestamp timestamps = 33; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; + +} + +message Blob { + // The URN of the blob. + // Example: `location/bucket-name/file-name`. + string urn = 1; + + // The name of the blob. + // Example: `file-name`. + string name = 2; + + // The source of the blob. + // Example: `gcs`. + string source = 3; + + // The description of the blob. + // Example: `This is a config file for x app` + string description = 4; + + // The length of the object content. + // Example: `300` + int64 size = 5; + + // Delete time of the blob object. + google.protobuf.Timestamp delete_time = 6; + + // Expire time of the blob object. + google.protobuf.Timestamp expire_time = 7; + + // The ownership of the blob. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 32; + + // The timestamp of the blob's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a blob. + odpf.assets.common.Timestamp timestamps = 33; +} \ No newline at end of file diff --git a/odpf/entities/common/event.proto b/odpf/assets/common/event.proto similarity index 50% rename from odpf/entities/common/event.proto rename to odpf/assets/common/event.proto index f07715ed..ea8ffc87 100644 --- a/odpf/entities/common/event.proto +++ b/odpf/assets/common/event.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -package odpf.entities.common; +package odpf.assets.common; -option java_package = "io.odpf.entities.common"; +option java_package = "io.odpf.assets.common"; option java_outer_classname = "EventProto"; -option go_package = "github.com/odpf/proton/entities/common"; +option go_package = "github.com/odpf/proton/assets/common"; import "google/protobuf/timestamp.proto"; @@ -12,14 +12,15 @@ import "google/protobuf/timestamp.proto"; // Event is majorly used to represent the state of the system in the form of events. // It can be used in any schema which intend to produce events to message bus. message Event { - // Required. The timestamp of the event. - // For example "2018-01-01T00:00:00Z". + // The timestamp of the event. + // Example: `2018-01-01T00:00:00Z`. google.protobuf.Timestamp timestamp = 1; - // Optional. The activity that created the event. - // For example "create", "update". + + // The activity that created the event. + // Example: `create`, `update`. string action = 2; - // Optional. The description of the event. - // For example "user is created from signup form". - string description = 3; + // The description of the event. + // Example: `user is created from signup form`. + string description = 3; } diff --git a/odpf/assets/common/resource.proto b/odpf/assets/common/resource.proto new file mode 100644 index 00000000..4ce080d6 --- /dev/null +++ b/odpf/assets/common/resource.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package odpf.assets.common; + +option java_package = "io.odpf.assets.common"; +option java_outer_classname = "ResourceProto"; +option go_package = "github.com/odpf/proton/assets/common"; + + +// Resource is a generic resource that represents a file or other resource. +// It can be a table, job, user or group. +message Resource { + // The unique identifier of the resource. + // Example: `user:jdoe` or `group:accounting`. + string urn = 1; + + // The name of the resource. + // Example: `John Doe` or `Accounting`. + string name = 2; + + // The source of the resource. + // Example: `github` or `bigquery`. + string source = 3; + + // The type of the asset. + // Example: `user` or `group`. + string type = 4; + + // The REST URL for accessing the resource. URL returns the resource itself. + // Example: `https://xyz.com/v1/users/user-123` + string resource_url = 5; +} diff --git a/odpf/assets/common/timestamp.proto b/odpf/assets/common/timestamp.proto new file mode 100644 index 00000000..df0e81d7 --- /dev/null +++ b/odpf/assets/common/timestamp.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package odpf.assets.common; + +option java_package = "io.odpf.assets.common"; +option java_outer_classname = "TimestampProto"; +option go_package = "github.com/odpf/proton/assets/common"; + +import "google/protobuf/timestamp.proto"; + +// Timestamp represents created and modified timestamps. +message Timestamp { + // The timestamp when the object was created. + google.protobuf.Timestamp create_time = 1; + + // The timestamp when the object was last modified. + google.protobuf.Timestamp update_time = 2; +} + +// A time window specified by its `start_time` and `end_time`. +message TimeWindow { + // Start time of the time window (exclusive). + google.protobuf.Timestamp start_time = 1; + + // End time of the time window (inclusive). If not specified, the current + // timestamp is used instead. + google.protobuf.Timestamp end_time = 2; +} \ No newline at end of file diff --git a/odpf/assets/dashboard.proto b/odpf/assets/dashboard.proto new file mode 100644 index 00000000..55a45f56 --- /dev/null +++ b/odpf/assets/dashboard.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "DashboardProto"; +option go_package = "github.com/odpf/proton/assets"; + +import "odpf/assets/facets/ownership.proto"; +import "odpf/assets/facets/lineage.proto"; +import "odpf/assets/facets/properties.proto"; + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +// Dashboard is a resource that represents a dashboard. +message Dashboard { + // The unique identifier for the dashboard. + // Example: "dashboard-1". + string urn = 1; + + // The name of the dashboard. + // Example: "Dashboard 1". + string name = 2; + + // The source of the dashboard. + // Example: "metabase". + string source = 3; + + // The description of the dashboard. + // Example: "This dashboard was created by the Metabase team." + string description = 4; + + // The url of the dashboard. + // Example: "https://metabase.com/dashboard/dashboard-1". + string url = 5; + + // The list of the charts in the dashboard. + // For an example, check the schema of the chart. + repeated Chart charts = 21; + + // The ownership of the topic. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 32; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 33; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; + +} + +message Chart { + // The URN of the chart. + // Example: `chart:1`. + string urn = 1; + + // The name of the chart. + // Example: `My Chart`. + string name = 2; + + // The type of the chart. + // Example: `line`. + string type = 3; + + // The source of the chart. + // Example: `metabase`. + string source = 4; + + // The description of the chart. + // Example: `This is a chart for my dashboard.` + string description = 5; + + // The url of the chart. + // Example: `http://metabase.com/charts/mychart`. + string url = 6; + + // The raw query of the chart. + // Example: `SELECT * FROM my_table`. + string raw_query = 7; + + // The source of the data. + // Example: `bigquery,graphite`. + string data_source = 8; + + // The dashboard ur of the chart. + // Example: `dashboard:1`. + string dashboard_urn = 9; + + // The source of the dashboard of the chart. + // Example: `metabase`. + string dashboard_source = 10; + + // The ownership of the dashboard. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // The lineage of the dashboard. + // For an example check out lineage schema. + odpf.assets.facets.Lineage lineage = 32; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a dashboard. + odpf.assets.facets.Properties properties = 33; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a dashboard. + odpf.assets.common.Timestamp timestamps = 34; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; + +} diff --git a/odpf/assets/facets/lineage.proto b/odpf/assets/facets/lineage.proto new file mode 100644 index 00000000..05366106 --- /dev/null +++ b/odpf/assets/facets/lineage.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package odpf.assets.facets; + +option java_package = "io.odpf.assets.facets"; +option java_outer_classname = "LineageProto"; +option go_package = "github.com/odpf/proton/assets/facets"; + +import "odpf/assets/common/resource.proto"; + +// Linage reprsents the relationship of resource to other resources. +// Relation is way of describing the relationship between two resources. +message Lineage { + // The resource that is the source of the relationship. + // Example: a resource that is the parent of another resource. + repeated odpf.assets.common.Resource upstreams = 1; + + // The resource that is the destination of the relationship. + // Example: a resource that is the child of another resource. + repeated odpf.assets.common.Resource downstreams = 2; +} diff --git a/odpf/entities/facets/ownership.proto b/odpf/assets/facets/ownership.proto similarity index 50% rename from odpf/entities/facets/ownership.proto rename to odpf/assets/facets/ownership.proto index a294e925..a3f57e21 100644 --- a/odpf/entities/facets/ownership.proto +++ b/odpf/assets/facets/ownership.proto @@ -1,12 +1,12 @@ syntax = "proto3"; -package odpf.entities.facets; +package odpf.assets.facets; -option java_package = "io.odpf.entities.facets"; +option java_package = "io.odpf.assets.facets"; option java_outer_classname = "OwnershipProto"; -option go_package = "github.com/odpf/proton/entities/facets"; +option go_package = "github.com/odpf/proton/assets/facets"; -import "odpf/entities/common/resource.proto"; +import "odpf/assets/common/resource.proto"; // Ownership is a facet that describes the ownership of a resource. message Ownership { @@ -17,13 +17,15 @@ message Ownership { // Owner is a facet that describes the owner of a resource. message Owner { - // Required: The name of the owner. - // For example: "user:johndoe". + // The name of the owner. + // Example: `user:johndoe`. + string urn = 1; - // Optional: The name of the owner. - // For example: "John Doe". + // The name of the owner. + // Example: `John Doe`. string name = 2; - // Optional: The role of the owner. - // For example: "admin", "steward". + + // The role of the owner. + // Example: `admin`, `steward`. string role = 3; } \ No newline at end of file diff --git a/odpf/assets/facets/properties.proto b/odpf/assets/facets/properties.proto new file mode 100644 index 00000000..d1a9fbb9 --- /dev/null +++ b/odpf/assets/facets/properties.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package odpf.assets.facets; + +option java_package = "io.odpf.assets.facets"; +option java_outer_classname = "PropertiesProto"; +option go_package = "github.com/odpf/proton/assets/facets"; + +import "google/protobuf/struct.proto"; + +message Properties { + + // Tags associated with this resource.. + repeated string tags = 1; + + // Labels associated with this resource. + map labels = 2; + + // Custom attributes associated with this resource. + google.protobuf.Struct attributes = 3; + +} \ No newline at end of file diff --git a/odpf/assets/facets/schema.proto b/odpf/assets/facets/schema.proto new file mode 100644 index 00000000..276c4b00 --- /dev/null +++ b/odpf/assets/facets/schema.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package odpf.assets.facets; + +option java_package = "io.odpf.assets.facets"; +option java_outer_classname = "SchemaProto"; +option go_package = "github.com/odpf/proton/assets/facets"; + + +import "odpf/assets/facets/properties.proto"; + +// Columns represents a list of columns. +// It is facet used to specify the schema of a table or a file. +message Columns { + // The list of columns. + repeated Column columns = 1; +} + +// Column represents a column in a table or a file. +message Column { + // The name of the column. + // Example: `customer_id`. + string name = 1; + + // The description of the column. + // Example: `The unique id of the customer` + string description = 2; + + // The data type of the column. + // Example: `INT64`. + + string data_type = 3; + // The format of the column. + // Example: `true`. + bool is_nullable = 4; + + // The length of the column. + // Example: `10`. + int64 length = 5; + + // The profile of the column. + ColumnProfile profile = 6; + + // Representation of the column properties. + // Properties facet can be used to set custom properties, tags and labels for a column. + odpf.assets.facets.Properties properties = 31; +} + +message ColumnProfile { + string min = 1; + string max = 2; + double avg = 3; + double med = 4; + int64 unique = 5; + int64 count = 6; + string top = 7; +} + +// TopicSchema represents a schema for message bus. +// It is facet used to specify the schema of a message bus. +message TopicSchema { + string schema_url = 1; + string format = 2; +} diff --git a/odpf/assets/group.proto b/odpf/assets/group.proto new file mode 100644 index 00000000..80605735 --- /dev/null +++ b/odpf/assets/group.proto @@ -0,0 +1,58 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "GroupProto"; +option go_package = "github.com/odpf/proton/assets"; + + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +import "odpf/assets/facets/properties.proto"; + +// Group represents a group of users and resources. +message Group { + // The unique identifier for the group. + // Example: `group:example`. + string urn = 1; + + // The name of the group. + // Example: `Example Group`. + string name = 2; + + // The email of the group. + // Example: `xyz@xyz.com` + string email = 3; + + // Source of the group. + // Example: `example.com`. + string source = 4; + + // The members of the group. + // For example look at schema of the member. + repeated Member members = 21; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 31; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 32; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; +} + +// Member represents a user. +message Member { + // The unique identifier for the user. + // Example: `user:example`. + string urn = 1; + // The role of the user. + // Example: `owner`. + string role = 2; +} \ No newline at end of file diff --git a/odpf/assets/job.proto b/odpf/assets/job.proto new file mode 100644 index 00000000..1df7ff7c --- /dev/null +++ b/odpf/assets/job.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "JobProto"; +option go_package = "github.com/odpf/proton/assets"; + +import "odpf/assets/facets/ownership.proto"; +import "odpf/assets/facets/lineage.proto"; +import "odpf/assets/facets/properties.proto"; + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +// Job is a resource that represents a job. +message Job { + // Required. The unique identifier of the job. + // Example: 'job_1'. + string urn = 1; + + // The name of the job. + // Example: 'Job 1'. + string name = 2; + + // The source of the job. + // Example: 'airflow'. + string source = 3; + + // The type of the job. + // Example: 'firehose', 'stream'. + string type = 4; + + // The description of the job. + // Example: 'This job is used to process data from a stream.' + string description = 5; + + // The ownership of the topic. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // The lineage of the topic. + // For an example check out lineage schema. + odpf.assets.facets.Lineage lineage = 32; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 33; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 34; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; +} \ No newline at end of file diff --git a/odpf/assets/table.proto b/odpf/assets/table.proto new file mode 100644 index 00000000..7a1d952e --- /dev/null +++ b/odpf/assets/table.proto @@ -0,0 +1,74 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "TableProto"; +option go_package = "github.com/odpf/proton/assets"; + +import "odpf/assets/facets/schema.proto"; +import "odpf/assets/facets/ownership.proto"; +import "odpf/assets/facets/lineage.proto"; +import "odpf/assets/facets/properties.proto"; + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +// Table is a table in a database. +// It can be a file, a table, a view, a materialized view, a temporary table, or a virtual table. +message Table { + // The URN of the table. + // Example: 'my-database.my-schema.my-table'. + string urn = 1; + // The name of the table. + // Example: 'my-table'. + string name = 2; + // The source of the table. + // Example: 'bigquery'. + string source = 3; + + // The description of the table. + // Example: 'My User table'. + string description = 4; + + // The metrics about the table. + // For example check the profile schem. + TableProfile profile = 21; + + // The columns of the table. + // Example: 'id', `name`, `age'. + odpf.assets.facets.Columns schema = 22; + + // The ownership of the topic. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 31; + + // The lineage of the topic. + // For an example check out lineage schema. + odpf.assets.facets.Lineage lineage = 32; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 33; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 34; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; +} + +// TableProfile is the metrics about the table. +message TableProfile { + // The number of rows in the table. + // Example: `100`. + int64 total_rows = 1; + + // The number of rows in the table that are not deleted. + // Example: `event_timestamp`. + string partition_key = 2; + + string partition_value = 3; +} diff --git a/odpf/assets/topic.proto b/odpf/assets/topic.proto new file mode 100644 index 00000000..e1f0b1a7 --- /dev/null +++ b/odpf/assets/topic.proto @@ -0,0 +1,74 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "TopicProto"; +option go_package = "github.com/odpf/proton/assets"; + +import "odpf/assets/facets/schema.proto"; +import "odpf/assets/facets/ownership.proto"; +import "odpf/assets/facets/lineage.proto"; +import "odpf/assets/facets/properties.proto"; + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +// Topic is resource that represents a logical group of messages +// in message bus like kafka, pubsub, pulsar etc. +message Topic { + // Required. The URN of the topic. + // Example: `urn:odpf:topic:my-topic`. + string urn = 1; + + // Required. The name of the topic. + // Example: `My Topic`. + string name = 2; + + // The source of the topic. + // Example: 'kafka', 'pubsub'. + string source = 3; + + // The description of the topic. + // Example: 'This is a topic for my application'. + string description = 4; + + // The metrics of the topic. + // For an example check out topic profile schema. + TopicProfile profile = 21; + + // The schama of the topic. + // For an example check out topic schema. + odpf.assets.facets.TopicSchema schema = 31; + + // The ownership of the topic. + // For an example check out ownership. + odpf.assets.facets.Ownership ownership = 32; + + // The lineage of the topic. + // For an example check out lineage schema. + odpf.assets.facets.Lineage lineage = 33; + + // List of the user's custom properties. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 34; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 35; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; +} + +// TopicProfile is the profile of the topic. +message TopicProfile { + // The thrroughput of the topic. + // Example: `1m/minute`. + string throughput = 1; + + // The number of partitions in the topic. + // Example: `12`. + int64 number_of_partitions = 2; +} diff --git a/odpf/assets/user.proto b/odpf/assets/user.proto new file mode 100644 index 00000000..4fb18709 --- /dev/null +++ b/odpf/assets/user.proto @@ -0,0 +1,107 @@ +syntax = "proto3"; + +package odpf.assets; + +option java_package = "io.odpf.assets"; +option java_outer_classname = "UserProto"; +option go_package = "github.com/odpf/proton/assets"; + + +import "odpf/assets/common/timestamp.proto"; +import "odpf/assets/common/event.proto"; + +import "odpf/assets/facets/properties.proto"; + +// User is a person who uses or operates something. +// It can be a user of the system, or a user of a device. +// User is a resource that represents a user. +message User { + // The unique identifier of the user + // Example: "user:jdoe" + string urn = 1; + + // The source of the user account. + // Example: `github` + string source = 2; + // The emai address of the user. + // Example: `job.deo@gmail.com` + string email = 3; + + // The username of the user. + // Example: `johndoe` + string username = 4; + + // The first name of the user. + // Example: `john` + string first_name = 5; + + // The last name of the user. + // Example: `doe` + string last_name = 6; + + // The full name of the user. + // Example: `john mayer doe` + string full_name = 7; + + // The display name of the user. + // Example: `John M. Doe` + string display_name = 8; + + // The job title of the user, + // Example: `data engineer` + string title = 9; + + // The status of the user. + // Example: `active` + string status = 10; + + // The email of the manger of the user. + // Example: `rambo.ryan@gmail.com` + string manager_email = 11; + + // List of the user social media accounts. + // For an example check out the profile schema. + repeated Profile profiles = 21; + + // List of the groups user belongs to. + // A user can be part of multiple groups and have a different role in every group. + repeated Membership memberships = 22; + + // Representation of custom properties of user. + // Properties facet can be used to set custom properties, tags and labels for a user. + odpf.assets.facets.Properties properties = 31; + + // The timestamp of the user's creation. + // Timstamp facet can be used to set the creation and updation timestamp of a user. + odpf.assets.common.Timestamp timestamps = 32; + + // The timestamp of the generated event. + // Event schemas is defined in the common event schema. + odpf.assets.common.Event event = 100; +} + +// Membership is a relationship between a user and a group. +message Membership { + // The unique identifier of the group. + // Example: `group:mygroup` + string group_urn = 1; + + // The role user has in the group. + // Example: "owner" + repeated string role = 2; +} + +// Profile is a social media account of the user. +message Profile { + // The unique identifier of the profile. + // Example: `profile:jdoe` + string id = 1; + + // The type of the profile. + // Example: `github` + string platform = 2; + + // The url of the profile. + // Example: `http://github.com/jdoe` + string url = 3; +} diff --git a/odpf/entities/common/resource.proto b/odpf/entities/common/resource.proto deleted file mode 100644 index de5b4c2f..00000000 --- a/odpf/entities/common/resource.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.common; - -option java_package = "io.odpf.entities.common"; -option java_outer_classname = "ResourceProto"; -option go_package = "github.com/odpf/proton/entities/common"; - - -// Resource is a generic resource that represents a file or other resource. -// It can be a data asset, job, user or group. -message Resource { - // Required. The unique identifier of the resource. - // For example. "user:jdoe" or "group:accounting". - string urn = 1; - // Required. The name of the resource. - // For example. "John Doe" or "Accounting". - string name = 2; - // Required. The source of the resource. - // For example. "github" or "bigquery". - string source = 3; - // Required. The type of the resource. - // For example. "user" or "group". - string type = 4; -} diff --git a/odpf/entities/common/timestamp.proto b/odpf/entities/common/timestamp.proto deleted file mode 100644 index 4908e419..00000000 --- a/odpf/entities/common/timestamp.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.common; - -option java_package = "io.odpf.entities.common"; -option java_outer_classname = "TimestampProto"; -option go_package = "github.com/odpf/proton/entities/common"; - -import "google/protobuf/timestamp.proto"; - -// Timestamp represents created and modified timestamps. -message Timestamp { - // Created is the timestamp when the object was created. - google.protobuf.Timestamp created_at = 1; - // Updated is the timestamp when the object was last modified. - google.protobuf.Timestamp updated_at = 2; -} diff --git a/odpf/entities/facets/columns.proto b/odpf/entities/facets/columns.proto deleted file mode 100644 index 162328f8..00000000 --- a/odpf/entities/facets/columns.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.facets; - -option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "ColumnsProto"; -option go_package = "github.com/odpf/proton/entities/facets"; - -import "odpf/entities/facets/properties.proto"; - -// Columns represents a list of columns. -// It is facet used to specify the schema of a table or a file. -message Columns { - // The list of columns. - repeated Column columns = 1; -} - -// Column represents a column in a table or a file. -message Column { - // Required. The name of the column. - // For example, "customer_id". - string name = 1; - // Optional. The description of the column. - // For example, "The unique id of the customer" - string description = 2; - // Required. The data type of the column. - // For example, "INT64". - string data_type = 3; - // Optional. The format of the column. - // For example, "true". - bool is_nullable = 4; - // Optional. The length of the column. - // For example, "10". - int64 length = 5; - // Optional. The profile of the column. - ColumnProfile profile = 6; - // Optional. List of the column's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a column. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 31; -} - -message ColumnProfile { - string min = 1; - string max = 2; - double avg = 3; - double med = 4; - int64 unique = 5; - int64 count = 6; - string top = 7; -} diff --git a/odpf/entities/facets/lineage.proto b/odpf/entities/facets/lineage.proto deleted file mode 100644 index 944e582c..00000000 --- a/odpf/entities/facets/lineage.proto +++ /dev/null @@ -1,20 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.facets; - -option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "LineageProto"; -option go_package = "github.com/odpf/proton/entities/facets"; - -import "odpf/entities/common/resource.proto"; - -// Linage reprsents the relationship of resource to other resources. -// Relation is way of describing the relationship between two resources. -message Lineage { - // The resource that is the source of the relationship. - // For example, a resource that is the parent of another resource. - repeated odpf.entities.common.Resource upstreams = 1; - // The resource that is the destination of the relationship. - // For example, a resource that is the child of another resource. - repeated odpf.entities.common.Resource downstreams = 2; -} diff --git a/odpf/entities/facets/properties.proto b/odpf/entities/facets/properties.proto deleted file mode 100644 index b9fbce1c..00000000 --- a/odpf/entities/facets/properties.proto +++ /dev/null @@ -1,22 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.facets; - -option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "PropertiesProto"; -option go_package = "github.com/odpf/proton/entities/facets"; - -import "google/protobuf/struct.proto"; - -message Properties { - - // Optional: List of tags the user has. - repeated string tags = 1; - - //Optional. List of labels the user has. - map labels = 2; - - // Optional: List of properties the user has. - google.protobuf.Struct fields = 3; - -} \ No newline at end of file diff --git a/odpf/entities/facets/schema.proto b/odpf/entities/facets/schema.proto deleted file mode 100644 index 431aa5c8..00000000 --- a/odpf/entities/facets/schema.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.facets; - -option java_package = "io.odpf.entities.facets"; -option java_outer_classname = "SchemaProto"; -option go_package = "github.com/odpf/proton/entities/facets"; - - -message TopicSchema { - string schema_url = 1; - string format = 2; -} diff --git a/odpf/entities/resources/bucket.proto b/odpf/entities/resources/bucket.proto deleted file mode 100644 index 69febf1e..00000000 --- a/odpf/entities/resources/bucket.proto +++ /dev/null @@ -1,77 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "BucketProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - -import "google/protobuf/timestamp.proto"; - -import "odpf/entities/facets/ownership.proto"; -import "odpf/entities/facets/lineage.proto"; -import "odpf/entities/facets/properties.proto"; - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -message Bucket { - // Required. The unique identifier for the bucket. - // For example, "bucket-1". - string urn = 1; - // Optional. The name of the bucket. - // For example, "bucket 1". - string name = 2; - // Required. The source of the bucket. - // For example, "metabase". - string source = 3; - // Optional. The description of the bucket. - // For example, "This bucket was created by the product team." - string description = 4; - // Optional. The location of the bucket. Can differ based on cloud storage used. (e.g. GCS, S3, etc) - // For example, "ASIA" - string location = 5; - // Optional. The type of the storage. Can differ based on cloud storage used. (e.g. GCS, S3, etc) - // For example, "STANDARD" - string storage_type = 6; - repeated Blob blobs = 7; - // Optional. The ownership of the bucket. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - odpf.entities.facets.Properties properties = 32; - // Optional. The timestamp of the bucket's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a bucket. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 33; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -message Blob { - // Required. The URN of the blob. - // For example, "location/bucket-name/file-name". - string urn = 1; - // Required. The name of the blob. - // For example, "file-name". - string name = 2; - // Optional. The source of the blob. - // For example, "gcs". - string source = 3; - // Optional. The description of the blob. - // For example, "This is a config file for x app" - string description = 4; - // Optional. The length of the object content. - // For example, 300 - int64 size = 5; - google.protobuf.Timestamp deleted_at = 6; - google.protobuf.Timestamp expired_at = 7; - // Optional. The ownership of the blob. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - odpf.entities.facets.Properties properties = 32; - // Optional. The timestamp of the blob's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a blob. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 33; -} \ No newline at end of file diff --git a/odpf/entities/resources/dashboard.proto b/odpf/entities/resources/dashboard.proto deleted file mode 100644 index 7c87adc4..00000000 --- a/odpf/entities/resources/dashboard.proto +++ /dev/null @@ -1,97 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "DashboardProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - -import "odpf/entities/facets/ownership.proto"; -import "odpf/entities/facets/lineage.proto"; -import "odpf/entities/facets/properties.proto"; - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -// Dashboard is a resource that represents a dashboard. -message Dashboard { - // Required. The unique identifier for the dashboard. - // For example, "dashboard-1". - string urn = 1; - // Optional. The name of the dashboard. - // For example, "Dashboard 1". - string name = 2; - // Required. The source of the dashboard. - // For example, "metabase". - string source = 3; - // Optional. The description of the dashboard. - // For example, "This dashboard was created by the Metabase team." - string description = 4; - // Required. The url of the dashboard. - // For example, "https://metabase.com/dashboard/dashboard-1". - string url = 5; - // Optional. The list of the charts in the dashboard. - // For an example, check the schema of the chart. - repeated Chart charts = 21; - // Optional. The ownership of the topic. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - odpf.entities.facets.Properties properties = 32; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 33; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -message Chart { - // Required. The URN of the chart. - // For example, chart:1. - string urn = 1; - // Required. The name of the chart. - // For example, "My Chart". - string name = 2; - // Optional. The type of the chart. - // For example, "line". - string type = 3; - // Optional. The source of the chart. - // For example, "metabase". - string source = 4; - // Optional. The description of the chart. - // For example, "This is a chart for my dashboard." - string description = 5; - // Optional. The url of the chart. - // For example, "http://metabase.com/charts/mychart". - string url = 6; - // Optional. The raw query of the chart. - // For example, "SELECT * FROM my_table". - string raw_query = 7; - // Optional. The source of the data. - // For example, "bigquery,graphite". - string data_source = 8; - // Optional. The dashboard ur of the chart. - // For example, "dashboard:1". - string dashboard_urn = 9; - // Optional. The source of the dashboard of the chart. - // For example, "metabase". - string dashboard_source = 10; - // Optional. The ownership of the dashboard. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - // Optional. The lineage of the dashboard. - // For an example check out lineage schema. - odpf.entities.facets.Lineage lineage = 32; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a dashboard. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 33; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a dashboard. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 34; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} diff --git a/odpf/entities/resources/group.proto b/odpf/entities/resources/group.proto deleted file mode 100644 index 585b3951..00000000 --- a/odpf/entities/resources/group.proto +++ /dev/null @@ -1,53 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "GroupProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -import "odpf/entities/facets/properties.proto"; - -// Group represents a group of users and resources. -message Group { - // Required. The unique identifier for the group. - // For example, "group:example". - string urn = 1; - // Optional. The name of the group. - // For example, "Example Group". - string name = 2; - // Optional. The email of the group. - // For example, "xyz@xyz.com" - string email = 3; - // Optional. Source of the group. - // For example, "example.com". - string source = 4; - // Required. The members of the group. - // For example look at schema of the member. - repeated Member members = 21; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 31; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 32; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -// Member represents a user. -message Member { - // Required. The unique identifier for the user. - // For example, "user:example". - string urn = 1; - // Optional. The role of the user. - // For example, "owner". - string role = 2; -} \ No newline at end of file diff --git a/odpf/entities/resources/job.proto b/odpf/entities/resources/job.proto deleted file mode 100644 index 36a97d01..00000000 --- a/odpf/entities/resources/job.proto +++ /dev/null @@ -1,50 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "JobProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - -import "odpf/entities/facets/ownership.proto"; -import "odpf/entities/facets/lineage.proto"; -import "odpf/entities/facets/properties.proto"; - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -// Job is a resource that represents a job. -message Job { - // Required. The unique identifier of the job. - // For example, "job_1". - string urn = 1; - // Optional. The name of the job. - // For example, "Job 1". - string name = 2; - // Optional. The source of the job. - // For example, "airflow". - string source = 3; - // Optional. The type of the job. - // For example, "firehose", "stream". - string type = 4; - // Optional. The description of the job. - // For example, "This job is used to process data from a stream." - string description = 5; - // Optional. The ownership of the topic. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - // Optional. The lineage of the topic. - // For an example check out lineage schema. - odpf.entities.facets.Lineage lineage = 32; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 33; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 34; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} \ No newline at end of file diff --git a/odpf/entities/resources/table.proto b/odpf/entities/resources/table.proto deleted file mode 100644 index bdda9bf1..00000000 --- a/odpf/entities/resources/table.proto +++ /dev/null @@ -1,66 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "TableProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - -import "odpf/entities/facets/columns.proto"; -import "odpf/entities/facets/ownership.proto"; -import "odpf/entities/facets/lineage.proto"; -import "odpf/entities/facets/properties.proto"; - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -// Table is a table in a database. -// It can be a file, a table, a view, a materialized view, a temporary table, or a virtual table. -message Table { - // Required. The URN of the table. - // For example, "my-database.my-schema.my-table". - string urn = 1; - // Required. The name of the table. - // For example, "my-table". - string name = 2; - // Optional. The source of the table. - // For example, "bigquery". - string source = 3; - // Optional. The description of the table. - // For example, "My User table". - string description = 4; - // Optional. The metrics about the table. - // For example check the profile schem. - TableProfile profile = 21; - // Optional. The columns of the table. - // For example, "id, name, age". - odpf.entities.facets.Columns schema = 22; - // Optional. The ownership of the topic. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 31; - // Optional. The lineage of the topic. - // For an example check out lineage schema. - odpf.entities.facets.Lineage lineage = 32; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 33; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 34; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -// TableProfile is the metrics about the table. -message TableProfile { - // Optional. The number of rows in the table. - // For example, "100". - int64 total_rows = 1; - // Optional. The number of rows in the table that are not deleted. - // For example, "event_timestamp". - string partition_key = 2; - string partition_value = 3; -} diff --git a/odpf/entities/resources/topic.proto b/odpf/entities/resources/topic.proto deleted file mode 100644 index 0a8c0cf8..00000000 --- a/odpf/entities/resources/topic.proto +++ /dev/null @@ -1,64 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "TopicProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - -import "odpf/entities/facets/schema.proto"; -import "odpf/entities/facets/ownership.proto"; -import "odpf/entities/facets/lineage.proto"; -import "odpf/entities/facets/properties.proto"; - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -// Topic is resource that represents a logical group of messages. -message Topic { - // Required. The URN of the topic. - // For example, "urn:odpf:topic:my-topic". - string urn = 1; - // Required. The name of the topic. - // For example, "My Topic". - string name = 2; - // Optional. The source of the topic. - // For example, "kafka", "pubsub". - string source = 3; - // Optional. The description of the topic. - // For example, "This is a topic for my application." - string description = 4; - // Optional. The metrics of the topic. - // For an example check out topic profile schema. - TopicProfile profile = 21; - // Optional. The schama of the topic. - // For an example check out topic schema. - odpf.entities.facets.TopicSchema schema = 31; - // Optional. The ownership of the topic. - // For an example check out ownership. - odpf.entities.facets.Ownership ownership = 32; - // Optional. The lineage of the topic. - // For an example check out lineage schema. - odpf.entities.facets.Lineage lineage = 33; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 34; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 35; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -// TopicProfile is the profile of the topic. -message TopicProfile { - // Optional. The thrroughput of the topic. - // For example: "1m/minute". - string throughput = 1; - // Optional. The number of partitions in the topic. - // For example: 12. - int64 number_of_partitions = 2; -} diff --git a/odpf/entities/resources/user.proto b/odpf/entities/resources/user.proto deleted file mode 100644 index bd7ef729..00000000 --- a/odpf/entities/resources/user.proto +++ /dev/null @@ -1,93 +0,0 @@ -syntax = "proto3"; - -package odpf.entities.resources; - -option java_package = "io.odpf.entities.resources"; -option java_outer_classname = "UserProto"; -option go_package = "github.com/odpf/proton/entities/resources"; - - -import "odpf/entities/common/timestamp.proto"; -import "odpf/entities/common/event.proto"; - -import "odpf/entities/facets/properties.proto"; - -// User is a person who uses or operates something. -// It can be a user of the system, or a user of a device. -// User is a resource that represents a user. -message User { - // Required. The unique identifier of the user - // For example "user:jdoe" - string urn = 1; - // Optional. The source of the user account. - // For example "github" - string source = 2; - // Required. The emai address of the user. - // For example "job.deo@gmail.com" - string email = 3; - // Required. The username of the user. - // For example "johndoe" - string username = 4; - // Optional. The first name of the user. - // For example "john" - string first_name = 5; - // Optional. The last name of the user. - // For example "doe" - string last_name = 6; - // Optional. The full name of the user. - // For example "john mayer doe" - string full_name = 7; - // Optional. The display name of the user. - // For example "John M. Doe" - string display_name = 8; - // Optional. The job title of the user, - // For example "data engineer" - string title = 9; - // Required. The status of the user. - // For example "true" - bool is_active = 10; - // Optional. The email of the manger of the user. - // For example "rambo.ryan@gmail.com" - string manager_email = 11; - // Optional. List of the user social media accounts. - // For an example check out the profile schema. - repeated Profile profiles = 21; - // Optional. List of the groups user belongs to. - // A user can be part of multiple groups and have a different role in every group. - // For an example check out the membership schema. - repeated Membership memberships = 22; - // Optional. List of the user's custom properties. - // Properties facet can be used to set custom properties, tags and labels for a user. - // Properties are defined in the properties schema. - odpf.entities.facets.Properties properties = 31; - // Optional. The timestamp of the user's creation. - // Timstamp facet can be used to set the creation and updation timestamp of a user. - // Timestamps are defined in the timestamp schema. - odpf.entities.common.Timestamp timestamps = 32; - // Optional. The timestamp of the generated event. - // Event schemas is defined in the common event schema. - odpf.entities.common.Event event = 100; -} - -// Membership is a relationship between a user and a group. -message Membership { - // Required. The unique identifier of the group. - // For example "group:mygroup" - string group_urn = 1; - // Required. The role user has in the group. - // For example "owner" - repeated string role = 2; -} - -// Profile is a social media account of the user. -message Profile { - // Required. The unique identifier of the profile. - // For example "profile:jdoe" - string id = 1; - // Required. The type of the profile. - // For example "github" - string platform = 2; - // Required. The url of the profile. - // For example "http://github.com/jdoe" - string url = 3; -} From bd95471ef8d05e29b99256e971aec71c02955a2a Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Thu, 26 Aug 2021 08:21:58 -0700 Subject: [PATCH 16/18] chore: cleanup --- .gitignore | 1 + .project | 28 ---------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 .project diff --git a/.gitignore b/.gitignore index e6c8f4ea..ca4a25fe 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ .gradle/ .idea/ +.project/ build/ generated-protos/ diff --git a/.project b/.project deleted file mode 100644 index cf127aba..00000000 --- a/.project +++ /dev/null @@ -1,28 +0,0 @@ - - - proton - Project proton created by Buildship. - - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - - - org.eclipse.buildship.core.gradleprojectnature - - - - 1625977453235 - - 30 - - org.eclipse.core.resources.regexFilterMatcher - node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ - - - - From f2c55a907a56d0f96f97f5c977c1ee68fe94b44d Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Fri, 27 Aug 2021 01:33:09 -0700 Subject: [PATCH 17/18] feat: extract resource --- odpf/assets/bucket.proto | 14 +++----------- odpf/assets/common/resource.proto | 4 ++-- odpf/assets/dashboard.proto | 18 +++--------------- odpf/assets/facets/properties.proto | 12 ++++++------ odpf/assets/group.proto | 14 +++----------- odpf/assets/job.proto | 18 +++--------------- odpf/assets/table.proto | 12 +++--------- odpf/assets/topic.proto | 14 +++----------- odpf/assets/user.proto | 9 +++------ 9 files changed, 29 insertions(+), 86 deletions(-) diff --git a/odpf/assets/bucket.proto b/odpf/assets/bucket.proto index d3b20071..a1fa851d 100644 --- a/odpf/assets/bucket.proto +++ b/odpf/assets/bucket.proto @@ -12,21 +12,13 @@ import "odpf/assets/facets/ownership.proto"; import "odpf/assets/facets/lineage.proto"; import "odpf/assets/facets/properties.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; message Bucket { - // The unique identifier for the bucket. - // Example: `bucket-1`. - string urn = 1; - - // The name of the bucket. - // Example: `bucket 1`. - string name = 2; - - // The source of the bucket. - // Example: `metabase`. - string source = 3; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The description of the bucket. // Example: `This bucket was created by the product team.` diff --git a/odpf/assets/common/resource.proto b/odpf/assets/common/resource.proto index 4ce080d6..405ff02e 100644 --- a/odpf/assets/common/resource.proto +++ b/odpf/assets/common/resource.proto @@ -20,7 +20,7 @@ message Resource { // The source of the resource. // Example: `github` or `bigquery`. - string source = 3; + string service = 3; // The type of the asset. // Example: `user` or `group`. @@ -28,5 +28,5 @@ message Resource { // The REST URL for accessing the resource. URL returns the resource itself. // Example: `https://xyz.com/v1/users/user-123` - string resource_url = 5; + string url = 5; } diff --git a/odpf/assets/dashboard.proto b/odpf/assets/dashboard.proto index 55a45f56..ea53be05 100644 --- a/odpf/assets/dashboard.proto +++ b/odpf/assets/dashboard.proto @@ -10,31 +10,19 @@ import "odpf/assets/facets/ownership.proto"; import "odpf/assets/facets/lineage.proto"; import "odpf/assets/facets/properties.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; // Dashboard is a resource that represents a dashboard. message Dashboard { - // The unique identifier for the dashboard. - // Example: "dashboard-1". - string urn = 1; - - // The name of the dashboard. - // Example: "Dashboard 1". - string name = 2; - - // The source of the dashboard. - // Example: "metabase". - string source = 3; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The description of the dashboard. // Example: "This dashboard was created by the Metabase team." string description = 4; - // The url of the dashboard. - // Example: "https://metabase.com/dashboard/dashboard-1". - string url = 5; - // The list of the charts in the dashboard. // For an example, check the schema of the chart. repeated Chart charts = 21; diff --git a/odpf/assets/facets/properties.proto b/odpf/assets/facets/properties.proto index d1a9fbb9..7f177636 100644 --- a/odpf/assets/facets/properties.proto +++ b/odpf/assets/facets/properties.proto @@ -1,22 +1,22 @@ syntax = "proto3"; -package odpf.assets.facets; +package odpf.entities.facets; -option java_package = "io.odpf.assets.facets"; +option java_package = "io.odpf.entities.facets"; option java_outer_classname = "PropertiesProto"; -option go_package = "github.com/odpf/proton/assets/facets"; +option go_package = "github.com/odpf/proton/entities/facets"; import "google/protobuf/struct.proto"; message Properties { - // Tags associated with this resource.. + // Optional: List of tags the user has. repeated string tags = 1; - // Labels associated with this resource. + //Optional. List of labels the user has. map labels = 2; - // Custom attributes associated with this resource. + // Optional: List of properties the user has. google.protobuf.Struct attributes = 3; } \ No newline at end of file diff --git a/odpf/assets/group.proto b/odpf/assets/group.proto index 80605735..b17678b4 100644 --- a/odpf/assets/group.proto +++ b/odpf/assets/group.proto @@ -9,27 +9,19 @@ option go_package = "github.com/odpf/proton/assets"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/facets/properties.proto"; // Group represents a group of users and resources. message Group { - // The unique identifier for the group. - // Example: `group:example`. - string urn = 1; - - // The name of the group. - // Example: `Example Group`. - string name = 2; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The email of the group. // Example: `xyz@xyz.com` string email = 3; - // Source of the group. - // Example: `example.com`. - string source = 4; - // The members of the group. // For example look at schema of the member. repeated Member members = 21; diff --git a/odpf/assets/job.proto b/odpf/assets/job.proto index 1df7ff7c..718315f9 100644 --- a/odpf/assets/job.proto +++ b/odpf/assets/job.proto @@ -10,26 +10,14 @@ import "odpf/assets/facets/ownership.proto"; import "odpf/assets/facets/lineage.proto"; import "odpf/assets/facets/properties.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; // Job is a resource that represents a job. message Job { - // Required. The unique identifier of the job. - // Example: 'job_1'. - string urn = 1; - - // The name of the job. - // Example: 'Job 1'. - string name = 2; - - // The source of the job. - // Example: 'airflow'. - string source = 3; - - // The type of the job. - // Example: 'firehose', 'stream'. - string type = 4; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The description of the job. // Example: 'This job is used to process data from a stream.' diff --git a/odpf/assets/table.proto b/odpf/assets/table.proto index 7a1d952e..f42b7581 100644 --- a/odpf/assets/table.proto +++ b/odpf/assets/table.proto @@ -11,21 +11,15 @@ import "odpf/assets/facets/ownership.proto"; import "odpf/assets/facets/lineage.proto"; import "odpf/assets/facets/properties.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; // Table is a table in a database. // It can be a file, a table, a view, a materialized view, a temporary table, or a virtual table. message Table { - // The URN of the table. - // Example: 'my-database.my-schema.my-table'. - string urn = 1; - // The name of the table. - // Example: 'my-table'. - string name = 2; - // The source of the table. - // Example: 'bigquery'. - string source = 3; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The description of the table. // Example: 'My User table'. diff --git a/odpf/assets/topic.proto b/odpf/assets/topic.proto index e1f0b1a7..dbcdf703 100644 --- a/odpf/assets/topic.proto +++ b/odpf/assets/topic.proto @@ -11,23 +11,15 @@ import "odpf/assets/facets/ownership.proto"; import "odpf/assets/facets/lineage.proto"; import "odpf/assets/facets/properties.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; // Topic is resource that represents a logical group of messages // in message bus like kafka, pubsub, pulsar etc. message Topic { - // Required. The URN of the topic. - // Example: `urn:odpf:topic:my-topic`. - string urn = 1; - - // Required. The name of the topic. - // Example: `My Topic`. - string name = 2; - - // The source of the topic. - // Example: 'kafka', 'pubsub'. - string source = 3; + // Representation of the resource + odpf.assets.common.Resource resource = 1; // The description of the topic. // Example: 'This is a topic for my application'. diff --git a/odpf/assets/user.proto b/odpf/assets/user.proto index 4fb18709..1a18db0c 100644 --- a/odpf/assets/user.proto +++ b/odpf/assets/user.proto @@ -9,6 +9,7 @@ option go_package = "github.com/odpf/proton/assets"; import "odpf/assets/common/timestamp.proto"; import "odpf/assets/common/event.proto"; +import "odpf/assets/common/resource.proto"; import "odpf/assets/facets/properties.proto"; @@ -16,13 +17,9 @@ import "odpf/assets/facets/properties.proto"; // It can be a user of the system, or a user of a device. // User is a resource that represents a user. message User { - // The unique identifier of the user - // Example: "user:jdoe" - string urn = 1; + // Representation of the resource + odpf.assets.common.Resource resource = 1; - // The source of the user account. - // Example: `github` - string source = 2; // The emai address of the user. // Example: `job.deo@gmail.com` string email = 3; From 9e21234f81a3df2a6be409f8ad5913601771f7bc Mon Sep 17 00:00:00 2001 From: Ravi Suhag Date: Fri, 27 Aug 2021 01:38:24 -0700 Subject: [PATCH 18/18] chore: fix typo --- odpf/assets/facets/properties.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/odpf/assets/facets/properties.proto b/odpf/assets/facets/properties.proto index 7f177636..1fc71522 100644 --- a/odpf/assets/facets/properties.proto +++ b/odpf/assets/facets/properties.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -package odpf.entities.facets; +package odpf.assets.facets; -option java_package = "io.odpf.entities.facets"; +option java_package = "io.odpf.assets.facets"; option java_outer_classname = "PropertiesProto"; -option go_package = "github.com/odpf/proton/entities/facets"; +option go_package = "github.com/odpf/proton/assets/facets"; import "google/protobuf/struct.proto";