feat: Added glue databases and tables (#1345)

* feat: Added glue databases and tables * mock generate
cloudquery · Aug 2, 2022 · 0284a37 · 0284a37
1 parent 562a6b3
commit 0284a37
Show file tree

Hide file tree

Showing 9 changed files with 503 additions and 0 deletions.
diff --git a/client/mocks/glue.go b/client/mocks/glue.go
diff --git a/client/services.go b/client/services.go
@@ -754,6 +754,8 @@ type GlueClient interface {
 	ListWorkflows(ctx context.Context, params *glue.ListWorkflowsInput, optFns ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error)
 	GetJobs(ctx context.Context, params *glue.GetJobsInput, optFns ...func(*glue.Options)) (*glue.GetJobsOutput, error)
 	GetJobRuns(ctx context.Context, params *glue.GetJobRunsInput, optFns ...func(*glue.Options)) (*glue.GetJobRunsOutput, error)
+	GetDatabases(ctx context.Context, params *glue.GetDatabasesInput, optFns ...func(*glue.Options)) (*glue.GetDatabasesOutput, error)
+	GetTables(ctx context.Context, params *glue.GetTablesInput, optFns ...func(*glue.Options)) (*glue.GetTablesOutput, error)
 }
 
 //go:generate mockgen -package=mocks -destination=./mocks/kinesis.go . KinesisClient

diff --git a/docs/tables/aws_glue_database_table_partition_keys.md b/docs/tables/aws_glue_database_table_partition_keys.md
@@ -0,0 +1,11 @@
+
+# Table: aws_glue_database_table_partition_keys
+A column in a Table
+## Columns
+| Name        | Type           | Description  |
+| ------------- | ------------- | -----  |
+|database_table_cq_id|uuid|Unique CloudQuery ID of aws_glue_database_tables table (FK)|
+|name|text|The name of the Column|
+|comment|text|A free-form text comment|
+|parameters|jsonb|These key-value pairs define properties associated with the column|
+|type|text|The data type of the Column|
diff --git a/docs/tables/aws_glue_database_tables.md b/docs/tables/aws_glue_database_tables.md
@@ -0,0 +1,28 @@
+
+# Table: aws_glue_database_tables
+Represents a collection of related data organized in columns and rows
+## Columns
+| Name        | Type           | Description  |
+| ------------- | ------------- | -----  |
+|database_cq_id|uuid|Unique CloudQuery ID of aws_glue_databases table (FK)|
+|name|text|The table name|
+|catalog_id|text|The ID of the Data Catalog in which the table resides|
+|create_time|timestamp without time zone|The time when the table definition was created in the Data Catalog|
+|created_by|text|The person or entity who created the table|
+|database_name|text|The name of the database where the table metadata resides|
+|description|text|A description of the table|
+|is_registered_with_lake_formation|boolean|Indicates whether the table has been registered with Lake Formation|
+|last_access_time|timestamp without time zone|The last time that the table was accessed|
+|last_analyzed_time|timestamp without time zone|The last time that column statistics were computed for this table|
+|owner|text|The owner of the table|
+|parameters|jsonb|These key-value pairs define properties associated with the table|
+|retention|bigint|The retention time for this table|
+|storage_descriptor|jsonb|A storage descriptor containing information about the physical storage of this table|
+|table_type|text|The type of this table (EXTERNAL_TABLE, VIRTUAL_VIEW, etc)|
+|target_table_catalog_id|text|The ID of the Data Catalog in which the table resides|
+|target_table_database_name|text|The name of the catalog database that contains the target table|
+|target_table_name|text|The name of the target table|
+|update_time|timestamp without time zone|The last time that the table was updated|
+|version_id|text|The ID of the table version|
+|view_expanded_text|text|If the table is a view, the expanded text of the view; otherwise null|
+|view_original_text|text|If the table is a view, the original text of the view; otherwise null|
diff --git a/docs/tables/aws_glue_databases.md b/docs/tables/aws_glue_databases.md
@@ -0,0 +1,18 @@
+
+# Table: aws_glue_databases
+The Database object represents a logical grouping of tables that might reside in a Hive metastore or an RDBMS
+## Columns
+| Name        | Type           | Description  |
+| ------------- | ------------- | -----  |
+|account_id|text|The AWS Account ID of the resource.|
+|region|text|The AWS Region of the resource.|
+|arn|text|The Amazon Resource Name (ARN) of the workflow.|
+|name|text|The name of the database|
+|catalog_id|text|The ID of the Data Catalog in which the database resides|
+|create_table_default_permissions|jsonb|Creates a set of default permissions on the table for principals|
+|create_time|timestamp without time zone|The time at which the metadata database was created in the catalog|
+|description|text|A description of the database|
+|location_uri|text|The location of the database (for example, an HDFS path)|
+|parameters|jsonb|These key-value pairs define parameters and properties of the database|
+|target_database_catalog_id|text|The ID of the Data Catalog in which the database resides|
+|target_database_name|text|The name of the catalog database|
diff --git a/resources/provider/provider.go b/resources/provider/provider.go
@@ -168,6 +168,7 @@ func Provider() *provider.Provider {
 			"emr.block_public_access_configs":         emr.EmrBlockPublicAccessConfigs(),
 			"emr.clusters":                            emr.EmrClusters(),
 			"fsx.backups":                             fsx.FsxBackups(),
+			"glue.databases":                          glue.Databases(),
 			"glue.jobs":                               glue.Jobs(),
 			"glue.workflows":                          glue.Workflows(),
 			"guardduty.detectors":                     guardduty.GuarddutyDetectors(),