From 2e8f24945d1d9ee99b7e7c5e51653ba2570a464c Mon Sep 17 00:00:00 2001 From: Patrick Decat Date: Mon, 28 Oct 2024 12:10:08 +0100 Subject: [PATCH] feat: add gcp_cloud_run_job table --- docs/tables/gcp_cloud_run_job.md | 219 +++++++++++++++++ gcp/plugin.go | 1 + gcp/table_gcp_cloud_run_job.go | 396 +++++++++++++++++++++++++++++++ 3 files changed, 616 insertions(+) create mode 100644 docs/tables/gcp_cloud_run_job.md create mode 100644 gcp/table_gcp_cloud_run_job.go diff --git a/docs/tables/gcp_cloud_run_job.md b/docs/tables/gcp_cloud_run_job.md new file mode 100644 index 00000000..3d1d6314 --- /dev/null +++ b/docs/tables/gcp_cloud_run_job.md @@ -0,0 +1,219 @@ +--- +title: "Steampipe Table: gcp_cloud_run_job - Query GCP Cloud Run Jobs using SQL" +description: "Allows users to query GCP Cloud Run Jobs, specifically the configurations and executions, providing insights into the application's deployment history." +--- + +# Table: gcp_cloud_run_job - Query GCP Cloud Run Jobs using SQL + +Google Cloud Run is a managed compute platform that enables you to run stateless containers that are invocable via HTTP requests. Cloud Run is serverless: it abstracts away all infrastructure management, so you can focus on what matters most — building great applications. It automatically scales up or down from zero to N depending on traffic. + +## Table Usage Guide + +The `gcp_cloud_run_job` table provides insights into Cloud Run jobs within Google Cloud Platform (GCP). As a developer or DevOps engineer, explore job-specific details through this table, including configurations and executions. Utilize it to uncover information about jobs, such as the application's traffic flow, deployment history, and the current state of the job. + +## Examples + +### Basic info +Explore the basic details of your Google Cloud Run jobs, including their names, and client versions. This information can help you understand the configuration and status of your jobs, which is useful for managing and optimizing your cloud resources. + +```sql+postgres +select + name, + client, + client_version, + create_time, + creator, + generation, + launch_stage +from + gcp_cloud_run_job; +``` + +```sql+sqlite +select + name, + client, + client_version, + create_time, + creator, + generation, + launch_stage +from + gcp_cloud_run_job; +``` + +### Count of jobs by launch stage +Determine the distribution of jobs based on their launch stages. This can help in understanding how many jobs are in different stages of their lifecycle, providing insights for resource allocation and strategic planning. + +```sql+postgres +select + launch_stage, + count(*) +from + gcp_cloud_run_job +group by + launch_stage; +``` + +```sql+sqlite +select + launch_stage, + count(*) +from + gcp_cloud_run_job +group by + launch_stage; +``` + +### List cloud-run jobs that are reconciling +Analyze the settings to understand which cloud-run jobs are currently in the process of reconciling. This can be useful for tracking and managing jobs that may be undergoing changes or updates. + +```sql+postgres +select + name, + client, + client_version, + create_time, + creator, + generation, + launch_stage, + reconciling +from + gcp_cloud_run_job +where + reconciling; +``` + +```sql+sqlite +select + name, + client, + client_version, + create_time, + creator, + generation, + launch_stage, + reconciling +from + gcp_cloud_run_job +where + reconciling = 1; +``` + +### List jobs created in the last 30 days +Discover the jobs that were established in the past 30 days to gain insights into recent activities and understand the context of their creation. This could be useful in tracking the growth of jobs over time or identifying any unexpected or unauthorized job creation. + +```sql+postgres +select + name, + create_time, + creator, + launch_stage +from + gcp_cloud_run_job +where + create_time >= now() - interval '30' day; +``` + +```sql+sqlite +select + name, + create_time, + creator, + launch_stage +from + gcp_cloud_run_job +where + create_time >= datetime('now', '-30 day'); +``` + +### Get condition details of jobs +This example allows you to gain insights into the status and condition details of various jobs in the Google Cloud Run environment. It can be used to understand the health of jobs, the reasons for their current state, and when they last transitioned, which can assist in troubleshooting and maintaining job stability. + +```sql+postgres +select + name, + c ->> 'ExecutionReason' as execution_reason, + c ->> 'LastTransitionTime' as last_transition_time, + c ->> 'Message' as message, + c ->> 'Reason' as reason, + c ->> 'RevisionReason' as revision_reason, + c ->> 'State' as state, + c ->> 'Type' as type +from + gcp_cloud_run_job, + jsonb_array_elements(conditions) as c; +``` + +```sql+sqlite +select + name, + json_extract(c.value, '$.ExecutionReason') as execution_reason, + json_extract(c.value, '$.LastTransitionTime') as last_transition_time, + json_extract(c.value, '$.Message') as message, + json_extract(c.value, '$.Reason') as reason, + json_extract(c.value, '$.RevisionReason') as revision_reason, + json_extract(c.value, '$.State') as state, + json_extract(c.value, '$.Type') as type +from + gcp_cloud_run_job, + json_each(conditions) as c; +``` + +### Get associated members or principals, with a role of jobs +Attaching an Identity and Access Management (IAM) policy to a Google Cloud Run job involves setting permissions for that particular job. Google Cloud Run jobs use IAM for access control, and by configuring IAM policies, you can define who has what type of access to your Cloud Run jobs. + +```sql+postgres +select + name, + i -> 'Condition' as condition, + i -> 'Members' as members, + i ->> 'Role' as role +from + gcp_cloud_run_job, + jsonb_array_elements(iam_policy -> 'Bindings') as i; +``` + +```sql+sqlite +select + name, + json_extract(i.value, '$.Condition') as condition, + json_extract(i.value, '$.Members') as members, + json_extract(i.value, '$.Role') as role +from + gcp_cloud_run_job, + json_each(json_extract(iam_policy, '$.Bindings')) as i; +``` + +### Get template details of jobs +Explore the various attributes of your cloud-based jobs, such as encryption keys, container details, and scaling parameters. This query is useful to gain an understanding of your job configurations and identify areas for potential adjustments or enhancements. + +```sql+postgres +select + name, + template ->> 'Containers' as containers, + template ->> 'EncryptionKey' as encryption_key, + template ->> 'ExecutionEnvironment' as execution_environment, + template ->> 'MaxRetries' as max_retries, + template ->> 'ServiceAccount' as service_account, + template ->> 'Timeout' as timeout, + template ->> 'Volumes' as volumes, + template ->> 'VpcAccess' as vpc_access +from + gcp_cloud_run_job; +``` + +```sql+sqlite +select + name, + json_extract(template, '$.Containers') as containers, + json_extract(template, '$.EncryptionKey') as encryption_key, + json_extract(template, '$.ExecutionEnvironment') as execution_environment, + json_extract(template, '$.MaxRetries') as max_retries, + json_extract(template, '$.ServiceAccount') as service_account, + json_extract(template, '$.Timeout') as timeout, + json_extract(template, '$.Volumes') as volumes, + json_extract(template, '$.VpcAccess') as vpc_access +from + gcp_cloud_run_job; +``` diff --git a/gcp/plugin.go b/gcp/plugin.go index 40f74507..f390c9f0 100644 --- a/gcp/plugin.go +++ b/gcp/plugin.go @@ -53,6 +53,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "gcp_cloud_identity_group": tableGcpCloudIdentityGroup(ctx), "gcp_cloud_identity_group_membership": tableGcpCloudIdentityGroupMembership(ctx), "gcp_cloudfunctions_function": tableGcpCloudfunctionFunction(ctx), + "gcp_cloud_run_job": tableGcpCloudRunJob(ctx), "gcp_cloud_run_service": tableGcpCloudRunService(ctx), "gcp_composer_environment": tableGcpComposerEnvironment(ctx), "gcp_compute_address": tableGcpComputeAddress(ctx), diff --git a/gcp/table_gcp_cloud_run_job.go b/gcp/table_gcp_cloud_run_job.go new file mode 100644 index 00000000..f8920a21 --- /dev/null +++ b/gcp/table_gcp_cloud_run_job.go @@ -0,0 +1,396 @@ +package gcp + +import ( + "context" + "strings" + + "github.com/turbot/go-kit/types" + "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin" + "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + + "google.golang.org/api/run/v2" +) + +//// TABLE DEFINITION + +func tableGcpCloudRunJob(ctx context.Context) *plugin.Table { + return &plugin.Table{ + Name: "gcp_cloud_run_job", + Description: "GCP Cloud Run Job", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "location"}), + Hydrate: getCloudRunJob, + }, + List: &plugin.ListConfig{ + Hydrate: listCloudRunJobs, + KeyColumns: plugin.KeyColumnSlice{ + { + Name: "location", + Require: plugin.Optional, + }, + }, + }, + GetMatrixItemFunc: BuildComputeLocationList, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The fully qualified name of this Job.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name").Transform(lastPathElement), + }, + { + Name: "client", + Description: "Arbitrary identifier for the API client.", + Type: proto.ColumnType_STRING, + }, + { + Name: "client_version", + Description: "Arbitrary version identifier for the API client.", + Type: proto.ColumnType_STRING, + }, + { + Name: "create_time", + Description: "The creation time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + { + Name: "delete_time", + Description: "The deletion time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + { + Name: "creator", + Description: "Email address of the authenticated creator.", + Type: proto.ColumnType_STRING, + }, + { + Name: "etag", + Description: "A system-generated fingerprint for this version of the resource.", + Type: proto.ColumnType_STRING, + }, + + { + Name: "execution_count", + Description: "Number of executions created for this job.", + Type: proto.ColumnType_INT, + }, + { + Name: "expire_time", + Description: "For a deleted resource, the time after which it will be permamently deleted.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + { + Name: "generation", + Description: "A number that monotonically increases every time the user modifies the desired state.", + Type: proto.ColumnType_INT, + }, + { + Name: "self_link", + Description: "The server-defined URL for the resource.", + Type: proto.ColumnType_STRING, + Hydrate: cloudRunJobSelfLink, + Transform: transform.FromValue(), + }, + { + Name: "last_modifier", + Description: "Email address of the last authenticated modifier.", + Type: proto.ColumnType_STRING, + }, + { + Name: "launch_stage", + Description: "The launch stage as defined by Google Cloud Platform Launch Stages (https://cloud.google.com/terms/launch-stages). Cloud Run supports `ALPHA`, `BETA`, and `GA`. If no value is specified, GA is assumed.", + Type: proto.ColumnType_STRING, + }, + { + Name: "observed_generation", + Description: "The generation of this Job.", + Type: proto.ColumnType_STRING, + }, + { + Name: "reconciling", + Description: "Returns true if the Job is currently being acted upon by the system to bring it into the desired state.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "run_execution_token", + Description: "A unique string used as a suffix for creating a new execution. The Job will become ready when the execution is successfully completed.", + Type: proto.ColumnType_STRING, + }, + { + Name: "satisfies_pzs", + Description: "Reserved for future use.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "start_execution_token", + Description: "A unique string used as a suffix creating a new execution. The Job will become ready when the execution is successfully started.", + Type: proto.ColumnType_STRING, + }, + { + Name: "uid", + Description: "Server assigned unique identifier for the trigger.", + Type: proto.ColumnType_STRING, + }, + { + Name: "update_time", + Description: "The last-modified time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromGo().NullIfZero(), + }, + + // JSON fields + { + Name: "annotations", + Description: "Unstructured key value map that may be set by external tools to store and arbitrary metadata.", + Type: proto.ColumnType_JSON, + }, + { + Name: "binary_authorization", + Description: "Settings for the Binary Authorization feature.", + Type: proto.ColumnType_JSON, + }, + { + Name: "conditions", + Description: "The Conditions of all other associated sub-resources.", + Type: proto.ColumnType_JSON, + }, + { + Name: "labels", + Description: "Unstructured key value map that can be used to organize and categorize objects.", + Type: proto.ColumnType_JSON, + }, + { + Name: "latest_created_execution", + Description: "The last created execution.", + Type: proto.ColumnType_JSON, + }, + { + Name: "template", + Description: "The template used to create executions for this Job.", + Type: proto.ColumnType_JSON, + }, + { + Name: "terminal_condition", + Description: "The Condition of this Job, containing its readiness status, and detailed error information in case it did not reach a serving state.", + Type: proto.ColumnType_JSON, + }, + { + Name: "iam_policy", + Description: "An Identity and Access Management (IAM) policy, which specifies access controls for Google Cloud resources.", + Type: proto.ColumnType_JSON, + Hydrate: getCloudRunJobIamPolicy, + Transform: transform.FromValue(), + }, + + // Standard steampipe columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(cloudRunJobData, "Title"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Labels"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromP(cloudRunJobData, "Akas"), + }, + + // Standard GCP columns + { + Name: "location", + Description: ColumnDescriptionLocation, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(cloudRunJobData, "Location"), + }, + { + Name: "project", + Description: ColumnDescriptionProject, + Type: proto.ColumnType_STRING, + Transform: transform.FromP(cloudRunJobData, "Project"), + }, + }, + } +} + +//// LIST FUNCTION + +func listCloudRunJobs(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + region := d.EqualsQualString("region") + + var location string + matrixLocation := d.EqualsQualString(matrixKeyLocation) + // Since, when the service API is disabled, matrixLocation value will be nil + if matrixLocation != "" { + location = matrixLocation + } + + // Minimize API call as per given location + if region != "" && region != location { + return nil, nil + } + + // Create Service Connection + service, err := CloudRunService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.listCloudRunJobs", "service_error", err) + return nil, err + } + + // Max limit is set as per documentation + pageSize := types.Int64(500) + limit := d.QueryContext.Limit + if d.QueryContext.Limit != nil { + if *limit < *pageSize { + pageSize = limit + } + } + + // Get project details + + projectId, err := getProject(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + input := "projects/" + project + "/locations/" + location + + resp := service.Projects.Locations.Jobs.List(input).PageSize(*pageSize) + if err := resp.Pages(ctx, func(page *run.GoogleCloudRunV2ListJobsResponse) error { + for _, item := range page.Jobs { + d.StreamListItem(ctx, item) + + // Check if context has been cancelled or if the limit has been hit (if specified) + // if there is a limit, it will return the number of rows required to reach this limit + if d.RowsRemaining(ctx) == 0 { + page.NextPageToken = "" + return nil + } + } + return nil + }); err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.listCloudRunJobs", "api_error", err) + return nil, err + } + + return nil, nil +} + +//// HYDRATE FUNCTIONS + +func getCloudRunJob(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + // Create Service Connection + service, err := CloudRunService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.getCloudRunJob", "service_error", err) + return nil, err + } + + // Get project details + + projectId, err := getProject(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + jobName := d.EqualsQuals["name"].GetStringValue() + location := d.EqualsQuals["location"].GetStringValue() + + // Empty Check + if jobName == "" || location == "" { + return nil, nil + } + + input := "projects/" + project + "/locations/" + location + "/jobs/" + jobName + + resp, err := service.Projects.Locations.Jobs.Get(input).Do() + if err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.getCloudRunJob", "api_error", err) + return nil, err + } + return resp, err +} + +func getCloudRunJobIamPolicy(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + + data := h.Item.(*run.GoogleCloudRunV2Job) + jobName := strings.Split(data.Name, "/")[5] + location := strings.Split(data.Name, "/")[3] + + // Create Service Connection + service, err := CloudRunService(ctx, d) + if err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.getCloudRunJobIamPolicy", "service_error", err) + return nil, err + } + + // Get project details + + projectId, err := getProject(ctx, d, h) + if err != nil { + return nil, err + } + project := projectId.(string) + + input := "projects/" + project + "/locations/" + location + "/jobs/" + jobName + + resp, err := service.Projects.Locations.Jobs.GetIamPolicy(input).Do() + if err != nil { + plugin.Logger(ctx).Error("gcp_cloud_run_job.getCloudRunJobIamPolicy", "api_error", err) + return nil, err + } + + return resp, err +} + +//// TRANSFORM FUNCTIONS + +func cloudRunJobSelfLink(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + data := h.Item.(*run.GoogleCloudRunV2Job) + + var location string + matrixLocation := d.EqualsQualString(matrixKeyLocation) + // Since, when the service API is disabled, matrixLocation value will be nil + if matrixLocation != "" { + location = matrixLocation + } + + projectID := strings.Split(data.Name, "/")[1] + name := strings.Split(data.Name, "/")[5] + + selfLink := "https://run.googleapis.com/v2/projects/" + projectID + "/regions/" + location + "/repositories/" + name + + return selfLink, nil +} + +//// TRANSFORM FUNCTIONS + +func cloudRunJobData(ctx context.Context, h *transform.TransformData) (interface{}, error) { + data := h.HydrateItem.(*run.GoogleCloudRunV2Job) + param := h.Param.(string) + + projectID := strings.Split(data.Name, "/")[1] + name := strings.Split(data.Name, "/")[5] + location := strings.Split(data.Name, "/")[3] + + turbotData := map[string]interface{}{ + "Project": projectID, + "Title": name, + "Location": location, + "Akas": []string{"gcp://run.googleapis.com/projects/" + projectID + "/repositories/" + name}, + } + + return turbotData[param], nil +}