diff --git a/docs/tables/gcp_logging_log_entry.md b/docs/tables/gcp_logging_log_entry.md index 26bd28e3..004723ca 100644 --- a/docs/tables/gcp_logging_log_entry.md +++ b/docs/tables/gcp_logging_log_entry.md @@ -22,8 +22,8 @@ The `gcp_logging_log_entry` table provides insights into Log Entries within Goog - `receive_timestamp` - `timestamp` - `trace` - - `log_entry_operation_id` - - `filter` + - `operation_id` + - `filter`: For additional details regarding the filter string, please refer to the documentation at https://cloud.google.com/logging/docs/view/logging-query-language. ## Examples @@ -45,8 +45,7 @@ from select log_name, insert_id, - log_entry_operation_first, - log_entry_operation_id, + operation_id, receive_timestamp from gcp_logging_log_entry; @@ -74,8 +73,6 @@ where select log_name, insert_id, - log_entry_operation_first, - log_entry_operation_last, resource_type, span_id, text_payload @@ -106,7 +103,6 @@ where select log_name, insert_id, - resource_type, severity, span_id, timestamp @@ -193,7 +189,7 @@ Explore the most recent activities in your system by checking the last entries i select log_name, insert_id, - log_entry_operation_last, + operation ->> 'Last' as log_entry_operation_last, receive_timestamp, resource_type, severity, @@ -201,14 +197,14 @@ select from gcp_logging_log_entry where - log_entry_operation_last; + (operation ->> 'Last')::boolean; ``` ```sql+sqlite select log_name, insert_id, - log_entry_operation_last, + json_extract(operation, '$.Last') as log_entry_operation_last, receive_timestamp, resource_type, severity, @@ -216,7 +212,7 @@ select from gcp_logging_log_entry where - log_entry_operation_last = 1; + json_extract(operation, '$.Last') = 'true'; ``` ### Filter log entries by log name @@ -252,6 +248,58 @@ where log_name = 'projects/parker-abbb/logs/cloudaudit.googleapis.com%2Factivity'; ``` +### Get split details of each log entry +Extracting detailed information about specific log entries in a structured and relational manner. It allows for a deeper analysis of the logs by providing contextual information like the sequence of the log entry + +```sql+postgres +select + log_name, + insert_id, + split ->> 'Index' as split_index, + split ->> 'TotalSplits' as total_splits, + split ->> 'Uid' as split_uid +from + gcp_logging_log_entry; +``` + +```sql+sqlite +select + log_name, + insert_id, + json_extract(split, '$.Index') as split_index, + json_extract(split, '$.TotalSplits') as total_splits, + json_extract(split, '$.Uid') as split_uid +from + gcp_logging_log_entry; +``` + +### Get operation details of each log entry +Retrieve the specifics of operation-related details from log entry records. This query can be instrumental in acquiring information regarding the initial operation, concluding operation, and the source of each operation. + +```sql+postgres +select + log_name, + insert_id, + operation_id, + operation ->> 'Producer' as operation_producer, + operation ->> 'First' as operation_first, + operation ->> 'Last' as operation_last +from + gcp_logging_log_entry; +``` + +```sql+sqlite +select + log_name, + insert_id, + operation_id, + json_extract(operation, '$.Producer') as operation_producer, + json_extract(operation, '$.First') as operation_first, + json_extract(operation, '$.Last') as operation_last +from + gcp_logging_log_entry; +``` + ## Filter examples For more information on Logging log entry filters, please refer to [Filter Pattern Syntax](https://cloud.google.com/logging/docs/view/logging-query-language). @@ -263,8 +311,6 @@ Discover the segments that have logged errors on your Google Compute Engine virt select log_name, insert_id, - log_entry_operation_first, - log_entry_operation_last, receive_timestamp, resource_type, severity @@ -299,8 +345,7 @@ select receive_timestamp, resource_type, severity, - timestamp, - resource_labels + timestamp from gcp_logging_log_entry where @@ -310,6 +355,39 @@ order by receive_timestamp asc; ``` +### Get proto payload details of each log entry +The query is useful for extracting specific information from log entries in a GCP logging system, particularly for entries related to Google Compute Engine (GCE) instances with errors. Extracting specific information from log entries in a GCP logging system, particularly for entries related to Google Compute Engine (GCE) instances with errors. + +```sql+postgres +select + insert_id, + log_name, + proto_payload -> 'authenticationInfo' as authentication_info, + proto_payload -> 'authorizationInfo' as authorization_info, + proto_payload -> 'serviceName' as service_name, + proto_payload -> 'resourceName' as resource_name, + proto_payload ->> '@type' as proto_payload_type, + proto_payload ->> 'methodName' as method_name, + proto_payload ->> 'callerIp' as caller_ip +from + gcp_logging_log_entry +where + filter = 'resource.type = "gce_instance" AND (severity = ERROR OR "error")'; +``` + ```sql+sqlite -Error: SQLite does not support CIDR operations. +select + insert_id, + log_name, + json_extract(proto_payload, '$.authenticationInfo') AS authentication_info, + json_extract(proto_payload, '$.authorizationInfo') AS authorization_info, + json_extract(proto_payload, '$.serviceName') AS service_name, + json_extract(proto_payload, '$.resourceName') AS resource_name, + json_extract(proto_payload, '$.@type') AS proto_payload_type, + json_extract(proto_payload, '$.methodName') AS method_name, + json_extract(proto_payload, '$.callerIp') AS caller_ip +from + gcp_logging_log_entry +where + filter = 'resource.type = "gce_instance" AND (severity = ERROR OR severity = "error")'; ``` \ No newline at end of file diff --git a/gcp/table_gcp_logging_log_entry.go b/gcp/table_gcp_logging_log_entry.go index 5ba2b644..a9c367df 100644 --- a/gcp/table_gcp_logging_log_entry.go +++ b/gcp/table_gcp_logging_log_entry.go @@ -2,6 +2,7 @@ package gcp import ( "context" + "encoding/json" "time" "github.com/turbot/go-kit/types" @@ -30,10 +31,10 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { {Name: "log_name", Require: plugin.Optional}, {Name: "span_id", Require: plugin.Optional}, {Name: "text_payload", Require: plugin.Optional}, - {Name: "receive_timestamp", Require: plugin.Optional}, - {Name: "timestamp", Require: plugin.Optional}, + {Name: "receive_timestamp", Require: plugin.Optional, Operators: []string{"=", ">", "<", ">=", "<="}}, + {Name: "timestamp", Require: plugin.Optional, Operators: []string{"=", ">", "<", ">=", "<="}}, {Name: "trace", Require: plugin.Optional}, - {Name: "log_entry_operation_id", Require: plugin.Optional}, + {Name: "operation_id", Require: plugin.Optional}, {Name: "filter", Require: plugin.Optional, CacheMatch: "exact"}, }, }, @@ -48,18 +49,6 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Description: "A unique identifier for the log entry.", Type: proto.ColumnType_STRING, }, - { - Name: "log_entry_operation_first", - Description: "Set this to True if this is the first log entry in the operation.", - Type: proto.ColumnType_BOOL, - Transform: transform.FromField("Operation.First"), - }, - { - Name: "log_entry_operation_last", - Description: "Set this to True if this is the last log entry in the operation.", - Type: proto.ColumnType_BOOL, - Transform: transform.FromField("Operation.Last"), - }, { Name: "filter", Type: proto.ColumnType_STRING, @@ -67,17 +56,11 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Transform: transform.FromQual("filter"), }, { - Name: "log_entry_operation_id", + Name: "operation_id", Description: "An arbitrary operation identifier. Log entries with the same identifier are assumed to be part of the same operation.", Type: proto.ColumnType_STRING, Transform: transform.FromField("Operation.Id"), }, - { - Name: "log_entry_operation_producer", - Description: "An arbitrary producer identifier.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("Operation.Producer"), - }, { Name: "receive_timestamp", Description: "The time the log entry was received by Logging.", @@ -120,34 +103,47 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Type: proto.ColumnType_BOOL, }, { - Name: "split_index", - Description: "The index of this LogEntry in the sequence of split log entries.", - Type: proto.ColumnType_INT, - Transform: transform.FromField("Split.Index"), + Name: "json_payload", + Description: "The log entry payload, represented as a structure that is expressed as a JSON object.", + Type: proto.ColumnType_JSON, + Transform: transform.FromP(covertLogEntryByteArrayToJsonObject, "JsonPayload"), }, { - Name: "total_splits", - Description: "The total number of log entries that the original LogEntry was split into.", - Type: proto.ColumnType_INT, - Transform: transform.FromField("Split.TotalSplits"), + Name: "proto_payload", + Description: "The log entry payload, represented as a protocol buffer. Some Google Cloud Platform services use this field for their log entry payloads. The following protocol buffer types are supported; user-defined types are not supported: 'type.googleapis.com/google.cloud.audit.AuditLog' 'type.googleapis.com/google.appengine.logging.v1.RequestLog'", + Type: proto.ColumnType_JSON, + Transform: transform.FromP(covertLogEntryByteArrayToJsonObject, "ProtoPayload"), }, { - Name: "split_uid", - Description: "A globally unique identifier for all log entries in a sequence of split log entries.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("Split.Uid"), + Name: "operation", + Description: "Information about an operation associated with the log entry, if applicable.", + Type: proto.ColumnType_JSON, + }, + { + Name: "resource", + Description: "The monitored resource that produced this log entry. Example: a log entry that reports a database error would be associated with the monitored resource designating the particular database that reported the error.", + Type: proto.ColumnType_JSON, }, { - Name: "resource_labels", - Description: "Values for all of the labels listed in the associated monitored resource descriptor.", + Name: "split", + Description: "Information indicating this LogEntry is part of a sequence of multiple log entries split from a single LogEntry.", Type: proto.ColumnType_JSON, - Transform: transform.FromField("Resource.Labels"), }, { Name: "source_location", Description: "Source code location information associated with the log entry, if any.", Type: proto.ColumnType_JSON, }, + { + Name: "metadata", + Description: "Auxiliary metadata for a MonitoredResource object. MonitoredResource objects contain the minimum set of information to uniquely identify a monitored resource instance.", + Type: proto.ColumnType_JSON, + }, + { + Name: "labels", + Description: "A map of key, value pairs that provides additional information about the log entry. The labels can be user-defined or system-defined.User-defined labels are arbitrary key, value pairs that you can use to classify logs. System-defined labels are defined by GCP services for platform logs.", + Type: proto.ColumnType_JSON, + }, // Standard steampipe columns { @@ -156,6 +152,12 @@ func tableGcpLoggingLogEntry(_ context.Context) *plugin.Table { Type: proto.ColumnType_STRING, Transform: transform.FromField("InsertId"), }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Labels"), + }, // Standard GCP columns { @@ -186,8 +188,9 @@ func listGcpLoggingLogEntries(ctx context.Context, d *plugin.QueryData, h *plugi } // Max limit isn't mentioned in the documentation - // Default limit is set as 1000 - pageSize := types.Int64(1000) + // Default limit is set as 10000 + // 10000 seems to be a balanced limit, based on initial tests for retrieving 140k log entries: 5000 (124s), 10000 (88s), 20000 (84s). + pageSize := types.Int64(10000) limit := d.QueryContext.Limit if d.QueryContext.Limit != nil { if *limit < *pageSize { @@ -301,7 +304,7 @@ func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string {"span_id", "spanId", "string"}, {"text_payload", "textPayload", "string"}, {"trace", "trace", "string"}, - {"log_entry_operation_id", "operation.id", "string"}, + {"operation_id", "operation.id", "string"}, {"receive_timestamp", "receiveTimestamp", "timestamp"}, {"timestamp", "timestamp", "timestamp"}, } @@ -330,10 +333,33 @@ func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetStringValue() + "\"" } case "timestamp": + propertyPath := filterQualItem.PropertyPath if filter == "" { - filter = filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + switch qual.Operator { + case "=": + filter = propertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case ">": + filter = propertyPath + " > \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case "<": + filter = propertyPath + " < \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case ">=": + filter = propertyPath + " >= \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case "<=": + filter = propertyPath + " <= \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } } else { - filter = filter + " AND " + filterQualItem.PropertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + switch qual.Operator { + case "=": + filter = filter + " AND " + propertyPath + " = \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case ">": + filter = filter + " AND " + propertyPath + " > \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case "<": + filter = filter + " AND " + propertyPath + " < \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case ">=": + filter = filter + " AND " + propertyPath + " >= \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + case "<=": + filter = filter + " AND " + propertyPath + " <= \"" + value.GetTimestampValue().AsTime().Format(time.RFC3339) + "\"" + } } } } @@ -341,3 +367,39 @@ func buildLoggingLogEntryFilterParam(equalQuals plugin.KeyColumnQualMap) string } return filter } + +//// TRANSFORM FUNCTION + +func covertLogEntryByteArrayToJsonObject(ctx context.Context, d *transform.TransformData) (interface{}, error) { + entry := d.HydrateItem.(*logging.LogEntry) + param := d.Param.(string) + + var protoPlayload interface{} + var jsonPayload interface{} + + a, err := entry.ProtoPayload.MarshalJSON() + if err != nil { + return nil, err + } + b, err := entry.JsonPayload.MarshalJSON() + if err != nil { + return nil, err + } + + err = json.Unmarshal(a, &protoPlayload) + if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.covertLogEntryByteArrayToJsonObject.protoPlayload", err) + } + + err = json.Unmarshal(b, &jsonPayload) + if err != nil { + plugin.Logger(ctx).Error("gcp_logging_log_entry.covertLogEntryByteArrayToJsonObject.jsonPayload", err) + } + + payload := map[string]interface{}{ + "JsonPayload": jsonPayload, + "ProtoPayload": protoPlayload, + } + + return payload[param], nil +}