Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add IngestionFailureEvent to the external proto #1984

Merged
merged 5 commits into from
Nov 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 57 additions & 0 deletions protos/google/pubsub/v1/pubsub.proto
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,63 @@ message PlatformLogsSettings {
Severity severity = 1 [(google.api.field_behavior) = OPTIONAL];
}

// Payload of the Platform Log entry sent when a failure is encountered while
// ingesting.
message IngestionFailureEvent {
// Specifies the reason why some data may have been left out of
// the desired Pub/Sub message due to the API message limits
// (https://cloud.google.com/pubsub/quotas#resource_limits). For example,
// when the number of attributes is larger than 100, the number of
// attributes is truncated to 100 to respect the limit on the attribute count.
// Other attribute limits are treated similarly. When the size of the desired
// message would've been larger than 10MB, the message won't be published at
// all, and ingestion of the subsequent messages will proceed as normal.
message ApiViolationReason {}

// Set when an Avro file is unsupported or its format is not valid. When this
// occurs, one or more Avro objects won't be ingested.
message AvroFailureReason {}

// Failure when ingesting from a Cloud Storage source.
message CloudStorageFailure {
// Optional. Name of the Cloud Storage bucket used for ingestion.
string bucket = 1 [(google.api.field_behavior) = OPTIONAL];

// Optional. Name of the Cloud Storage object which contained the section
// that couldn't be ingested.
string object_name = 2 [(google.api.field_behavior) = OPTIONAL];

// Optional. Generation of the Cloud Storage object which contained the
// section that couldn't be ingested.
int64 object_generation = 3 [(google.api.field_behavior) = OPTIONAL];

// Reason why ingestion failed for the specified object.
oneof reason {
// Optional. Failure encountered when parsing an Avro file.
AvroFailureReason avro_failure_reason = 5
[(google.api.field_behavior) = OPTIONAL];

// Optional. The Pub/Sub API limits prevented the desired message from
// being published.
ApiViolationReason api_violation_reason = 6
[(google.api.field_behavior) = OPTIONAL];
}
}

// Required. Name of the import topic. Format is:
// projects/{project_name}/topics/{topic_name}.
string topic = 1 [(google.api.field_behavior) = REQUIRED];

// Required. Error details explaining why ingestion to Pub/Sub has failed.
string error_message = 2 [(google.api.field_behavior) = REQUIRED];

oneof failure {
// Optional. Failure when ingesting from Cloud Storage.
CloudStorageFailure cloud_storage_failure = 3
[(google.api.field_behavior) = OPTIONAL];
}
}

// A topic resource.
message Topic {
option (google.api.resource) = {
Expand Down
Loading
Loading