Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: [google-cloud-orchestration-airflow] add airflow_database_retention_days and airflow_metadata_retention_config to message DataRetentionConfig #12893

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "1.12.2" # {x-release-please-version}
__version__ = "0.0.0" # {x-release-please-version}
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "1.12.2" # {x-release-please-version}
__version__ = "0.0.0" # {x-release-please-version}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from .services.environments import EnvironmentsAsyncClient, EnvironmentsClient
from .services.image_versions import ImageVersionsAsyncClient, ImageVersionsClient
from .types.environments import (
AirflowMetadataRetentionPolicyConfig,
CheckUpgradeRequest,
CheckUpgradeResponse,
CloudDataLineageIntegration,
Expand Down Expand Up @@ -94,6 +95,7 @@
__all__ = (
"EnvironmentsAsyncClient",
"ImageVersionsAsyncClient",
"AirflowMetadataRetentionPolicyConfig",
"CheckUpgradeRequest",
"CheckUpgradeResponse",
"CloudDataLineageIntegration",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "1.12.2" # {x-release-please-version}
__version__ = "0.0.0" # {x-release-please-version}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# limitations under the License.
#
from .environments import (
AirflowMetadataRetentionPolicyConfig,
CheckUpgradeRequest,
CheckUpgradeResponse,
CloudDataLineageIntegration,
Expand Down Expand Up @@ -83,6 +84,7 @@
from .operations import OperationMetadata

__all__ = (
"AirflowMetadataRetentionPolicyConfig",
"CheckUpgradeRequest",
"CheckUpgradeResponse",
"CloudDataLineageIntegration",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
"WorkloadsConfig",
"DataRetentionConfig",
"TaskLogsRetentionConfig",
"AirflowMetadataRetentionPolicyConfig",
"StorageConfig",
"RecoveryConfig",
"ScheduledSnapshotsConfig",
Expand Down Expand Up @@ -1443,9 +1444,11 @@ class EnvironmentConfig(proto.Message):
multiple chunks, each with a size of at least 4
hours.

If this value is omitted, Cloud Composer
components may be subject to maintenance at any
time.
If this value is omitted, the default value for
maintenance window is applied. By default,
maintenance windows are from 00:00:00 to
04:00:00 (GMT) on Friday, Saturday, and Sunday
every week.
workloads_config (google.cloud.orchestration.airflow.service_v1beta1.types.WorkloadsConfig):
Optional. The workloads configuration settings for the GKE
cluster associated with the Cloud Composer environment. The
Expand Down Expand Up @@ -2608,6 +2611,9 @@ class TriggererResource(proto.Message):
class DagProcessorResource(proto.Message):
r"""Configuration for resources used by Airflow DAG processors.

This field is supported for Cloud Composer environments in versions
composer-3.\ *.*-airflow-*.*.\* and newer.

Attributes:
cpu (float):
Optional. CPU request and limit for a single
Expand Down Expand Up @@ -2673,16 +2679,36 @@ class DataRetentionConfig(proto.Message):
mechanism.

Attributes:
airflow_database_retention_days (int):
Optional. The number of days describing for
how long to store event-based records in airflow
database. If the retention mechanism is enabled
this value must be a positive integer otherwise,
value should be set to 0.
task_logs_retention_config (google.cloud.orchestration.airflow.service_v1beta1.types.TaskLogsRetentionConfig):
Optional. The configuration settings for task
logs retention
airflow_metadata_retention_config (google.cloud.orchestration.airflow.service_v1beta1.types.AirflowMetadataRetentionPolicyConfig):
Optional. The retention policy for airflow
metadata database.
"""

airflow_database_retention_days: int = proto.Field(
proto.INT32,
number=1,
)
task_logs_retention_config: "TaskLogsRetentionConfig" = proto.Field(
proto.MESSAGE,
number=4,
message="TaskLogsRetentionConfig",
)
airflow_metadata_retention_config: "AirflowMetadataRetentionPolicyConfig" = (
proto.Field(
proto.MESSAGE,
number=5,
message="AirflowMetadataRetentionPolicyConfig",
)
)


class TaskLogsRetentionConfig(proto.Message):
Expand All @@ -2691,8 +2717,7 @@ class TaskLogsRetentionConfig(proto.Message):
Attributes:
storage_mode (google.cloud.orchestration.airflow.service_v1beta1.types.TaskLogsRetentionConfig.TaskLogsStorageMode):
Optional. The mode of storage for Airflow
workers task logs. For details, see
go/composer-store-task-logs-in-cloud-logging-only-design-doc
workers task logs.
"""

class TaskLogsStorageMode(proto.Enum):
Expand All @@ -2719,6 +2744,45 @@ class TaskLogsStorageMode(proto.Enum):
)


class AirflowMetadataRetentionPolicyConfig(proto.Message):
r"""The policy for airflow metadata database retention.

Attributes:
retention_mode (google.cloud.orchestration.airflow.service_v1beta1.types.AirflowMetadataRetentionPolicyConfig.RetentionMode):
Optional. Retention can be either enabled or
disabled.
retention_days (int):
Optional. How many days data should be
retained for.
"""

class RetentionMode(proto.Enum):
r"""Describes retention policy.

Values:
RETENTION_MODE_UNSPECIFIED (0):
Default mode doesn't change environment
parameters.
RETENTION_MODE_ENABLED (1):
Retention policy is enabled.
RETENTION_MODE_DISABLED (2):
Retention policy is disabled.
"""
RETENTION_MODE_UNSPECIFIED = 0
RETENTION_MODE_ENABLED = 1
RETENTION_MODE_DISABLED = 2

retention_mode: RetentionMode = proto.Field(
proto.ENUM,
number=1,
enum=RetentionMode,
)
retention_days: int = proto.Field(
proto.INT32,
number=2,
)


class StorageConfig(proto.Message):
r"""The configuration for data storage in the environment.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
],
"language": "PYTHON",
"name": "google-cloud-orchestration-airflow",
"version": "1.12.2"
"version": "0.1.0"
},
"snippets": [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
],
"language": "PYTHON",
"name": "google-cloud-orchestration-airflow-service",
"version": "1.12.2"
"version": "0.1.0"
},
"snippets": [
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10931,7 +10931,12 @@ def test_create_environment_rest(request_type):
}
},
"data_retention_config": {
"task_logs_retention_config": {"storage_mode": 1}
"airflow_database_retention_days": 3279,
"task_logs_retention_config": {"storage_mode": 1},
"airflow_metadata_retention_config": {
"retention_mode": 1,
"retention_days": 1512,
},
},
"resilience_mode": 1,
},
Expand Down Expand Up @@ -11856,7 +11861,12 @@ def test_update_environment_rest(request_type):
}
},
"data_retention_config": {
"task_logs_retention_config": {"storage_mode": 1}
"airflow_database_retention_days": 3279,
"task_logs_retention_config": {"storage_mode": 1},
"airflow_metadata_retention_config": {
"retention_mode": 1,
"retention_days": 1512,
},
},
"resilience_mode": 1,
},
Expand Down
Loading