From abfa63d01ff62ea3d896f7c8d734270ff62cdc66 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 25 Oct 2024 11:10:59 +0200 Subject: [PATCH] Consistent LOG_LEVEL for ECS and ECS tasks --- .../environment/tasks/env_stacks_updater.py | 2 +- backend/dataall/core/stacks/tasks/cdkproxy.py | 2 +- .../catalog/tasks/catalog_indexer_task.py | 2 +- .../omics/tasks/omics_workflows_fetcher.py | 2 +- .../s3_datasets/tasks/tables_syncer.py | 2 +- .../tasks/dataset_subscription_task.py | 2 +- .../tasks/subscriptions/sqs_poller.py | 2 +- .../tasks/persistent_email_reminders_task.py | 2 +- .../tasks/share_expiration_task.py | 2 +- .../shares_base/tasks/share_manager_task.py | 2 +- .../shares_base/tasks/share_reapplier_task.py | 2 +- .../shares_base/tasks/share_verifier_task.py | 2 +- deploy/stacks/container.py | 19 ++++++++++--------- 13 files changed, 22 insertions(+), 21 deletions(-) diff --git a/backend/dataall/core/environment/tasks/env_stacks_updater.py b/backend/dataall/core/environment/tasks/env_stacks_updater.py index 013702238..ecf6b72f9 100644 --- a/backend/dataall/core/environment/tasks/env_stacks_updater.py +++ b/backend/dataall/core/environment/tasks/env_stacks_updater.py @@ -13,10 +13,10 @@ from dataall.base.utils import Parameter root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) RETRIES = 30 SLEEP_TIME = 30 diff --git a/backend/dataall/core/stacks/tasks/cdkproxy.py b/backend/dataall/core/stacks/tasks/cdkproxy.py index 62262dbc9..198f80081 100644 --- a/backend/dataall/core/stacks/tasks/cdkproxy.py +++ b/backend/dataall/core/stacks/tasks/cdkproxy.py @@ -6,10 +6,10 @@ from dataall.base.db import get_engine root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) logger = logging.getLogger(__name__) +logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) if __name__ == '__main__': diff --git a/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py b/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py index 7fd628465..3a5c8ef74 100644 --- a/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py +++ b/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py @@ -10,10 +10,10 @@ from dataall.base.utils.alarm_service import AlarmService root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) class CatalogIndexerTask: diff --git a/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py b/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py index 0db79e2c9..ecfff37db 100644 --- a/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py +++ b/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py @@ -12,10 +12,10 @@ root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def fetch_omics_workflows(engine): diff --git a/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py index e530a9e6a..bfd4feaad 100644 --- a/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py +++ b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py @@ -17,10 +17,10 @@ from dataall.modules.s3_datasets.services.dataset_alarm_service import DatasetAlarmService root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def sync_tables(engine): diff --git a/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py b/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py index e5a29c904..cf6a9ec05 100644 --- a/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py +++ b/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py @@ -23,10 +23,10 @@ from dataall.modules.shares_base.services.share_notification_service import DataSharingNotificationType root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) # TODO: review this task usage and remove if not needed diff --git a/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py b/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py index 544345fe6..a122b8915 100644 --- a/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py +++ b/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py @@ -7,10 +7,10 @@ from botocore.exceptions import ClientError root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) ENVNAME = os.getenv('envname', 'local') region = os.getenv('AWS_REGION', 'eu-west-1') diff --git a/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py b/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py index 861f317f6..be307bd73 100644 --- a/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py +++ b/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py @@ -10,10 +10,10 @@ root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def persistent_email_reminders(engine): diff --git a/backend/dataall/modules/shares_base/tasks/share_expiration_task.py b/backend/dataall/modules/shares_base/tasks/share_expiration_task.py index 037d5fe17..310b89255 100644 --- a/backend/dataall/modules/shares_base/tasks/share_expiration_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_expiration_task.py @@ -13,10 +13,10 @@ from dataall.modules.shares_base.services.sharing_service import SharingService root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def share_expiration_checker(engine): diff --git a/backend/dataall/modules/shares_base/tasks/share_manager_task.py b/backend/dataall/modules/shares_base/tasks/share_manager_task.py index f71a7dc50..9b8b8c70d 100644 --- a/backend/dataall/modules/shares_base/tasks/share_manager_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_manager_task.py @@ -7,10 +7,10 @@ from dataall.base.loader import load_modules, ImportMode root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) if __name__ == '__main__': diff --git a/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py b/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py index 053d53dd2..32eebd5ab 100644 --- a/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py @@ -12,10 +12,10 @@ from dataall.base.loader import load_modules, ImportMode root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) class EcsBulkShareRepplyService: diff --git a/backend/dataall/modules/shares_base/tasks/share_verifier_task.py b/backend/dataall/modules/shares_base/tasks/share_verifier_task.py index e96ca22f8..9213ab9c8 100644 --- a/backend/dataall/modules/shares_base/tasks/share_verifier_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_verifier_task.py @@ -12,10 +12,10 @@ from dataall.base.loader import load_modules, ImportMode root = logging.getLogger() -root.setLevel(logging.INFO) if not root.hasHandlers(): root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) +log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def verify_shares(engine): diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index 0f875cabf..009d80613 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -52,6 +52,7 @@ def __init__( self._ecr_repository = ecr_repository self._vpc = vpc self._prod_sizing = prod_sizing + self._log_level = 'INFO' if prod_sizing else 'DEBUG' (self.scheduled_tasks_sg, self.share_manager_sg) = self.create_ecs_security_groups( envname, resource_prefix, vpc, vpce_connection, s3_prefix_list, lambdas @@ -216,7 +217,7 @@ def add_catalog_indexer_task(self): command=['python3.9', '-m', 'dataall.modules.catalog.tasks.catalog_indexer_task'], container_id=container_id, ecr_repository=self._ecr_repository, - environment=self._create_env('INFO'), + environment=self._create_env(), image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group(self._envname, self._resource_prefix, log_group_name='catalog-indexer'), schedule_expression=Schedule.expression('rate(6 hours)'), @@ -260,7 +261,7 @@ def add_share_management_task(self): f'ShareManagementTaskContainer{self._envname}', container_name='container', image=ecs.ContainerImage.from_ecr_repository(repository=self._ecr_repository, tag=self._cdkproxy_image_tag), - environment=self._create_env('DEBUG'), + environment=self._create_env(), command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_manager_task'], logging=ecs.LogDriver.aws_logs( stream_prefix='task', @@ -291,7 +292,7 @@ def add_share_verifier_task(self): command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_verifier_task'], container_id='container', ecr_repository=self._ecr_repository, - environment=self._create_env('INFO'), + environment=self._create_env(), image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group(self._envname, self._resource_prefix, log_group_name='share-verifier'), schedule_expression=Schedule.expression('rate(7 days)'), @@ -320,7 +321,7 @@ def add_share_reapplier_task(self): f'ShareReapplierTaskContainer{self._envname}', container_name='container', image=ecs.ContainerImage.from_ecr_repository(repository=self._ecr_repository, tag=self._cdkproxy_image_tag), - environment=self._create_env('INFO'), + environment=self._create_env(), command=['python3.9', '-m', 'dataall.modules.shares_base.tasks.share_reapplier_task'], logging=ecs.LogDriver.aws_logs( stream_prefix='task', @@ -382,7 +383,7 @@ def add_subscription_task(self): ], container_id='container', ecr_repository=self._ecr_repository, - environment=self._create_env('INFO'), + environment=self._create_env(), image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group(self._envname, self._resource_prefix, log_group_name='subscriptions'), schedule_expression=Schedule.expression('rate(15 minutes)'), @@ -402,7 +403,7 @@ def add_sync_dataset_table_task(self): command=['python3.9', '-m', 'dataall.modules.s3_datasets.tasks.tables_syncer'], container_id='container', ecr_repository=self._ecr_repository, - environment=self._create_env('INFO'), + environment=self._create_env(), image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group(self._envname, self._resource_prefix, log_group_name='tables-syncer'), schedule_expression=Schedule.expression('rate(15 minutes)'), @@ -422,7 +423,7 @@ def add_omics_fetch_workflows_task(self): command=['python3.9', '-m', 'dataall.modules.omics.tasks.omics_workflows_fetcher'], container_id='container', ecr_repository=self._ecr_repository, - environment=self._create_env('DEBUG'), + environment=self._create_env(), image_tag=self._cdkproxy_image_tag, log_group=self.create_log_group( self._envname, self._resource_prefix, log_group_name='omics-workflows-fetcher' @@ -822,10 +823,10 @@ def set_scheduled_task( def ecs_task_role(self) -> iam.Role: return self.task_role - def _create_env(self, log_lvl) -> Dict: + def _create_env(self) -> Dict: return { 'AWS_REGION': self.region, 'envname': self._envname, - 'LOGLEVEL': log_lvl, + 'LOG_LEVEL': self._log_level, 'config_location': '/config.json', }