diff --git a/backend/dataall/__init__.py b/backend/dataall/__init__.py index a6387d880..5a78164b2 100644 --- a/backend/dataall/__init__.py +++ b/backend/dataall/__init__.py @@ -1,2 +1,13 @@ from . import core, version from .base import utils, db, api +import logging +import os +import sys + +logging.basicConfig( + level=os.environ.get('LOG_LEVEL', 'INFO'), + handlers=[logging.StreamHandler(sys.stdout)], + format='[%(levelname)s] %(message)s', +) +for name in ['boto3', 's3transfer', 'botocore', 'boto', 'urllib3']: + logging.getLogger(name).setLevel(logging.ERROR) diff --git a/backend/dataall/core/environment/tasks/env_stacks_updater.py b/backend/dataall/core/environment/tasks/env_stacks_updater.py index ecf6b72f9..40b9a14c6 100644 --- a/backend/dataall/core/environment/tasks/env_stacks_updater.py +++ b/backend/dataall/core/environment/tasks/env_stacks_updater.py @@ -12,11 +12,8 @@ from dataall.base.db import get_engine from dataall.base.utils import Parameter -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) + RETRIES = 30 SLEEP_TIME = 30 diff --git a/backend/dataall/core/stacks/tasks/cdkproxy.py b/backend/dataall/core/stacks/tasks/cdkproxy.py index 198f80081..f0aa7d6d5 100644 --- a/backend/dataall/core/stacks/tasks/cdkproxy.py +++ b/backend/dataall/core/stacks/tasks/cdkproxy.py @@ -5,11 +5,7 @@ from dataall.base.cdkproxy.cdk_cli_wrapper import deploy_cdk_stack from dataall.base.db import get_engine -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) logger = logging.getLogger(__name__) -logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) if __name__ == '__main__': diff --git a/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py b/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py index 032739db9..807b712a3 100644 --- a/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py +++ b/backend/dataall/modules/catalog/tasks/catalog_indexer_task.py @@ -9,11 +9,7 @@ from dataall.base.loader import load_modules, ImportMode from dataall.base.utils.alarm_service import AlarmService -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) class CatalogIndexerTask: diff --git a/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py b/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py index ecfff37db..d20d69cfc 100644 --- a/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py +++ b/backend/dataall/modules/omics/tasks/omics_workflows_fetcher.py @@ -11,11 +11,7 @@ from dataall.modules.omics.db.omics_repository import OmicsRepository -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def fetch_omics_workflows(engine): diff --git a/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py index bfd4feaad..9bd47dd3c 100644 --- a/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py +++ b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py @@ -16,11 +16,7 @@ from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer from dataall.modules.s3_datasets.services.dataset_alarm_service import DatasetAlarmService -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def sync_tables(engine): diff --git a/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py b/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py index cf6a9ec05..a08fb5cf8 100644 --- a/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py +++ b/backend/dataall/modules/s3_datasets_shares/tasks/dataset_subscription_task.py @@ -22,11 +22,7 @@ from dataall.modules.shares_base.db.share_object_models import ShareObject from dataall.modules.shares_base.services.share_notification_service import DataSharingNotificationType -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) # TODO: review this task usage and remove if not needed diff --git a/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py b/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py index a122b8915..89497e62a 100644 --- a/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py +++ b/backend/dataall/modules/s3_datasets_shares/tasks/subscriptions/sqs_poller.py @@ -6,11 +6,8 @@ import boto3 from botocore.exceptions import ClientError -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) + ENVNAME = os.getenv('envname', 'local') region = os.getenv('AWS_REGION', 'eu-west-1') diff --git a/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py b/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py index be307bd73..42635f6f9 100644 --- a/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py +++ b/backend/dataall/modules/shares_base/tasks/persistent_email_reminders_task.py @@ -9,11 +9,7 @@ from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def persistent_email_reminders(engine): diff --git a/backend/dataall/modules/shares_base/tasks/share_expiration_task.py b/backend/dataall/modules/shares_base/tasks/share_expiration_task.py index 310b89255..51328e55a 100644 --- a/backend/dataall/modules/shares_base/tasks/share_expiration_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_expiration_task.py @@ -12,11 +12,7 @@ from dataall.modules.shares_base.services.shares_enums import ShareObjectActions from dataall.modules.shares_base.services.sharing_service import SharingService -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def share_expiration_checker(engine): diff --git a/backend/dataall/modules/shares_base/tasks/share_manager_task.py b/backend/dataall/modules/shares_base/tasks/share_manager_task.py index 9b8b8c70d..d120251c9 100644 --- a/backend/dataall/modules/shares_base/tasks/share_manager_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_manager_task.py @@ -6,12 +6,7 @@ from dataall.base.db import get_engine from dataall.base.loader import load_modules, ImportMode -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) - if __name__ == '__main__': try: diff --git a/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py b/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py index 32eebd5ab..225f069bd 100644 --- a/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_reapplier_task.py @@ -11,11 +11,7 @@ from dataall.base.loader import load_modules, ImportMode -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) class EcsBulkShareRepplyService: diff --git a/backend/dataall/modules/shares_base/tasks/share_verifier_task.py b/backend/dataall/modules/shares_base/tasks/share_verifier_task.py index 9213ab9c8..a731a5756 100644 --- a/backend/dataall/modules/shares_base/tasks/share_verifier_task.py +++ b/backend/dataall/modules/shares_base/tasks/share_verifier_task.py @@ -11,11 +11,7 @@ from dataall.base.loader import load_modules, ImportMode -root = logging.getLogger() -if not root.hasHandlers(): - root.addHandler(logging.StreamHandler(sys.stdout)) log = logging.getLogger(__name__) -log.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) def verify_shares(engine):