From a6ead086ff992a8b9a445d84a52bab06008a6487 Mon Sep 17 00:00:00 2001 From: Omar Soriano Date: Wed, 31 Jul 2024 03:34:59 +0000 Subject: [PATCH 1/3] (trivial) fix ws --- deploy-agent/deployd/client/base_client.py | 8 ++++---- deploy-agent/deployd/common/caller.py | 6 +++--- deploy-agent/deployd/common/executor.py | 4 ++-- deploy-agent/deployd/common/single_instance.py | 6 +++--- deploy-agent/deployd/download/http_download_helper.py | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deploy-agent/deployd/client/base_client.py b/deploy-agent/deployd/client/base_client.py index 2bb5130c0e..493cb2893a 100644 --- a/deploy-agent/deployd/client/base_client.py +++ b/deploy-agent/deployd/client/base_client.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,13 +20,13 @@ class BaseClient(with_metaclass(ABCMeta, object)): """This class plays a role as an interface defining methods for agent to communicate with teletraan service. """ - + @abstractmethod def send_reports(self, env_reports=None): """Args: env_reports: a dict with env name as key and DeployStatus as value. Returns: - PingResponse describing next action for deploy agent. + PingResponse describing next action for deploy agent. """ pass diff --git a/deploy-agent/deployd/common/caller.py b/deploy-agent/deployd/common/caller.py index cbc008022b..f162b1c94d 100644 --- a/deploy-agent/deployd/common/caller.py +++ b/deploy-agent/deployd/common/caller.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,7 +15,7 @@ import subprocess import traceback import logging -import time +import time from typing import Optional, Tuple from future.utils import PY3 diff --git a/deploy-agent/deployd/common/executor.py b/deploy-agent/deployd/common/executor.py index 90b0736b8a..b1ce905bd0 100644 --- a/deploy-agent/deployd/common/executor.py +++ b/deploy-agent/deployd/common/executor.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/deploy-agent/deployd/common/single_instance.py b/deploy-agent/deployd/common/single_instance.py index 33713f8939..19a696b030 100644 --- a/deploy-agent/deployd/common/single_instance.py +++ b/deploy-agent/deployd/common/single_instance.py @@ -32,12 +32,12 @@ def __init__(self) -> None: appname = 'deploy-agent' lockfile_name = '.{}.lock'.format(appname) self._create_lock_dir() - # Backward compatibility as old deploy agent versions use lock file in /tmp. - # Use the old lock file if it exists + # Backward compatibility as old deploy agent versions use lock file in /tmp. + # Use the old lock file if it exists tmp_lockfile_path = os.path.join(tempfile.gettempdir(), lockfile_name) if os.path.exists(tmp_lockfile_path): lockfile_path = tmp_lockfile_path - else: + else: lockfile_path = os.path.join(LOCKFILE_DIR, lockfile_name) lockfile_flags = os.O_WRONLY | os.O_CREAT # This is 0o222, i.e. 146, --w--w--w- diff --git a/deploy-agent/deployd/download/http_download_helper.py b/deploy-agent/deployd/download/http_download_helper.py index 5f0dd3d18e..985a76cc3a 100644 --- a/deploy-agent/deployd/download/http_download_helper.py +++ b/deploy-agent/deployd/download/http_download_helper.py @@ -98,4 +98,4 @@ def validate_source(self) -> bool: return True else: log.error(f"{domain} is not in the allow list: {allow_list}.") - return False \ No newline at end of file + return False From 7a353754f7f2aee5d8bab9ba9449edc3b0c7a573 Mon Sep 17 00:00:00 2001 From: Omar Soriano Date: Wed, 31 Jul 2024 03:21:58 +0000 Subject: [PATCH 2/3] Update deploy-agent future deps The "future" packages may have been useful for python 2/3 compatibility, but the current targets are: 3.8, 3.12 Drop the unnecessary dependencies: - print_function - Feature is included in 3.0 - absolute_import - Feature is included in 3.0 - with_metaclass - In 3.0, inheriting object makes no difference - PY3 - Evaluates to True in 3.0 References: - https://docs.python.org/3/library/__future__.html#module-contents --- deploy-agent/deployd/client/base_client.py | 5 ++-- deploy-agent/deployd/common/caller.py | 10 ++------ deploy-agent/deployd/common/config.py | 1 - deploy-agent/deployd/common/executor.py | 24 +++++-------------- .../deployd/common/single_instance.py | 14 +---------- deploy-agent/deployd/common/utils.py | 1 - .../download/download_helper_factory.py | 2 +- .../deployd/download/http_download_helper.py | 1 - deploy-agent/deployd/staging/stager.py | 1 - deploy-agent/setup.py | 3 +-- .../unit/deploy/client/test_base_client.py | 5 ++-- 11 files changed, 15 insertions(+), 52 deletions(-) diff --git a/deploy-agent/deployd/client/base_client.py b/deploy-agent/deployd/client/base_client.py index 493cb2893a..4bac47f22b 100644 --- a/deploy-agent/deployd/client/base_client.py +++ b/deploy-agent/deployd/client/base_client.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from abc import ABCMeta, abstractmethod -from future.utils import with_metaclass +from abc import ABC, abstractmethod -class BaseClient(with_metaclass(ABCMeta, object)): +class BaseClient(ABC): """This class plays a role as an interface defining methods for agent to communicate with teletraan service. """ diff --git a/deploy-agent/deployd/common/caller.py b/deploy-agent/deployd/common/caller.py index f162b1c94d..dae2d19a0b 100644 --- a/deploy-agent/deployd/common/caller.py +++ b/deploy-agent/deployd/common/caller.py @@ -18,8 +18,6 @@ import time from typing import Optional, Tuple -from future.utils import PY3 - log = logging.getLogger(__name__) @@ -32,12 +30,8 @@ def call_and_log(cmd, **kwargs) -> Tuple[Optional[str], str, Optional[int]]: output = "" start = time.time() try: - if PY3: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, encoding='utf-8', **kwargs) - else: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, **kwargs) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding='utf-8', **kwargs) while process.poll() is None: line = process.stdout.readline() if line: diff --git a/deploy-agent/deployd/common/config.py b/deploy-agent/deployd/common/config.py index 4e6f22659c..3d94a0131e 100644 --- a/deploy-agent/deployd/common/config.py +++ b/deploy-agent/deployd/common/config.py @@ -1,4 +1,3 @@ -from __future__ import print_function # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/deployd/common/executor.py b/deploy-agent/deployd/common/executor.py index b1ce905bd0..13aad17d98 100644 --- a/deploy-agent/deployd/common/executor.py +++ b/deploy-agent/deployd/common/executor.py @@ -22,8 +22,6 @@ import traceback from typing import Tuple -from future.utils import PY3 - from deployd.common.types import DeployReport, PingStatus, PRE_STAGE_STEPS, AgentStatus log = logging.getLogger(__name__) @@ -111,14 +109,11 @@ def run_cmd(self, cmd, **kw) -> DeployReport: # sleep some seconds before next poll sleep_time = self._get_sleep_interval(start, self.PROCESS_POLL_INTERVAL) - if PY3: - # Wait up to sleep_time for the process to terminate (new in Python 3.3) - try: - process.wait(sleep_time) - except subprocess.TimeoutExpired: - pass - else: - time.sleep(sleep_time) + # Wait up to sleep_time for the process to terminate (new in Python 3.3) + try: + process.wait(sleep_time) + except subprocess.TimeoutExpired: + pass # finish executing sub process deploy_report.error_code = process.returncode @@ -202,14 +197,7 @@ def _graceful_shutdown(self, process) -> None: try: log.info('Gracefully shutdown currently running process with timeout {}'.format(self.TERMINATE_TIMEOUT)) os.killpg(process.pid, signal.SIGTERM) - if PY3: - process.wait(self.TERMINATE_TIMEOUT) - else: - start_time = datetime.datetime.now() - while process.poll() is None: - if (datetime.datetime.now() - start_time).seconds > self.TERMINATE_TIMEOUT: - raise Exception('Timed out while waiting for the process to shutdown') - time.sleep(min(self.PROCESS_POLL_INTERVAL, self.TERMINATE_TIMEOUT)) + process.wait(self.TERMINATE_TIMEOUT) except Exception as e: log.debug('Failed to gracefully shutdown: {}'.format(e)) Executor._kill_process(process) diff --git a/deploy-agent/deployd/common/single_instance.py b/deploy-agent/deployd/common/single_instance.py index 19a696b030..9a345dd8ae 100644 --- a/deploy-agent/deployd/common/single_instance.py +++ b/deploy-agent/deployd/common/single_instance.py @@ -1,5 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +15,8 @@ import logging import os import stat -import errno import fcntl from . import utils -from future.utils import PY3 import tempfile log = logging.getLogger(__name__) LOCKFILE_DIR = '/var/lock' @@ -61,12 +57,4 @@ def __init__(self) -> None: utils.exit_abruptly(1) def _create_lock_dir(self) -> None: - if PY3: - os.makedirs(LOCKFILE_DIR, exist_ok=True) - else: - # Need to handle the case when lock dir exists in py2 - try: - os.makedirs(LOCKFILE_DIR) # py2 - except OSError as e: - if e.errno != errno.EEXIST: - raise + os.makedirs(LOCKFILE_DIR, exist_ok=True) diff --git a/deploy-agent/deployd/common/utils.py b/deploy-agent/deployd/common/utils.py index b6b63e9294..192feadfe9 100644 --- a/deploy-agent/deployd/common/utils.py +++ b/deploy-agent/deployd/common/utils.py @@ -1,4 +1,3 @@ -from __future__ import print_function # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/deployd/download/download_helper_factory.py b/deploy-agent/deployd/download/download_helper_factory.py index 9db07046f2..78ee85a262 100644 --- a/deploy-agent/deployd/download/download_helper_factory.py +++ b/deploy-agent/deployd/download/download_helper_factory.py @@ -16,7 +16,7 @@ from typing import Optional import boto3 -from future.moves.urllib.parse import urlparse +from urllib.parse import urlparse from deployd.download.download_helper import DownloadHelper from deployd.download.s3_download_helper import S3DownloadHelper diff --git a/deploy-agent/deployd/download/http_download_helper.py b/deploy-agent/deployd/download/http_download_helper.py index 985a76cc3a..96a9cdc33e 100644 --- a/deploy-agent/deployd/download/http_download_helper.py +++ b/deploy-agent/deployd/download/http_download_helper.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/deployd/staging/stager.py b/deploy-agent/deployd/staging/stager.py index 935ff76576..73c0456185 100644 --- a/deploy-agent/deployd/staging/stager.py +++ b/deploy-agent/deployd/staging/stager.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import # Copyright 2016 Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/deploy-agent/setup.py b/deploy-agent/setup.py index 060191e834..fd2937ce52 100644 --- a/deploy-agent/setup.py +++ b/deploy-agent/setup.py @@ -30,8 +30,7 @@ "gevent==24.2.1; python_version >= '3.12'", "lockfile==0.10.2", "boto3==1.34.134", - "python-daemon==2.0.6", - "future==0.18.2" + "python-daemon==2.0.6" ] setup( diff --git a/deploy-agent/tests/unit/deploy/client/test_base_client.py b/deploy-agent/tests/unit/deploy/client/test_base_client.py index c4aada1cdc..74c8fd5e48 100644 --- a/deploy-agent/tests/unit/deploy/client/test_base_client.py +++ b/deploy-agent/tests/unit/deploy/client/test_base_client.py @@ -13,8 +13,7 @@ # limitations under the License. import unittest -from abc import ABCMeta, abstractmethod -from future.utils import with_metaclass +from abc import ABC, abstractmethod from tests import TestCase from deployd.client.base_client import BaseClient @@ -34,7 +33,7 @@ def test_abc_equivalent_to_old(self): """ Make sure that new changes to base client extend the original class """ - class OldBaseClient(with_metaclass(ABCMeta, object)): + class OldBaseClient(ABC): @abstractmethod def send_reports(self, env_reports=None): pass From 798509a4422b16c8e1e684a92ea1d86b2c238caf Mon Sep 17 00:00:00 2001 From: Omar Soriano Date: Wed, 31 Jul 2024 18:03:24 +0000 Subject: [PATCH 3/3] bump deploy-agent version --- deploy-agent/deployd/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy-agent/deployd/__init__.py b/deploy-agent/deployd/__init__.py index 68023a79ca..a5ec14b2eb 100644 --- a/deploy-agent/deployd/__init__.py +++ b/deploy-agent/deployd/__init__.py @@ -27,4 +27,4 @@ # 2: puppet applied successfully with changes PUPPET_SUCCESS_EXIT_CODES = [0, 2] -__version__ = '1.2.63' +__version__ = '1.2.64'