From b34c613d33de3400aebf436249942774b1a27dd2 Mon Sep 17 00:00:00 2001 From: dijiachen Date: Mon, 1 Jul 2024 11:58:49 +0800 Subject: [PATCH 1/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apiserver/paasng/paasng/platform/smart_app/services/detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index ca8d15764a..339ca92f00 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -185,7 +185,7 @@ def _load_logo(archive: Union[ZipClient, BinaryTarClient], relative_path: str) - # A: 因为 meta_info 会被归档存储进数据库, bytes 类型无法序列化成 json logo_b64data = "base64," + base64.b64encode(archive.read_file(relative_path + "logo.png")).decode() except (RuntimeError, KeyError): - logger.exception("Can't read logo.png.") + logger.info("Can't read logo.png.") return logo_b64data From af29a428c7a9d52ba0020c6c01e99e265f62a109 Mon Sep 17 00:00:00 2001 From: dijiachen Date: Thu, 11 Jul 2024 11:22:33 +0800 Subject: [PATCH 2/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../paasng/paasng/platform/smart_app/services/detector.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index 339ca92f00..a4e0667e08 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -184,8 +184,11 @@ def _load_logo(archive: Union[ZipClient, BinaryTarClient], relative_path: str) - # Q: 为什么需要进行 base64 编码? # A: 因为 meta_info 会被归档存储进数据库, bytes 类型无法序列化成 json logo_b64data = "base64," + base64.b64encode(archive.read_file(relative_path + "logo.png")).decode() - except (RuntimeError, KeyError): - logger.info("Can't read logo.png.") + except (RuntimeError, KeyError) as e: + if "Don't exists" in str(e) or "Not found in archive" in str(e): + logger.info("The logo.png does not exist,using default logo.") + else: + logger.exception("Can't read logo.png.") return logo_b64data From 41142d41200704ae285172618ef5bd6c023b219c Mon Sep 17 00:00:00 2001 From: dijiachen Date: Thu, 11 Jul 2024 11:26:02 +0800 Subject: [PATCH 3/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apiserver/paasng/paasng/platform/smart_app/services/detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index a4e0667e08..96bbaab523 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -186,7 +186,7 @@ def _load_logo(archive: Union[ZipClient, BinaryTarClient], relative_path: str) - logo_b64data = "base64," + base64.b64encode(archive.read_file(relative_path + "logo.png")).decode() except (RuntimeError, KeyError) as e: if "Don't exists" in str(e) or "Not found in archive" in str(e): - logger.info("The logo.png does not exist,using default logo.") + logger.info("The logo.png does not exist, using default logo.") else: logger.exception("Can't read logo.png.") return logo_b64data From a2665644951fb36e7bd36824c657330ed47987c6 Mon Sep 17 00:00:00 2001 From: dijiachen Date: Fri, 12 Jul 2024 15:28:56 +0800 Subject: [PATCH 4/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../paasng/platform/smart_app/services/detector.py | 9 ++++----- .../paasng/paasng/platform/sourcectl/package/client.py | 7 +++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index 96bbaab523..eefcc8a765 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -184,11 +184,10 @@ def _load_logo(archive: Union[ZipClient, BinaryTarClient], relative_path: str) - # Q: 为什么需要进行 base64 编码? # A: 因为 meta_info 会被归档存储进数据库, bytes 类型无法序列化成 json logo_b64data = "base64," + base64.b64encode(archive.read_file(relative_path + "logo.png")).decode() - except (RuntimeError, KeyError) as e: - if "Don't exists" in str(e) or "Not found in archive" in str(e): - logger.info("The logo.png does not exist, using default logo.") - else: - logger.exception("Can't read logo.png.") + except FileNotFoundError: + logger.info("The logo.png does not exist, using default logo.") + except (RuntimeError, KeyError): + logger.exception("Can't read logo.png.") return logo_b64data diff --git a/apiserver/paasng/paasng/platform/sourcectl/package/client.py b/apiserver/paasng/paasng/platform/sourcectl/package/client.py index 38718b8f29..d0b471c6c2 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/package/client.py +++ b/apiserver/paasng/paasng/platform/sourcectl/package/client.py @@ -143,7 +143,10 @@ def read_file(self, filename) -> bytes: if p.returncode != 0: if self._is_invalid_file_format_error(stderr): raise InvalidPackageFileFormatError() - raise RuntimeError(f"Failed to extractfile from the tarball, error: {stderr!r}") + if "Not found in archive" in stderr: + raise FileNotFoundError(f"Failed to extractfile from the tarball, error: {stderr!r}") + else: + raise RuntimeError(f"Failed to extractfile from the tarball, error: {stderr!r}") return (temp_dir / filename).read_bytes() def export(self, local_path: str): @@ -220,7 +223,7 @@ def read_file(self, file_path: str) -> bytes: try: info = self.zip_.getinfo(key) except KeyError as e: - raise KeyError(f"filename: {file_path} Don't exists.") from e + raise FileNotFoundError(f"filename: {file_path} Don't exists.") from e return self.zip_.read(info) From 8c59271e4851aed800dbe9cd420d54f520dc233f Mon Sep 17 00:00:00 2001 From: dijiachen Date: Fri, 12 Jul 2024 16:56:18 +0800 Subject: [PATCH 5/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../paasng/platform/sourcectl/package/client.py | 10 +++++++--- .../platform/sourcectl/packages/test_client.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apiserver/paasng/paasng/platform/sourcectl/package/client.py b/apiserver/paasng/paasng/platform/sourcectl/package/client.py index d0b471c6c2..d2ca6c91e0 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/package/client.py +++ b/apiserver/paasng/paasng/platform/sourcectl/package/client.py @@ -42,6 +42,10 @@ class InvalidPackageFileFormatError(Exception): """The package file is not in a valid format, it might be corrupt.""" +class FileDoesNotExistError(Exception): + """The file does not exist.""" + + class BasePackageClient(metaclass=abc.ABCMeta): @abc.abstractmethod def read_file(self, file_path: str) -> bytes: @@ -99,7 +103,7 @@ def read_file(self, file_path: str) -> bytes: key = os.path.join(self.relative_path, key) file = self.tar.extractfile(key) if not file: - raise KeyError(f"filename: {file_path} Don't exists.") + raise FileDoesNotExistError(f"filename: {file_path} Don't exists.") return file.read() def export(self, local_path: str): @@ -144,7 +148,7 @@ def read_file(self, filename) -> bytes: if self._is_invalid_file_format_error(stderr): raise InvalidPackageFileFormatError() if "Not found in archive" in stderr: - raise FileNotFoundError(f"Failed to extractfile from the tarball, error: {stderr!r}") + raise FileDoesNotExistError(f"Failed to extractfile from the tarball, error: {stderr!r}") else: raise RuntimeError(f"Failed to extractfile from the tarball, error: {stderr!r}") return (temp_dir / filename).read_bytes() @@ -223,7 +227,7 @@ def read_file(self, file_path: str) -> bytes: try: info = self.zip_.getinfo(key) except KeyError as e: - raise FileNotFoundError(f"filename: {file_path} Don't exists.") from e + raise FileDoesNotExistError(f"filename: {file_path} Don't exists.") from e return self.zip_.read(info) diff --git a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py index 9ffd9ace23..87cf01efda 100644 --- a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py +++ b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py @@ -26,6 +26,7 @@ from paasng.platform.sourcectl.package.client import ( BasePackageClient, BinaryTarClient, + FileDoesNotExistError, GenericLocalClient, GenericRemoteClient, InvalidPackageFileFormatError, @@ -100,7 +101,7 @@ class TestLocalClient: [ (dict(Procfile="web: npm run dev\n"), "Procfile", does_not_raise(), b"web: npm run dev\n"), (dict(Procfile="web: npm run dev\n"), "./Procfile", does_not_raise(), b"web: npm run dev\n"), - (dict(Procfile="web: npm run dev\n"), "procfile", pytest.raises(KeyError), None), + (dict(Procfile="web: npm run dev\n"), "procfile", pytest.raises((KeyError, FileDoesNotExistError)), None), ], ) def test_read_file(self, client_cls, archive_maker, contents, filename, ctx, expected): @@ -119,16 +120,22 @@ def test_read_file(self, client_cls, archive_maker, contents, filename, ctx, exp ({"foo/foo": "1"}, "./", "./foo/foo", does_not_raise(), b"1"), ({"foo/foo": "1"}, "./foo", "foo", does_not_raise(), b"1"), ({"foo/foo": "1"}, "./foo", "./foo", does_not_raise(), b"1"), - ({"foo/foo": "1"}, "./foo", "./foo/foo", pytest.raises(KeyError), b"1"), - ({"foo/foo": "1"}, "./foo", "foo/foo", pytest.raises(KeyError), b"1"), + ({"foo/foo": "1"}, "./foo", "./foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"1"), + ({"foo/foo": "1"}, "./foo", "foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"1"), ({"foo/foo/foo": "3"}, "./", "foo/foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./", "./foo/foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/", "foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/", "./foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/foo", "foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/foo", "./foo", does_not_raise(), b"3"), - ({"foo/foo/foo": "3"}, "./foo/foo", "foo/foo/foo", pytest.raises(KeyError), b"3"), - ({"foo/foo/foo": "3"}, "./foo/foo", "./foo/foo/foo", pytest.raises(KeyError), b"3"), + ({"foo/foo/foo": "3"}, "./foo/foo", "foo/foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"3"), + ( + {"foo/foo/foo": "3"}, + "./foo/foo", + "./foo/foo/foo", + pytest.raises((KeyError, FileDoesNotExistError)), + b"3", + ), ], ) def test_read_file_with_relative_path( From dc11d8ce57213d6f9d46134fbb52cc36e530da82 Mon Sep 17 00:00:00 2001 From: dijiachen Date: Fri, 12 Jul 2024 17:36:11 +0800 Subject: [PATCH 6/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/smart_app/services/detector.py | 9 +++++++-- .../paasng/platform/sourcectl/package/client.py | 2 +- .../platform/sourcectl/packages/test_client.py | 17 +++++------------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/apiserver/paasng/paasng/platform/smart_app/services/detector.py b/apiserver/paasng/paasng/platform/smart_app/services/detector.py index eefcc8a765..b61de13255 100644 --- a/apiserver/paasng/paasng/platform/smart_app/services/detector.py +++ b/apiserver/paasng/paasng/platform/smart_app/services/detector.py @@ -38,7 +38,12 @@ from paasng.platform.declarative.handlers import get_desc_handler from paasng.platform.smart_app.services.path import PathProtocol from paasng.platform.sourcectl.models import SPStat -from paasng.platform.sourcectl.package.client import BinaryTarClient, InvalidPackageFileFormatError, ZipClient +from paasng.platform.sourcectl.package.client import ( + BinaryTarClient, + FileDoesNotExistError, + InvalidPackageFileFormatError, + ZipClient, +) logger = logging.getLogger(__name__) @@ -184,7 +189,7 @@ def _load_logo(archive: Union[ZipClient, BinaryTarClient], relative_path: str) - # Q: 为什么需要进行 base64 编码? # A: 因为 meta_info 会被归档存储进数据库, bytes 类型无法序列化成 json logo_b64data = "base64," + base64.b64encode(archive.read_file(relative_path + "logo.png")).decode() - except FileNotFoundError: + except FileDoesNotExistError: logger.info("The logo.png does not exist, using default logo.") except (RuntimeError, KeyError): logger.exception("Can't read logo.png.") diff --git a/apiserver/paasng/paasng/platform/sourcectl/package/client.py b/apiserver/paasng/paasng/platform/sourcectl/package/client.py index d2ca6c91e0..d371a9c6e7 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/package/client.py +++ b/apiserver/paasng/paasng/platform/sourcectl/package/client.py @@ -42,7 +42,7 @@ class InvalidPackageFileFormatError(Exception): """The package file is not in a valid format, it might be corrupt.""" -class FileDoesNotExistError(Exception): +class FileDoesNotExistError(KeyError, RuntimeError): """The file does not exist.""" diff --git a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py index 87cf01efda..9ffd9ace23 100644 --- a/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py +++ b/apiserver/paasng/tests/paasng/platform/sourcectl/packages/test_client.py @@ -26,7 +26,6 @@ from paasng.platform.sourcectl.package.client import ( BasePackageClient, BinaryTarClient, - FileDoesNotExistError, GenericLocalClient, GenericRemoteClient, InvalidPackageFileFormatError, @@ -101,7 +100,7 @@ class TestLocalClient: [ (dict(Procfile="web: npm run dev\n"), "Procfile", does_not_raise(), b"web: npm run dev\n"), (dict(Procfile="web: npm run dev\n"), "./Procfile", does_not_raise(), b"web: npm run dev\n"), - (dict(Procfile="web: npm run dev\n"), "procfile", pytest.raises((KeyError, FileDoesNotExistError)), None), + (dict(Procfile="web: npm run dev\n"), "procfile", pytest.raises(KeyError), None), ], ) def test_read_file(self, client_cls, archive_maker, contents, filename, ctx, expected): @@ -120,22 +119,16 @@ def test_read_file(self, client_cls, archive_maker, contents, filename, ctx, exp ({"foo/foo": "1"}, "./", "./foo/foo", does_not_raise(), b"1"), ({"foo/foo": "1"}, "./foo", "foo", does_not_raise(), b"1"), ({"foo/foo": "1"}, "./foo", "./foo", does_not_raise(), b"1"), - ({"foo/foo": "1"}, "./foo", "./foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"1"), - ({"foo/foo": "1"}, "./foo", "foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"1"), + ({"foo/foo": "1"}, "./foo", "./foo/foo", pytest.raises(KeyError), b"1"), + ({"foo/foo": "1"}, "./foo", "foo/foo", pytest.raises(KeyError), b"1"), ({"foo/foo/foo": "3"}, "./", "foo/foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./", "./foo/foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/", "foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/", "./foo/foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/foo", "foo", does_not_raise(), b"3"), ({"foo/foo/foo": "3"}, "./foo/foo", "./foo", does_not_raise(), b"3"), - ({"foo/foo/foo": "3"}, "./foo/foo", "foo/foo/foo", pytest.raises((KeyError, FileDoesNotExistError)), b"3"), - ( - {"foo/foo/foo": "3"}, - "./foo/foo", - "./foo/foo/foo", - pytest.raises((KeyError, FileDoesNotExistError)), - b"3", - ), + ({"foo/foo/foo": "3"}, "./foo/foo", "foo/foo/foo", pytest.raises(KeyError), b"3"), + ({"foo/foo/foo": "3"}, "./foo/foo", "./foo/foo/foo", pytest.raises(KeyError), b"3"), ], ) def test_read_file_with_relative_path( From 17bf5866aed9eddd13978fca3c1e796d569f3230 Mon Sep 17 00:00:00 2001 From: dijiachen Date: Tue, 16 Jul 2024 11:28:14 +0800 Subject: [PATCH 7/7] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=20logo=20=E5=A4=B1=E8=B4=A5=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=BA=A7=E5=88=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../paasng/platform/sourcectl/package/client.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apiserver/paasng/paasng/platform/sourcectl/package/client.py b/apiserver/paasng/paasng/platform/sourcectl/package/client.py index d371a9c6e7..2f81975de5 100644 --- a/apiserver/paasng/paasng/platform/sourcectl/package/client.py +++ b/apiserver/paasng/paasng/platform/sourcectl/package/client.py @@ -43,7 +43,11 @@ class InvalidPackageFileFormatError(Exception): class FileDoesNotExistError(KeyError, RuntimeError): - """The file does not exist.""" + """The file does not exist. + + This exception is used to maintain compatibility with existing code that handles missing files. + TODO: Consider unifying exception handling for read_file. + """ class BasePackageClient(metaclass=abc.ABCMeta): @@ -147,7 +151,7 @@ def read_file(self, filename) -> bytes: if p.returncode != 0: if self._is_invalid_file_format_error(stderr): raise InvalidPackageFileFormatError() - if "Not found in archive" in stderr: + if self._is_not_found_error(stderr): raise FileDoesNotExistError(f"Failed to extractfile from the tarball, error: {stderr!r}") else: raise RuntimeError(f"Failed to extractfile from the tarball, error: {stderr!r}") @@ -192,6 +196,13 @@ def _is_invalid_file_format_error(message: str) -> bool: return True return False + @staticmethod + def _is_not_found_error(message: str) -> bool: + """Check if the stderr message indicates a file not found.""" + if re.search(r"tar:.*: Not found in archive", message): + return True + return False + class ZipClient(BasePackageClient): """基于本地 zip 包初始化的 client"""