From e3cb2c62b588d7d4a9cb8464d0b18bd9ea4fef95 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 10 Aug 2021 14:34:27 -0700 Subject: [PATCH 1/4] fix: Import registered command to pyinstaller hook, add unit tests --- installer/__init__.py | 0 installer/pyinstaller/__init__.py | 0 installer/pyinstaller/hidden_imports.py | 15 ++++++++++ installer/pyinstaller/hook-samcli.py | 32 +++------------------- samcli/cli/command.py | 14 ++++++++++ tests/unit/cli/test_pyinstaller_imports.py | 24 ++++++++++++++++ 6 files changed, 57 insertions(+), 28 deletions(-) create mode 100644 installer/__init__.py create mode 100644 installer/pyinstaller/__init__.py create mode 100644 installer/pyinstaller/hidden_imports.py create mode 100644 tests/unit/cli/test_pyinstaller_imports.py diff --git a/installer/__init__.py b/installer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/installer/pyinstaller/__init__.py b/installer/pyinstaller/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/installer/pyinstaller/hidden_imports.py b/installer/pyinstaller/hidden_imports.py new file mode 100644 index 0000000000..9733a6a214 --- /dev/null +++ b/installer/pyinstaller/hidden_imports.py @@ -0,0 +1,15 @@ +from samcli.cli.command import _SAM_CLI_COMMAND_PACKAGES + +SAM_CLI_HIDDEN_IMPORTS = _SAM_CLI_COMMAND_PACKAGES + [ + "cookiecutter.extensions", + "jinja2_time", + "text_unidecode", + "samtranslator", + # default hidden import 'pkg_resources.py2_warn' is added + # since pyInstaller 4.0. + "pkg_resources.py2_warn", + "aws_lambda_builders.workflows", + "configparser", +] + +SAM_CLI_COLLECT_DATA_PACKAGES = ["samcli", "samtranslator", "aws_lambda_builders", "text_unidecode"] diff --git a/installer/pyinstaller/hook-samcli.py b/installer/pyinstaller/hook-samcli.py index 72a939a9b2..9f7d528205 100644 --- a/installer/pyinstaller/hook-samcli.py +++ b/installer/pyinstaller/hook-samcli.py @@ -1,30 +1,6 @@ from PyInstaller.utils import hooks +from .hidden_imports import SAM_CLI_HIDDEN_IMPORTS, SAM_CLI_COLLECT_DATA_PACKAGES -hiddenimports = [ - "cookiecutter.extensions", - "jinja2_time", - "text_unidecode", - "samtranslator", - "samcli.commands.init", - "samcli.commands.validate.validate", - "samcli.commands.build", - "samcli.commands.local.local", - "samcli.commands.package", - "samcli.commands.deploy", - "samcli.commands.logs", - "samcli.commands.publish", - "samcli.commands.pipeline.pipeline", - "samcli.commands.pipeline.init", - "samcli.commands.pipeline.bootstrap", - # default hidden import 'pkg_resources.py2_warn' is added - # since pyInstaller 4.0. - "pkg_resources.py2_warn", - "aws_lambda_builders.workflows", - "configparser", -] -datas = ( - hooks.collect_data_files("samcli") - + hooks.collect_data_files("samtranslator") - + hooks.collect_data_files("aws_lambda_builders") - + hooks.collect_data_files("text_unidecode") -) +hiddenimports = SAM_CLI_HIDDEN_IMPORTS + +datas = sum(hooks.collect_data_files(package) for package in SAM_CLI_COLLECT_DATA_PACKAGES) diff --git a/samcli/cli/command.py b/samcli/cli/command.py index c135400586..c34a587b25 100644 --- a/samcli/cli/command.py +++ b/samcli/cli/command.py @@ -8,6 +8,8 @@ import click +from installer.pyinstaller.hidden_imports import SAM_CLI_HIDDEN_IMPORTS + logger = logging.getLogger(__name__) # Commands that are bundled with the CLI by default in app life-cycle order. @@ -62,6 +64,15 @@ def __init__(self, *args, cmd_packages=None, **kwargs): if not cmd_packages: cmd_packages = _SAM_CLI_COMMAND_PACKAGES + elif isinstance(cmd_packages, list): + for package in cmd_packages: + if package not in SAM_CLI_HIDDEN_IMPORTS: + logger.warning( + "Package %s is not registered as default hidden imports. " + "Consider adding it to /installer/pyinstaller/hidden_imports.py, " + "Otherwise the PyInstaller package will not import it.", + package, + ) self._commands = {} self._commands = BaseCommand._set_commands(cmd_packages) @@ -118,3 +129,6 @@ def get_command(self, ctx, cmd_name): return None return mod.cli + + def get_command_packages(self): + return self._commands.values() diff --git a/tests/unit/cli/test_pyinstaller_imports.py b/tests/unit/cli/test_pyinstaller_imports.py new file mode 100644 index 0000000000..430f26828d --- /dev/null +++ b/tests/unit/cli/test_pyinstaller_imports.py @@ -0,0 +1,24 @@ +import click + +from unittest import TestCase +from installer.pyinstaller import hidden_imports +from samcli.cli.command import BaseCommand + + +class TestPyinstallerImportCommands(TestCase): + def setUp(self): + pass + + def test_hook_contains_all_default_command_packages(self): + cmd = BaseCommand() + command_package_names = cmd.get_command_packages() + + for name in command_package_names: + self.assertIn(name, hidden_imports.SAM_CLI_HIDDEN_IMPORTS) + + def test_hook_not_contain_self_defined_command_packages(self): + cmd = BaseCommand(cmd_packages=["my.self.defined.package"]) + command_package_names = cmd.get_command_packages() + + for name in command_package_names: + self.assertNotIn(name, hidden_imports.SAM_CLI_HIDDEN_IMPORTS) From b596a3986ea54ca179d457812cb814f576e83788 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 10 Aug 2021 17:02:02 -0700 Subject: [PATCH 2/4] fix cicular import --- samcli/cli/command.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/samcli/cli/command.py b/samcli/cli/command.py index c34a587b25..2bdafd6ece 100644 --- a/samcli/cli/command.py +++ b/samcli/cli/command.py @@ -8,8 +8,6 @@ import click -from installer.pyinstaller.hidden_imports import SAM_CLI_HIDDEN_IMPORTS - logger = logging.getLogger(__name__) # Commands that are bundled with the CLI by default in app life-cycle order. @@ -66,10 +64,10 @@ def __init__(self, *args, cmd_packages=None, **kwargs): cmd_packages = _SAM_CLI_COMMAND_PACKAGES elif isinstance(cmd_packages, list): for package in cmd_packages: - if package not in SAM_CLI_HIDDEN_IMPORTS: + if package not in _SAM_CLI_COMMAND_PACKAGES: logger.warning( - "Package %s is not registered as default hidden imports. " - "Consider adding it to /installer/pyinstaller/hidden_imports.py, " + "Command line package %s is not registered as default hidden imports. " + "Consider adding it to /installer/pyinstaller/hidden_imports.py, or _SAM_CLI_COMMAND_PACKAGES" "Otherwise the PyInstaller package will not import it.", package, ) From 3f834bb30b352025cab4f6d6459deecf919178f3 Mon Sep 17 00:00:00 2001 From: Raymond Wang Date: Tue, 17 Aug 2021 19:08:19 +0000 Subject: [PATCH 3/4] update hook imports --- installer/pyinstaller/hidden_imports.py | 2 -- installer/pyinstaller/hook-samcli.py | 9 +++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/installer/pyinstaller/hidden_imports.py b/installer/pyinstaller/hidden_imports.py index 9733a6a214..6758eeee87 100644 --- a/installer/pyinstaller/hidden_imports.py +++ b/installer/pyinstaller/hidden_imports.py @@ -11,5 +11,3 @@ "aws_lambda_builders.workflows", "configparser", ] - -SAM_CLI_COLLECT_DATA_PACKAGES = ["samcli", "samtranslator", "aws_lambda_builders", "text_unidecode"] diff --git a/installer/pyinstaller/hook-samcli.py b/installer/pyinstaller/hook-samcli.py index 9f7d528205..1855250020 100644 --- a/installer/pyinstaller/hook-samcli.py +++ b/installer/pyinstaller/hook-samcli.py @@ -1,6 +1,11 @@ from PyInstaller.utils import hooks -from .hidden_imports import SAM_CLI_HIDDEN_IMPORTS, SAM_CLI_COLLECT_DATA_PACKAGES +from installer.pyinstaller.hidden_imports import SAM_CLI_HIDDEN_IMPORTS hiddenimports = SAM_CLI_HIDDEN_IMPORTS -datas = sum(hooks.collect_data_files(package) for package in SAM_CLI_COLLECT_DATA_PACKAGES) +datas = ( + hooks.collect_data_files("samcli") + + hooks.collect_data_files("samtranslator") + + hooks.collect_data_files("aws_lambda_builders") + + hooks.collect_data_files("text_unidecode") +) From f7bebb3f96a47713323a99b8cd1c0b1379642496 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Wed, 18 Aug 2021 08:37:31 -0700 Subject: [PATCH 4/4] Remove test purpose only code --- samcli/cli/command.py | 12 ------------ tests/unit/cli/test_pyinstaller_imports.py | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/samcli/cli/command.py b/samcli/cli/command.py index 1f72f0d04a..0741cda84f 100644 --- a/samcli/cli/command.py +++ b/samcli/cli/command.py @@ -63,15 +63,6 @@ def __init__(self, *args, cmd_packages=None, **kwargs): if not cmd_packages: cmd_packages = _SAM_CLI_COMMAND_PACKAGES - elif isinstance(cmd_packages, list): - for package in cmd_packages: - if package not in _SAM_CLI_COMMAND_PACKAGES: - logger.warning( - "Command line package %s is not registered as default hidden imports. " - "Consider adding it to /installer/pyinstaller/hidden_imports.py, or _SAM_CLI_COMMAND_PACKAGES" - "Otherwise the PyInstaller package will not import it.", - package, - ) self._commands = {} self._commands = BaseCommand._set_commands(cmd_packages) @@ -128,6 +119,3 @@ def get_command(self, ctx, cmd_name): return None return mod.cli - - def get_command_packages(self): - return self._commands.values() diff --git a/tests/unit/cli/test_pyinstaller_imports.py b/tests/unit/cli/test_pyinstaller_imports.py index 430f26828d..0fb215dd0a 100644 --- a/tests/unit/cli/test_pyinstaller_imports.py +++ b/tests/unit/cli/test_pyinstaller_imports.py @@ -11,14 +11,14 @@ def setUp(self): def test_hook_contains_all_default_command_packages(self): cmd = BaseCommand() - command_package_names = cmd.get_command_packages() + command_package_names = cmd._commands.values() for name in command_package_names: self.assertIn(name, hidden_imports.SAM_CLI_HIDDEN_IMPORTS) def test_hook_not_contain_self_defined_command_packages(self): cmd = BaseCommand(cmd_packages=["my.self.defined.package"]) - command_package_names = cmd.get_command_packages() + command_package_names = cmd._commands.values() for name in command_package_names: self.assertNotIn(name, hidden_imports.SAM_CLI_HIDDEN_IMPORTS)