From fdf27ec9c7c2350f7cc991b6ec920569509b34f6 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Wed, 22 Jul 2020 17:06:38 -0700 Subject: [PATCH 1/6] Added Dynamic Encoding Selection Based on Windows System Default --- .gitignore | 1 + .../workflows/dotnet_clipackage/dotnetcli.py | 14 +++++++++++--- .../workflows/dotnet_clipackage/utils.py | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index c7f990167..b031dced1 100644 --- a/.gitignore +++ b/.gitignore @@ -187,6 +187,7 @@ typings/ # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff: +.vscode # Sensitive or high-churn files: diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index b1fe104f1..dd570f5e4 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -49,14 +49,22 @@ def run(self, args, cwd=None): LOG.debug("executing dotnet: %s", invoke_dotnet) - p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd) + #TODO Support for other OS + encoding, encoding_err = self.os_utils.popen(["powershell.exe", + "[system.console]::OutputEncoding.CodePage"], + stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, encoding="ascii").communicate() + + encoding = encoding.strip() + LOG.info("Encoding: %s" % encoding) + + p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd, encoding=encoding) out, err = p.communicate() # The package command contains lots of useful information on how the package was created and # information when the package command was not successful. For that reason the output is # always written to the output to help developers diagnose issues. - LOG.info(out.decode("utf8").strip()) + LOG.info(out.strip()) if p.returncode != 0: - raise DotnetCLIExecutionError(message=err.decode("utf8").strip()) + raise DotnetCLIExecutionError(message=err.strip()) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py index 0df1a3fcd..fffeca068 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py @@ -15,8 +15,8 @@ class OSUtils(object): Convenience wrapper around common system functions """ - def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): - p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd) + def popen(self, command, stdout=None, stderr=None, env=None, cwd=None, encoding=None): + p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd, encoding=encoding) return p def is_windows(self): From bf0cb25a5e908286061883aa78fd192674e294d6 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Fri, 24 Jul 2020 13:48:35 -0700 Subject: [PATCH 2/6] Fixed dotnet output system dependent encoding --- .../workflows/dotnet_clipackage/dotnetcli.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index dd570f5e4..d0083ff70 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -4,6 +4,7 @@ import sys import logging +import locale from .utils import OSUtils @@ -49,14 +50,8 @@ def run(self, args, cwd=None): LOG.debug("executing dotnet: %s", invoke_dotnet) - #TODO Support for other OS - encoding, encoding_err = self.os_utils.popen(["powershell.exe", - "[system.console]::OutputEncoding.CodePage"], - stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, encoding="ascii").communicate() - - encoding = encoding.strip() - LOG.info("Encoding: %s" % encoding) - + # DotNet output is in system locale dependent encoding + encoding = locale.getpreferredencoding() p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd, encoding=encoding) out, err = p.communicate() From 04b91a8fc659d07ff6b16b96e7baef4ebf2d7222 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Fri, 24 Jul 2020 15:45:58 -0700 Subject: [PATCH 3/6] Updated test_custom_make urllib3 Version --- tests/integration/workflows/custom_make/test_custom_make.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/workflows/custom_make/test_custom_make.py b/tests/integration/workflows/custom_make/test_custom_make.py index 8b4e59fdd..6eec92c9f 100644 --- a/tests/integration/workflows/custom_make/test_custom_make.py +++ b/tests/integration/workflows/custom_make/test_custom_make.py @@ -44,7 +44,7 @@ def test_must_build_python_project_through_makefile(self): "chardet", "urllib3", "idna", - "urllib3-1.25.9.dist-info", + "urllib3-1.25.10.dist-info", "chardet-3.0.4.dist-info", "certifi-2020.4.5.2.dist-info", "certifi", From 991c4b8a5c918b0277f10b0f866248eb412845be Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Fri, 24 Jul 2020 16:24:46 -0700 Subject: [PATCH 4/6] Reformatted with black --- aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index d0083ff70..e9ab15f68 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -52,7 +52,9 @@ def run(self, args, cwd=None): # DotNet output is in system locale dependent encoding encoding = locale.getpreferredencoding() - p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd, encoding=encoding) + p = self.os_utils.popen( + invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd, encoding=encoding + ) out, err = p.communicate() From a11a6ead4ecabf33cebe2122f39c9fb69fe95536 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Mon, 27 Jul 2020 11:11:17 -0700 Subject: [PATCH 5/6] Rollback Changes to popen for Python 2 Support --- .../workflows/dotnet_clipackage/dotnetcli.py | 8 +++----- aws_lambda_builders/workflows/dotnet_clipackage/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index e9ab15f68..6b7226129 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -52,16 +52,14 @@ def run(self, args, cwd=None): # DotNet output is in system locale dependent encoding encoding = locale.getpreferredencoding() - p = self.os_utils.popen( - invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd, encoding=encoding - ) + p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd) out, err = p.communicate() # The package command contains lots of useful information on how the package was created and # information when the package command was not successful. For that reason the output is # always written to the output to help developers diagnose issues. - LOG.info(out.strip()) + LOG.info(out.decode(encoding).strip()) if p.returncode != 0: - raise DotnetCLIExecutionError(message=err.strip()) + raise DotnetCLIExecutionError(message=err.decode(encoding).strip()) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py index fffeca068..0df1a3fcd 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/utils.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/utils.py @@ -15,8 +15,8 @@ class OSUtils(object): Convenience wrapper around common system functions """ - def popen(self, command, stdout=None, stderr=None, env=None, cwd=None, encoding=None): - p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd, encoding=encoding) + def popen(self, command, stdout=None, stderr=None, env=None, cwd=None): + p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd) return p def is_windows(self): From 29dbbfd349c7c95c7d34ce5ec45e9caa86519064 Mon Sep 17 00:00:00 2001 From: Wilton Wang Date: Tue, 4 Aug 2020 14:43:18 -0700 Subject: [PATCH 6/6] Added Doc Page for DotNet Output Encoding --- aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py index 6b7226129..5986b3656 100644 --- a/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py +++ b/aws_lambda_builders/workflows/dotnet_clipackage/dotnetcli.py @@ -51,6 +51,8 @@ def run(self, args, cwd=None): LOG.debug("executing dotnet: %s", invoke_dotnet) # DotNet output is in system locale dependent encoding + # https://docs.microsoft.com/en-us/dotnet/api/system.console.outputencoding?view=netcore-3.1#remarks + # "The default code page that the console uses is determined by the system locale." encoding = locale.getpreferredencoding() p = self.os_utils.popen(invoke_dotnet, stdout=self.os_utils.pipe, stderr=self.os_utils.pipe, cwd=cwd)