From 5203db3b3aaa53353b765f64b34fb30ee019319e Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Fri, 30 Aug 2019 13:45:15 -0700 Subject: [PATCH 1/3] feat: Provide pinned requirements to allow for reproducible builds --- Makefile | 7 +++- appveyor.yml | 1 + requirements/isolated.txt | 37 ++++++++++++++++++++ scripts/check-isolated-needs-update.py | 48 ++++++++++++++++++++++++++ scripts/check-requirements.py | 43 +++++++++++++++++++++++ 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 requirements/isolated.txt create mode 100644 scripts/check-isolated-needs-update.py create mode 100755 scripts/check-requirements.py diff --git a/Makefile b/Makefile index 9d042790bd..affc53c7b6 100644 --- a/Makefile +++ b/Makefile @@ -38,4 +38,9 @@ black-check: black --check samcli/* tests/* # Verifications to run before sending a pull request -pr: init dev black-check \ No newline at end of file +pr: init dev black-check + +update-isolated-req: + pipenv --three + pipenv run pip install -r requirements/base.txt + pipenv run pip freeze > requirements/isolated.txt diff --git a/appveyor.yml b/appveyor.yml index 3b70592f86..f5668d48be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,6 +82,7 @@ test_script: # Runs only in Linux - sh: "pytest -vv tests/integration" - sh: "/tmp/black --check setup.py tests samcli" + - sh: "python scripts/check-isolated-needs-update.py" # Smoke tests run in parallel - it runs on both Linux & Windows # Presence of the RUN_SMOKE envvar will run the smoke tests diff --git a/requirements/isolated.txt b/requirements/isolated.txt new file mode 100644 index 0000000000..c42a9ccddc --- /dev/null +++ b/requirements/isolated.txt @@ -0,0 +1,37 @@ +arrow==0.14.6 +aws-lambda-builders==0.4.0 +aws-sam-translator==1.11.0 +binaryornot==0.4.4 +boto3==1.9.220 +botocore==1.12.220 +certifi==2019.6.16 +chardet==3.0.4 +chevron==0.13.1 +Click==7.0 +cookiecutter==1.6.0 +dateparser==0.7.1 +docker==4.0.2 +docutils==0.15.2 +Flask==1.0.4 +future==0.17.1 +idna==2.8 +itsdangerous==1.1.0 +Jinja2==2.10.1 +jinja2-time==0.2.0 +jmespath==0.9.4 +jsonschema==2.6.0 +MarkupSafe==1.1.1 +poyo==0.5.0 +python-dateutil==2.8.0 +pytz==2019.2 +PyYAML==5.1.2 +regex==2019.8.19 +requests==2.22.0 +s3transfer==0.2.1 +serverlessrepo==0.1.9 +six==1.11.0 +tzlocal==2.0.0 +urllib3==1.25.3 +websocket-client==0.56.0 +Werkzeug==0.15.5 +whichcraft==0.6.0 diff --git a/scripts/check-isolated-needs-update.py b/scripts/check-isolated-needs-update.py new file mode 100644 index 0000000000..33c4dec874 --- /dev/null +++ b/scripts/check-isolated-needs-update.py @@ -0,0 +1,48 @@ +import io +import os +from subprocess import Popen, PIPE + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", os.linesep) + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +def get_requirements_list(content): + pkgs_versions = [] + for line in content.split(os.linesep): + if line: + # remove markers from the line, which are seperated by ';' + pkgs_versions.append(line.split(";")[0]) + + return pkgs_versions + + +isolated_req_content = read(os.path.join("requirements", "isolated.txt")) +base_req_content = read(os.path.join("requirements", "base.txt")) + +isolated_req_list = get_requirements_list(isolated_req_content) +base_req_list = get_requirements_list(base_req_content) + +process = Popen(["pip", "freeze"], stdout=PIPE) + +all_installed_pkgs_list = [] +for package in process.stdout.readlines(): + package = package.decode("utf-8").strip(os.linesep) + all_installed_pkgs_list.append(package) + +for installed_pkg in all_installed_pkgs_list: + for base_req in base_req_list: + # a base requirement can be defined with different specifiers (>, <, ==, etc.). Instead of doing tons of string parsing, + # brute force the check by assuming the installed_pkgs will have == as a specifier. This is true due to how pip freeze + # works. So check to make sure the installed pakcage we are looking at is in the base.txt file, if so make sure the + # full requirement==version is within the isolated list. + if base_req.startswith(installed_pkg.split("==")[0]): + assert installed_pkg in isolated_req_list, "{} is in base.txt but not in isolated.txt".format(installed_pkg) + print("{} is in the isolated.txt file".format(installed_pkg)) + break diff --git a/scripts/check-requirements.py b/scripts/check-requirements.py new file mode 100755 index 0000000000..4d4b522db7 --- /dev/null +++ b/scripts/check-requirements.py @@ -0,0 +1,43 @@ +import io +import os +from subprocess import Popen, PIPE + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", os.linesep) + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +exclude_packages = ("setuptools", "wheel", "pip", "aws-sam-cli") + +all_pkgs_list = [] +process = Popen(["pip", "freeze"], stdout=PIPE) + +for package in process.stdout.readlines(): + package = package.decode("utf-8").strip(os.linesep) + if package.split("==")[0] not in exclude_packages: + all_pkgs_list.append(package) +all_pkgs_list = sorted(all_pkgs_list) +print("installed package/versions" + os.linesep) +print(",".join(all_pkgs_list)) +print(os.linesep) + +content = read(os.path.join("requirements", "isolated.txt")) + +locked_pkgs = [] +for line in content.split(os.linesep): + if line: + locked_pkgs.append(line) + +locked_pkgs = sorted(locked_pkgs) +print("locked package/versions" + os.linesep) +print(",".join(locked_pkgs)) +print(os.linesep) + +assert len(locked_pkgs) == len(all_pkgs_list), "Number of expected dependencies do not match the number installed" +assert locked_pkgs == all_pkgs_list, "The list of expected dependencies do not match what is installed" From f48c1acbe2121d7f7d5d6822b91b7412f599e448 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Tue, 3 Sep 2019 10:20:34 -0700 Subject: [PATCH 2/3] Fix isolated needs update script to ignore py2 and handle py library as a special case --- scripts/check-isolated-needs-update.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/check-isolated-needs-update.py b/scripts/check-isolated-needs-update.py index 33c4dec874..490db283e1 100644 --- a/scripts/check-isolated-needs-update.py +++ b/scripts/check-isolated-needs-update.py @@ -1,4 +1,5 @@ import io +import sys import os from subprocess import Popen, PIPE @@ -22,6 +23,10 @@ def get_requirements_list(content): return pkgs_versions +# Don't try and compare the isolated list with the Python2 version. SAM CLI installers +# all use Python3.6+ and Python2.7 is going EOL +if sys.version_info[0] < 3: + sys.exit(0) isolated_req_content = read(os.path.join("requirements", "isolated.txt")) base_req_content = read(os.path.join("requirements", "base.txt")) @@ -36,13 +41,15 @@ def get_requirements_list(content): package = package.decode("utf-8").strip(os.linesep) all_installed_pkgs_list.append(package) -for installed_pkg in all_installed_pkgs_list: +for installed_pkg_version in all_installed_pkgs_list: for base_req in base_req_list: # a base requirement can be defined with different specifiers (>, <, ==, etc.). Instead of doing tons of string parsing, # brute force the check by assuming the installed_pkgs will have == as a specifier. This is true due to how pip freeze # works. So check to make sure the installed pakcage we are looking at is in the base.txt file, if so make sure the # full requirement==version is within the isolated list. - if base_req.startswith(installed_pkg.split("==")[0]): - assert installed_pkg in isolated_req_list, "{} is in base.txt but not in isolated.txt".format(installed_pkg) - print("{} is in the isolated.txt file".format(installed_pkg)) + installed_pkg = installed_pkg_version.split("==")[0] + # There is a py library we use but due to how we are comparing requirements, we need to handle this as a special case. :( + if installed_pkg != 'py' and base_req.startswith(installed_pkg): + assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format(installed_pkg_version) + print("{} is in the isolated.txt file".format(installed_pkg_version)) break From 6e0bbdbd312ad3a427780073988b216a096bab21 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Fri, 6 Sep 2019 09:16:15 -0700 Subject: [PATCH 3/3] Ignore boto3, blackify scripts/* --- Makefile | 4 ++-- appveyor.yml | 2 +- scripts/check-isolated-needs-update.py | 9 ++++++--- scripts/check-requirements.py | 12 ++++++------ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index affc53c7b6..03b4bdb3ba 100644 --- a/Makefile +++ b/Makefile @@ -32,10 +32,10 @@ lint: dev: lint test black: - black samcli/* tests/* + black samcli/* tests/* scripts/* black-check: - black --check samcli/* tests/* + black --check samcli/* tests/* scripts/* # Verifications to run before sending a pull request pr: init dev black-check diff --git a/appveyor.yml b/appveyor.yml index f5668d48be..2947271d3d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -81,7 +81,7 @@ test_script: # Runs only in Linux - sh: "pytest -vv tests/integration" - - sh: "/tmp/black --check setup.py tests samcli" + - sh: "/tmp/black --check setup.py tests samcli scripts" - sh: "python scripts/check-isolated-needs-update.py" # Smoke tests run in parallel - it runs on both Linux & Windows diff --git a/scripts/check-isolated-needs-update.py b/scripts/check-isolated-needs-update.py index 490db283e1..c4325f2e59 100644 --- a/scripts/check-isolated-needs-update.py +++ b/scripts/check-isolated-needs-update.py @@ -23,6 +23,7 @@ def get_requirements_list(content): return pkgs_versions + # Don't try and compare the isolated list with the Python2 version. SAM CLI installers # all use Python3.6+ and Python2.7 is going EOL if sys.version_info[0] < 3: @@ -49,7 +50,9 @@ def get_requirements_list(content): # full requirement==version is within the isolated list. installed_pkg = installed_pkg_version.split("==")[0] # There is a py library we use but due to how we are comparing requirements, we need to handle this as a special case. :( - if installed_pkg != 'py' and base_req.startswith(installed_pkg): - assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format(installed_pkg_version) - print("{} is in the isolated.txt file".format(installed_pkg_version)) + if installed_pkg not in ("py", "boto3") and base_req.startswith(installed_pkg): + assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format( + installed_pkg_version + ) + print ("{} is in the isolated.txt file".format(installed_pkg_version)) break diff --git a/scripts/check-requirements.py b/scripts/check-requirements.py index 4d4b522db7..07dd425e4d 100755 --- a/scripts/check-requirements.py +++ b/scripts/check-requirements.py @@ -23,9 +23,9 @@ def read(*filenames, **kwargs): if package.split("==")[0] not in exclude_packages: all_pkgs_list.append(package) all_pkgs_list = sorted(all_pkgs_list) -print("installed package/versions" + os.linesep) -print(",".join(all_pkgs_list)) -print(os.linesep) +print ("installed package/versions" + os.linesep) +print (",".join(all_pkgs_list)) +print (os.linesep) content = read(os.path.join("requirements", "isolated.txt")) @@ -35,9 +35,9 @@ def read(*filenames, **kwargs): locked_pkgs.append(line) locked_pkgs = sorted(locked_pkgs) -print("locked package/versions" + os.linesep) -print(",".join(locked_pkgs)) -print(os.linesep) +print ("locked package/versions" + os.linesep) +print (",".join(locked_pkgs)) +print (os.linesep) assert len(locked_pkgs) == len(all_pkgs_list), "Number of expected dependencies do not match the number installed" assert locked_pkgs == all_pkgs_list, "The list of expected dependencies do not match what is installed"