diff --git a/Makefile b/Makefile index 9d042790bd..03b4bdb3ba 100644 --- a/Makefile +++ b/Makefile @@ -32,10 +32,15 @@ lint: dev: lint test black: - black samcli/* tests/* + black samcli/* tests/* scripts/* black-check: - black --check samcli/* tests/* + black --check samcli/* tests/* scripts/* # Verifications to run before sending a pull request -pr: init dev black-check \ No newline at end of file +pr: init dev black-check + +update-isolated-req: + pipenv --three + pipenv run pip install -r requirements/base.txt + pipenv run pip freeze > requirements/isolated.txt diff --git a/appveyor.yml b/appveyor.yml index 3b70592f86..2947271d3d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -81,7 +81,8 @@ test_script: # Runs only in Linux - sh: "pytest -vv tests/integration" - - sh: "/tmp/black --check setup.py tests samcli" + - sh: "/tmp/black --check setup.py tests samcli scripts" + - sh: "python scripts/check-isolated-needs-update.py" # Smoke tests run in parallel - it runs on both Linux & Windows # Presence of the RUN_SMOKE envvar will run the smoke tests diff --git a/requirements/isolated.txt b/requirements/isolated.txt new file mode 100644 index 0000000000..c42a9ccddc --- /dev/null +++ b/requirements/isolated.txt @@ -0,0 +1,37 @@ +arrow==0.14.6 +aws-lambda-builders==0.4.0 +aws-sam-translator==1.11.0 +binaryornot==0.4.4 +boto3==1.9.220 +botocore==1.12.220 +certifi==2019.6.16 +chardet==3.0.4 +chevron==0.13.1 +Click==7.0 +cookiecutter==1.6.0 +dateparser==0.7.1 +docker==4.0.2 +docutils==0.15.2 +Flask==1.0.4 +future==0.17.1 +idna==2.8 +itsdangerous==1.1.0 +Jinja2==2.10.1 +jinja2-time==0.2.0 +jmespath==0.9.4 +jsonschema==2.6.0 +MarkupSafe==1.1.1 +poyo==0.5.0 +python-dateutil==2.8.0 +pytz==2019.2 +PyYAML==5.1.2 +regex==2019.8.19 +requests==2.22.0 +s3transfer==0.2.1 +serverlessrepo==0.1.9 +six==1.11.0 +tzlocal==2.0.0 +urllib3==1.25.3 +websocket-client==0.56.0 +Werkzeug==0.15.5 +whichcraft==0.6.0 diff --git a/scripts/check-isolated-needs-update.py b/scripts/check-isolated-needs-update.py new file mode 100644 index 0000000000..c4325f2e59 --- /dev/null +++ b/scripts/check-isolated-needs-update.py @@ -0,0 +1,58 @@ +import io +import sys +import os +from subprocess import Popen, PIPE + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", os.linesep) + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +def get_requirements_list(content): + pkgs_versions = [] + for line in content.split(os.linesep): + if line: + # remove markers from the line, which are seperated by ';' + pkgs_versions.append(line.split(";")[0]) + + return pkgs_versions + + +# Don't try and compare the isolated list with the Python2 version. SAM CLI installers +# all use Python3.6+ and Python2.7 is going EOL +if sys.version_info[0] < 3: + sys.exit(0) + +isolated_req_content = read(os.path.join("requirements", "isolated.txt")) +base_req_content = read(os.path.join("requirements", "base.txt")) + +isolated_req_list = get_requirements_list(isolated_req_content) +base_req_list = get_requirements_list(base_req_content) + +process = Popen(["pip", "freeze"], stdout=PIPE) + +all_installed_pkgs_list = [] +for package in process.stdout.readlines(): + package = package.decode("utf-8").strip(os.linesep) + all_installed_pkgs_list.append(package) + +for installed_pkg_version in all_installed_pkgs_list: + for base_req in base_req_list: + # a base requirement can be defined with different specifiers (>, <, ==, etc.). Instead of doing tons of string parsing, + # brute force the check by assuming the installed_pkgs will have == as a specifier. This is true due to how pip freeze + # works. So check to make sure the installed pakcage we are looking at is in the base.txt file, if so make sure the + # full requirement==version is within the isolated list. + installed_pkg = installed_pkg_version.split("==")[0] + # There is a py library we use but due to how we are comparing requirements, we need to handle this as a special case. :( + if installed_pkg not in ("py", "boto3") and base_req.startswith(installed_pkg): + assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format( + installed_pkg_version + ) + print ("{} is in the isolated.txt file".format(installed_pkg_version)) + break diff --git a/scripts/check-requirements.py b/scripts/check-requirements.py new file mode 100755 index 0000000000..07dd425e4d --- /dev/null +++ b/scripts/check-requirements.py @@ -0,0 +1,43 @@ +import io +import os +from subprocess import Popen, PIPE + + +def read(*filenames, **kwargs): + encoding = kwargs.get("encoding", "utf-8") + sep = kwargs.get("sep", os.linesep) + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +exclude_packages = ("setuptools", "wheel", "pip", "aws-sam-cli") + +all_pkgs_list = [] +process = Popen(["pip", "freeze"], stdout=PIPE) + +for package in process.stdout.readlines(): + package = package.decode("utf-8").strip(os.linesep) + if package.split("==")[0] not in exclude_packages: + all_pkgs_list.append(package) +all_pkgs_list = sorted(all_pkgs_list) +print ("installed package/versions" + os.linesep) +print (",".join(all_pkgs_list)) +print (os.linesep) + +content = read(os.path.join("requirements", "isolated.txt")) + +locked_pkgs = [] +for line in content.split(os.linesep): + if line: + locked_pkgs.append(line) + +locked_pkgs = sorted(locked_pkgs) +print ("locked package/versions" + os.linesep) +print (",".join(locked_pkgs)) +print (os.linesep) + +assert len(locked_pkgs) == len(all_pkgs_list), "Number of expected dependencies do not match the number installed" +assert locked_pkgs == all_pkgs_list, "The list of expected dependencies do not match what is installed"