(aws-lambda-python): Poetry requirements export fails when path + pypi dependencies due to lack of hashes on paths

[huonw]

What is the problem?

We're starting to use PythonFunction from @aws-cdk/aws-lambda-python-alpha with poetry (a pyproject.toml and poetry.lock file in the directory specified by entry), and it works pretty well, thanks.

However, the exported requirements.txt includes hashes for external dependencies but not path deps, and if there's any hashes in the file, then pip requires a hash for every dep:

  --require-hashes            Require a hash to check each requirement
                              against, for repeatable installs. This option is
                              implied when any package in a requirements file
                              has a --hash option.

This means that if a Poetry package depends on both a path and pypi package, PythonFunction won't be able to bundle the requirements.

This is arguably a poetry and/or pip issue, but poetry export does provide a --without-hashes argument to emit the hashes, which could be added to:

aws-cdk/packages/@aws-cdk/aws-lambda-python/lib/packaging.ts

Line 49 in 6dfc254

exportCommand: `poetry export --with-credentials --format ${DependenciesFile.PIP} --output ${DependenciesFile.PIP}`,

Reproduction Steps

import * as cdk from 'aws-cdk-lib';
import * as lambdaPython from '@aws-cdk/aws-lambda-python-alpha';

export const app = new cdk.App();
const stack = new cdk.Stack(app, 'Stack', {});

new lambdaPython.PythonFunction(stack, 'Func', {
  entry: 'parent',
  index: 'parent/__init__.py',
  runtime: cdk.aws_lambda.Runtime.PYTHON_3_9,
});

Directory tree of parent:

parent
├── child
│   ├── child
│   │   └── __init__.py
│   └── pyproject.toml
├── parent
│   └── __init__.py
├── poetry.lock
└── pyproject.toml

parent/pyproject.toml:

[tool.poetry]
name = "parent"
version = "0.1.0"
description = ""
authors = []

[tool.poetry.dependencies]
python = "^3.9"
child = { path = "child" }
boto3 = "^1.21.12"

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

parent/poetry.lock:

[[package]]
name = "boto3"
version = "1.21.12"
description = "The AWS SDK for Python"
category = "main"
optional = false
python-versions = ">= 3.6"

[package.dependencies]
botocore = ">=1.24.12,<1.25.0"
jmespath = ">=0.7.1,<1.0.0"
s3transfer = ">=0.5.0,<0.6.0"

[package.extras]
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]

[[package]]
name = "botocore"
version = "1.24.12"
description = "Low-level, data-driven core of boto 3."
category = "main"
optional = false
python-versions = ">= 3.6"

[package.dependencies]
jmespath = ">=0.7.1,<1.0.0"
python-dateutil = ">=2.1,<3.0.0"
urllib3 = ">=1.25.4,<1.27"

[package.extras]
crt = ["awscrt (==0.12.5)"]

[[package]]
name = "child"
version = "0.1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
develop = false

[package.source]
type = "directory"
url = "child"

[[package]]
name = "jmespath"
version = "0.10.0"
description = "JSON Matching Expressions"
category = "main"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"

[[package]]
name = "python-dateutil"
version = "2.8.2"
description = "Extensions to the standard Python datetime module"
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"

[package.dependencies]
six = ">=1.5"

[[package]]
name = "s3transfer"
version = "0.5.2"
description = "An Amazon S3 Transfer Manager"
category = "main"
optional = false
python-versions = ">= 3.6"

[package.dependencies]
botocore = ">=1.12.36,<2.0a.0"

[package.extras]
crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]

[[package]]
name = "six"
version = "1.16.0"
description = "Python 2 and 3 compatibility utilities"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"

[[package]]
name = "urllib3"
version = "1.26.8"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"

[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]

[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "8ae88458c4b21dd26abae2f44d834b102dcde4f235634e676d62eddbc563e93f"

[metadata.files]
boto3 = [
    {file = "boto3-1.21.12-py3-none-any.whl", hash = "sha256:75709628320cea8ce137975dc33b75213c2e4f6e7cd09e55290de7245e2c79e2"},
    {file = "boto3-1.21.12.tar.gz", hash = "sha256:c92ec20a670721b5a1bc013b305a84db2b7f9c716653b3056ce7e2fbd2a180ef"},
]
botocore = [
    {file = "botocore-1.24.12-py3-none-any.whl", hash = "sha256:0cd7395311a3fef4aad8df8f511b4f7d221c24ae30934bd5c03458b0fc096d0c"},
    {file = "botocore-1.24.12.tar.gz", hash = "sha256:0174999a04b0a2e42457106093ace9b36fa94772a442d9bcf60750263d1d073e"},
]
child = []
jmespath = [
    {file = "jmespath-0.10.0-py2.py3-none-any.whl", hash = "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"},
    {file = "jmespath-0.10.0.tar.gz", hash = "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9"},
]
python-dateutil = [
    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
]
s3transfer = [
    {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
    {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
]
six = [
    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
urllib3 = [
    {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
    {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
]

parent/child/pyproject.toml

[tool.poetry]
name = "child"
version = "0.1.0"
description = ""
authors = []

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

What did you expect to happen?

The bundling should work when there's both pypi/external and path dependencies, similar to if there's only pypi deps.

What actually happened?

The bundling computes a requirements file like the following:

boto3==1.21.12; python_version >= "3.6" \
    --hash=sha256:75709628320cea8ce137975dc33b75213c2e4f6e7cd09e55290de7245e2c79e2 \
    --hash=sha256:c92ec20a670721b5a1bc013b305a84db2b7f9c716653b3056ce7e2fbd2a180ef
botocore==1.24.12; python_version >= "3.6" \
    --hash=sha256:0cd7395311a3fef4aad8df8f511b4f7d221c24ae30934bd5c03458b0fc096d0c \
    --hash=sha256:0174999a04b0a2e42457106093ace9b36fa94772a442d9bcf60750263d1d073e
child @ file:///asset-input/child
jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f \
    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9
python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
    --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
    --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
s3transfer==0.5.2; python_version >= "3.6" \
    --hash=sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971 \
    --hash=sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed
six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
urllib3==1.26.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" \
    --hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \
    --hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c

Note how child has no --hash argument. The bundling then fails with output including:

Processing ./child
ERROR: Can't verify hashes for these file:// requirements because they point to directories:
    child@ file:///asset-input/child from file:///asset-input/child (from -r requirements.txt (line 7))

CDK CLI Version

2.14.0 (build 762d71b)

Framework Version

No response

Node.js Version

v14.17.5

OS

macOS

Language

Typescript

Language Version

TypeScript 4.1.6

Other information

No response

[corymhall]

@huonw looks like this might be an easy fix.

I am unassigning and marking this issue as p2, which means that we are unable to work on this immediately, but would definitely welcome contributions.

We use +1s to help prioritize our work, and are happy to revaluate this issue based on community feedback. You can reach out to the cdk.dev community on Slack to solicit support for reprioritization.

[github-actions]

⚠️COMMENT VISIBILITY WARNING⚠️

Comments on closed issues are hard for our team to see.
If you need more assistance, please either tag a team member or open a new issue that references this one.
If you wish to keep having a conversation with other community members under this issue feel free to do so.