diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2696ead3..d1f5026b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,16 +2,22 @@ ## Development setup -Install dependencies with +Install the correct version of poetry: ``` -pipenv install +pip3 install poetry==1.0.10 +``` + +Install dependencies with poetry + +``` +poetry install ``` Run the agent with ``` -pipenv shell +poetry shell export PYTHONPATH=$(pwd)/src python -m semgrep_agent --config p/r2c ``` @@ -29,3 +35,9 @@ Connect to semgrep-app with the `--publish-deployment` & `--publish-token` flags Let CI pass on GitHub Actions before releasing. The following command will change all action runs to use your current `HEAD`: ```make release``` + +### Publishing to pypi + +``` +poetry publish +``` diff --git a/Dockerfile b/Dockerfile index 1224a225..a9b44cfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM returntocorp/semgrep:0.22.0@sha256:f9c9f9712557599b54658d0ccde32b70f36c26caaf5c64591eabad857b205caa AS semgrep -FROM python:3.8-alpine +FROM python:3.7-alpine WORKDIR /app -COPY Pipfile* ./ +COPY poetry.lock ./ +COPY pyproject.toml ./ ENV INSTALLED_SEMGREP_VERSION=0.22.0 @@ -10,10 +11,13 @@ COPY --from=semgrep /usr/local/bin/semgrep-core /tmp/semgrep-core RUN apk add --no-cache --virtual=.build-deps build-base libffi-dev openssl-dev &&\ apk add --no-cache --virtual=.run-deps bash git less libffi openssl &&\ - pip install --no-cache-dir pipenv==2020.5.28 &&\ - pipenv install --system &&\ + pip install --no-cache-dir poetry==1.0.10 &&\ + pip install --no-cache-dir pipx &&\ PRECOMPILED_LOCATION=/tmp/semgrep-core pipx install semgrep==${INSTALLED_SEMGREP_VERSION} &&\ - pip uninstall -y pipenv &&\ + poetry config virtualenvs.create false &&\ + # Don't install dev dependencies or semgrep-agent + poetry install --no-dev --no-root &&\ + pip uninstall -y poetry &&\ apk del .build-deps &&\ rm -rf /root/.cache/* /tmp/* diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 7fafd7f9..00000000 --- a/Pipfile +++ /dev/null @@ -1,22 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -requests = "*" -click = "*" -sh = "*" -boltons = "*" -gitpython = "~=2.1" -glom = "*" -pipx = "*" -pymmh3 = "*" - -[requires] -python_version = "3.8" - -[pipenv] -allow_prereleases = true diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 8394af88..00000000 --- a/Pipfile.lock +++ /dev/null @@ -1,166 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "2aa65e47faa32a4b0d117256c3d0a044e625697a89b2bd847cf78d6ca54737f1" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "argcomplete": { - "hashes": [ - "sha256:2fbe5ed09fd2c1d727d4199feca96569a5b50d44c71b16da9c742201f7cc295c", - "sha256:91dc7f9c7f6281d5a0dce5e73d2e33283aaef083495c13974a7dd197a1cdc949" - ], - "version": "==1.12.0" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==19.3.0" - }, - "boltons": { - "hashes": [ - "sha256:1567a4ab991a52be4e9a778c6c433e50237018b322759146b07732b71e57ef67", - "sha256:d367506c0b32042bb1ee3bf7899f2dcc8492dceb42ce3727b89e174d85bffe6e" - ], - "index": "pypi", - "version": "==20.2.0" - }, - "certifi": { - "hashes": [ - "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3", - "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41" - ], - "version": "==2020.6.20" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", - "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" - ], - "index": "pypi", - "version": "==7.1.2" - }, - "face": { - "hashes": [ - "sha256:3790311a7329e4b0d90baee346eecad54b337629576edf3a246683a5f0d24446", - "sha256:7d59ca5ba341316e58cf72c6aff85cca2541cf5056c4af45cb63af9a814bed3e" - ], - "version": "==20.1.1" - }, - "gitdb2": { - "hashes": [ - "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350", - "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.0.6" - }, - "gitpython": { - "hashes": [ - "sha256:23b4de99c5fc1564701301cead04e16aa3e47019034668327206d1b52e1e54f7", - "sha256:838c38d882a97b2fd146dbd4eb9baaf857cb65ebaa64e4c06fca48aaa491e1c6" - ], - "index": "pypi", - "version": "==2.1.15" - }, - "glom": { - "hashes": [ - "sha256:953cf65080dbe01db03b4d4fa0743f9a10e6e35f5404807fdc59425ca7af19e6", - "sha256:e753d2e8d16647ffcd9f0f99ac85d3db523ff0a1f097cf0a154a60702bca7e42" - ], - "index": "pypi", - "version": "==20.5.0" - }, - "idna": { - "hashes": [ - "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", - "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.10" - }, - "pipx": { - "hashes": [ - "sha256:6ae400380708ac60d7d2c300fc5002b60f8c31877169d12a1c288b6155f4b1cc", - "sha256:b90285a1c6eddf45ba4e48110b81d7c1e285b127e82fa99a61168937f98e5cb5" - ], - "index": "pypi", - "version": "==0.15.4.0" - }, - "pymmh3": { - "hashes": [ - "sha256:c47e9e735518501a3c9b719dbd795aa5d5a1dba48d9c85bae1777ab9ae6c7c45" - ], - "index": "pypi", - "version": "==0.0.5" - }, - "requests": { - "hashes": [ - "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", - "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" - ], - "index": "pypi", - "version": "==2.24.0" - }, - "sh": { - "hashes": [ - "sha256:6f792e45b45d039b423081558904981e8ab49572b0c38009fcc65feaab06bcda", - "sha256:97a3d2205e3c6a842d87ebbc9ae93acae5a352b1bc4609b428d0fd5bb9e286a3" - ], - "index": "pypi", - "version": "==1.13.1" - }, - "smmap": { - "hashes": [ - "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4", - "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.0.4" - }, - "smmap2": { - "hashes": [ - "sha256:0cb6ea470b1ad9a65a02ca7f4c7ae601861f7dd24a43812ca51cfca2892bb524", - "sha256:44cc8bdaf96442dbb9a8e2e14377d074b3d0eea292eee3c95c8c449b6c92c557" - ], - "version": "==3.0.1" - }, - "urllib3": { - "hashes": [ - "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", - "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.25.10" - }, - "userpath": { - "hashes": [ - "sha256:211544ea02d8715fdc06f429cf66cd18c9877a31751d966d6de11b24faaed255", - "sha256:801bbfed9c9de8bf33e615a2017031280bd5c85436d27a061801c594bbe99f63" - ], - "version": "==1.4.1" - } - }, - "develop": {} -} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..470b8e24 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,237 @@ +[[package]] +category = "main" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.1.0" + +[package.extras] +dev = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[[package]] +category = "main" +description = "When they're not builtins, they're boltons." +name = "boltons" +optional = false +python-versions = "*" +version = "20.2.1" + +[[package]] +category = "main" +description = "Python package for providing Mozilla's CA Bundle." +name = "certifi" +optional = false +python-versions = "*" +version = "2020.6.20" + +[[package]] +category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + +[[package]] +category = "main" +description = "Composable command line interface toolkit" +name = "click" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "7.1.2" + +[[package]] +category = "main" +description = "A command-line application framework (and CLI parser). Friendly for users, full-featured for developers." +name = "face" +optional = false +python-versions = "*" +version = "20.1.1" + +[package.dependencies] +boltons = ">=20.0.0" + +[[package]] +category = "main" +description = "Git Object Database" +name = "gitdb2" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.0.6" + +[package.dependencies] +smmap2 = ">=2.0.0" + +[[package]] +category = "main" +description = "Python Git Library" +name = "gitpython" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.1.15" + +[package.dependencies] +gitdb2 = ">=2,<3" + +[[package]] +category = "main" +description = "A declarative object transformer and formatter, for conglomerating nested data." +name = "glom" +optional = false +python-versions = "*" +version = "20.8.0" + +[package.dependencies] +attrs = "*" +boltons = ">=19.3.0" +face = ">=20.1.0" + +[package.extras] +yaml = ["pyyaml"] + +[[package]] +category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.10" + +[[package]] +category = "main" +description = "Pure python mmh3 implementation compatible with python2 and python3" +name = "pymmh3" +optional = false +python-versions = "*" +version = "0.0.5" + +[[package]] +category = "main" +description = "Python HTTP for Humans." +name = "requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "2.24.0" + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<4" +idna = ">=2.5,<3" +urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] + +[[package]] +category = "main" +description = "Python subprocess replacement" +name = "sh" +optional = false +python-versions = "*" +version = "1.14.0" + +[[package]] +category = "main" +description = "A pure Python implementation of a sliding window memory map manager" +name = "smmap" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.0.4" + +[[package]] +category = "main" +description = "A mirror package for smmap" +name = "smmap2" +optional = false +python-versions = "*" +version = "3.0.1" + +[package.dependencies] +smmap = ">=3.0.1" + +[[package]] +category = "main" +description = "HTTP library with thread-safe connection pooling, file post, and more." +name = "urllib3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "1.25.10" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"] +socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] + +[metadata] +content-hash = "ba665b35743d7f26d077de845d49d017bef53f2b9fc5ca6c0b2b9d0e18edcb50" +lock-version = "1.0" +python-versions = "^3.7" + +[metadata.files] +attrs = [ + {file = "attrs-20.1.0-py2.py3-none-any.whl", hash = "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff"}, + {file = "attrs-20.1.0.tar.gz", hash = "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a"}, +] +boltons = [ + {file = "boltons-20.2.1-py2.py3-none-any.whl", hash = "sha256:3dd8a8e3c1886e7f7ba3422b50f55a66e1700161bf01b919d098e7d96dd2d9b6"}, + {file = "boltons-20.2.1.tar.gz", hash = "sha256:dd362291a460cc1e0c2e91cc6a60da3036ced77099b623112e8f833e6734bdc5"}, +] +certifi = [ + {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, + {file = "certifi-2020.6.20.tar.gz", hash = "sha256:5930595817496dd21bb8dc35dad090f1c2cd0adfaf21204bf6732ca5d8ee34d3"}, +] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] +click = [ + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, +] +face = [ + {file = "face-20.1.1-py2-none-any.whl", hash = "sha256:3790311a7329e4b0d90baee346eecad54b337629576edf3a246683a5f0d24446"}, + {file = "face-20.1.1.tar.gz", hash = "sha256:7d59ca5ba341316e58cf72c6aff85cca2541cf5056c4af45cb63af9a814bed3e"}, +] +gitdb2 = [ + {file = "gitdb2-2.0.6-py2.py3-none-any.whl", hash = "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b"}, + {file = "gitdb2-2.0.6.tar.gz", hash = "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350"}, +] +gitpython = [ + {file = "GitPython-2.1.15-py2.py3-none-any.whl", hash = "sha256:23b4de99c5fc1564701301cead04e16aa3e47019034668327206d1b52e1e54f7"}, + {file = "GitPython-2.1.15.tar.gz", hash = "sha256:838c38d882a97b2fd146dbd4eb9baaf857cb65ebaa64e4c06fca48aaa491e1c6"}, +] +glom = [ + {file = "glom-20.8.0-py2.py3-none-any.whl", hash = "sha256:6422e442ef17fe84037570ac1fba1a5e47d22743d9766dbf0f36964405428bb0"}, + {file = "glom-20.8.0.tar.gz", hash = "sha256:5fa3a9d99c7f3e5410a810fa8a158c0f71e39036c47b77745c7f2e4630372f82"}, +] +idna = [ + {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, + {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, +] +pymmh3 = [ + {file = "pymmh3-0.0.5-py2.py3-none-any.whl", hash = "sha256:c47e9e735518501a3c9b719dbd795aa5d5a1dba48d9c85bae1777ab9ae6c7c45"}, +] +requests = [ + {file = "requests-2.24.0-py2.py3-none-any.whl", hash = "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898"}, + {file = "requests-2.24.0.tar.gz", hash = "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b"}, +] +sh = [ + {file = "sh-1.14.0-py2.py3-none-any.whl", hash = "sha256:4219d338349581a07cd1358c1d7de0fa57bb5d44e547cc87b0da7d5349705030"}, + {file = "sh-1.14.0.tar.gz", hash = "sha256:05c7e520cdf70f70a7228a03b589da9f96c6e0d06fc487ab21fc62b26a592e59"}, +] +smmap = [ + {file = "smmap-3.0.4-py2.py3-none-any.whl", hash = "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4"}, + {file = "smmap-3.0.4.tar.gz", hash = "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24"}, +] +smmap2 = [ + {file = "smmap2-3.0.1-py3-none-any.whl", hash = "sha256:0cb6ea470b1ad9a65a02ca7f4c7ae601861f7dd24a43812ca51cfca2892bb524"}, + {file = "smmap2-3.0.1.tar.gz", hash = "sha256:44cc8bdaf96442dbb9a8e2e14377d074b3d0eea292eee3c95c8c449b6c92c557"}, +] +urllib3 = [ + {file = "urllib3-1.25.10-py2.py3-none-any.whl", hash = "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"}, + {file = "urllib3-1.25.10.tar.gz", hash = "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..579c48df --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[tool.poetry] +name = "semgrep-agent" +version = "0.1.0b1" +description = "" +authors = ["Return To Corporation "] +license = "Proprietary" +packages = [ + { include = "semgrep_agent", from = "src" }, +] + +[tool.poetry.dependencies] +python = "^3.7" +requests = "^2.24.0" +click = "^7.1.2" +sh = "^1.13.1" +boltons = "^20.2.1" +gitpython = "^2.1.15" +glom = "^20.8.0" +pymmh3 = "^0.0.5" + +[tool.poetry.dev-dependencies] + +[tool.poetry.scripts] +semgrep-agent = "semgrep_agent.__main__:main" + +[build-system] +requires = ["poetry>=1.0.10"] +build-backend = "poetry.masonry.api" diff --git a/src/semgrep_agent/meta.py b/src/semgrep_agent/meta.py index de85c12d..6d670040 100644 --- a/src/semgrep_agent/meta.py +++ b/src/semgrep_agent/meta.py @@ -96,7 +96,8 @@ def glom_event(self, spec: TType) -> Any: @cachedproperty def event(self) -> Dict[str, Any]: - if value := os.getenv("GITHUB_EVENT_PATH"): + value = os.getenv("GITHUB_EVENT_PATH") + if value: debug_echo(f"found github event data at {value}") return json.loads(Path(value).read_text()) # type: ignore return {} @@ -136,7 +137,8 @@ def ci_actor(self) -> Optional[str]: @cachedproperty def ci_job_url(self) -> Optional[str]: - if self.repo_url and (value := os.getenv("GITHUB_RUN_ID")): + value = os.getenv("GITHUB_RUN_ID") + if self.repo_url and value: return f"{self.repo_url}/actions/runs/{value}" return None diff --git a/src/semgrep_agent/semgrep.py b/src/semgrep_agent/semgrep.py index 81b76f5e..90e0a99f 100644 --- a/src/semgrep_agent/semgrep.py +++ b/src/semgrep_agent/semgrep.py @@ -80,7 +80,8 @@ def get_semgrepignore(scan: "Scan") -> TextIO: ) semgrepignore.write((TEMPLATES_DIR / ".semgrepignore").read_text()) - if scan_patterns := scan.ignore_patterns: + scan_patterns = scan.ignore_patterns + if scan_patterns: click.echo( "| adding further path ignore rules configured on the web UI", err=True ) diff --git a/src/semgrep_agent/utils.py b/src/semgrep_agent/utils.py index a791da40..76e86125 100644 --- a/src/semgrep_agent/utils.py +++ b/src/semgrep_agent/utils.py @@ -47,7 +47,8 @@ def get_git_repo(path: Optional[Path] = None) -> Optional[gitpython.Repo]: # ty def zsplit(s: str) -> List[str]: """Split a string on null characters.""" - if s := s.strip("\0"): + s = s.strip("\0") + if s: return s.split("\0") else: return []