From 77d7d8094b204b82117c4421dc2180f2daccf4fc Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 18 May 2021 10:59:39 +0300 Subject: [PATCH 1/6] tox, poetry, ci, first cut --- .circleci/config.yml | 113 ++++++++++++++++++++++++++++++------------ README.md | 16 ++++-- pyproject.toml | 43 ++++++++++++++++ redisgraph/util.py | 2 +- requirements.txt | 2 - setup.cfg | 2 - setup.py | 31 ------------ test-requirements.txt | 10 ---- tox.ini | 30 ++--------- 9 files changed, 140 insertions(+), 109 deletions(-) create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py delete mode 100644 test-requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index ffb39d0..1846d00 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,27 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# version: 2.1 commands: + + abort_for_docs: + steps: + - run: + name: Avoid tests for docs + command: | + if [[ $CIRCLE_BRANCH == *docs ]]; then + echo "Identifies as documents PR, no testing required" + circleci step halt + fi + + abort_for_noci: + steps: + - run: + name: Ignore CI for specific branches + command: | + if [[ $CIRCLE_BRANCH == *noci ]]; then + echo "Identifies as actively ignoring CI, no testing required." + circleci step halt + fi + + early_return_for_forked_pull_requests: description: >- If this build is from a fork, stop executing the current job and return success. @@ -12,33 +30,31 @@ commands: - run: name: Early return if this build is from a forked PR command: | - if [ -n "$CIRCLE_PR_NUMBER" ]; then + if [[ -n "$CIRCLE_PR_NUMBER" ]]; then echo "Nothing to do for forked PRs, so marking this step successful" circleci step halt fi -jobs: - build: - docker: - - image: circleci/python:3.6.1 - - - image: redislabs/redisgraph:edge - - working_directory: ~/repo + build_and_test: steps: + - abort_for_docs + - abort_for_noci - checkout - - run: - name: Install tox - command: sudo pip install tox + name: install tox dependencies + command: | + pip install --user --quiet -r requirements.txt - run: - name: Test package build - command: tox -e sdist + name: build + command: | + poetry build --format sdist + poetry build --format wheel - run: - name: Run code styles - command: tox -e pep8 + name: lint + command: | + tox -e linters - run: name: Run unittest with coverage @@ -49,7 +65,6 @@ jobs: command: tox -e func - early_return_for_forked_pull_requests - - run: name: codecove command: | @@ -57,19 +72,53 @@ jobs: codecov --file .tox/cover/report/coverage.xml --name ${CODECOV_NAME}-unittests codecov --file .tox/func/report/coverage.xml --name ${CODECOV_NAME}-functional +jobs: + build: + parameters: + python_version: + type: string + docker: + - image: circleci/python:<> + - image: redislabs/redisgraph:edge + + working_directory: ~/repo + + steps: + - build_and_test + +on-any-branch: &on-any-branch + filters: + branches: + only: + - /.*/ + tags: + ignore: /.*/ + +python-versions: &python-versions + matrix: + parameters: + python_version: + - "3.6.9" + - "3.7.9" + - "3.8.9" + - "3.9.4" + workflows: version: 2 commit: jobs: - - build - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - build + - build: + <<: *on-any-branch + <<: *python-versions + +# nightly: +# triggers: +# - schedule: +# cron: "0 0 * * *" +# filters: +# branches: +# only: +# - master +# jobs: +# - build diff --git a/README.md b/README.md index 5716c9e..d73f436 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![PyPI version](https://badge.fury.io/py/redisgraph.svg)](https://badge.fury.io/py/redisgraph) [![GitHub issues](https://img.shields.io/github/release/RedisGraph/redisgraph-py.svg)](https://github.com/RedisGraph/redisgraph-py/releases/latest) [![Codecov](https://codecov.io/gh/RedisGraph/redisgraph-py/branch/master/graph/badge.svg)](https://codecov.io/gh/RedisGraph/redisgraph-py) -[![Known Vulnerabilities](https://snyk.io/test/github/RedisGraph/redisgraph-py/badge.svg?targetFile=requirements.txt)](https://snyk.io/test/github/RedisGraph/redisgraph-py?targetFile=requirements.txt) +[![Known Vulnerabilities](https://snyk.io/test/github/RedisGraph/redisgraph-py/badge.svg?targetFile=pyproject.toml)](https://snyk.io/test/github/RedisGraph/redisgraph-py?targetFile=pyproject.toml) [![Total alerts](https://img.shields.io/lgtm/alerts/g/RedisGraph/redisgraph-py.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/RedisGraph/redisgraph-py/alerts/) # redisgraph-py @@ -91,7 +91,13 @@ pip install git+https://github.com/RedisGraph/redisgraph-py.git@master ### Install for development in env -``` -tox -e env -source ./tox/env/bin/activate -``` \ No newline at end of file +1. Create a virtualenv to manage your python dependencies, and ensure it's active. + ```virtualenv -v venv; source venv/bin/activate``` + +2. Install [pypoetry](https://python-poetry.org/) to manage your dependencies. + ```pip install poetry``` + +3. Install dependencies. + ```poetry install``` + +[tox](https://tox.readthedocs.io/en/latest/) runs all code linters as its default target. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a81db7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "redisgraph" +version = "2.3.0" +description = "RedisGraph Python Client" +authors = ["RedisLabs "] +license = "BSD-3-Clause" +readme = "README.md" +keywords = ["Redis", "Graph"] +classifiers=[ + 'Topic :: Database', + 'Programming Language :: Python', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Development Status :: 5 - Production/Stable' +] + +[tool.poetry.urls] +url = "https://redisgraph.io" +repository = "https://github.com/RedisGraph/redisgraph-py" + +[tool.poetry.dependencies] +python = "^3.6" +redis = "^3.5.3" +prettytable = "^2.1.0" +pip = "^21.1.0" +poetry = "^1.1.6" + +[tool.poetry.dev-dependencies] +tox = "^3.23.1" +pytest = "^6.2.4" +pytest-cov = "^2.12.0" +pytest-html = "^3.1.1" +testtools = "^2.4.0" +mock = "^4.0.3" +codecov = "^2.1.11" +flake8 = "^3.9.2" +tox-poetry = "^0.3.0" +vulture = "^2.3" +bandit = "^1.7.0" + +[build-system] +requires = ["poetry-core>=1.1.16"] +build-backend = "poetry.core.masonry.api" diff --git a/redisgraph/util.py b/redisgraph/util.py index 117c186..21e8a68 100644 --- a/redisgraph/util.py +++ b/redisgraph/util.py @@ -8,7 +8,7 @@ def random_string(length=10): """ Returns a random N character long string. """ - return ''.join(random.choice(string.ascii_lowercase) for x in range(length)) + return ''.join(random.choice(string.ascii_lowercase) for x in range(length)) # nosec def quote_string(v): diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 05412e5..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -redis -prettytable diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 224a779..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 45a384f..0000000 --- a/setup.py +++ /dev/null @@ -1,31 +0,0 @@ -from setuptools import setup, find_packages -import io - -def read_all(f): - with io.open(f, encoding="utf-8") as I: - return I.read() - - -requirements = list(map(str.strip, open("requirements.txt").readlines())) - - -setup( - name='redisgraph', - version='2.3.0', - description='RedisGraph Python Client', - long_description=read_all("README.md"), - long_description_content_type='text/markdown', - url='https://github.com/RedisGraph/redisgraph-py', - packages=find_packages(), - install_requires=requirements, - classifiers=[ - 'Topic :: Database', - 'Programming Language :: Python', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Development Status :: 5 - Production/Stable' - ], - author='RedisLabs', - keywords='Redis Graph', - author_email='oss@redislabs.com' -) diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index 0e6e551..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -flake8 - -pytest -pytest-cov -pytest-html - -testtools>=1.4.0 -mock - -codecov diff --git a/tox.ini b/tox.ini index 139cab2..777eb7e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,35 +1,20 @@ [tox] minversion = 1.6 skipsdist = True -envlist = py3,pep8,cover,func +envlist = linters,cover,func [testenv] -setenv = VIRTUAL_ENV={envdir} - LANG=en_US.UTF-8 - LANGUAGE=en_US:en - LC_ALL=C - PYTHONHASHSEED=0 - TOX_ENV_NAME={envname} whitelist_externals = find -deps = -r{toxinidir}/test-requirements.txt -install_command = pip install -U {opts} {packages} -usedevelop = True commands_pre = find . -type f -name "*.pyc" -delete commands = py.test -v --html={envdir}/report/index.html --durations=10 {posargs:tests/unit} -distribute = false -basepython = python3 -[testenv:pep8] +[testenv:linters] commands = flake8 --show-source - -[testenv:py37] -basepython = python3.7 - -[testenv:py38] -basepython = python3.8 + vulture redisgraph --min-confidence 80 + bandit redisgraph/** [testenv:cover] commands = py.test --cov=redisgraph tests/unit/ --cov-report=xml:{envdir}/report/coverage.xml --cov-report=html:{envdir}/report/html @@ -37,13 +22,6 @@ commands = py.test --cov=redisgraph tests/unit/ --cov-report=xml:{envdir}/report [testenv:func] commands = py.test --cov=redisgraph tests/functional/ --cov-report=xml:{envdir}/report/coverage.xml --cov-report=html:{envdir}/report/html -[testenv:venv] -commands = {posargs} - -[testenv:sdist] -deps = -commands = python setup.py sdist - [flake8] show-source = true # TODO(boris-42): Enable E226 and E501 rules From 4d21fc38da3665da7969687e62c97125b9e45027 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 18 May 2021 14:16:01 +0300 Subject: [PATCH 2/6] master ci restored --- .circleci/config.yml | 55 ++++++++++++++++++++++++++++++++++---------- Dockerfile | 17 ++++++++++++++ pyproject.toml | 6 ++--- 3 files changed, 62 insertions(+), 16 deletions(-) create mode 100644 Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 1846d00..24b760d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,6 +72,22 @@ commands: codecov --file .tox/cover/report/coverage.xml --name ${CODECOV_NAME}-unittests codecov --file .tox/func/report/coverage.xml --name ${CODECOV_NAME}-functional + docker: + parameters: + docker_version: + type: string + default: "edge" + steps: + - setup_remote_docker + - run: + name: dockers + description: Build and release docker + command: | + bash <(curl -fsSL https://get.docker.com) + docker login -u redisfab -p $DOCKER_REDISFAB_PWD + docker build -t redisgraph:<> . + docker pusH + jobs: build: parameters: @@ -80,11 +96,22 @@ jobs: docker: - image: circleci/python:<> - image: redislabs/redisgraph:edge + steps: + - build_and_test - working_directory: ~/repo + # since this is used by cron, we by default build against latest + build_and_edge: + parameters: + docker_version: + type: string + default: "edge" + docker: + - image: circleci/python:latest + - image: redislabs/redisgraph:edge steps: - build_and_test + - docker on-any-branch: &on-any-branch filters: @@ -94,6 +121,13 @@ on-any-branch: &on-any-branch tags: ignore: /.*/ +on-master: &on-master + filters: + branches: + only: + - master + +# the is to build and test, per commit against all supported python versions python-versions: &python-versions matrix: parameters: @@ -102,7 +136,7 @@ python-versions: &python-versions - "3.7.9" - "3.8.9" - "3.9.4" - + - "latest" workflows: version: 2 @@ -112,13 +146,10 @@ workflows: <<: *on-any-branch <<: *python-versions -# nightly: -# triggers: -# - schedule: -# cron: "0 0 * * *" -# filters: -# branches: -# only: -# - master -# jobs: -# - build + nightly: + triggers: + - schedule: + cron: "0 0 * * *" + <<: *on-master + jobs: + - build_and_edge diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..02e6673 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM redislabs/redisgraph:edge as builder + +RUN apt update && apt install -y python3 python3-pip +ADD . /build +WORKDIR /build +RUN pip3 install poetry +RUN poetry config virtualenvs.create false +RUN poetry build + +### clean docker stage +FROM redislabs/redisgraph:edge as runner + +RUN apt update && apt install -y python3 python3-pip +RUN rm -rf /var/cache/apt/ + +COPY --from=builder /build/dist/redisgraph*.tar.gz /tmp +RUN pip3 install /tmp/redisgraph*.tar.gz diff --git a/pyproject.toml b/pyproject.toml index 8a81db7..239a7ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisgraph" -version = "2.3.0" +version = "2.4.0" description = "RedisGraph Python Client" authors = ["RedisLabs "] license = "BSD-3-Clause" @@ -22,8 +22,6 @@ repository = "https://github.com/RedisGraph/redisgraph-py" python = "^3.6" redis = "^3.5.3" prettytable = "^2.1.0" -pip = "^21.1.0" -poetry = "^1.1.6" [tool.poetry.dev-dependencies] tox = "^3.23.1" @@ -39,5 +37,5 @@ vulture = "^2.3" bandit = "^1.7.0" [build-system] -requires = ["poetry-core>=1.1.16"] +requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From 383eba3932bfbfec8bd214f2de78a4516a4da893 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 18 May 2021 14:22:51 +0300 Subject: [PATCH 3/6] pypoetry for pypi releases --- .github/workflows/publish-pypi.yml | 55 +++++++++++++----------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 9ace164..7caeccd 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -1,50 +1,43 @@ name: Publish Pypi on: release: - types: [published] + types: [ published ] jobs: - publish: - name: publish + pytest: + name: Publish to PyPi runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true steps: - uses: actions/checkout@master - - name: Set up Python 2.7 + - name: Set up Python 3.7 uses: actions/setup-python@v1 with: - python-version: 2.7 + python-version: 3.7 - - name: Install twine - run: | - pip install twine - - - name: Install wheel - run: | - pip install wheel - - - name: Create a source distribution - run: | - python setup.py sdist + - name: Install Poetry + uses: dschep/install-poetry-action@v1.3 - - name: Create a wheel - run: | - python setup.py bdist_wheel + - name: Cache Poetry virtualenv + uses: actions/cache@v1 + id: cache + with: + path: ~/.virtualenvs + key: poetry-${{ hashFiles('**/poetry.lock') }} + restore-keys: | + poetry-${{ hashFiles('**/poetry.lock') }} - - name: Create a .pypirc + - name: Set Poetry config run: | - echo -e "[pypi]" >> ~/.pypirc - echo -e "username = __token__" >> ~/.pypirc - echo -e "password = ${{ secrets.PYPI_TOKEN }}" >> ~/.pypirc - echo -e "[testpypi]" >> ~/.pypirc - echo -e "username = __token__" >> ~/.pypirc - echo -e "password = ${{ secrets.TESTPYPI_TOKEN }}" >> ~/.pypirc + poetry config virtualenvs.in-project false + poetry config virtualenvs.path ~/.virtualenvs - - name: Publish to Test PyPI - if: github.event_name == 'release' - run: | - twine upload --skip-existing -r testpypi dist/* + - name: Install Dependencies + run: poetry install + if: steps.cache.outputs.cache-hit != 'true' - name: Publish to PyPI if: github.event_name == 'release' run: | - twine upload -r pypi dist/* + poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }} --build From e9d30acc04aa4daa7a38c5f4b042c920ba63ad30 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Tue, 18 May 2021 14:47:38 +0300 Subject: [PATCH 4/6] removing docker warnings --- .circleci/config.yml | 16 ++++++++++------ Dockerfile | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 24b760d..9ebb0ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,9 +41,9 @@ commands: - abort_for_noci - checkout - run: - name: install tox dependencies + name: install poetry command: | - pip install --user --quiet -r requirements.txt + pip install --user --quiet poetry - run: name: build @@ -51,6 +51,11 @@ commands: poetry build --format sdist poetry build --format wheel + - save_cache: + paths: + - ./.tox + key: v1-dependencies-{{ checksum "pyproject.toml" }} + - run: name: lint command: | @@ -86,7 +91,7 @@ commands: bash <(curl -fsSL https://get.docker.com) docker login -u redisfab -p $DOCKER_REDISFAB_PWD docker build -t redisgraph:<> . - docker pusH + docker push jobs: build: @@ -100,7 +105,7 @@ jobs: - build_and_test # since this is used by cron, we by default build against latest - build_and_edge: + build_and_publish: parameters: docker_version: type: string @@ -139,7 +144,6 @@ python-versions: &python-versions - "latest" workflows: - version: 2 commit: jobs: - build: @@ -152,4 +156,4 @@ workflows: cron: "0 0 * * *" <<: *on-master jobs: - - build_and_edge + - build_and_publish diff --git a/Dockerfile b/Dockerfile index 02e6673..be30667 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,5 +13,5 @@ FROM redislabs/redisgraph:edge as runner RUN apt update && apt install -y python3 python3-pip RUN rm -rf /var/cache/apt/ -COPY --from=builder /build/dist/redisgraph*.tar.gz /tmp +COPY --from=builder /build/dist/redisgraph*.tar.gz /tmp/ RUN pip3 install /tmp/redisgraph*.tar.gz From d2da7f1303fba79d2b3a5c60ca7b405dcf7b2066 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 19 May 2021 08:33:23 +0300 Subject: [PATCH 5/6] lifting bloom changes to redisgraph-py, circle deps come from the cirlce folder --- .circleci/circle_requirements.txt | 3 +++ .circleci/config.yml | 25 +++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 .circleci/circle_requirements.txt diff --git a/.circleci/circle_requirements.txt b/.circleci/circle_requirements.txt new file mode 100644 index 0000000..1c010d2 --- /dev/null +++ b/.circleci/circle_requirements.txt @@ -0,0 +1,3 @@ +poetry>=1.1.6 +tox>=3.23.1 +tox-poetry>=0.3.0 diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ebb0ed..37ca17a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,22 +40,31 @@ commands: - abort_for_docs - abort_for_noci - checkout - - run: - name: install poetry - command: | - pip install --user --quiet poetry + + - restore_cache: # Download and cache dependencies + keys: + - v1-dependencies-{{ checksum "pyproject.toml" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- - run: - name: build + name: install tox dependencies command: | - poetry build --format sdist - poetry build --format wheel + pip install --user --quiet -r .circleci/circle_requirements.txt - save_cache: paths: - ./.tox + - ~/.cache/pip key: v1-dependencies-{{ checksum "pyproject.toml" }} + + - run: + name: build + command: | + poetry build --format sdist + poetry build --format wheel + - run: name: lint command: | @@ -71,7 +80,7 @@ commands: - early_return_for_forked_pull_requests - run: - name: codecove + name: codecov command: | . .tox/func/bin/activate codecov --file .tox/cover/report/coverage.xml --name ${CODECOV_NAME}-unittests From 9a697e86314c74790516fe2ad706c47f04d234aa Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 19 May 2021 11:28:55 +0300 Subject: [PATCH 6/6] unifying the pypi settings removing second poetry build call, since they can be done in one --- .circleci/config.yml | 5 ++--- pyproject.toml | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 37ca17a..5780379 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,10 +60,9 @@ commands: - run: - name: build + name: build sdist and wheels command: | - poetry build --format sdist - poetry build --format wheel + poetry build - run: name: lint diff --git a/pyproject.toml b/pyproject.toml index 239a7ce..f01693c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,10 @@ classifiers=[ 'Topic :: Database', 'Programming Language :: Python', 'Intended Audience :: Developers', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: BSD License', 'Development Status :: 5 - Production/Stable' ]