diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index fd1696174..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Tests - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - workflow_dispatch: - schedule: - - cron: '0 2 * * *' - - -jobs: - test: - name: "Python: ${{ matrix.python-version }} - SQLA: ${{ matrix.sqla-version }} - CrateDB: ${{ matrix.crate-version }} - on ${{ matrix.os }}" - runs-on: ${{ matrix.os }} - strategy: - matrix: - crate-version: [nightly] - os: [ubuntu-latest] - sqla-version: ['1.1.18', '1.2.19', '1.3.23'] - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] - fail-fast: false - - steps: - - uses: actions/checkout@master - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - # Workaround needed for Python 3.5 - python -m pip install --upgrade "setuptools>=31,<51" - - pip install zc.buildout==2.13.4 - - # replace SQLAlchemy version - sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg - - # replace CrateDB version - if [ ${{ matrix.crate-version }} = "nightly" ]; then - sed -ir 's/releases/releases\/nightly/g' base.cfg - sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg - else - sed -ir 's/crate-/crate_/g' base.cfg - sed -ir 's/crate_server.*/crate_server = ${{ matrix.crate-version }}/g' versions.cfg - fi - - buildout -n -c base.cfg - - - name: Test - run: | - bin/flake8 - bin/coverage run bin/test -vv1 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..e81cfaa18 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,38 @@ +name: Nightly + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' + + +jobs: + nightly: + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['nightly'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + fail-fast: false + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} + + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..883185ae1 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + + +jobs: + test: + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['4.5.0'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + fail-fast: false + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Adjust environment for macOS + if: matrix.os == 'macos-latest' + run: | + brew install gnu-getopt + echo "/usr/local/opt/gnu-getopt/bin" >> $GITHUB_PATH + + - name: Install dependencies + run: | + ./devtools/setup_ci.sh --cratedb-version=${{ matrix.cratedb-version }} --sqlalchemy-version=${{ matrix.sqla-version }} + + - name: Invoke tests + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/base.cfg b/base.cfg index f0df080e0..e470e47bd 100644 --- a/base.cfg +++ b/base.cfg @@ -19,11 +19,21 @@ eggs = crate recipe = zc.recipe.egg eggs = createcoverage -[crate] +[crate:linux] recipe = hexagonit.recipe.download url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz strip-top-level-dir = true +[crate:macosx] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz +strip-top-level-dir = true + +[crate:windows] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip +strip-top-level-dir = true + [test] relative-paths = true recipe = zc.recipe.testrunner diff --git a/devtools/setup_ci.sh b/devtools/setup_ci.sh new file mode 100755 index 000000000..ec70f22d5 --- /dev/null +++ b/devtools/setup_ci.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +function args() { + options=$(getopt --long cratedb-version: --long sqlalchemy-version: -- "$@") + [ $? -eq 0 ] || { + echo "Incorrect options provided" + exit 1 + } + eval set -- "$options" + while true; do + case "$1" in + --cratedb-version) + shift; + cratedb_version=$1 + ;; + --sqlalchemy-version) + shift; + sqlalchemy_version=$1 + ;; + --) + shift + break + ;; + esac + shift + done +} + +function main() { + + # Read command line arguments. + args $0 "$@" + + # Sanity checks. + [ -z ${cratedb_version} ] && { + echo "--cratedb-version must be given" + exit 1 + } + [ -z ${sqlalchemy_version} ] && { + echo "--sqlalchemy-version must be given" + exit 1 + } + + # Let's go. + echo "Invoking tests with CrateDB ${cratedb_version} and SQLAlchemy ${sqlalchemy_version}" + + python -m pip install --upgrade pip + + # Workaround needed for Python 3.5 + python -m pip install --upgrade "setuptools>=31,<51" + + pip install zc.buildout==2.13.4 + + # Replace SQLAlchemy version. + sed -ir "s/SQLAlchemy.*/SQLAlchemy = ${sqlalchemy_version}/g" versions.cfg + + # Replace CrateDB version. + if [ ${cratedb_version} = "nightly" ]; then + sed -ir "s/releases/releases\/nightly/g" base.cfg + sed -ir "s/crate_server.*/crate_server = latest/g" versions.cfg + else + sed -ir "s/crate_server.*/crate_server = ${cratedb_version}/g" versions.cfg + fi + + buildout -n -c base.cfg + +} + +main "$@" diff --git a/src/crate/testing/doctests/layer.txt b/src/crate/testing/doctests/layer.txt index 6a17dffde..2e4e67b4e 100644 --- a/src/crate/testing/doctests/layer.txt +++ b/src/crate/testing/doctests/layer.txt @@ -225,26 +225,9 @@ We might have to wait a moment before the cluster is finally created:: From Uri -------- -The CrateLayer can also be created by providing a URI that points to a Crate +The CrateLayer can also be created by providing a URI that points to a CrateDB tarball:: - >>> import urllib.request - >>> import json - >>> with urllib.request.urlopen('http://crate.io/versions.json') as response: - ... versions = json.loads(response.read().decode()) - ... version = versions['crate_testing'] - - >>> uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) - >>> tmpdir = tempfile.mkdtemp() - >>> layer = CrateLayer.from_uri( - ... uri, name='crate-uri', http_port=42203, directory=tmpdir) - >>> layer.setUp() - - >>> work_dir = os.path.join(tmpdir, 'crate-' + version) - >>> os.path.exists(work_dir) - True - - >>> layer.tearDown() - - >>> os.path.exists(work_dir) - False + uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) + layer = CrateLayer.from_uri( + uri, name='crate-uri', http_port=42203, directory=tmpdir) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 47578b828..3bd3fc992 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -72,6 +72,8 @@ def prepend_http(host): def _download_and_extract(uri, directory): + sys.stderr.write("\nINFO: Downloading CrateDB archive from {} into {}".format(uri, directory)) + sys.stderr.flush() with io.BytesIO(urlopen(uri).read()) as tmpfile: with tarfile.open(fileobj=tmpfile) as t: t.extractall(directory) @@ -160,7 +162,8 @@ def from_uri(uri, crate_home = os.path.join(directory, crate_dir) if os.path.exists(crate_home): - sys.stderr.write('Not extracting Crate tarball because folder already exists') + sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists") + sys.stderr.flush() else: _download_and_extract(uri, directory) diff --git a/src/crate/testing/test_layer.py b/src/crate/testing/test_layer.py index f2bcfc4b0..ca55a2f92 100644 --- a/src/crate/testing/test_layer.py +++ b/src/crate/testing/test_layer.py @@ -18,11 +18,15 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. - +import json import os import tempfile +import urllib +from distutils.version import LooseVersion from unittest import TestCase, mock from io import BytesIO + +import crate from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url @@ -58,6 +62,22 @@ def test_wait_for_http(self): addr = wait_for_http_url(log=log, timeout=1) self.assertEqual(None, addr) + @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None) + def test_layer_from_uri(self): + """ + The CrateLayer can also be created by providing an URI that points to + a CrateDB tarball. + """ + with urllib.request.urlopen("https://crate.io/versions.json") as response: + versions = json.loads(response.read().decode()) + version = versions["crate_testing"] + + self.assertGreaterEqual(LooseVersion(version), LooseVersion("4.5.0")) + + uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version) + layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203) + self.assertIsInstance(layer, CrateLayer) + @mock.patch.dict('os.environ', {}, clear=True) def test_java_home_env_not_set(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/versions.cfg b/versions.cfg index 1ff59bf64..b2a01b5b2 100644 --- a/versions.cfg +++ b/versions.cfg @@ -1,5 +1,5 @@ [versions] -crate_server = 4.4.2 +crate_server = 4.5.0 flake8 = 3.7.9 mccabe = 0.6.1