diff --git a/.github/workflows/influxdb.yml b/.github/workflows/influxdb.yml new file mode 100644 index 00000000..d6deb03c --- /dev/null +++ b/.github/workflows/influxdb.yml @@ -0,0 +1,90 @@ +--- +name: "Tests: InfluxDB" + +on: + pull_request: + branches: ~ + paths: + - '.github/workflows/influxdb.yml' + - 'cratedb_toolkit/io/influxdb/**' + - 'pyproject.toml' + push: + branches: [ main ] + paths: + - '.github/workflows/influxdb.yml' + - 'cratedb_toolkit/io/influxdb/**' + - 'pyproject.toml' + + # Allow job to be triggered manually. + workflow_dispatch: + + # Run job each night after CrateDB nightly has been published. + schedule: + - cron: '0 3 * * *' + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + tests: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + # TODO: yarl, dependency of influxio, is currently not available on Python 3.12. + # https://github.com/aio-libs/yarl/pull/942 + python-version: ["3.8", "3.11"] + influxdb-version: ["2.6", "2.7"] + + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + INFLUXDB_VERSION: ${{ matrix.influxdb-version }} + # Do not tear down Testcontainers + TC_KEEPALIVE: true + + name: " + Python ${{ matrix.python-version }}, + InfluxDB ${{ matrix.influxdb-version }}, + OS ${{ matrix.os }} + " + steps: + + - name: Acquire sources + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + cache: 'pip' + cache-dependency-path: 'pyproject.toml' + + - name: Setup project + run: | + + # `setuptools 0.64.0` adds support for editable install hooks (PEP 660). + # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400 + pip install "setuptools>=64" --upgrade + + # Install package in editable mode. + pip install --use-pep517 --prefer-binary --editable=.[influxdb,io,test,develop] + + - name: Run linter and software tests + run: | + poe check + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: ./coverage.xml + flags: influxdb + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: false diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc07a479..a1d8a974 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,7 +65,7 @@ jobs: pip install "setuptools>=64" --upgrade # Install package in editable mode. - pip install --use-pep517 --prefer-binary --editable=.[io,influxdb,test,develop] + pip install --use-pep517 --prefer-binary --editable=.[io,test,develop] - name: Run linter and software tests run: | diff --git a/.gitignore b/.gitignore index 0f24e1a4..efc0473d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,5 @@ __pycache__ *.pyc dist -.coverage +.coverage* coverage.xml diff --git a/cratedb_toolkit/io/cli.py b/cratedb_toolkit/io/cli.py index c975edda..004841b8 100644 --- a/cratedb_toolkit/io/cli.py +++ b/cratedb_toolkit/io/cli.py @@ -146,10 +146,10 @@ def load_table_cratedb(sqlalchemy_url: str, resource_url: str): ctk load table influxdb2://example:token@localhost:8086/testdrive/demo """ if resource_url.startswith("influxdb"): - from influxio.core import copy + from cratedb_toolkit.io.influxdb import influxdb_copy source_url = resource_url.replace("influxdb2", "http") target_url = sqlalchemy_url - copy(source_url, target_url, progress=True) + influxdb_copy(source_url, target_url, progress=True) else: raise NotImplementedError("Importing resource not implemented yet") diff --git a/cratedb_toolkit/io/influxdb.py b/cratedb_toolkit/io/influxdb.py new file mode 100644 index 00000000..ec71f394 --- /dev/null +++ b/cratedb_toolkit/io/influxdb.py @@ -0,0 +1,5 @@ +from influxio.core import copy + + +def influxdb_copy(source_url, target_url, progress: bool = False): + copy(source_url, target_url, progress=progress) diff --git a/cratedb_toolkit/testing/testcontainers/influxdb2.py b/cratedb_toolkit/testing/testcontainers/influxdb2.py index 1f240a80..8bc490bd 100644 --- a/cratedb_toolkit/testing/testcontainers/influxdb2.py +++ b/cratedb_toolkit/testing/testcontainers/influxdb2.py @@ -10,6 +10,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import os + from influxdb_client import InfluxDBClient from testcontainers.core.config import MAX_TRIES from testcontainers.core.generic import DbContainer @@ -29,14 +31,15 @@ class InfluxDB2Container(KeepaliveContainer, DbContainer): The example spins up an InfluxDB2 database instance. """ + INFLUXDB_VERSION = os.environ.get("INFLUXDB_VERSION", "latest") + ORGANIZATION = "example" TOKEN = "token" # noqa: S105 # TODO: Dual-port use with 8083+8086. def __init__( self, - # TODO: Use `influxdb:latest` by default? - image: str = "influxdb:2.7", + image: str = f"influxdb:{INFLUXDB_VERSION}", port: int = 8086, dialect: str = "influxdb2", **kwargs, diff --git a/tests/io/influxdb/conftest.py b/tests/io/influxdb/conftest.py index 01c2bb0b..cdbeb89c 100644 --- a/tests/io/influxdb/conftest.py +++ b/tests/io/influxdb/conftest.py @@ -1,9 +1,6 @@ import logging import pytest -from influxdb_client import InfluxDBClient - -from cratedb_toolkit.testing.testcontainers.influxdb2 import InfluxDB2Container logger = logging.getLogger(__name__) @@ -20,12 +17,16 @@ class InfluxDB2Fixture: """ def __init__(self): + from influxdb_client import InfluxDBClient + self.container = None self.client: InfluxDBClient = None self.setup() def setup(self): # TODO: Make image name configurable. + from cratedb_toolkit.testing.testcontainers.influxdb2 import InfluxDB2Container + self.container = InfluxDB2Container() self.container.start() self.client = self.container.get_connection_client() diff --git a/tests/io/influxdb/test_cli.py b/tests/io/influxdb/test_cli.py index 2acd3ae5..ee937fe8 100644 --- a/tests/io/influxdb/test_cli.py +++ b/tests/io/influxdb/test_cli.py @@ -1,3 +1,11 @@ +# ruff: noqa: E402 +import pytest + +pytest.importorskip("influxio", reason="Skipping InfluxDB tests because 'influxio' package is not installed") +pytest.importorskip( + "influxdb_client", reason="Skipping InfluxDB tests because 'influxdb-client' package is not installed" +) + from click.testing import CliRunner from influxio.model import InfluxDbAdapter from influxio.testdata import DataFrameFactory