InfluxDB: Improve CI

.github/workflows/influxdb.yml

@@ -0,0 +1,90 @@
+---
+name: "Tests: InfluxDB"
+
+on:
+ pull_request:
+ branches: ~
+ paths:
+ - '.github/workflows/influxdb.yml'
+ - 'cratedb_toolkit/io/influxdb/**'
+ - 'pyproject.toml'
+ push:
+ branches: [ main ]
+ paths:
+ - '.github/workflows/influxdb.yml'
+ - 'cratedb_toolkit/io/influxdb/**'
+ - 'pyproject.toml'
+
+ # Allow job to be triggered manually.
+ workflow_dispatch:
+
+ # Run job each night after CrateDB nightly has been published.
+ schedule:
+ - cron: '0 3 * * *'
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+ cancel-in-progress: true
+ group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+
+ tests:
+
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: ["ubuntu-latest"]
+ # TODO: yarl, dependency of influxio, is currently not available on Python 3.12.
+ # https://github.com/aio-libs/yarl/pull/942
+ python-version: ["3.8", "3.11"]
+ influxdb-version: ["2.6", "2.7"]
+
+ env:
+ OS: ${{ matrix.os }}
+ PYTHON: ${{ matrix.python-version }}
+ INFLUXDB_VERSION: ${{ matrix.influxdb-version }}
+ # Do not tear down Testcontainers
+ TC_KEEPALIVE: true
+
+ name: "
+ Python ${{ matrix.python-version }}, 
+ InfluxDB ${{ matrix.influxdb-version }}, 
+ OS ${{ matrix.os }}
+ "
+ steps:
+
+ - name: Acquire sources
+ uses: actions/checkout@v4
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ matrix.python-version }}
+ architecture: x64
+ cache: 'pip'
+ cache-dependency-path: 'pyproject.toml'
+
+ - name: Setup project
+ run: |
+
+ # `setuptools 0.64.0` adds support for editable install hooks (PEP 660).
+ # https://github.com/pypa/setuptools/blob/main/CHANGES.rst#v6400
+ pip install "setuptools>=64" --upgrade
+
+ # Install package in editable mode.
+ pip install --use-pep517 --prefer-binary --editable=.[influxdb,io,test,develop]
+
+ - name: Run linter and software tests
+ run: |
+ poe check
+
+ - name: Upload coverage to Codecov
+ uses: codecov/codecov-action@v3
+ with:
+ files: ./coverage.xml
+ flags: influxdb
+ env_vars: OS,PYTHON
+ name: codecov-umbrella
+ fail_ci_if_error: false

.github/workflows/main.yml

@@ -65,7 +65,7 @@ jobs:
  pip install "setuptools>=64" --upgrade
 
  # Install package in editable mode.
- pip install --use-pep517 --prefer-binary --editable=.[io,influxdb,test,develop]
+ pip install --use-pep517 --prefer-binary --editable=.[io,test,develop]
 
  - name: Run linter and software tests
  run: |

.gitignore

@@ -5,5 +5,5 @@
 __pycache__
 *.pyc
 dist
-.coverage
+.coverage*
 coverage.xml

cratedb_toolkit/io/cli.py

@@ -146,10 +146,10 @@ def load_table_cratedb(sqlalchemy_url: str, resource_url: str):
  ctk load table influxdb2://example:token@localhost:8086/testdrive/demo
  """
  if resource_url.startswith("influxdb"):
- from influxio.core import copy
+ from cratedb_toolkit.io.influxdb import influxdb_copy
 
  source_url = resource_url.replace("influxdb2", "http")
  target_url = sqlalchemy_url
- copy(source_url, target_url, progress=True)
+ influxdb_copy(source_url, target_url, progress=True)
  else:
  raise NotImplementedError("Importing resource not implemented yet")

cratedb_toolkit/io/influxdb.py

@@ -0,0 +1,5 @@
+from influxio.core import copy
+
+
+def influxdb_copy(source_url, target_url, progress: bool = False):
+ copy(source_url, target_url, progress=progress)

cratedb_toolkit/testing/testcontainers/influxdb2.py

@@ -10,6 +10,8 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import os
+
 from influxdb_client import InfluxDBClient
 from testcontainers.core.config import MAX_TRIES
 from testcontainers.core.generic import DbContainer
@@ -29,14 +31,15 @@ class InfluxDB2Container(KeepaliveContainer, DbContainer):
  The example spins up an InfluxDB2 database instance.
  """
 
+ INFLUXDB_VERSION = os.environ.get("INFLUXDB_VERSION", "latest")
+
  ORGANIZATION = "example"
  TOKEN = "token" # noqa: S105
 
  # TODO: Dual-port use with 8083+8086.
  def __init__(
  self,
- # TODO: Use `influxdb:latest` by default?
- image: str = "influxdb:2.7",
+ image: str = f"influxdb:{INFLUXDB_VERSION}",
  port: int = 8086,
  dialect: str = "influxdb2",
  **kwargs,

tests/io/influxdb/conftest.py

@@ -1,9 +1,6 @@
 import logging
 
 import pytest
-from influxdb_client import InfluxDBClient
-
-from cratedb_toolkit.testing.testcontainers.influxdb2 import InfluxDB2Container
 
 logger = logging.getLogger(__name__)
 
@@ -20,12 +17,16 @@ class InfluxDB2Fixture:
  """
 
  def __init__(self):
+ from influxdb_client import InfluxDBClient
+
  self.container = None
  self.client: InfluxDBClient = None
  self.setup()
 
  def setup(self):
  # TODO: Make image name configurable.
+ from cratedb_toolkit.testing.testcontainers.influxdb2 import InfluxDB2Container
+
  self.container = InfluxDB2Container()
  self.container.start()
  self.client = self.container.get_connection_client()

tests/io/influxdb/test_cli.py

@@ -1,3 +1,11 @@
+# ruff: noqa: E402
+import pytest
+
+pytest.importorskip("influxio", reason="Skipping InfluxDB tests because 'influxio' package is not installed")
+pytest.importorskip(
+ "influxdb_client", reason="Skipping InfluxDB tests because 'influxdb-client' package is not installed"
+)
+
 from click.testing import CliRunner
 from influxio.model import InfluxDbAdapter
 from influxio.testdata import DataFrameFactory