adds tests to build containers, removes psutil

.dockerignore

@@ -9,5 +9,11 @@ __pycache__
 .egg-info
 _storage
 _test_storage
+.vscode
 Dockerfile
-.md
+.md
+_secrets
+docs
+tests
+logs
+experiments

.github/workflows/test_build_images.yml

@@ -0,0 +1,37 @@
+name: test build docker images
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+
+jobs:
+  get_docs_changes:
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_airflow:
+    name: Build alpine and airflow images
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.x"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.2
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Build images
+        run: make test-build-images

Makefile

@@ -68,13 +68,15 @@ reset-test-storage:
 	mkdir _storage
 	python3 tests/tools/create_storages.py
 
-recreate-compiled-deps:
-	poetry export -f requirements.txt --output _gen_requirements.txt --without-hashes --extras gcp --extras redshift
-	grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
-
 build-library: dev
 	poetry version
 	poetry build
 
 publish-library: build-library
 	poetry publish
+
+test-build-images: build-library
+	poetry export -f requirements.txt --output _gen_requirements.txt --without-hashes --extras gcp --extras redshift
+	grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
+	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
+	docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .

deploy/dlt/Dockerfile

@@ -38,7 +38,7 @@ ENV IMAGE_VERSION=${IMAGE_VERSION}
 
 # install exactly the same version of the library we used to build
 COPY dist/dlt-${IMAGE_VERSION}.tar.gz .
-RUN pip3 install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[gcp,redshift]
+RUN pip3 install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[gcp,redshift,duckdb]
 
 WORKDIR /
 RUN rm -r /tmp/pydlt

deploy/dlt/Dockerfile.airflow

@@ -0,0 +1,27 @@
+FROM apache/airflow:2.5.2-python3.8
+
+# Metadata
+LABEL org.label-schema.vendor="dltHub" \
+    org.label-schema.url="https://dlthub.com" \
+    org.label-schema.name="dlt" \
+    org.label-schema.description="**data load tool (dlt)** is a simple, open source Python library that makes data loading easy."
+
+# prepare dirs to install dlt
+RUN mkdir -p /tmp/pydlt
+
+WORKDIR /tmp/pydlt
+
+# add build labels and envs
+ARG COMMIT_SHA=""
+ARG IMAGE_VERSION=""
+LABEL commit_sha = ${COMMIT_SHA}
+LABEL version=${IMAGE_VERSION}
+ENV COMMIT_SHA=${COMMIT_SHA}
+ENV IMAGE_VERSION=${IMAGE_VERSION}
+
+# install exactly the same version of the library we used to build
+COPY dist/dlt-${IMAGE_VERSION}.tar.gz .
+RUN pip3 install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[gcp,redshift,duckdb]
+
+WORKDIR /
+RUN rm -r /tmp/pydlt

dlt/common/data_writers/buffered.py

@@ -106,7 +106,7 @@ def _rotate_file(self) -> None:
 
     def _flush_items(self) -> None:
         if len(self._buffered_items) > 0:
-            # we only open a writer when there are any files in the buffer and first flush is requested
+            # we only open a writer when there are any items in the buffer and first flush is requested
             if not self._writer:
                 # create new writer and write header
                 if self._file_format_spec.is_binary_format:

dlt/common/runtime/collector.py

@@ -2,7 +2,6 @@
 import sys
 import logging
 import time
-import psutil
 from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, ContextManager, Dict, Type, TYPE_CHECKING, DefaultDict, NamedTuple, Optional, Union, TextIO, TypeVar
@@ -120,6 +119,12 @@ def __init__(self, log_period: float = 1.0, logger: Union[logging.Logger, TextIO
         self.counters: DefaultDict[str, int] = None
         self.counter_info: Dict[str, LogCollector.CounterInfo] = None
         self.messages: Dict[str, Optional[str]] = None
+        if dump_system_stats:
+            try:
+                import psutil
+            except ImportError:
+                self._log(logging.WARNING, "psutil dependency is not installed and mem stats will not be available. add psutil to your environment or pass dump_system_stats argument as False to disable warning.")
+                dump_system_stats = False
         self.dump_system_stats = dump_system_stats
         self.last_log_time: float = None
 
@@ -168,20 +173,25 @@ def dump_counters(self) -> None:
             log_lines.append(counter_line.strip())
 
         if self.dump_system_stats:
+            import psutil
+
             process = psutil.Process(os.getpid())
             mem_info = process.memory_info()
             current_mem = mem_info.rss / (1024 ** 2)  # Convert to MB
             mem_percent = psutil.virtual_memory().percent
             cpu_percent = process.cpu_percent()
-
             log_lines.append(f"Memory usage: {current_mem:.2f} MB ({mem_percent:.2f}%) | CPU usage: {cpu_percent:.2f}%")
+
         log_lines.append("")
         log_message = "\n".join(log_lines)
         if not self.logger:
             # try to attach dlt logger
             self.logger = dlt_logger.LOGGER
+        self._log(self.log_level, log_message)
+
+    def _log(self, log_level: int, log_message: str) -> None:
         if isinstance(self.logger, logging.Logger):
-            self.logger.log(self.log_level, log_message)
+            self.logger.log(log_level, log_message)
         else:
             print(log_message, file=self.logger or sys.stdout)