Merge pull request #9 from keboola/ondra-migrate-large-tables

Migrate large tables (replication mode)

.github/workflows/push.yml

@@ -26,7 +26,7 @@ jobs:
         steps:
             -
                 name: 'Check out the repo'
-                uses: actions/checkout@v2
+                uses: actions/checkout@v4
             -
                 name: 'Print Docker version'
                 run: 'docker -v'
@@ -44,8 +44,8 @@ jobs:
                     TAG="${GITHUB_REF##*/}"
                     IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
                     echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
-                    echo "::set-output name=app_image_tag::$TAG"
-                    echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG"
+                    echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT
+                    echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT
             -
                 name: 'Push image to ECR'
                 uses: keboola/action-push-to-ecr@master
@@ -63,7 +63,7 @@ jobs:
         steps:
             -
                 name: 'Check out the repo'
-                uses: actions/checkout@v2
+                uses: actions/checkout@v4
             -
                 name: 'Pull image from ECR'
                 uses: keboola/action-pull-from-ecr@master

Dockerfile

@@ -1,32 +1,73 @@
-FROM php:7.4-cli
+FROM php:8.2-cli-buster
 
 ARG COMPOSER_FLAGS="--prefer-dist --no-interaction"
 ARG DEBIAN_FRONTEND=noninteractive
+ARG SNOWFLAKE_ODBC_VERSION=2.25.12
+ARG SNOWFLAKE_GPG_KEY=630D9F3CAB551AF3
 ENV COMPOSER_ALLOW_SUPERUSER 1
 ENV COMPOSER_PROCESS_TIMEOUT 3600
+ENV LANGUAGE=en_US.UTF-8
+ENV LANG=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
 
 WORKDIR /code/
 
-COPY docker/php-prod.ini /usr/local/etc/php/php.ini
+COPY docker/php/php-prod.ini /usr/local/etc/php/php.ini
+COPY docker/php/xdebug.ini /usr/local/etc/php/conf.d/
 COPY docker/composer-install.sh /tmp/composer-install.sh
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         git \
         locales \
         unzip \
+        unixodbc \
+        unixodbc-dev \
+        gnupg \
+        debsig-verify \
+        gpg-agent \
 	&& rm -r /var/lib/apt/lists/* \
 	&& sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \
 	&& locale-gen \
 	&& chmod +x /tmp/composer-install.sh \
 	&& /tmp/composer-install.sh
 
-ENV LANGUAGE=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LC_ALL=en_US.UTF-8
+# Install PHP odbc extension
+# https://github.com/docker-library/php/issues/103
+RUN set -x \
+    && docker-php-source extract \
+    && cd /usr/src/php/ext/odbc \
+    && phpize \
+    && sed -ri 's@^ *test +"\$PHP_.*" *= *"no" *&& *PHP_.*=yes *$@#&@g' configure \
+    && ./configure --with-unixODBC=shared,/usr \
+    && docker-php-ext-install odbc \
+    && docker-php-source delete
+
+#snoflake download + verify package
+COPY docker/snowflake/snowflake-policy.pol /etc/debsig/policies/$SNOWFLAKE_GPG_KEY/generic.pol
+COPY docker/snowflake/simba.snowflake.ini /usr/lib/snowflake/odbc/lib/simba.snowflake.ini
+ADD https://sfc-repo.azure.snowflakecomputing.com/odbc/linux/$SNOWFLAKE_ODBC_VERSION/snowflake-odbc-$SNOWFLAKE_ODBC_VERSION.x86_64.deb /tmp/snowflake-odbc.deb
+
+RUN mkdir -p ~/.gnupg \
+    && chmod 700 ~/.gnupg \
+    && echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf \
+    && mkdir -p /usr/share/debsig/keyrings/$SNOWFLAKE_GPG_KEY \
+    && if ! gpg --keyserver hkp://keys.gnupg.net --recv-keys $SNOWFLAKE_GPG_KEY; then \
+        gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $SNOWFLAKE_GPG_KEY;  \
+    fi \
+    && gpg --export $SNOWFLAKE_GPG_KEY > /usr/share/debsig/keyrings/$SNOWFLAKE_GPG_KEY/debsig.gpg \
+    && curl https://sfc-repo.snowflakecomputing.com/odbc/linux/$SNOWFLAKE_ODBC_VERSION/snowflake-odbc-$SNOWFLAKE_ODBC_VERSION.x86_64.deb --output /tmp/snowflake-odbc.deb \
+    && debsig-verify /tmp/snowflake-odbc.deb \
+    && gpg --batch --delete-key --yes $SNOWFLAKE_GPG_KEY \
+    && dpkg -i /tmp/snowflake-odbc.deb \
+    && rm /tmp/snowflake-odbc.deb
+
+RUN pecl install xdebug \
+    && docker-php-ext-enable xdebug
 
 ## Composer - deps always cached unless changed
 # First copy only composer files
 COPY composer.* /code/
+COPY patches /code/patches
 
 # Download dependencies, but don't run scripts or init autoloaders as the app is missing
 RUN composer install $COMPOSER_FLAGS --no-scripts --no-autoloader

composer.json

@@ -3,30 +3,32 @@
     "description": "MyComponent description",
     "license": "MIT",
     "require": {
-        "php": "^7.4",
-        "keboola/php-component": "^8.1",
+        "php": "^8.2",
+        "keboola/db-adapter-snowflake": "^1.4",
+        "keboola/php-component": "^9.4",
         "keboola/storage-api-client": "^14.16"
     },
     "require-dev": {
+        "cweagans/composer-patches": "^1.7",
         "keboola/coding-standard": ">=7.0.2",
         "keboola/csv": "^1.1",
         "keboola/datadir-tests": "^5.2",
         "keboola/php-temp": "^2.0",
         "php-parallel-lint/php-parallel-lint": "^1.2",
-        "phpstan/phpstan": "^0.12.14",
+        "phpstan/phpstan": "^1.10",
         "phpunit/phpunit": "^9.0",
         "symfony/finder": "^5.4",
         "symfony/process": "^5.0"
     },
     "autoload": {
         "psr-4": {
-            "AppProjectMigrateLargeTables\\": "src/"
+            "Keboola\\AppProjectMigrateLargeTables\\": "src/"
         }
     },
     "autoload-dev": {
         "psr-4": {
-            "AppProjectMigrateLargeTables\\Tests\\": "tests/phpunit/",
-            "AppProjectMigrateLargeTables\\FunctionalTests\\": "tests/functional/"
+            "Keboola\\AppProjectMigrateLargeTables\\Tests\\": "tests/phpunit/",
+            "Keboola\\AppProjectMigrateLargeTables\\FunctionalTests\\": "tests/functional/"
         }
     },
     "scripts": {
@@ -56,7 +58,15 @@
         "sort-packages": true,
         "optimize-autoloader": true,
         "allow-plugins": {
-            "dealerdirect/phpcodesniffer-composer-installer": true
+            "dealerdirect/phpcodesniffer-composer-installer": true,
+            "cweagans/composer-patches": true
+        }
+    },
+    "extra": {
+        "patches": {
+            "microsoft/azure-storage-common": [
+                "patches/microsoft-azure-storage-common-src-common-internal-servicerestproxy-php.patch"
+            ]
         }
     }
 }