Merge pull request #176 from keboola/adamvyborny-PST-1420

New Native Types support
keboola · Jul 5, 2024 · 9d3595f · 9d3595f
2 parents aef9349 + 98a684f
commit 9d3595f
Show file tree

Hide file tree

Showing 38 changed files with 1,710 additions and 1,462 deletions.
diff --git a/composer.json b/composer.json
@@ -14,11 +14,11 @@
         "ext-json": "*",
         "ext-mbstring": "*",
         "ext-pdo": "*",
-        "keboola/db-extractor-adapter": "^1.13",
-        "keboola/db-extractor-common": "^16.2",
-        "keboola/db-extractor-config": "^1.14",
+        "keboola/db-extractor-adapter": "^1.14",
+        "keboola/db-extractor-common": "^17.0",
+        "keboola/db-extractor-config": "^1.15",
         "keboola/db-extractor-table-format": "^3.8",
-        "keboola/php-component": "^9.4",
+        "keboola/php-component": "^10.1",
         "keboola/php-datatypes": "^7.2",
         "keboola/php-temp": "^2.0",
         "keboola/php-utils": "^4.1",

diff --git a/composer.lock b/composer.lock
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,11 +1,11 @@
-version: "3"
 services:
   app: &app
     build: .
     image: keboola/db-extractor-mysql
     command: composer ci
     environment:
       KBC_COMPONENT_RUN_MODE: run
+      KBC_DATA_TYPE_SUPPORT: none
       MYSQL_DB_HOST: mysql
       MYSQL_DB_SSL_HOST: mysql-ssl
       MYSQL_DB_SSL_BAD_CN_HOST: mysql-different-cn
@@ -32,17 +32,20 @@ services:
 
   mysql:
     image: mysql:${MYSQL_VERSION}
-    command: mysqld --local-infile --port=3306 --default-authentication-plugin=mysql_native_password
+    entrypoint: /usr/local/bin/entrypoint.sh
+    volumes:
+      - ./docker/mysql-entrypoint.sh:/usr/local/bin/entrypoint.sh
     environment:
       MYSQL_DATABASE: test
       MYSQL_ROOT_PASSWORD: rootpassword
 
   mysql-ssl:
     image: mysql:${MYSQL_VERSION}
-    command: mysqld --local-infile --port=3306 --default-authentication-plugin=mysql_native_password
+    entrypoint: /usr/local/bin/entrypoint.sh
     volumes:
       - ./docker/ssl/certificates:/ssl-cert
       - ./docker/mysql-ssl/${MYSQL_VERSION}/conf.d:/etc/mysql/conf.d
+      - ./docker/mysql-entrypoint.sh:/usr/local/bin/entrypoint.sh
     environment:
       MYSQL_DATABASE: test
       MYSQL_ROOT_PASSWORD: rootpassword

diff --git a/docker/mysql-entrypoint.sh b/docker/mysql-entrypoint.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "$MYSQL_VERSION" =~ ^5\.6 ]]; then
+  exec docker-entrypoint.sh mysqld --local-infile --port=3306 --default-authentication-plugin=mysql_native_password
+else
+  exec docker-entrypoint.sh mysqld --local-infile --port=3306 --mysql-native-password=ON
+fi
diff --git a/phpstan.neon b/phpstan.neon
@@ -1,3 +1,3 @@
 parameters:
-    checkMissingIterableValueType: false
     ignoreErrors:
+        - identifier: missingType.iterableValue
diff --git a/src/Keboola/DbExtractor/Extractor/MySQLMetadataProvider.php b/src/Keboola/DbExtractor/Extractor/MySQLMetadataProvider.php
@@ -38,7 +38,7 @@ public function getTable(InputTable $table): Table
 
     /**
      * @param array|InputTable[] $whitelist
-     * @param bool $loadColumns if false, columns metadata are NOT loaded, useful useful if there are a lot of tables
+     * @param bool $loadColumns if false, columns metadata are NOT loaded, useful if there are a lot of tables
      */
     public function listTables(array $whitelist = [], bool $loadColumns = true): TableCollection
     {

diff --git a/...etime-with-state/expected/data/out/tables/in.c-main.auto-increment-timestamp.csv.manifest b/...etime-with-state/expected/data/out/tables/in.c-main.auto-increment-timestamp.csv.manifest
@@ -1,9 +1,6 @@
 {
     "destination": "in.c-main.auto-increment-timestamp",
     "incremental": true,
-    "primary_key": [
-        "Weir_d_I_D"
-    ],
     "metadata": [
         {
             "key": "KBC.name",
@@ -26,12 +23,19 @@
             "value": 6
         }
     ],
+    "columns": [
+        "Weir_d_I_D",
+        "Weir_d_Na_me",
+        "someInteger",
+        "someDecimal",
+        "type",
+        "datetime"
+    ],
+    "primary_key": [
+        "Weir_d_I_D"
+    ],
     "column_metadata": {
         "Weir_d_I_D": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "int"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": false
@@ -52,10 +56,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "Weir_d_I_D"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": true
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": false
@@ -78,10 +78,6 @@
             }
         ],
         "Weir_d_Na_me": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "varchar"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": false
@@ -106,10 +102,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "Weir_d_Na_me"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": false
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": true
@@ -124,10 +116,6 @@
             }
         ],
         "someInteger": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "int"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": true
@@ -152,10 +140,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "someInteger"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": false
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": false
@@ -166,10 +150,6 @@
             }
         ],
         "someDecimal": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "decimal"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": true
@@ -194,10 +174,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "someDecimal"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": false
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": false
@@ -208,10 +184,6 @@
             }
         ],
         "type": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "varchar"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": true
@@ -236,10 +208,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "type"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": false
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": false
@@ -250,10 +218,6 @@
             }
         ],
         "datetime": [
-            {
-                "key": "KBC.datatype.type",
-                "value": "datetime"
-            },
             {
                 "key": "KBC.datatype.nullable",
                 "value": false
@@ -270,10 +234,6 @@
                 "key": "KBC.sanitizedName",
                 "value": "datetime"
             },
-            {
-                "key": "KBC.primaryKey",
-                "value": false
-            },
             {
                 "key": "KBC.uniqueKey",
                 "value": false
@@ -283,13 +243,5 @@
                 "value": 6
             }
         ]
-    },
-    "columns": [
-        "Weir_d_I_D",
-        "Weir_d_Na_me",
-        "someInteger",
-        "someDecimal",
-        "type",
-        "datetime"
-    ]
+    }
 }