Merge branch 'master' into centos7node

snowflakedb · Aug 4, 2023 · 7c3b729 · 7c3b729
2 parents 4a950e1 + 63ea42a
commit 7c3b729
Show file tree

Hide file tree

Showing 24 changed files with 686 additions and 156 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -107,17 +107,12 @@ jobs:
         name: Tests on Linux
         runs-on: ubuntu-latest
         strategy:
+            fail-fast: false
             matrix:
-                image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12', 'nodejs-centos7-fips' ]
+                image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12', 'nodejs-centos7-node14', 'nodejs-centos7-fips' ]
                 cloud: [ 'AWS' ]
         steps:
             - uses: actions/checkout@v1
-            - uses: actions/setup-node@v1
-              with:
-                node-version: '12.x'
-            - uses: actions/setup-python@v1
-              with:
-                python-version: '3.7'
             - name: Download Build Artifacts
               uses: actions/download-artifact@v1
               with:
@@ -131,10 +126,11 @@ jobs:
               run: ./ci/test.sh
 
     test-mac-optional:
-        needs: test-mac-required
+        needs: build
         name: Tests on Mac
         runs-on: macos-latest
         strategy:
+            fail-fast: false
             matrix:
                 cloud: [ 'AZURE', 'GCP' ]
         steps:
@@ -160,10 +156,11 @@ jobs:
               run: /usr/local/bin/bash ./ci/test_mac.sh
 
     test-windows-optional:
-        needs: test-windows-required
+        needs: build
         name: Tests on Windows
         runs-on: windows-latest
         strategy:
+            fail-fast: false
             matrix:
                 cloud: [ 'AZURE', 'GCP' ]
         steps:
@@ -187,23 +184,18 @@ jobs:
               run: ci\\test_windows.bat
 
     test-linux-optional:
-        needs: test-linux-required
+        needs: build
         name: Tests on Linux
         runs-on: ubuntu-latest
         strategy:
+            fail-fast: false
             matrix:
-                image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12', 'nodejs-centos7-fips' ]
+                image: [ 'nodejs-centos6-default', 'nodejs-centos7-node12' , 'nodejs-centos7-node14', 'nodejs-centos7-fips']
                 #REMOVE AZURE for now
                 #cloud: [ 'AZURE', 'GCP' ]
                 cloud: [ 'GCP' ]
         steps:
             - uses: actions/checkout@v1
-            - uses: actions/setup-node@v1
-              with:
-                node-version: '12.x'
-            - uses: actions/setup-python@v1
-              with:
-                python-version: '3.7'
             - name: Download Build Artifacts
               uses: actions/download-artifact@v1
               with:
@@ -221,24 +213,16 @@ jobs:
         name: Upload code coverage
         runs-on: ubuntu-latest
         steps:
-          - name: Checkout # it is still recommended to checkout https://about.codecov.io/blog/uploading-code-coverage-in-a-separate-job-on-github-actions/
+          - name: Checkout # it is still recommended to check out https://about.codecov.io/blog/uploading-code-coverage-in-a-separate-job-on-github-actions/
             uses: actions/checkout@v3
           - name: Download code coverage result
             uses: actions/download-artifact@v3
             with:
               name: codeCoverage
               path: coverage
-          # codecov action sometimes fails but retry works - until action provides retry let's retry on our own https://github.com/codecov/codecov-action/issues/926
-          - name: Upload coverage reports to Codecov with GitHub Action (take 1)
-            id: uploadToCodeCovTake1
+          - name: Upload coverage reports to Codecov
             uses: codecov/codecov-action@v3
-            continue-on-error: true
             with:
+              # without the token code cov may fail because of Github limits https://github.com/codecov/codecov-action/issues/557
+              token: ${{ secrets.CODE_COV_UPLOAD_TOKEN }}
               fail_ci_if_error: true
-          - name: Upload coverage reports to Codecov with GitHub Action (take 2)
-            id: uploadToCodeCovTake2
-            # only run when take 1 failed
-            if: steps.uploadToCodeCovTake1.outcome == 'failure'
-            uses: codecov/codecov-action@v3
-            with:
-              fail_ci_if_error: true
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -28,7 +28,7 @@ timestamps {
       string(name: 'branch', value: 'main'),
       string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT),
       string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH),
-      string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'nodejs-centos6-default'),
+      string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'nodejs-centos7-node14'),
       string(name: 'parent_job', value: env.JOB_NAME),
       string(name: 'parent_build_number', value: env.BUILD_NUMBER)
     ]

diff --git a/README.md b/README.md
@@ -15,19 +15,7 @@ NodeJS Driver for Snowflake
 Install
 ======================================================================
 
-Include ``snowflake-sdk`` in ``dependencies`` section in ``package.json``
-```
-{
-  "name": "<your_application_name>",
-  "version": "<your_application_version>",
-  "dependencies": {
-    "...": "...",        
-    "snowflake-sdk": "^1.6.22",
-    "...": "..."
-  }
-}
-```
-And run the <code>npm install</code>
+Run `npm i snowflake-sdk` in your existing NodeJs project.
 
 Docs
 ======================================================================

diff --git a/ci/_init.sh b/ci/_init.sh
@@ -22,13 +22,16 @@ TEST_IMAGE_VERSION=2
 
 declare -A BUILD_IMAGE_NAMES=(
     [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-build:$BUILD_IMAGE_VERSION
+    [$DRIVER_NAME-centos7-node14]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-node14-build:$BUILD_IMAGE_VERSION
 )
 export BUILD_IMAGE_NAMES
 
 declare -A TEST_IMAGE_NAMES=(
     [$DRIVER_NAME-centos6-default]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos6-default-test:$BUILD_IMAGE_VERSION
     [$DRIVER_NAME-centos7-node12]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-node12-test:$BUILD_IMAGE_VERSION
     [$DRIVER_NAME-centos7-fips]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-fips-test:$BUILD_IMAGE_VERSION
+    [$DRIVER_NAME-centos7-node14]=$DOCKER_REGISTRY_NAME/client-$DRIVER_NAME-centos7-node14-test:$BUILD_IMAGE_VERSION
+
 )
 export TEST_IMAGE_NAMES
 
diff --git a/ci/image/Dockerfile.nodejs-centos7-node14-build b/ci/image/Dockerfile.nodejs-centos7-node14-build
@@ -0,0 +1,61 @@
+FROM centos:7
+# Dockerfile.nodejs-centos7-node14-build is equal to Dockerfile.nodejs-centos7-node14-test
+
+# update OS
+RUN yum -y update && \
+    yum -y install epel-release && \
+    yum -y install centos-release-scl
+
+# nvm environment variables
+ENV NVM_DIR /usr/local/nvm
+ENV NODE_VERSION 14.21.3
+
+# node
+RUN mkdir -p $NVM_DIR && \
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.4/install.sh | bash && \
+    source $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+COPY scripts/npmrc /root/.npmrc
+
+# python
+RUN yum -y install rh-python36
+COPY scripts/python3.6.sh /usr/local/bin/python3.6
+COPY scripts/python3.6.sh /usr/local/bin/python3
+RUN chmod a+x /usr/local/bin/python3.6 && \
+    chmod a+x /usr/local/bin/python3
+COPY scripts/pip.sh /usr/local/bin/pip
+RUN chmod a+x /usr/local/bin/pip && \
+    pip install -U pip && \
+    pip install -U snowflake-connector-python
+
+# aws
+RUN pip install -U awscli
+COPY scripts/aws.sh /usr/local/bin/aws
+RUN chmod a+x /usr/local/bin/aws
+
+# Development tools + git + zstd + jq + gosu
+RUN yum -y groupinstall "Development Tools" && \
+    yum -y install zlib-devel && \
+    curl -o - https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar xfz - && \
+    cd git-2.26.0 && \
+    ./configure --prefix=/opt/git && make && make install && \
+    ln -s /opt/git/bin/git /usr/local/bin/git && \
+    yum -y install zstd && \
+    yum -y install jq && \
+    curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu
+
+# workspace
+RUN mkdir -p /home/user && \
+    chmod 777 /home/user
+WORKDIR /home/user
+
+# entry point
+COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/ci/image/Dockerfile.nodejs-centos7-node14-test b/ci/image/Dockerfile.nodejs-centos7-node14-test
@@ -0,0 +1,61 @@
+FROM centos:7
+# Dockerfile.nodejs-centos7-node14-build is equal to Dockerfile.nodejs-centos7-node14-test
+
+# update OS
+RUN yum -y update && \
+    yum -y install epel-release && \
+    yum -y install centos-release-scl
+
+# nvm environment variables
+ENV NVM_DIR /usr/local/nvm
+ENV NODE_VERSION 14.21.3
+
+# node
+RUN mkdir -p $NVM_DIR && \
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.4/install.sh | bash && \
+    source $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+COPY scripts/npmrc /root/.npmrc
+
+# python
+RUN yum -y install rh-python36
+COPY scripts/python3.6.sh /usr/local/bin/python3.6
+COPY scripts/python3.6.sh /usr/local/bin/python3
+RUN chmod a+x /usr/local/bin/python3.6 && \
+    chmod a+x /usr/local/bin/python3
+COPY scripts/pip.sh /usr/local/bin/pip
+RUN chmod a+x /usr/local/bin/pip && \
+    pip install -U pip && \
+    pip install -U snowflake-connector-python
+
+# aws
+RUN pip install -U awscli
+COPY scripts/aws.sh /usr/local/bin/aws
+RUN chmod a+x /usr/local/bin/aws
+
+# Development tools + git + zstd + jq + gosu
+RUN yum -y groupinstall "Development Tools" && \
+    yum -y install zlib-devel && \
+    curl -o - https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar xfz - && \
+    cd git-2.26.0 && \
+    ./configure --prefix=/opt/git && make && make install && \
+    ln -s /opt/git/bin/git /usr/local/bin/git && \
+    yum -y install zstd && \
+    yum -y install jq && \
+    curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.11/gosu-amd64" && \
+    chmod +x /usr/local/bin/gosu
+
+# workspace
+RUN mkdir -p /home/user && \
+    chmod 777 /home/user
+WORKDIR /home/user
+
+# entry point
+COPY scripts/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/lib/connection/connection_config.js b/lib/connection/connection_config.js
@@ -11,6 +11,7 @@ const NativeTypes = require('./result/data_types').NativeTypes;
 const GlobalConfig = require('../global_config');
 const authenticationTypes = require('../authentication/authentication').authenticationTypes;
 const stringSimilarity = require("string-similarity");
+const RowMode = require('./../constants/row_mode');
 const WAIT_FOR_BROWSER_ACTION_TIMEOUT = 120000;
 const DEFAULT_PARAMS =
 [
@@ -33,6 +34,7 @@ const DEFAULT_PARAMS =
   'database',
   'schema',
   'role',
+  'rowMode',
   'streamResult',
   'fetchAsString',
   'clientSessionKeepAlive',
@@ -145,9 +147,10 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo)
   // only validate credentials if necessary
   if (validateCredentials)
   {
-    // username is not required for oauth authenticator
+    // username is not required for oauth and external browser authenticators
     if (!Util.exists(options.authenticator) ||
-      options.authenticator !== authenticationTypes.OAUTH_AUTHENTICATOR) 
+      (options.authenticator !== authenticationTypes.OAUTH_AUTHENTICATOR &&
+      options.authenticator !== authenticationTypes.EXTERNAL_BROWSER_AUTHENTICATOR))
     {
       // check for missing username
       Errors.checkArgumentExists(Util.exists(options.username),
@@ -362,6 +365,11 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo)
       ErrorCodes.ERR_CONN_CREATE_INVALID_FETCH_AS_STRING_VALUES,
       JSON.stringify(fetchAsString[invalidValueIndex]));
   }
+  // Row mode is optional, can be undefined
+  const rowMode = options.rowMode;
+  if (Util.exists(rowMode)) {
+    RowMode.checkRowModeValid(rowMode);
+  }
 
   // check for invalid clientSessionKeepAlive
   var clientSessionKeepAlive = options.clientSessionKeepAlive;
@@ -620,6 +628,15 @@ function ConnectionConfig(options, validateCredentials, qaMode, clientInfo)
     return fetchAsString;
   };
 
+  /**
+   * Returns the rowMode string value ('array', 'object' or 'object_with_renamed_duplicated_columns'). Could be null or undefined.
+   *
+   * @returns  {String}
+   */
+  this.getRowMode = function () {
+    return rowMode ;
+  };
+
   /**
    * Returns the client type.
    *

diff --git a/lib/connection/result/column.js b/lib/connection/result/column.js
@@ -25,7 +25,7 @@ var NULL_UPPERCASE = 'NULL';
  */
 function Column(options, index, statementParameters, resultVersion)
 {
-  var name = options.name;
+  var name = options.overriddenName || options.name;
   var nullable = options.nullable;
   var scale = options.scale;
   var type = options.type;

diff --git a/lib/connection/result/result.js b/lib/connection/result/result.js
@@ -9,17 +9,17 @@ var Chunk = require('./chunk');
 var ResultStream = require('./result_stream');
 var ChunkCache = require('./chunk_cache');
 var Column = require('./column');
-var Parameters = require('../../parameters');
 var StatementType = require('./statement_type');
+const ColumnNamesCreator = require('./unique_column_name_creator');
+const RowMode = require('../../constants/row_mode');
 
 /**
  * Creates a new Result.
  *
  * @param {Object} options
  * @constructor
  */
-function Result(options)
-{
+function Result(options) {
   var data;
   var chunkHeaders;
   var parametersMap;
@@ -56,8 +56,7 @@ function Result(options)
   // if no chunk headers were specified, but a query-result-master-key (qrmk)
   // was specified, build the chunk headers from the qrmk
   chunkHeaders = data.chunkHeaders;
-  if (!Util.isObject(chunkHeaders) && Util.isString(data.qrmk))
-  {
+  if (!Util.isObject(chunkHeaders) && Util.isString(data.qrmk)) {
     chunkHeaders =
       {
         'x-amz-server-side-encryption-customer-algorithm': 'AES256',
@@ -74,8 +73,7 @@ function Result(options)
   // convert the parameters array to a map
   parametersMap = {};
   parametersArray = data.parameters;
-  for (index = 0, length = parametersArray.length; index < length; index++)
-  {
+  for (index = 0, length = parametersArray.length; index < length; index++) {
     parameter = parametersArray[index];
     parametersMap[parameter.name] = parameter.value;
   }
@@ -95,8 +93,14 @@ function Result(options)
   // index map in which the keys are the column names and the values are the
   // indices of the columns with the corresponding names
   this._mapColumnNameToIndices = mapColumnNameToIndices = {};
-  for (index = 0; index < numColumns; index++)
-  {
+
+  const rowMode = options.rowMode;
+  if (rowMode === RowMode.OBJECT_WITH_RENAMED_DUPLICATED_COLUMNS) {
+    ColumnNamesCreator.addOverridenNamesForDuplicatedColumns(rowtype);
+  }
+
+  for (let index = 0; index < numColumns; index++) {
+
     // create a new column and add it to the columns array
     columns[index] = column =
       new Column(rowtype[index], index, parametersMap, version);