Merge branch 'upmaster' into SPARK-36406

Author: LuciferYang

.github/workflows/benchmark.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Run benchmarks
 
 on:

.github/workflows/build_and_test.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Build and test
 
 on:
@@ -686,7 +705,7 @@ jobs:
       HIVE_PROFILE: hive2.3
       GITHUB_PREV_SHA: ${{ github.event.before }}
       SPARK_LOCAL_IP: localhost
-      ORACLE_DOCKER_IMAGE_NAME: oracle/database:18.4.0-xe
+      ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:18.4.0
       SKIP_MIMA: true
     steps:
     - name: Checkout Spark repository
@@ -724,24 +743,6 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 8
-    - name: Cache Oracle docker-images repository
-      id: cache-oracle-docker-images
-      uses: actions/cache@v2
-      with:
-        path: ./oracle/docker-images
-        # key should contains the commit hash of the Oracle docker images to be checkout.
-        key: oracle-docker-images-3f422c4a35b423dfcdbcc57a84f01db6c82eb6c1
-    - name: Checkout Oracle docker-images repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        repository: oracle/docker-images
-        ref: 3f422c4a35b423dfcdbcc57a84f01db6c82eb6c1
-        path: ./oracle/docker-images
-    - name: Install Oracle Docker image
-      run: |
-        cd oracle/docker-images/OracleDatabase/SingleInstance/dockerfiles
-        ./buildContainerImage.sh -v 18.4.0 -x
     - name: Run tests
       run: |
         ./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest

.github/workflows/cancel_duplicate_workflow_runs.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Cancelling Duplicates
 on:
   workflow_run:

.github/workflows/publish_snapshot.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Publish Snapshot
 
 on:

.github/workflows/stale.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Close stale PRs
 
 on:

.github/workflows/test_report.yml

@@ -1,3 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
 name: Report test results
 on:
   workflow_run:

R/pkg/R/sparkR.R

@@ -655,6 +655,40 @@ sparkCheckInstall <- function(sparkHome, master, deployMode) {
     } else {
       if (interactive() || isMasterLocal(master)) {
         message("Spark not found in SPARK_HOME: ", sparkHome)
+        # If EXISTING_SPARKR_BACKEND_PORT environment variable is set, assume
+        # that we're in Spark submit. spark-submit always sets Spark home
+        # so this case should not happen. This is just a safeguard.
+        isSparkRSubmit <- Sys.getenv("EXISTING_SPARKR_BACKEND_PORT", "") != ""
+
+        # SPARKR_ASK_INSTALLATION is an internal environment variable in case
+        # users want to disable this behavior. This environment variable should
+        # be removed if no user complains. This environment variable was added
+        # in case other notebook projects are affected.
+        if (!isSparkRSubmit && Sys.getenv("SPARKR_ASK_INSTALLATION", "TRUE") == "TRUE") {
+          # Finally, we're either plain R shell or Rscript.
+          msg <- paste0(
+            "Will you download and install (or reuse if it exists) Spark package ",
+            "under the cache [", sparkCachePath(), "]? (y/n): ")
+
+          answer <- NA
+          while (is.na(answer) || (answer != "y" && answer != "n")) {
+            # Dispatch on R shell in case readLines does not work in RStudio
+            # See https://stackoverflow.com/questions/30191232/use-stdin-from-within-r-studio
+            if (interactive()) {
+              answer <- readline(prompt = msg)
+            } else {
+              cat(msg)
+              answer <- readLines("stdin", n = 1)
+            }
+          }
+          if (answer == "n") {
+            stop(paste0(
+             "Please make sure Spark package is installed in this machine.\n",
+             "  - If there is one, set the path in sparkHome parameter or ",
+             "environment variable SPARK_HOME.\n",
+             "  - If not, you may run install.spark function to do the job."))
+          }
+        }
         packageLocalDir <- install.spark()
         packageLocalDir
       } else if (isClientMode(master) || deployMode == "client") {

common/kvstore/src/main/java/org/apache/spark/util/kvstore/LevelDB.java

@@ -19,7 +19,8 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.ref.SoftReference;
+import java.lang.ref.Reference;
+import java.lang.ref.WeakReference;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -69,10 +70,10 @@ public class LevelDB implements KVStore {
 
   /**
    * Trying to close a JNI LevelDB handle with a closed DB causes JVM crashes. This is used to
-   * ensure that all iterators are correctly closed before LevelDB is closed. Use soft reference
+   * ensure that all iterators are correctly closed before LevelDB is closed. Use weak references
    * to ensure that the iterator can be GCed, when it is only referenced here.
    */
-  private final ConcurrentLinkedQueue<SoftReference<LevelDBIterator<?>>> iteratorTracker;
+  private final ConcurrentLinkedQueue<Reference<LevelDBIterator<?>>> iteratorTracker;
 
   public LevelDB(File path) throws Exception {
     this(path, new KVStoreSerializer());
@@ -250,7 +251,7 @@ public <T> KVStoreView<T> view(Class<T> type) throws Exception {
       public Iterator<T> iterator() {
         try {
           LevelDBIterator<T> it = new LevelDBIterator<>(type, LevelDB.this, this);
-          iteratorTracker.add(new SoftReference<>(it));
+          iteratorTracker.add(new WeakReference<>(it));
           return it;
         } catch (Exception e) {
           throw Throwables.propagate(e);
@@ -301,7 +302,7 @@ public void close() throws IOException {
 
       try {
         if (iteratorTracker != null) {
-          for (SoftReference<LevelDBIterator<?>> ref: iteratorTracker) {
+          for (Reference<LevelDBIterator<?>> ref: iteratorTracker) {
             LevelDBIterator<?> it = ref.get();
             if (it != null) {
               it.close();

core/src/main/resources/error/error-classes.json

@@ -3,6 +3,9 @@
     "message" : [ "Field name %s is ambiguous and has %s matching fields in the struct." ],
     "sqlState" : "42000"
   },
+  "CONCURRENT_QUERY_ERROR" : {
+    "message" : [ "Another instance of this query was just started by a concurrent session." ]
+  },
   "DIVIDE_BY_ZERO" : {
     "message" : [ "divide by zero" ],
     "sqlState" : "22012"
@@ -11,6 +14,13 @@
     "message" : [ "Found duplicate keys '%s'" ],
     "sqlState" : "23000"
   },
+  "FAILED_RENAME_PATH" : {
+    "message" : [ "Failed to rename %s to %s as destination already exists" ],
+    "sqlState" : "22023"
+  },
+  "FAILED_SET_ORIGINAL_PERMISSION_BACK" : {
+    "message" : [ "Failed to set original permission %s back to the created path: %s. Exception: %s" ]
+  },
   "GROUPING_COLUMN_MISMATCH" : {
     "message" : [ "Column of grouping (%s) can't be found in grouping columns %s" ],
     "sqlState" : "42000"
@@ -29,17 +39,32 @@
     "message" : [ "Invalid pivot column '%s'. Pivot columns must be comparable." ],
     "sqlState" : "42000"
   },
+  "INCOMPATIBLE_DATASOURCE_REGISTER" : {
+    "message" : [ "Detected an incompatible DataSourceRegister. Please remove the incompatible library from classpath or upgrade it. Error: %s" ]
+  },
+  "INDEX_OUT_OF_BOUNDS" : {
+    "message" : [ "Index %s must be between 0 and the length of the ArrayData." ],
+    "sqlState" : "22023"
+  },
   "INVALID_FIELD_NAME" : {
     "message" : [ "Field name %s is invalid: %s is not a struct." ],
     "sqlState" : "42000"
   },
+  "INVALID_FRACTION_OF_SECOND" : {
+    "message" : [ "The fraction of sec must be zero. Valid range is [0, 60]." ],
+    "sqlState" : "22023"
+  },
   "INVALID_JSON_SCHEMA_MAPTYPE" : {
     "message" : [ "Input schema %s can only contain StringType as a key type for a MapType." ]
   },
   "MISSING_COLUMN" : {
     "message" : [ "cannot resolve '%s' given input columns: [%s]" ],
     "sqlState" : "42000"
   },
+  "MISSING_METHOD" : {
+    "message" : [ "A method named \"%s\" is not declared in any enclosing class nor any supertype" ],
+    "sqlState" : "42000"
+  },
   "MISSING_STATIC_PARTITION_COLUMN" : {
     "message" : [ "Unknown static partition column: %s" ],
     "sqlState" : "42000"
@@ -56,13 +81,29 @@
     "message" : [ "Invalid pivot value '%s': value data type %s does not match pivot column data type %s" ],
     "sqlState" : "42000"
   },
+  "RENAME_SRC_PATH_NOT_FOUND" : {
+    "message" : [ "Failed to rename as %s was not found" ],
+    "sqlState" : "22023"
+  },
   "SECOND_FUNCTION_ARGUMENT_NOT_INTEGER" : {
     "message" : [ "The second argument of '%s' function needs to be an integer." ],
     "sqlState" : "22023"
   },
   "UNABLE_TO_ACQUIRE_MEMORY" : {
     "message" : [ "Unable to acquire %s bytes of memory, got %s" ]
   },
+  "UNRECOGNIZED_SQL_TYPE" : {
+    "message" : [ "Unrecognized SQL type %s" ],
+    "sqlState" : "42000"
+  },
+  "UNSUPPORTED_LITERAL_TYPE" : {
+    "message" : [ "Unsupported literal type %s %s" ],
+    "sqlState" : "0A000"
+  },
+  "UNSUPPORTED_TRANSACTION_BY_JDBC_SERVER" : {
+    "message" : [ "The target JDBC server does not support transaction and can only support ALTER TABLE with a single action." ],
+    "sqlState" : "0A000"
+  },
   "WRITING_JOB_ABORTED" : {
     "message" : [ "Writing job aborted" ],
     "sqlState" : "40000"