From 5d854a6f7014885b375a20ad5bf5d1c92b742561 Mon Sep 17 00:00:00 2001 From: Muhammet Orazov Date: Wed, 6 Apr 2022 13:38:59 +0200 Subject: [PATCH] #195: Fixed bug with Azure Data Lake Gen2 missing class (#204) Fixed #195 --- .github/workflows/ci-build.yml | 2 + dependencies.md | 186 +++++++++--------- doc/changes/changes_2.3.1.md | 18 +- pom.xml | 29 ++- .../cloudetl/it/BaseS3IntegrationTest.scala | 3 +- 5 files changed, 130 insertions(+), 108 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index ee0c28c2..8f09c590 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -39,6 +39,8 @@ jobs: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar + - name: Enable Testcontainers Reuse + run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - name: Run scalafix linting run: mvn --batch-mode clean compile test-compile scalafix:scalafix - name: Run tests and build with Maven diff --git a/dependencies.md b/dependencies.md index 0c558617..9f9c09eb 100644 --- a/dependencies.md +++ b/dependencies.md @@ -20,133 +20,135 @@ | Apache Hadoop Azure Data Lake support | [Apache License, Version 2.0][3] | | Apache Hadoop HDFS | [Apache License, Version 2.0][3] | | [Alluxio Core - Client - HDFS][23] | [Apache License][24] | -| [Protocol Buffers [Core]][25] | [3-Clause BSD License][26] | +| [Protocol Buffers [Core]][25] | [BSD-3-Clause][26] | | [gcs-connector-hadoop3][27] | [Apache License, Version 2.0][5] | | [ORC Core][29] | [Apache License, Version 2.0][3] | | [Apache Avro][31] | [Apache License, Version 2.0][3] | | [delta-core][33] | [Apache-2.0][34] | -| [Spark Project SQL][35] | [Apache 2.0 License][36] | -| [Parquet for Java][37] | [MIT][15] | -| [JUL to SLF4J bridge][39] | [MIT License][40] | -| [SLF4J LOG4J-12 Binding relocated][39] | [MIT License][40] | -| [Apache Log4j API][43] | [Apache License, Version 2.0][3] | -| [Apache Log4j 1.x Compatibility API][45] | [Apache License, Version 2.0][3] | -| [scala-logging][47] | [Apache 2.0 License][36] | +| [Data Mapper for Jackson][35] | [The Apache Software License, Version 2.0][5] | +| [Spark Project SQL][37] | [Apache 2.0 License][38] | +| [Parquet for Java][39] | [MIT][15] | +| [JUL to SLF4J bridge][41] | [MIT License][42] | +| [SLF4J LOG4J-12 Binding relocated][41] | [MIT License][42] | +| [Apache Log4j API][45] | [Apache License, Version 2.0][3] | +| [Apache Log4j 1.x Compatibility API][47] | [Apache License, Version 2.0][3] | +| [scala-logging][49] | [Apache 2.0 License][38] | ## Test Dependencies | Dependency | License | | ------------------------------------------ | ----------------------------------------- | -| [scalatest][49] | [the Apache License, ASL Version 2.0][34] | -| [scalatestplus-mockito][51] | [Apache-2.0][34] | -| [mockito-core][53] | [The MIT License][54] | -| [Hamcrest][55] | [BSD License 3][56] | -| [testcontainers-scala-scalatest][57] | [The MIT License (MIT)][15] | -| [Testcontainers :: Localstack][59] | [MIT][60] | -| [Test containers for Exasol on Docker][61] | [MIT][15] | -| [Test Database Builder for Java][63] | [MIT][15] | -| [Matcher for SQL Result Sets][65] | [MIT][15] | +| [scalatest][51] | [the Apache License, ASL Version 2.0][34] | +| [scalatestplus-mockito][53] | [Apache-2.0][34] | +| [mockito-core][55] | [The MIT License][56] | +| [Hamcrest][57] | [BSD License 3][58] | +| [testcontainers-scala-scalatest][59] | [The MIT License (MIT)][15] | +| [Testcontainers :: Localstack][61] | [MIT][62] | +| [Test containers for Exasol on Docker][63] | [MIT][15] | +| [Test Database Builder for Java][65] | [MIT][15] | +| [Matcher for SQL Result Sets][67] | [MIT][15] | ## Plugin Dependencies | Dependency | License | | ------------------------------------------------------- | --------------------------------------------- | -| [scala-maven-plugin][67] | [Public domain (Unlicense)][68] | -| [Apache Maven Compiler Plugin][69] | [Apache License, Version 2.0][3] | -| [ScalaTest Maven Plugin][71] | [the Apache License, ASL Version 2.0][34] | -| [Apache Maven Enforcer Plugin][73] | [Apache License, Version 2.0][3] | -| [Apache Maven GPG Plugin][75] | [Apache License, Version 2.0][3] | -| [Apache Maven Source Plugin][77] | [Apache License, Version 2.0][3] | -| [Apache Maven Javadoc Plugin][79] | [Apache License, Version 2.0][3] | -| [Apache Maven JAR Plugin][81] | [Apache License, Version 2.0][3] | -| [Apache Maven Assembly Plugin][83] | [Apache License, Version 2.0][3] | -| [Versions Maven Plugin][85] | [Apache License, Version 2.0][3] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][87] | [ASL2][5] | -| [Reproducible Build Maven Plugin][89] | [Apache 2.0][5] | -| [Project keeper maven plugin][91] | [MIT][15] | -| [JaCoCo :: Maven Plugin][93] | [Eclipse Public License 2.0][94] | -| [error-code-crawler-maven-plugin][95] | [MIT][15] | -| [Artifact reference checker and unifier][97] | [MIT][15] | -| [OpenFastTrace Maven Plugin][99] | [GNU General Public License v3.0][100] | -| [Scalastyle Maven Plugin][101] | [Apache 2.0][36] | -| [spotless-maven-plugin][103] | [The Apache Software License, Version 2.0][3] | -| [scalafix-maven-plugin][105] | [BSD-3-Clause][26] | -| [Apache Maven Clean Plugin][107] | [Apache License, Version 2.0][3] | -| [Apache Maven Resources Plugin][109] | [Apache License, Version 2.0][3] | -| [Maven Surefire Plugin][111] | [The Apache Software License, Version 2.0][5] | -| [Apache Maven Install Plugin][113] | [Apache License, Version 2.0][5] | -| [Maven Deploy Plugin][115] | [The Apache Software License, Version 2.0][5] | -| [Apache Maven Site Plugin][117] | [Apache License, Version 2.0][3] | +| [scala-maven-plugin][69] | [Public domain (Unlicense)][70] | +| [Apache Maven Compiler Plugin][71] | [Apache License, Version 2.0][3] | +| [ScalaTest Maven Plugin][73] | [the Apache License, ASL Version 2.0][34] | +| [Apache Maven Enforcer Plugin][75] | [Apache License, Version 2.0][3] | +| [Apache Maven GPG Plugin][77] | [Apache License, Version 2.0][3] | +| [Apache Maven Source Plugin][79] | [Apache License, Version 2.0][3] | +| [Apache Maven Javadoc Plugin][81] | [Apache License, Version 2.0][3] | +| [Apache Maven JAR Plugin][83] | [Apache License, Version 2.0][3] | +| [Apache Maven Assembly Plugin][85] | [Apache License, Version 2.0][3] | +| [Versions Maven Plugin][87] | [Apache License, Version 2.0][3] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][89] | [ASL2][5] | +| [Reproducible Build Maven Plugin][91] | [Apache 2.0][5] | +| [Project keeper maven plugin][93] | [MIT][15] | +| [JaCoCo :: Maven Plugin][95] | [Eclipse Public License 2.0][96] | +| [error-code-crawler-maven-plugin][97] | [MIT][15] | +| [Artifact reference checker and unifier][99] | [MIT][15] | +| [OpenFastTrace Maven Plugin][101] | [GNU General Public License v3.0][102] | +| [Scalastyle Maven Plugin][103] | [Apache 2.0][38] | +| [spotless-maven-plugin][105] | [The Apache Software License, Version 2.0][3] | +| [scalafix-maven-plugin][107] | [BSD-3-Clause][26] | +| [Apache Maven Clean Plugin][109] | [Apache License, Version 2.0][3] | +| [Apache Maven Resources Plugin][111] | [Apache License, Version 2.0][3] | +| [Maven Surefire Plugin][113] | [The Apache Software License, Version 2.0][5] | +| [Apache Maven Install Plugin][115] | [Apache License, Version 2.0][5] | +| [Maven Deploy Plugin][117] | [The Apache Software License, Version 2.0][5] | +| [Apache Maven Site Plugin][119] | [Apache License, Version 2.0][3] | -[91]: https://github.com/exasol/project-keeper-maven-plugin -[51]: https://github.com/scalatest/scalatestplus-mockito +[93]: https://github.com/exasol/project-keeper-maven-plugin +[53]: https://github.com/scalatest/scalatestplus-mockito [16]: https://github.com/exasol/error-reporting-java [8]: http://wiki.fasterxml.com/JacksonModuleScala -[37]: https://github.com/exasol/parquet-io-java +[39]: https://github.com/exasol/parquet-io-java [5]: http://www.apache.org/licenses/LICENSE-2.0.txt -[101]: http://www.scalastyle.org -[103]: https://github.com/diffplug/spotless +[103]: http://www.scalastyle.org +[105]: https://github.com/diffplug/spotless [12]: https://netty.io/netty-all/ [14]: https://github.com/exasol/import-export-udf-common-scala [15]: https://opensource.org/licenses/MIT -[53]: https://github.com/mockito/mockito +[55]: https://github.com/mockito/mockito [27]: https://github.com/GoogleCloudPlatform/BigData-interop/gcs-connector/ [11]: https://opensource.org/licenses/Apache-2.0 [33]: https://delta.io/ -[85]: http://www.mojohaus.org/versions-maven-plugin/ -[56]: http://opensource.org/licenses/BSD-3-Clause -[69]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[109]: https://maven.apache.org/plugins/maven-resources-plugin/ -[99]: https://github.com/itsallcode/openfasttrace-maven-plugin -[107]: https://maven.apache.org/plugins/maven-clean-plugin/ -[94]: https://www.eclipse.org/legal/epl-2.0/ +[87]: http://www.mojohaus.org/versions-maven-plugin/ +[58]: http://opensource.org/licenses/BSD-3-Clause +[71]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[111]: https://maven.apache.org/plugins/maven-resources-plugin/ +[101]: https://github.com/itsallcode/openfasttrace-maven-plugin +[109]: https://maven.apache.org/plugins/maven-clean-plugin/ +[96]: https://www.eclipse.org/legal/epl-2.0/ [6]: http://github.com/FasterXML/jackson -[68]: http://unlicense.org/ +[70]: http://unlicense.org/ [1]: https://www.apache.org/licenses/LICENSE-2.0 -[93]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[54]: https://github.com/mockito/mockito/blob/main/LICENSE -[65]: https://github.com/exasol/hamcrest-resultset-matcher -[89]: http://zlika.github.io/reproducible-build-maven-plugin -[40]: http://www.opensource.org/licenses/mit-license.php -[111]: http://maven.apache.org/surefire/maven-surefire-plugin +[95]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[56]: https://github.com/mockito/mockito/blob/main/LICENSE +[67]: https://github.com/exasol/hamcrest-resultset-matcher +[91]: http://zlika.github.io/reproducible-build-maven-plugin +[42]: http://www.opensource.org/licenses/mit-license.php +[113]: http://maven.apache.org/surefire/maven-surefire-plugin [26]: https://opensource.org/licenses/BSD-3-Clause [23]: https://www.alluxio.io/alluxio-core/alluxio-core-client/alluxio-core-client-hdfs/ [24]: https://github.com/alluxio/alluxio/blob/master/LICENSE [31]: https://avro.apache.org -[47]: https://github.com/lightbend/scala-logging -[77]: https://maven.apache.org/plugins/maven-source-plugin/ +[35]: http://jackson.codehaus.org +[49]: https://github.com/lightbend/scala-logging +[79]: https://maven.apache.org/plugins/maven-source-plugin/ [10]: https://github.com/grpc/grpc-java [29]: https://orc.apache.org/orc-core -[55]: http://hamcrest.org/JavaHamcrest/ -[39]: http://www.slf4j.org -[97]: https://github.com/exasol/artifact-reference-checker-maven-plugin -[105]: https://github.com/evis/scalafix-maven-plugin -[81]: https://maven.apache.org/plugins/maven-jar-plugin/ +[57]: http://hamcrest.org/JavaHamcrest/ +[41]: http://www.slf4j.org +[99]: https://github.com/exasol/artifact-reference-checker-maven-plugin +[107]: https://github.com/evis/scalafix-maven-plugin +[83]: https://maven.apache.org/plugins/maven-jar-plugin/ [34]: http://www.apache.org/licenses/LICENSE-2.0 -[57]: https://github.com/testcontainers/testcontainers-scala +[59]: https://github.com/testcontainers/testcontainers-scala [4]: https://github.com/google/guava -[36]: http://www.apache.org/licenses/LICENSE-2.0.html -[49]: http://www.scalatest.org -[43]: https://logging.apache.org/log4j/2.x/log4j-api/ -[45]: https://logging.apache.org/log4j/2.x/log4j-1.2-api/ -[63]: https://github.com/exasol/test-db-builder-java +[38]: http://www.apache.org/licenses/LICENSE-2.0.html +[51]: http://www.scalatest.org +[45]: https://logging.apache.org/log4j/2.x/log4j-api/ +[47]: https://logging.apache.org/log4j/2.x/log4j-1.2-api/ +[65]: https://github.com/exasol/test-db-builder-java [2]: https://commons.apache.org/proper/commons-lang/ -[60]: http://opensource.org/licenses/MIT +[62]: http://opensource.org/licenses/MIT [0]: https://www.scala-lang.org/ -[61]: https://github.com/exasol/exasol-testcontainers -[117]: https://maven.apache.org/plugins/maven-site-plugin/ -[100]: https://www.gnu.org/licenses/gpl-3.0.html +[63]: https://github.com/exasol/exasol-testcontainers +[119]: https://maven.apache.org/plugins/maven-site-plugin/ +[102]: https://www.gnu.org/licenses/gpl-3.0.html [3]: https://www.apache.org/licenses/LICENSE-2.0.txt -[71]: http://nexus.sonatype.org/oss-repository-hosting.html/scalatest-maven-plugin -[73]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[113]: http://maven.apache.org/plugins/maven-install-plugin/ -[87]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[75]: https://maven.apache.org/plugins/maven-gpg-plugin/ -[59]: https://testcontainers.org -[67]: http://github.com/davidB/scala-maven-plugin -[35]: http://spark.apache.org/ -[115]: http://maven.apache.org/plugins/maven-deploy-plugin/ +[73]: http://nexus.sonatype.org/oss-repository-hosting.html/scalatest-maven-plugin +[75]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[115]: http://maven.apache.org/plugins/maven-install-plugin/ +[89]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[77]: https://maven.apache.org/plugins/maven-gpg-plugin/ +[61]: https://testcontainers.org +[69]: http://github.com/davidB/scala-maven-plugin +[37]: http://spark.apache.org/ +[117]: http://maven.apache.org/plugins/maven-deploy-plugin/ [25]: https://developers.google.com/protocol-buffers/protobuf-java/ -[79]: https://maven.apache.org/plugins/maven-javadoc-plugin/ -[95]: https://github.com/exasol/error-code-crawler-maven-plugin -[83]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[81]: https://maven.apache.org/plugins/maven-javadoc-plugin/ +[97]: https://github.com/exasol/error-code-crawler-maven-plugin +[85]: https://maven.apache.org/plugins/maven-assembly-plugin/ diff --git a/doc/changes/changes_2.3.1.md b/doc/changes/changes_2.3.1.md index f4c231dc..c1ecd375 100644 --- a/doc/changes/changes_2.3.1.md +++ b/doc/changes/changes_2.3.1.md @@ -1,6 +1,6 @@ -# Cloud Storage Extension 2.3.1, released 2022-04-?? +# Cloud Storage Extension 2.3.1, released 2022-04-06 -Code name: +Code name: 2.3.1: String Truncation and Bug Fixing ## Summary @@ -15,6 +15,7 @@ In this release we added a new parameter `TIMEZONE_UTC`, when set to `true`, the * #190: Added S3 bucket name validation * #193: Fixed bug reading delta formatted data * #194: Added timezone `UTC` parameter +* #195: Fixed bug with missing class reading from Azure Data Lake Gen2 storage ## Refactoring @@ -31,12 +32,12 @@ In this release we added a new parameter `TIMEZONE_UTC`, when set to `true`, the * Added `com.fasterxml.jackson.module:jackson-module-scala_2.13:2.13.2` * Added `com.google.cloud.bigdataoss:gcs-connector:1.9.4-hadoop3` * Added `com.google.guava:guava:31.1-jre` -* Added `com.google.protobuf:protobuf-java:3.19.4` +* Added `com.google.protobuf:protobuf-java:3.20.0` * Added `com.typesafe.scala-logging:scala-logging_2.13:3.9.4` * Added `io.delta:delta-core_2.13:1.1.0` * Added `io.grpc:grpc-netty:1.45.1` * Added `io.netty:netty-all:4.1.75.Final` -* Added `org.alluxio:alluxio-core-client-hdfs:2.7.3` +* Added `org.alluxio:alluxio-core-client-hdfs:2.7.4` * Added `org.apache.avro:avro:1.11.0` * Added `org.apache.commons:commons-lang3:3.12.0` * Added `org.apache.hadoop:hadoop-aws:3.3.2` @@ -48,13 +49,14 @@ In this release we added a new parameter `TIMEZONE_UTC`, when set to `true`, the * Added `org.apache.logging.log4j:log4j-api:2.17.2` * Added `org.apache.orc:orc-core:1.7.3` * Added `org.apache.spark:spark-sql_2.13:3.2.1` +* Added `org.codehaus.jackson:jackson-mapper-asl:1.9.13` * Added `org.scala-lang:scala-library:2.13.8` * Added `org.slf4j:jul-to-slf4j:1.7.36` * Added `org.slf4j:slf4j-log4j12:1.7.36` ### Test Dependency Updates -* Added `com.dimafeng:testcontainers-scala-scalatest_2.13:0.40.4` +* Added `com.dimafeng:testcontainers-scala-scalatest_2.13:0.40.5` * Added `com.exasol:exasol-testcontainers:6.1.1` * Added `com.exasol:hamcrest-resultset-matcher:1.5.1` * Added `com.exasol:test-db-builder-java:3.3.1` @@ -68,13 +70,13 @@ In this release we added a new parameter `TIMEZONE_UTC`, when set to `true`, the * Added `com.diffplug.spotless:spotless-maven-plugin:2.22.0` * Added `com.exasol:artifact-reference-checker-maven-plugin:0.4.1` -* Added `com.exasol:error-code-crawler-maven-plugin:1.1.0` +* Added `com.exasol:error-code-crawler-maven-plugin:1.1.1` * Added `com.exasol:project-keeper-maven-plugin:1.3.4` * Added `io.github.evis:scalafix-maven-plugin_2.13:0.1.4_0.9.33` * Added `io.github.zlika:reproducible-build-maven-plugin:0.15` * Added `net.alchim31.maven:scala-maven-plugin:4.6.1` * Added `org.apache.maven.plugins:maven-assembly-plugin:3.3.0` -* Added `org.apache.maven.plugins:maven-clean-plugin:3.1.0` +* Added `org.apache.maven.plugins:maven-clean-plugin:3.2.0` * Added `org.apache.maven.plugins:maven-compiler-plugin:3.10.1` * Added `org.apache.maven.plugins:maven-deploy-plugin:2.7` * Added `org.apache.maven.plugins:maven-enforcer-plugin:3.0.0` @@ -88,7 +90,7 @@ In this release we added a new parameter `TIMEZONE_UTC`, when set to `true`, the * Added `org.apache.maven.plugins:maven-surefire-plugin:2.12.4` * Added `org.codehaus.mojo:versions-maven-plugin:2.10.0` * Added `org.itsallcode:openfasttrace-maven-plugin:1.5.0` -* Added `org.jacoco:jacoco-maven-plugin:0.8.7` +* Added `org.jacoco:jacoco-maven-plugin:0.8.8` * Added `org.scalastyle:scalastyle-maven-plugin:1.0.0` * Added `org.scalatest:scalatest-maven-plugin:2.0.2` * Added `org.sonatype.ossindex.maven:ossindex-maven-plugin:3.2.0` diff --git a/pom.xml b/pom.xml index c57076a0..ee327731 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ org.alluxio alluxio-core-client-hdfs - 2.7.3 + 2.7.4 log4j @@ -242,7 +242,7 @@ com.google.protobuf protobuf-java - 3.19.4 + 3.20.0 com.google.cloud.bigdataoss @@ -264,6 +264,11 @@ delta-core_${scala.compat.version} 1.1.0 + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + org.apache.spark spark-sql_${scala.compat.version} @@ -356,7 +361,7 @@ com.dimafeng testcontainers-scala-scalatest_${scala.compat.version} - 0.40.4 + 0.40.5 test @@ -640,6 +645,18 @@ 3.2.0 ${ossindex.skip} + + + 3f596fc0-9615-4b93-b30a-d4e0532e667f + af22d349-929c-41ae-ba6c-2f7b507026f0 + 1205a1ec-0837-406f-b081-623b9fb02992 + b85a00e3-7d9b-49cf-9b19-b73f8ee60275 + 5113110f-3321-491d-9506-447a3361f9cd + fc1e8802-77e5-458f-b987-eb778c6ac2fc + 3242fdc1-bfe9-46a6-af0c-0b8f57f56eb7 + 6ce886d0-2dfd-4cef-b9a4-2fb400baf5ef + 292c11e9-cf66-4d76-aaf7-b63a091f8891 + @@ -693,7 +710,7 @@ org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.8 prepare-agent @@ -740,7 +757,7 @@ com.exasol error-code-crawler-maven-plugin - 1.1.0 + 1.1.1 @@ -846,7 +863,7 @@ maven-clean-plugin - 3.1.0 + 3.2.0 maven-install-plugin diff --git a/src/test/scala/com/exasol/cloudetl/it/BaseS3IntegrationTest.scala b/src/test/scala/com/exasol/cloudetl/it/BaseS3IntegrationTest.scala index 3c9bfb6b..84b8bde0 100644 --- a/src/test/scala/com/exasol/cloudetl/it/BaseS3IntegrationTest.scala +++ b/src/test/scala/com/exasol/cloudetl/it/BaseS3IntegrationTest.scala @@ -19,8 +19,7 @@ trait BaseS3IntegrationTest extends BaseIntegrationTest { val LONG_MIN = Long.MIN_VALUE val LONG_MAX = Long.MAX_VALUE - val LOCALSTACK_DOCKER_IMAGE = DockerImageName.parse("localstack/localstack:0.14") - val s3Container = new LocalStackContainer(LOCALSTACK_DOCKER_IMAGE) + val s3Container = new LocalStackContainer(DockerImageName.parse("localstack/localstack:0.14")) .withServices(S3) .withReuse(true) var s3: AmazonS3 = _