Skip to content
This repository has been archived by the owner on Jun 3, 2021. It is now read-only.

Excavator: Upgrades Baseline to the latest version #58

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 2 additions & 19 deletions .baseline/checkstyle/checkstyle-suppressions.xml
Original file line number Diff line number Diff line change
@@ -1,24 +1,7 @@
<?xml version="1.0"?>

<!--
~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<!DOCTYPE suppressions PUBLIC
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
"-//Checkstyle//DTD SuppressionFilter Configuration 1.2//EN"
"https://checkstyle.org/dtds/suppressions_1_2.dtd">

<!-- IMPORTANT ECLIPSE NOTE: If you change this file, you must restart Eclipse
for your changes to take effect in its Checkstyle integration. -->
Expand Down
47 changes: 18 additions & 29 deletions .baseline/checkstyle/checkstyle.xml
Original file line number Diff line number Diff line change
@@ -1,23 +1,7 @@
<?xml version="1.0"?>
<!--
~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
"-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
"https://checkstyle.org/dtds/configuration_1_3.dtd">

<!--
Palantir Baseline Checkstyle configuration.
Expand Down Expand Up @@ -69,6 +53,9 @@
<property name="optional" value="true"/>
</module>
<module name="SuppressWarningsFilter"/> <!-- baseline-gradle: README.md -->
<module name="BeforeExecutionExclusionFileFilter">
<property name="fileNamePattern" value="module\-info\.java$"/>
</module>
<module name="TreeWalker">
<module name="SuppressionCommentFilter"/> <!-- baseline-gradle: README.md -->
<module name="SuppressionCommentFilter">
Expand Down Expand Up @@ -100,11 +87,13 @@
<module name="AvoidStarImport"/> <!-- Java Style Guide: No wildcard imports -->
<module name="AvoidStaticImport"> <!-- Java Style Guide: No static imports -->
<property name="excludes" value="
com.google.common.base.Preconditions.*,
com.palantir.logsafe.Preconditions.*,
java.util.Collections.*,
java.util.stream.Collectors.*,
com.palantir.logsafe.Preconditions.*,
com.google.common.base.Preconditions.*,
org.apache.commons.lang3.Validate.*"/>
org.apache.commons.lang3.Validate.*,
org.assertj.core.api.Assertions.*,
org.mockito.Mockito.*"/>
</module>
<module name="ClassTypeParameterName"> <!-- Java Style Guide: Type variable names -->
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
Expand All @@ -126,7 +115,6 @@
<module name="EmptyStatement"/> <!-- Java Style Guide: One statement per line -->
<module name="EqualsHashCode"/>
<module name="FallThrough"/> <!-- Java Style Guide: Fall-through: commented -->
<module name="FinalClass"/> <!-- Java Coding Guidelines: Private constructors -->
<module name="GenericWhitespace"> <!-- Java Style Guide: Horizontal whitespace -->
<message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
<message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
Expand All @@ -143,6 +131,11 @@
<property name="illegalPkgs" value="junit.framework"/>
<message key="import.illegal" value="Use JUnit 4-style (org.junit.*) test classes and assertions instead of JUnit 3 (junit.framework.*)."/>
</module>
<module name="IllegalImport"> <!-- Only relevant for pre-Java 11 because javafx is gone completely in Java 11 -->
<property name="id" value="BanJavafx"/>
<property name="illegalPkgs" value="javafx"/>
<message key="import.illegal" value="Must not import javafx classes because some OpenJDK builds do not include javafx."/>
</module>
<module name="IllegalImport"> <!-- Java Coding Guidelines: Import the canonical package -->
<property name="illegalPkgs" value="org.elasticsearch.common.base, com.clearspring.analytics.util, org.spark_project.guava"/>
<message key="import.illegal" value="Must not import repackaged classes."/>
Expand Down Expand Up @@ -380,10 +373,6 @@
<property name="format" value="\bIOUtils\.toString\("/>
<property name="message" value="Prefer Guava''s [CharStreams,Files,Resources].toString to avoid charset/stream closing issues."/>
</module>
<module name="RegexpSinglelineJava">
<property name="format" value="static enum"/>
<property name="message" value="Redundant ''static'' modifier."/>
</module>
<module name="RegexpSinglelineJava">
<property name="format" value="\/\/TODO|\/\/ TODO(?!\([^()\s]+\): )"/>
<property name="message" value="TODO format: // TODO(#issue): explanation"/>
Expand Down Expand Up @@ -433,6 +422,7 @@
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyTypes" value="true"/>
<property name="allowEmptyLoops" value="true"/>
<property name="allowEmptyLambdas" value="true"/>
<property name="ignoreEnhancedForColon" value="false"/>
<message key="ws.notFollowed" value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
<message key="ws.notPreceded" value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
Expand All @@ -446,7 +436,7 @@
</module>
<module name="CyclomaticComplexity"/> <!-- Java Coding Guidelines: Reduce Cyclomatic Complexity -->
<module name="DesignForExtension"> <!-- Java Coding Guidelines: Design for extension -->
<property name="ignoredAnnotations" value="Test, Before, BeforeEach, After, AfterEach, BeforeClass, BeforeAll, AfterClass, AfterAll"/>
<property name="ignoredAnnotations" value="ParameterizedTest, Test, Before, BeforeEach, After, AfterEach, BeforeClass, BeforeAll, AfterClass, AfterAll"/>
</module>
<module name="JavadocMethod"> <!-- Java Style Guide: Where Javadoc is used -->
<property name="scope" value="public"/>
Expand Down Expand Up @@ -479,10 +469,9 @@
<module name="NestedTryDepth"/> <!-- Java Coding Guide: Try/catch blocks: never nested -->
<module name="NonEmptyAtclauseDescription"/> <!-- Java Style Guide: At-clauses -->
<module name="ParameterName"> <!-- Java Style Guide: Parameter names -->
<property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
<property name="format" value="^_?[a-z][a-zA-Z0-9]+$"/>
<message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="SingleLineJavadoc"/> <!-- Java Style Guide: General form -->
<module name="SummaryJavadocCheck"> <!-- Java Coding Guidelines: Javadoc -->
<property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
</module>
Expand Down
16 changes: 0 additions & 16 deletions .baseline/idea/intellij-java-palantir-style.xml
Original file line number Diff line number Diff line change
@@ -1,19 +1,3 @@
<!--
~ (c) Copyright 2019 Palantir Technologies Inc. All rights reserved.
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<project version="4">
<component name="ProjectCodeStyleSettingsManager">
<option name="PER_PROJECT_SETTINGS">
Expand Down
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ buildscript {

dependencies {
classpath 'com.palantir.gradle.consistentversions:gradle-consistent-versions:0.15.0'
classpath 'com.palantir.baseline:gradle-baseline-java:0.50.0'
classpath 'com.palantir.baseline:gradle-baseline-java:2.49.1'
classpath 'com.palantir.gradle.conjure:gradle-conjure:4.7.2'
classpath 'com.palantir.sls-packaging:gradle-sls-packaging:3.3.0'
classpath 'gradle.plugin.com.palantir.gradle.gitversion:gradle-git-version:0.11.0'
Expand Down
1 change: 1 addition & 0 deletions spark-tpcds-benchmark-runner/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ dependencies {
compile 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8'
compile 'com.fasterxml.jackson.datatype:jackson-datatype-guava'
compile 'com.google.guava:guava'
compile 'com.palantir.safe-logging:preconditions'
compile 'org.apache.spark:spark-sql_2.11'

testCompile 'junit:junit'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.datatype.guava.GuavaModule;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.google.common.base.Preconditions;
import com.palantir.logsafe.Preconditions;
import com.palantir.spark.tpcds.immutables.ImmutablesConfigStyle;
import java.io.IOException;
import java.nio.file.Path;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,8 @@ private ListenableFuture<?> generateAndUploadDataForScale(
throw new IllegalStateException(
String.format("Dsdgen failed with return code %d", returnCode));
}
log.info("Finished running dsdgen for the following data scale.",
log.info(
"Finished running dsdgen for the following data scale.",
SafeArg.of("scale", scale));
log.info(
"Uploading tpcds data from the following location.",
Expand Down Expand Up @@ -225,7 +226,8 @@ private void saveTablesAsParquet(int scale) {
paths.tableParquetLocation(scale, table));
});
saveAsParquetTask.addListener(() -> {
log.info("Saved a table as parquet at the following scale.",
log.info(
"Saved a table as parquet at the following scale.",
SafeArg.of("table", table),
SafeArg.of("scale", scale));
}, dataGeneratorThreadPool);
Expand All @@ -250,11 +252,15 @@ private void uploadCsvs(org.apache.hadoop.fs.Path rootDataPath, File tpcdsTempDi
throw new RuntimeException(e);
}
});
uploadCsvTask.addListener(() ->
log.info("Finished uploading CSV to the Hadoop File System.",
uploadCsvTask.addListener(
() ->
log.info(
"Finished uploading CSV to the Hadoop File System.",
SafeArg.of("localFilePath", file),
SafeArg.of("destination",
new org.apache.hadoop.fs.Path(rootDataPath,
SafeArg.of(
"destination",
new org.apache.hadoop.fs.Path(
rootDataPath,
file.getName()))),
dataGeneratorThreadPool);
return uploadCsvTask;
Expand Down Expand Up @@ -318,7 +324,6 @@ private Path findDsdgenTgz() throws FileNotFoundException {
String.format(
"Dsdgen tarball not found at %s; was this benchmark runner"
+ " packaged correctly?", dsdgenTgzPath));

}
return dsdgenTgzPath;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@

package com.palantir.spark.tpcds.metrics;

import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.palantir.logsafe.Preconditions;
import com.palantir.spark.tpcds.config.TpcdsBenchmarkConfig;
import com.palantir.spark.tpcds.immutables.ImmutablesStyle;
import com.palantir.spark.tpcds.paths.TpcdsPaths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.spark.sql.Row;
Expand All @@ -34,7 +34,7 @@
public final class TpcdsBenchmarkMetrics {

private final TpcdsBenchmarkConfig config;
private final List<Row> metrics = Lists.newArrayList();
private final List<Row> metrics = new ArrayList<>();
private final TpcdsPaths paths;
private final SparkSession spark;
private RunningQuery currentRunningQuery;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,10 @@
package com.palantir.spark.tpcds.schemas;

import com.google.common.base.CharMatcher;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Suppliers;
import com.google.common.collect.Maps;
import com.google.common.io.CharStreams;
import com.palantir.logsafe.Preconditions;
import com.palantir.spark.tpcds.constants.TpcdsTable;
import com.palantir.spark.tpcds.datagen.TpcdsDataGenerator;
import java.io.IOException;
Expand All @@ -30,6 +29,7 @@
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand All @@ -42,7 +42,7 @@
public final class TpcdsSchemas {
private static final Pattern DECIMAL_PATTERN = Pattern.compile("decimal\\((\\d+),(\\d+)\\)");

private final Map<TpcdsTable, StructType> schemas = Maps.newConcurrentMap();
private final Map<TpcdsTable, StructType> schemas = new ConcurrentHashMap<>();
private final Supplier<String> cachedSqlSchemaDefinition =
Suppliers.memoize(TpcdsSchemas::getSqlSchemaDefinition);

Expand Down
5 changes: 3 additions & 2 deletions versions.lock
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ com.github.luben:zstd-jni:1.3.5-3 (2 constraints: 9922e77c)
com.github.stephenc.jcip:jcip-annotations:1.0-1 (1 constraints: 930cbb09)
com.google.code.findbugs:jsr305:3.0.2 (7 constraints: 7f65ecaf)
com.google.code.gson:gson:2.2.4 (1 constraints: 8c0d3f2f)
com.google.errorprone:error_prone_annotations:2.2.0 (1 constraints: 160aebb4)
com.google.errorprone:error_prone_annotations:2.3.3 (2 constraints: 191b3159)
com.google.flatbuffers:flatbuffers-java:1.9.0 (2 constraints: e5199714)
com.google.guava:failureaccess:1.0.1 (1 constraints: 140ae1b4)
com.google.guava:guava:27.0.1-jre (14 constraints: 18e224d1)
Expand All @@ -31,7 +31,8 @@ com.jcraft:jsch:0.1.54 (1 constraints: be0df13c)
com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7 (1 constraints: de137765)
com.nimbusds:nimbus-jose-jwt:4.41.1 (1 constraints: e70c4e1d)
com.ning:compress-lzf:1.0.4 (2 constraints: d1213f05)
com.palantir.safe-logging:safe-logging:1.5.1 (1 constraints: 180dc736)
com.palantir.safe-logging:preconditions:1.12.2 (1 constraints: 3805313b)
com.palantir.safe-logging:safe-logging:1.12.2 (2 constraints: 491e20c9)
com.squareup.okhttp:okhttp:2.7.5 (2 constraints: 1e24b4d7)
com.squareup.okio:okio:1.15.0 (2 constraints: 0b21a1a6)
com.thoughtworks.paranamer:paranamer:2.8 (5 constraints: c84a936c)
Expand Down
1 change: 1 addition & 0 deletions versions.props
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
com.google.guava:guava = 27.0.1-jre
com.palantir.immutables.style:* = 1.2.0
com.palantir.safe-logging:preconditions:1.12.2
io.dropwizard.metrics:* = 3.2.6
javax.servlet.jsp:jsp-api:2.2
org.immutables:* = 2.7.5
Expand Down