From 7b8728b4cfa5b35d6f19cbdc23e9bb5d0c8aaf6a Mon Sep 17 00:00:00 2001 From: kokila Date: Thu, 29 Feb 2024 16:20:19 +0530 Subject: [PATCH] HIVE-27925: Addendum: HiveConf: unify ConfVars enum and use underscore for better readability --- .../org/apache/hadoop/hive/conf/HiveConf.java | 8 +-- .../mr/hive/TestHiveIcebergStatistics.java | 18 +++---- .../apache/iceberg/mr/hive/TestHiveShell.java | 2 +- .../hadoop/hive/hooks/TestHs2Hooks.java | 2 +- .../tools/metatool/TestHiveMetaTool.java | 3 +- .../ql/parse/TestReplicationScenarios.java | 2 +- .../parse/TestStatsReplicationScenarios.java | 2 +- .../ql/txn/compactor/TestCompactorBase.java | 2 +- .../txn/compactor/TestCrudCompactorOnTez.java | 14 ++--- .../txn/compactor/TestMRCompactorOnTez.java | 2 +- .../apache/hive/jdbc/TestJdbcWithMiniHS2.java | 2 +- .../auth/jwt/TestHttpJwtAuthentication.java | 2 +- .../auth/saml/TestHttpSamlAuthentication.java | 7 +-- .../service/server/TestGracefulStopHS2.java | 2 +- .../ql/ddl/table/create/CreateTableDesc.java | 2 +- .../create/like/CreateTableLikeOperation.java | 2 +- .../misc/truncate/TruncateTableAnalyzer.java | 2 +- .../add/AbstractAddPartitionAnalyzer.java | 3 +- .../AlterTableConcatenateAnalyzer.java | 4 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 8 +-- .../hive/ql/optimizer/GenMapRedUtils.java | 2 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 2 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 2 +- .../hive/ql/parse/SemanticAnalyzer.java | 16 +++--- .../hadoop/hive/ql/stats/BasicStatsTask.java | 2 +- .../hadoop/hive/ql/stats/StatsFactory.java | 4 +- .../hive/ql/txn/compactor/StatsUpdater.java | 2 +- .../hive/ql/TxnCommandsBaseForTests.java | 2 +- .../hive/ql/exec/TestFileSinkOperator.java | 2 +- .../ql/exec/tez/TestTezOutputCommitter.java | 2 +- .../hive/ql/stats/TestStatsUpdaterThread.java | 54 +++++++++---------- 32 files changed, 86 insertions(+), 95 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b9eccc19e1e6..a1df4493faa8 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1523,7 +1523,7 @@ public static enum ConfVars { "This limits the number of partitions that can be requested from the metastore for a given table.\n" + "The default value \"-1\" means no limit."), - NEWTABLEDEFAULTPARA("hive.table.parameters.default", "", + NEW_TABLE_DEFAULT_PARA("hive.table.parameters.default", "", "Default property values for newly created tables"), /** * @deprecated With HIVE-25813 table properties of source tables will not be copied over to dest table. @@ -2813,11 +2813,11 @@ public static enum ConfVars { "Set this to true to use approximation based logic to adjust ndv after join."), HIVE_STATS_NUM_NULLS_ESTIMATE_PERC("hive.stats.num.nulls.estimate.percent", (float)5, "This many percentage of rows will be estimated as number of nulls in absence of statistics."), - HIVESTATSAUTOGATHER("hive.stats.autogather", true, + HIVE_STATS_AUTOGATHER("hive.stats.autogather", true, "A flag to gather statistics (only basic) automatically during the INSERT OVERWRITE command."), - HIVESTATSCOLAUTOGATHER("hive.stats.column.autogather", true, + HIVE_STATS_COL_AUTOGATHER("hive.stats.column.autogather", true, "A flag to gather column statistics automatically."), - HIVESTATSDBCLASS("hive.stats.dbclass", "fs", new PatternSet("custom", "fs"), + HIVE_STATS_DBCLASS("hive.stats.dbclass", "fs", new PatternSet("custom", "fs"), "The storage that stores temporary Hive statistics. In filesystem based statistics collection ('fs'), \n" + "each task writes statistics it has collected in a file on the filesystem, which will be aggregated \n" + "after the job has finished. Supported values are fs (filesystem) and custom as defined in StatsSetupConst.java."), // StatsSetupConst.StatDB diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java index d31ebcf80d1e..f377b523be5f 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java @@ -81,7 +81,7 @@ public void testAnalyzeTableComputeStatisticsEmptyTable() throws IOException, TE public void testStatsWithInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); @@ -107,7 +107,7 @@ public void testStatsWithInsert() { public void testStatsWithInsertOverwrite() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); @@ -125,7 +125,7 @@ public void testStatsWithPartitionedInsert() { PartitionSpec spec = PartitionSpec.builderFor(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA) .identity("last_name").build(); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, spec, fileFormat, ImmutableList.of()); @@ -150,7 +150,7 @@ public void testStatsWithCTAS() { shell.executeStatement(testTables.getInsertQuery( HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, TableIdentifier.of("default", "source"), false)); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); shell.executeStatement(String.format( "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source", testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")), @@ -169,7 +169,7 @@ public void testStatsWithPartitionedCTAS() { shell.executeStatement(testTables.getInsertQuery( HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, TableIdentifier.of("default", "source"), false)); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); shell.executeStatement(String.format( "CREATE TABLE target PARTITIONED BY (dept, name) STORED BY ICEBERG %s AS SELECT * FROM source s", testTables.propertiesForCreateTableSQL( @@ -189,7 +189,7 @@ public void testStatsRemoved() throws IOException { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); @@ -215,7 +215,7 @@ public void testStatsRemoved() throws IOException { public void testColumnStatsAccurate() throws Exception { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); @@ -236,7 +236,7 @@ public void testColumnStatsAccurate() throws Exception { public void testMergeStatsWithInsert() { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); @@ -267,7 +267,7 @@ public void testMergeStatsWithInsert() { public void testIcebergColStatsPath() throws IOException { TableIdentifier identifier = TableIdentifier.of("default", "customers"); - shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true); + shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, true); Table table = testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA, PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of()); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java index 29941f809d09..4898ce813383 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveShell.java @@ -202,7 +202,7 @@ private HiveConf initializeConf() { // Speed up test execution hiveConf.setLongVar(HiveConf.ConfVars.HIVE_COUNTERS_PULL_INTERVAL, 1L); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); // Resource configuration hiveConf.setInt("mapreduce.map.memory.mb", 1024); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java index add09aec5da2..6526c8c82835 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/hooks/TestHs2Hooks.java @@ -146,7 +146,7 @@ public static void setUpBeforeClass() throws Exception { hiveConf.setVar(ConfVars.SEMANTIC_ANALYZER_HOOK, SemanticAnalysisHook.class.getName()); hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - hiveConf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(ConfVars.HIVE_STATS_COL_AUTOGATHER, false); hiveServer2 = new HiveServer2(); hiveServer2.init(hiveConf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java index bc874161f971..def816c41d66 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/tools/metatool/TestHiveMetaTool.java @@ -20,7 +20,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.nio.file.Files; @@ -117,7 +116,7 @@ public void setUp() throws Exception { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.SPLIT_UPDATE, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); hiveConf.setBoolean("mapred.input.dir.recursive", true); TestTxnDbUtil.setConfValues(hiveConf); txnHandler = TxnUtils.getTxnStore(hiveConf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 60788ad75ca4..6ffaecab38d0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -221,7 +221,7 @@ static void internalBeforeClassSetup(Map additionalProperties) "org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore"); hconf.set(HiveConf.ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL.varname, "/tmp/warehouse/external"); hconf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_METADATA_QUERIES, true); - hconf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true); + hconf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, true); hconf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE, true); hconf.setBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET, false); hconf.setBoolVar(HiveConf.ConfVars.REPL_BATCH_INCREMENTAL_EVENTS, false); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java index d842385f48ed..58fb5958b734 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java @@ -112,7 +112,7 @@ static void internalBeforeClassSetup(Map primaryOverrides, // Run with autogather false on primary if requested Map sourceOverrides = new HashMap<>(); hasAutogather = autogather; - additionalOverrides.put(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, + additionalOverrides.put(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname, autogather ? "true" : "false"); sourceOverrides.putAll(additionalOverrides); sourceOverrides.putAll(primaryOverrides); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java index 3027028a0798..b7c689e9effc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactorBase.java @@ -85,7 +85,7 @@ public void setup() throws Exception { hiveConf.setVar(HiveConf.ConfVars.POST_EXEC_HOOKS, ""); hiveConf.setVar(HiveConf.ConfVars.METASTORE_WAREHOUSE, TEST_WAREHOUSE_DIR); hiveConf.setVar(HiveConf.ConfVars.HIVE_INPUT_FORMAT, HiveInputFormat.class.getName()); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_OPTIMIZE_METADATA_QUERIES, false); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_INITIATOR_ON, true); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_CLEANER_ON, true); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index a1885423a9d9..f64fac9038d9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -105,7 +105,7 @@ public void testRebalanceCompactionWithParallelDeleteAsSecondPessimisticLock() t private void testRebalanceCompactionWithParallelDeleteAsSecond(boolean optimisticLock) throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); conf.setBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK, optimisticLock); //set grouping size to have 3 buckets, and re-create driver with the new config @@ -204,7 +204,7 @@ private void testRebalanceCompactionWithParallelDeleteAsSecond(boolean optimisti public void testRebalanceCompactionOfNotPartitionedImplicitlyBucketedTableWithOrder() throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); //set grouping size to have 3 buckets, and re-create driver with the new config conf.set("tez.grouping.min-size", "400"); @@ -260,7 +260,7 @@ public void testRebalanceCompactionOfNotPartitionedImplicitlyBucketedTableWithOr public void testRebalanceCompactionOfNotPartitionedImplicitlyBucketedTable() throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); //set grouping size to have 3 buckets, and re-create driver with the new config conf.set("tez.grouping.min-size", "400"); @@ -314,7 +314,7 @@ public void testRebalanceCompactionOfNotPartitionedImplicitlyBucketedTable() thr public void testRebalanceCompactionOfPartitionedImplicitlyBucketedTable() throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); //set grouping size to have 3 buckets, and re-create driver with the new config conf.set("tez.grouping.min-size", "1"); @@ -423,7 +423,7 @@ public void testRebalanceCompactionOfPartitionedImplicitlyBucketedTable() throws public void testRebalanceCompactionOfNotPartitionedExplicitlyBucketedTable() throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); final String tableName = "rebalance_test"; executeStatementOnDriver("drop table if exists " + tableName, driver); @@ -445,7 +445,7 @@ public void testRebalanceCompactionOfNotPartitionedExplicitlyBucketedTable() thr public void testRebalanceCompactionNotPartitionedExplicitBucketNumbers() throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); conf.setBoolVar(HiveConf.ConfVars.HIVE_COMPACTOR_GATHER_STATS, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); //set grouping size to have 3 buckets, and re-create driver with the new config conf.set("tez.grouping.min-size", "400"); @@ -3098,7 +3098,7 @@ public void testStatsAfterCompactionPartTblForQueryBasedCompaction() throws Exce public void testStatsAfterCompactionPartTbl(boolean isQueryBased, boolean isAutoGatherStats, CompactionType compactionType) throws Exception { conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, isQueryBased); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, isAutoGatherStats); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, isAutoGatherStats); String dbName = "default"; String tblName = "minor_compaction_test"; IMetaStoreClient msClient = new HiveMetaStoreClient(conf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorOnTez.java index 90d3cb1cc55a..e79ce9560d98 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestMRCompactorOnTez.java @@ -41,7 +41,7 @@ public void testCompactorGatherStats() throws Exception{ conf.setBoolVar(HiveConf.ConfVars.HIVE_WRITE_ACID_VERSION_FILE, true); conf.setVar(HiveConf.ConfVars.COMPACTOR_JOB_QUEUE, CUSTOM_COMPACTION_QUEUE); conf.setBoolVar(HiveConf.ConfVars.HIVE_MR_COMPACTOR_GATHER_STATS, true); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); conf.setVar(HiveConf.ConfVars.HIVE_PROTO_EVENTS_BASE_PATH, tmpFolder); String dbName = "default"; diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index fcf666da13df..7f03c0ee0c5f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -215,7 +215,7 @@ private static void startMiniHS2(HiveConf conf) throws Exception { private static void startMiniHS2(HiveConf conf, boolean httpMode) throws Exception { conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); - conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); + conf.setBoolVar(ConfVars.HIVE_STATS_COL_AUTOGATHER, false); // store post-exec hooks calls so we can look at them later conf.setVar(ConfVars.POST_EXEC_HOOKS, ReadableHook.class.getName() + "," + LineageLogger.class.getName()); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java index 202ff0d481cf..7aa7adb67e98 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java @@ -116,7 +116,7 @@ public static void setupHS2() throws Exception { HiveConf conf = new HiveConf(); conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); - conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); + conf.setBoolVar(ConfVars.HIVE_STATS_COL_AUTOGATHER, false); conf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, "JWT"); // the content of the URL below is the same as jwtVerificationJWKSFile conf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION_JWT_JWKS_URL, "http://localhost:" + MOCK_JWKS_SERVER_PORT + diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java index d71fdedf7ea9..7d119e9372c2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/saml/TestHttpSamlAuthentication.java @@ -33,9 +33,7 @@ import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.ServerSocket; -import java.net.Socket; import java.nio.charset.StandardCharsets; import java.sql.ResultSet; import java.sql.SQLException; @@ -53,16 +51,13 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; -import org.apache.hadoop.hive.ql.metadata.TestHive; import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; import org.apache.hive.jdbc.miniHS2.MiniHS2; -import org.apache.hive.jdbc.saml.HiveJdbcBrowserClient; import org.apache.hive.jdbc.saml.IJdbcBrowserClient; import org.apache.hive.jdbc.saml.IJdbcBrowserClient.HiveJdbcBrowserException; import org.apache.hive.jdbc.saml.IJdbcBrowserClientFactory; import org.apache.hive.jdbc.saml.SimpleSAMLPhpTestBrowserClient; -import org.apache.thrift.TException; import org.junit.After; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -108,7 +103,7 @@ public static void setupHS2() throws Exception { HiveConf conf = new HiveConf(); conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); - conf.setBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER, false); + conf.setBoolVar(ConfVars.HIVE_STATS_COL_AUTOGATHER, false); conf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, "SAML"); conf.setVar(ConfVars.HIVE_SERVER2_SAML_IDP_METADATA, idpMetadataFile.getAbsolutePath()); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java index 164a63315e41..983e8afd6cd9 100755 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java @@ -51,7 +51,7 @@ public static void setupBeforeClass() throws Exception { HiveConf conf = new HiveConf(); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); conf.setTimeVar(HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, 60, TimeUnit.SECONDS); MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false); miniHS2 = builder.build(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java index 2310f806c5f0..0cdbdce9d71f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java @@ -929,7 +929,7 @@ public Table toTable(HiveConf conf) throws HiveException { StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null, StatsSetupConst.FALSE); if (!this.isCTAS && !tbl.isPartitioned() && !tbl.isTemporary() && - conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { // Put the flag into the dictionary in order not to pollute the table, // ObjectDictionary is meant to convey repeatitive messages. ObjectDictionary dictionary = tbl.getTTable().isSetDictionary() ? diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java index 6534d70e3268..7adb306131ab 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/like/CreateTableLikeOperation.java @@ -70,7 +70,7 @@ public int execute() throws HiveException { } if (desc.getLocation() == null && !tbl.isPartitioned() && - context.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + context.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java index 1b6a1c6e4ec2..cc92b99c8429 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/truncate/TruncateTableAnalyzer.java @@ -360,7 +360,7 @@ private void addMoveTask(ASTNode root, Table table, Map partitio private void addStatTask(ASTNode root, Table table, Path oldPartitionLocation, Path newPartitionLocation, LoadTableDesc loadTableDesc, Task moveTask) throws SemanticException { // Recalculate the HDFS stats if auto gather stats is set - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { BasicStatsWork basicStatsWork; if (oldPartitionLocation.equals(newPartitionLocation)) { // If we're merging to the same location, we can avoid some metastore calls diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AbstractAddPartitionAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AbstractAddPartitionAnalyzer.java index b8433d261707..4d73782aaee4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AbstractAddPartitionAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/add/AbstractAddPartitionAnalyzer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.ddl.DDLWork; -import org.apache.hadoop.hive.ql.ddl.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer; import org.apache.hadoop.hive.ql.ddl.table.AlterTableType; import org.apache.hadoop.hive.ql.ddl.table.partition.PartitionUtils; @@ -119,7 +118,7 @@ private List createPartitions(ASTNode private AlterTableAddPartitionDesc.PartitionDesc createPartitionDesc(Table table, String location, Map partitionSpec) { Map params = null; - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER) && location == null) { + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER) && location == null) { params = new HashMap(); StatsSetupConst.setStatsStateForCreateTable(params, MetaStoreUtils.getColumnNames(table.getCols()), StatsSetupConst.TRUE); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java index 8e4a9d1b6fd1..1322ace7e918 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateAnalyzer.java @@ -41,8 +41,6 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.hooks.ReadEntity; -import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; @@ -214,7 +212,7 @@ private void addMoveTask(TableName tableName, Table table, Map p private void addStatTask(TableName tableName, Table table, Map partitionSpec, Path oldLocation, Path newLocation, LoadTableDesc loadTableDesc, Task moveTask) throws SemanticException { - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { BasicStatsWork basicStatsWork; if (oldLocation.equals(newLocation)) { // If we're merging to the same location, we can avoid some metastore calls diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 6d38f1d5afca..b247e276cb19 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -4199,7 +4199,7 @@ public static int getDPColOffset(FileSinkDesc conf) { public static List getStatsTmpDirs(BaseWork work, Configuration conf) { List statsTmpDirs = new ArrayList<>(); - if (!StatsSetupConst.StatDB.fs.name().equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVESTATSDBCLASS))) { + if (!StatsSetupConst.StatDB.fs.name().equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_STATS_DBCLASS))) { // no-op for non-fs stats collection return statsTmpDirs; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 11b52c51a730..23aea1d31810 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2801,7 +2801,7 @@ private Partition loadPartitionInternal(Path loadPath, Table tbl, Map()); - if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (this.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { StatsSetupConst.setStatsStateForCreateTable(newTPart.getParameters(), MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE); } @@ -3020,7 +3020,7 @@ private void setStatsPropAndAlterPartitions(boolean resetStatistics, Table tbl, List partitions, AcidUtils.TableSnapshot tableSnapshot) throws TException { - if (partitions.isEmpty() || conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER)) { + if (partitions.isEmpty() || conf.getBoolVar(ConfVars.HIVE_STATS_AUTOGATHER)) { return; } EnvironmentContext ec = new EnvironmentContext(); @@ -3520,7 +3520,7 @@ public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType // If there is no column stats gather stage present in the plan. So we don't know the accuracy of the stats or // auto gather stats is turn off explicitly. We need to reset the stats in both cases. - if (resetStatistics || !this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (resetStatistics || !this.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { LOG.debug("Clear table column statistics and set basic statistics to false for " + tbl.getCompleteName()); StatsSetupConst.setBasicStatsState(tbl.getParameters(), StatsSetupConst.FALSE); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index 175f868bb82d..525ecfbb13e7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -1894,7 +1894,7 @@ public static boolean isMergeRequired(List> mvTasks, HiveConf hco fsOp.getConf().isMmTable(), fsOp.getConf().isDirectInsert(), fsOp.getConf().getMoveTaskId(), fsOp.getConf().getAcidOperation()); // TODO: wtf?!! why is this in this method? This has nothing to do with anything. - if (isInsertTable && hconf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) + if (isInsertTable && hconf.getBoolVar(ConfVars.HIVE_STATS_AUTOGATHER) && !fsOp.getConf().isMaterialization()) { // mark the MapredWork and FileSinkOperator for gathering stats fsOp.getConf().setGatherStats(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java index 5d4768000a96..7424d4e8ea86 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java @@ -123,7 +123,7 @@ private void handlePartialPartitionSpec(Map partSpec, ColumnStat partValsSpecified += partSpec.get(partKey) == null ? 0 : 1; } try { - // for static partition, it may not exist when HIVESTATSCOLAUTOGATHER is + // for static partition, it may not exist when HIVE_STATS_COL_AUTOGATHER is // set to true if (context == null) { if ((partValsSpecified == tbl.getPartitionKeys().size()) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index ce036a2b63e8..271b384aba4e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -426,7 +426,7 @@ private void analyzeLoad(ASTNode ast) throws SemanticException { // However, some other stats, like number of files, do not require a complete scan // Update the stats which do not require a complete scan. Task statTask = null; - if (conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { BasicStatsWork basicStatsWork = new BasicStatsWork(loadTableWork); basicStatsWork.setNoStatsAggregator(true); basicStatsWork.setClearAggregatorStats(true); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index b66ae3cba482..227baef5ce48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -24,7 +24,7 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.DYNAMIC_PARTITION_CONVERT; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_ARCHIVE_ENABLED; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_STATS_DBCLASS; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; @@ -2563,7 +2563,7 @@ private void getMetaData(QB qb, ReadEntity parentInput) } } } - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.getTableName().getTable().toLowerCase(), ts); } @@ -2625,7 +2625,7 @@ private void getMetaData(QB qb, ReadEntity parentInput) throw new SemanticException( generateErrorMessage(ast, "Error creating temporary folder on: " + location.toString()), e); } - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_AUTOGATHER)) { TableSpec ts = new TableSpec(db, conf, this.ast); // Add the table spec for the destination table. qb.getParseInfo().addTableSpec(ts.getTableName().getTable().toLowerCase(), ts); @@ -8199,8 +8199,8 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) // the following code is used to collect column stats when // hive.stats.autogather=true // and it is an insert overwrite or insert into table - if (conf.getBoolVar(ConfVars.HIVESTATSAUTOGATHER) - && conf.getBoolVar(ConfVars.HIVESTATSCOLAUTOGATHER) + if (conf.getBoolVar(ConfVars.HIVE_STATS_AUTOGATHER) + && conf.getBoolVar(ConfVars.HIVE_STATS_COL_AUTOGATHER) && enableColumnStatsCollecting() && destinationTable != null && (!destinationTable.isNonNative() || destinationTable.getStorageHandler().commitInMoveTask()) @@ -8540,7 +8540,7 @@ private FileSinkDesc createFileSinkDesc(String dest, TableDesc table_desc, // it should be the same as the MoveWork's sourceDir. fileSinkDesc.setStatsAggPrefix(fileSinkDesc.getDirName().toString()); if (!destTableIsMaterialization && - HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { + HiveConf.getVar(conf, HIVE_STATS_DBCLASS).equalsIgnoreCase(StatDB.fs.name())) { String statsTmpLoc = ctx.getTempDirForInterimJobPath(dest_path).toString(); fileSinkDesc.setStatsTmpDir(statsTmpLoc); LOG.debug("Set stats collection dir : " + statsTmpLoc); @@ -12190,7 +12190,7 @@ private void setupStats(TableScanDesc tsDesc, QBParseInfo qbp, Table tab, String return; } - if (HiveConf.getVar(conf, HIVESTATSDBCLASS).equalsIgnoreCase(StatDB.fs.name())) { + if (HiveConf.getVar(conf, HIVE_STATS_DBCLASS).equalsIgnoreCase(StatDB.fs.name())) { String statsTmpLoc = ctx.getTempDirForInterimJobPath(tab.getPath()).toString(); LOG.debug("Set stats collection dir : " + statsTmpLoc); tsDesc.setTmpStatsDir(statsTmpLoc); @@ -13781,7 +13781,7 @@ private Map validateAndAddDefaultProperties( boolean isTemporaryTable, boolean isTransactional, boolean isManaged, String[] qualifiedTabName, boolean isTableTypeChanged) throws SemanticException { Map retValue = Optional.ofNullable(tblProp).orElseGet(HashMap::new); - String paraString = HiveConf.getVar(conf, ConfVars.NEWTABLEDEFAULTPARA); + String paraString = HiveConf.getVar(conf, ConfVars.NEW_TABLE_DEFAULT_PARA); if (paraString != null && !paraString.isEmpty()) { for (String keyValuePair : paraString.split(",")) { String[] keyValue = keyValuePair.split("=", 2); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index 96f43c426c81..4abb77641fe7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -431,7 +431,7 @@ private ExecutorService buildBasicStatsExecutor() { } private StatsAggregator createStatsAggregator(StatsCollectionContext scc, HiveConf conf) throws HiveException { - String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVESTATSDBCLASS); + String statsImpl = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_STATS_DBCLASS); StatsFactory factory = StatsFactory.newFactory(statsImpl, conf); if (factory == null) { throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java index 2e2abb6cb2fe..8d79f6a2c95f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsFactory.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.util.ReflectionUtils; -import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_STATS_DBCLASS; /** * A factory of stats publisher and aggregator implementations of the @@ -43,7 +43,7 @@ public final class StatsFactory { private final Configuration jobConf; public static StatsFactory newFactory(Configuration conf) { - return newFactory(HiveConf.getVar(conf, HIVESTATSDBCLASS), conf); + return newFactory(HiveConf.getVar(conf, HIVE_STATS_DBCLASS), conf); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java index ce0998158dc3..c5b7b52b2a9b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java @@ -77,7 +77,7 @@ public void gatherStats(CompactionInfo ci, HiveConf conf, sb.append(")"); } sb.append(" compute statistics"); - if (!conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER) && ci.isMajorCompaction()) { + if (!conf.getBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER) && ci.isMajorCompaction()) { List columnList = msc.findColumnsWithStats(CompactionInfo.compactionInfoToStruct(ci)); if (!columnList.isEmpty()) { sb.append(" for columns ").append(String.join(",", columnList)); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java index b7acefc531c4..a2dd2d5c739b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java @@ -128,7 +128,7 @@ void setUpInternal() throws Exception { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); HiveConf.setBoolVar(hiveConf, HiveConf.ConfVars.SPLIT_UPDATE, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); hiveConf.setBoolean("mapred.input.dir.recursive", true); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_INITIATOR_ON, true); MetastoreConf.setBoolVar(hiveConf, MetastoreConf.ConfVars.COMPACTOR_CLEANER_ON, true); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java index eae7f69fbca0..ce6a495a37ea 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java @@ -313,7 +313,7 @@ public void setup() throws Exception { TFSOStatsPublisher.class.getName()); jc.set(HiveConf.ConfVars.HIVE_STATS_DEFAULT_AGGREGATOR.varname, TFSOStatsAggregator.class.getName()); - jc.set(HiveConf.ConfVars.HIVESTATSDBCLASS.varname, "custom"); + jc.set(HiveConf.ConfVars.HIVE_STATS_DBCLASS.varname, "custom"); } @After diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezOutputCommitter.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezOutputCommitter.java index 01df5bc326d9..29f04dc3ba9d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezOutputCommitter.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezOutputCommitter.java @@ -122,7 +122,7 @@ private IDriver getDriverWithCommitter(String committerClass) { conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - conf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); conf.setInt("tez.am.task.max.failed.attempts", MAX_TASK_ATTEMPTS); conf.set("mapred.output.committer.class", committerClass); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java index e2bfa9763b7f..d37b54437832 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java @@ -81,8 +81,8 @@ public void setUp() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); // hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, true); hiveConf.set(MetastoreConf.ConfVars.STATS_AUTO_UPDATE.getVarname(), "all"); TestTxnDbUtil.setConfValues(hiveConf); TestTxnDbUtil.prepDb(hiveConf); @@ -299,7 +299,7 @@ public void testTxnDynamicPartitions() throws Exception { StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); executeQuery("create table simple_stats (s string) partitioned by (i int)" + " stored as orc " + " TBLPROPERTIES (\"transactional\"=\"true\")"); @@ -326,8 +326,8 @@ public void testExistingOnly() throws Exception { IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); executeQuery("create table simple_stats (i int, s string)"); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("insert into simple_stats (i, s) values (1, 'test')"); executeQuery("analyze table simple_stats compute statistics for columns i"); verifyStatsUpToDate("simple_stats", Lists.newArrayList("s"), msClient, false); @@ -350,8 +350,8 @@ public void testQueueingWithThreads() throws Exception { StatsUpdaterThread su = createUpdater(); su.startWorkers(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string) partitioned by (i int)"); for (int i = 0; i < PART_COUNT; ++i) { executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')"); @@ -378,8 +378,8 @@ public void testAllPartitions() throws Exception { final int PART_COUNT = 3; StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string) partitioned by (i int)"); for (int i = 0; i < PART_COUNT; ++i) { executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')"); @@ -401,14 +401,14 @@ public void testPartitionSubset() throws Exception { final int NONSTAT_PART_COUNT = 3; StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string) partitioned by (i int)"); for (int i = 0; i < NONSTAT_PART_COUNT; ++i) { executeQuery("insert into simple_stats partition(i='" + i + "') values ('test')"); } - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, true); executeQuery("insert into simple_stats partition(i='" + NONSTAT_PART_COUNT + "') values ('test')"); verifyPartStatsUpToDate(NONSTAT_PART_COUNT, 0, msClient, "simple_stats", false); @@ -435,8 +435,8 @@ public void testPartitionSubset() throws Exception { public void testPartitionsWithDifferentColsAll() throws Exception { StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string, t string, u string) partitioned by (i int)"); executeQuery("insert into simple_stats partition(i=0) values ('test', '0', 'foo')"); executeQuery("insert into simple_stats partition(i=1) values ('test', '1', 'bar')"); @@ -465,8 +465,8 @@ public void testPartitionsWithDifferentColsExistingOnly() throws Exception { hiveConf.set(MetastoreConf.ConfVars.STATS_AUTO_UPDATE.getVarname(), "existing"); StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string, t string, u string) partitioned by (i int)"); executeQuery("insert into simple_stats partition(i=0) values ('test', '0', 'foo')"); executeQuery("insert into simple_stats partition(i=1) values ('test', '1', 'bar')"); @@ -502,16 +502,16 @@ public void testParallelOps() throws Exception { hiveConf.setInt(MetastoreConf.ConfVars.STATS_AUTO_UPDATE_WORKER_COUNT.getVarname(), 4); StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table simple_stats (s string)"); executeQuery("create table simple_stats2 (s string) partitioned by (i int)"); executeQuery("create table simple_stats3 (s string) partitioned by (i int)"); executeQuery("insert into simple_stats values ('test')"); executeQuery("insert into simple_stats2 partition(i=0) values ('test')"); executeQuery("insert into simple_stats3 partition(i=0) values ('test')"); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, true); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, true); executeQuery("insert into simple_stats3 partition(i=1) values ('test')"); assertTrue(su.runOneIteration()); @@ -526,8 +526,8 @@ public void testParallelOps() throws Exception { assertEquals(3, su.getQueueLength()); // Nothing new added to the queue while analyze runs. // Add another partition without stats. - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("insert into simple_stats3 partition(i=2) values ('test')"); assertTrue(su.runOneIteration()); @@ -569,8 +569,8 @@ private void testNoStatsUpdateForReplTable(String tblNamePrefix, String txnPrope executeQuery("alter database " + dbName + " set dbproperties('" + ReplConst.TARGET_OF_REPLICATION + "'='true')"); StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table " + tblWOStats + "(i int, s string) " + txnProperty); executeQuery("insert into " + tblWOStats + "(i, s) values (1, 'test')"); @@ -636,8 +636,8 @@ private void testNoStatsUpdateForFailoverDb(String tblNamePrefix, String txnProp String dbName = ss.getCurrentDatabase(); StatsUpdaterThread su = createUpdater(); IMetaStoreClient msClient = new HiveMetaStoreClient(hiveConf); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, false); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER, false); + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COL_AUTOGATHER, false); executeQuery("create table " + tblWOStats + "(i int, s string) " + txnProperty); executeQuery("insert into " + tblWOStats + "(i, s) values (1, 'test')");