From 26d22d084a4a60e7c8e2e39c0652db67d38de5e9 Mon Sep 17 00:00:00 2001 From: Henrib Date: Fri, 10 Jan 2025 15:35:02 +0100 Subject: [PATCH 01/40] HIVE-28059 : major rebase stage 1; --- .../hive/TestHiveIcebergBranchOperation.java | 4 +- .../TestHiveIcebergStorageHandlerNoScan.java | 4 +- ...n_partition_evolution_w_id_spec_w_filter.q | 19 +-- .../src/test/queries/positive/iceberg_stats.q | 22 --- .../test/results/positive/iceberg_stats.q.out | 159 ------------------ ...rtition_evolution_w_id_spec_w_filter.q.out | 112 +++--------- .../hive/metastore/HiveMetaStoreClient.java | 6 +- .../hive/metastore/conf/MetastoreConf.java | 2 + .../hadoop/hive/metastore/HiveMetaStore.java | 10 ++ 9 files changed, 42 insertions(+), 296 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java index d62f83eb6fcc..c5eb59987fb0 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java @@ -231,12 +231,12 @@ public void testCreateBranchFromTag() throws IOException, InterruptedException { // Create a branch based on a tag which doesn't exist will fail. Assertions.assertThatThrownBy(() -> shell.executeStatement(String.format( "ALTER TABLE customers CREATE BRANCH %s FOR TAG AS OF %s", branchName2, nonExistTag))) - .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("does not exist"); + .isInstanceOf(IllegalArgumentException.class).hasMessageEndingWith("does not exist"); // Create a branch based on a branch will fail. Assertions.assertThatThrownBy(() -> shell.executeStatement(String.format( "ALTER TABLE customers CREATE BRANCH %s FOR TAG AS OF %s", branchName2, branchName1))) - .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("does not exist"); + .isInstanceOf(IllegalArgumentException.class).hasMessageEndingWith("does not exist"); } @Test diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index a8072f45afe6..70c5f6880418 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -700,7 +700,7 @@ public void testCreateTableError() { "')")) .isInstanceOf(IllegalArgumentException.class) .hasMessageStartingWith("Failed to execute Hive query") - .hasMessageContaining("Table location not set"); + .hasMessageEndingWith("Table location not set"); } } @@ -775,7 +775,7 @@ public void testCreatePartitionedTableWithPropertiesAndWithColumnSpecification() "')")) .isInstanceOf(IllegalArgumentException.class) .hasMessageStartingWith("Failed to execute Hive query") - .hasMessageContaining( + .hasMessageEndingWith( "Provide only one of the following: Hive partition transform specification, " + "or the iceberg.mr.table.partition.spec property"); } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q index 53e915d09ca2..7d0576343aea 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q @@ -65,26 +65,11 @@ delete from ice_orc where last_name in ('ln1', 'ln9'); delete from ice_orc where last_name in ('ln3', 'ln11'); delete from ice_orc where last_name in ('ln5', 'ln13'); -alter table ice_orc set partition spec(team_id); -insert into ice_orc VALUES - ('fn17', 'ln17', 1, 10, 100), - ('fn18','ln18', 1, 10, 100); -insert into ice_orc VALUES - ('fn19','ln19', 2, 11, 100), - ('fn20','ln20', 2, 11, 100); -insert into ice_orc VALUES - ('fn21','ln21', 3, 12, 100), - ('fn22','ln22', 3, 12, 100); -insert into ice_orc VALUES - ('fn23','ln23', 4, 13, 100), - ('fn24','ln24', 4, 13, 100); - - select * from ice_orc; describe formatted ice_orc; -explain alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2); -alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2); +explain alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15'); +alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15'); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q index 6fc965e17456..de88018f32e0 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q @@ -28,27 +28,5 @@ select count(*) from ice01; insert overwrite table ice01 select * from ice01; explain select count(*) from ice01; --- false means that count(*) query won't use row count stored in HMS -set iceberg.hive.keep.stats=false; - -create external table ice03 (id int, key int) Stored by Iceberg stored as ORC - TBLPROPERTIES('format-version'='2'); - -insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1); --- Iceberg table can utilize fetch task to directly retrieve the row count from iceberg SnapshotSummary -explain select count(*) from ice03; -select count(*) from ice03; - --- delete some values -delete from ice03 where id in (2,4); - -explain select count(*) from ice03; -select count(*) from ice03; - --- iow -insert overwrite table ice03 select * from ice03; -explain select count(*) from ice03; - drop table ice01; drop table ice02; -drop table ice03; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out index 4e5b70945016..33c60b54608d 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out @@ -192,155 +192,6 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: create external table ice03 (id int, key int) Stored by Iceberg stored as ORC - TBLPROPERTIES('format-version'='2') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ice03 -POSTHOOK: query: create external table ice03 (id int, key int) Stored by Iceberg stored as ORC - TBLPROPERTIES('format-version'='2') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ice03 -PREHOOK: query: insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice03 -POSTHOOK: query: insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice03 -PREHOOK: query: explain select count(*) from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(*) from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: hdfs://### HDFS PATH ### -5 -PREHOOK: query: delete from ice03 where id in (2,4) -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: default@ice03 -POSTHOOK: query: delete from ice03 where id in (2,4) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: default@ice03 -PREHOOK: query: explain select count(*) from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(*) from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: ice03 - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - Select Operator - Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - Group By Operator - aggregations: count() - minReductionHashAggr: 0.6666666 - mode: hash - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - Reduce Output Operator - null sort order: - sort order: - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - value expressions: _col0 (type: bigint) - Execution mode: vectorized - Reducer 2 - Execution mode: vectorized - Reduce Operator Tree: - Group By Operator - aggregations: count(VALUE._col0) - mode: mergepartial - outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: select count(*) from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: select count(*) from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: hdfs://### HDFS PATH ### -3 -PREHOOK: query: insert overwrite table ice03 select * from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: default@ice03 -POSTHOOK: query: insert overwrite table ice03 select * from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: default@ice03 -PREHOOK: query: explain select count(*) from ice03 -PREHOOK: type: QUERY -PREHOOK: Input: default@ice03 -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: explain select count(*) from ice03 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: hdfs://### HDFS PATH ### -STAGE DEPENDENCIES: - Stage-0 is a root stage - -STAGE PLANS: - Stage: Stage-0 - Fetch Operator - limit: 1 - Processor Tree: - ListSink - PREHOOK: query: drop table ice01 PREHOOK: type: DROPTABLE PREHOOK: Input: default@ice01 @@ -361,13 +212,3 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@ice02 POSTHOOK: Output: database:default POSTHOOK: Output: default@ice02 -PREHOOK: query: drop table ice03 -PREHOOK: type: DROPTABLE -PREHOOK: Input: default@ice03 -PREHOOK: Output: database:default -PREHOOK: Output: default@ice03 -POSTHOOK: query: drop table ice03 -POSTHOOK: type: DROPTABLE -POSTHOOK: Input: default@ice03 -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ice03 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out index 7df4035b818a..95a7ef33c919 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q.out @@ -149,61 +149,6 @@ POSTHOOK: query: delete from ice_orc where last_name in ('ln5', 'ln13') POSTHOOK: type: QUERY POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc -PREHOOK: query: alter table ice_orc set partition spec(team_id) -PREHOOK: type: ALTERTABLE_SETPARTSPEC -PREHOOK: Input: default@ice_orc -POSTHOOK: query: alter table ice_orc set partition spec(team_id) -POSTHOOK: type: ALTERTABLE_SETPARTSPEC -POSTHOOK: Input: default@ice_orc -POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES - ('fn17', 'ln17', 1, 10, 100), - ('fn18','ln18', 1, 10, 100) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES - ('fn17', 'ln17', 1, 10, 100), - ('fn18','ln18', 1, 10, 100) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES - ('fn19','ln19', 2, 11, 100), - ('fn20','ln20', 2, 11, 100) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES - ('fn19','ln19', 2, 11, 100), - ('fn20','ln20', 2, 11, 100) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES - ('fn21','ln21', 3, 12, 100), - ('fn22','ln22', 3, 12, 100) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES - ('fn21','ln21', 3, 12, 100), - ('fn22','ln22', 3, 12, 100) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc -PREHOOK: query: insert into ice_orc VALUES - ('fn23','ln23', 4, 13, 100), - ('fn24','ln24', 4, 13, 100) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: insert into ice_orc VALUES - ('fn23','ln23', 4, 13, 100), - ('fn24','ln24', 4, 13, 100) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc PREHOOK: query: select * from ice_orc PREHOOK: type: QUERY PREHOOK: Input: default@ice_orc @@ -217,15 +162,7 @@ fn12 ln12 2 11 100 fn14 ln14 3 12 100 fn15 ln15 4 13 100 fn16 ln16 4 13 100 -fn17 ln17 1 10 100 -fn18 ln18 1 10 100 -fn19 ln19 2 11 100 fn2 ln2 1 10 100 -fn20 ln20 2 11 100 -fn21 ln21 3 12 100 -fn22 ln22 3 12 100 -fn23 ln23 4 13 100 -fn24 ln24 4 13 100 fn4 ln4 2 11 100 fn6 ln6 3 12 100 fn7 ln7 4 13 100 @@ -245,7 +182,8 @@ company_id bigint # Partition Transform Information # col_name transform_type -team_id IDENTITY +company_id IDENTITY +dept_id IDENTITY # Detailed Table Information Database: default @@ -254,24 +192,24 @@ Retention: 0 #### A masked pattern was here #### Table Type: EXTERNAL_TABLE Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"company_id\":\"true\",\"dept_id\":\"true\",\"first_name\":\"true\",\"last_name\":\"true\",\"team_id\":\"true\"}} + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} EXTERNAL TRUE bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"24\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"12\",\"total-delete-files\":\"6\",\"total-position-deletes\":\"6\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"added-position-delete-files\":\"2\",\"added-delete-files\":\"2\",\"added-files-size\":\"#Masked#\",\"added-position-deletes\":\"2\",\"changed-partition-count\":\"2\",\"total-records\":\"16\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"6\",\"total-position-deletes\":\"6\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# - default-partition-spec {\"spec-id\":2,\"fields\":[{\"name\":\"team_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1002}]} + default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 iceberg.orc.files.only true #### A masked pattern was here #### - numFiles 12 - numRows 18 + numFiles 8 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 15 + snapshot-count 11 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -288,11 +226,11 @@ InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat Compressed: No Sort Columns: [] -PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2) +PREHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15') PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: default@ice_orc PREHOOK: Output: default@ice_orc -POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2) +POSTHOOK: query: explain alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15') POSTHOOK: type: ALTERTABLE_COMPACT POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc @@ -308,11 +246,11 @@ STAGE PLANS: table name: default.ice_orc blocking: true -PREHOOK: query: alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2) +PREHOOK: query: alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15') PREHOOK: type: ALTERTABLE_COMPACT PREHOOK: Input: default@ice_orc PREHOOK: Output: default@ice_orc -POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2) +POSTHOOK: query: alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15') POSTHOOK: type: ALTERTABLE_COMPACT POSTHOOK: Input: default@ice_orc POSTHOOK: Output: default@ice_orc @@ -329,15 +267,7 @@ fn12 ln12 2 11 100 fn14 ln14 3 12 100 fn15 ln15 4 13 100 fn16 ln16 4 13 100 -fn17 ln17 1 10 100 -fn18 ln18 1 10 100 -fn19 ln19 2 11 100 fn2 ln2 1 10 100 -fn20 ln20 2 11 100 -fn21 ln21 3 12 100 -fn22 ln22 3 12 100 -fn23 ln23 4 13 100 -fn24 ln24 4 13 100 fn4 ln4 2 11 100 fn6 ln6 3 12 100 fn7 ln7 4 13 100 @@ -357,7 +287,8 @@ company_id bigint # Partition Transform Information # col_name transform_type -team_id IDENTITY +company_id IDENTITY +dept_id IDENTITY # Detailed Table Information Database: default @@ -371,19 +302,19 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"added-data-files\":\"4\",\"deleted-data-files\":\"8\",\"removed-position-delete-files\":\"6\",\"removed-delete-files\":\"6\",\"added-records\":\"10\",\"deleted-records\":\"16\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"6\",\"changed-partition-count\":\"9\",\"total-records\":\"18\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} + current-snapshot-summary {\"added-data-files\":\"4\",\"deleted-data-files\":\"4\",\"removed-position-delete-files\":\"3\",\"removed-delete-files\":\"3\",\"added-records\":\"5\",\"deleted-records\":\"8\",\"added-files-size\":\"#Masked#\",\"removed-files-size\":\"#Masked#\",\"removed-position-deletes\":\"3\",\"changed-partition-count\":\"5\",\"total-records\":\"11\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"8\",\"total-delete-files\":\"1\",\"total-position-deletes\":\"1\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"} current-snapshot-timestamp-ms #Masked# - default-partition-spec {\"spec-id\":2,\"fields\":[{\"name\":\"team_id\",\"transform\":\"identity\",\"source-id\":4,\"field-id\":1002}]} + default-partition-spec {\"spec-id\":1,\"fields\":[{\"name\":\"company_id\",\"transform\":\"identity\",\"source-id\":5,\"field-id\":1000},{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1001}]} format-version 2 iceberg.orc.files.only true #### A masked pattern was here #### numFiles 8 - numRows 18 + numRows 10 parquet.compression zstd #### A masked pattern was here #### rawDataSize 0 serialization.format 1 - snapshot-count 20 + snapshot-count 15 storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG totalSize #Masked# @@ -405,8 +336,7 @@ PREHOOK: type: SHOW COMPACTIONS POSTHOOK: query: show compactions order by 'partition' POSTHOOK: type: SHOW COMPACTIONS CompactionId Database Table Partition Type State Worker host Worker Enqueue Time Start Time Duration(ms) HadoopJobId Error message Initiator host Initiator Pool name TxnId Next TxnId Commit Time Highest WriteId -#Masked# default ice_orc team_id=10 MAJOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default ice_orc team_id=11 MAJOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default ice_orc team_id=12 MAJOR succeeded #Masked# manual default 0 0 0 --- -#Masked# default ice_orc team_id=13 MAJOR succeeded #Masked# manual default 0 0 0 --- +#Masked# default ice_orc company_id=100/dept_id=1 MAJOR succeeded #Masked# manual default 0 0 0 --- +#Masked# default ice_orc company_id=100/dept_id=2 MAJOR succeeded #Masked# manual default 0 0 0 --- +#Masked# default ice_orc company_id=100/dept_id=4 MAJOR succeeded #Masked# manual default 0 0 0 --- #Masked# default ice_orc --- MAJOR succeeded #Masked# manual default 0 0 0 --- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 780b0c025e01..e0b64f2c4927 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -282,7 +282,7 @@ public Void run() throws Exception { open(); } - /** + /** * Instantiate the metastore server handler directly instead of connecting * through the network * @@ -304,8 +304,8 @@ static ThriftHiveMetastore.Iface callEmbeddedMetastore(Configuration conf) throw try { Class clazz = Class.forName(HIVE_METASTORE_CLASS); //noinspection JavaReflectionMemberAccess - Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, - Configuration.class); + String methodName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_CREATE); + Method method = clazz.getDeclaredMethod(methodName,Configuration.class); method.setAccessible(true); return (ThriftHiveMetastore.Iface) method.invoke(null, conf); } catch (InvocationTargetException e) { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index c5e5fc3fd753..544f26dd9d88 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -911,6 +911,8 @@ public enum ConfVars { HMS_HANDLER_PROXY_CLASS("metastore.hmshandler.proxy", "hive.metastore.hmshandler.proxy", METASTORE_RETRYING_HANDLER_CLASS, "The proxy class name of HMSHandler, default is RetryingHMSHandler."), + HMS_HANDLER_CREATE("metastore.hmshandler.create", "metastore.hmshandler.create","newHMSHandler", + "The method name to create new HMSHandler"), IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus.identifierFactory", "datanucleus1", "Name of the identifier factory to use when generating table/column names etc. \n" + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 986bb2e4e848..0fd998b2df57 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -152,6 +152,16 @@ static Iface newHMSHandler(Configuration conf) HMSHandler baseHandler = new HMSHandler("hive client", conf); return HMSHandlerProxyFactory.getProxy(conf, baseHandler, true); } + + static Iface newHMSRetryingLocalHandler(Configuration conf) + throws MetaException { + HMSHandler baseHandler = new HMSHandler("hive client", conf); + RetryingHMSHandler handler = new RetryingHMSHandler(conf, baseHandler, true); + return (IHMSHandler) java.lang.reflect.Proxy.newProxyInstance( + RetryingHMSHandler.class.getClassLoader(), + new Class[] { IHMSHandler.class }, handler); + } + /** * Discard a current delegation token. From 23a3c909e8a621a4ccd5b6af47bcc2b9c89b3be8 Mon Sep 17 00:00:00 2001 From: Henrib Date: Fri, 31 Jan 2025 19:02:06 +0100 Subject: [PATCH 02/40] HIVE-28059 : major rebase and simpler code; --- .../metastore-catalog/data/conf/README.txt | 1 + .../metastore-catalog/pom.xml | 340 +++++++++ .../apache/iceberg/HiveCachingCatalog.java | 331 +++++++++ .../iceberg/rest/HMSCatalogAdapter.java | 672 ++++++++++++++++++ .../apache/iceberg/rest/HMSCatalogServer.java | 169 +++++ .../iceberg/rest/HMSCatalogServlet.java | 288 ++++++++ .../org/apache/iceberg/hive/HiveUtil.java | 68 ++ .../org/apache/iceberg/rest/HMSTestBase.java | 405 +++++++++++ .../apache/iceberg/rest/TestHMSCatalog.java | 158 ++++ .../auth/jwt/jwt-authorized-key.json | 12 + .../auth/jwt/jwt-unauthorized-key.json | 12 + .../auth/jwt/jwt-verification-jwks.json | 20 + .../src/test/resources/hive-log4j2.properties | 39 + .../hive/metastore/conf/MetastoreConf.java | 14 + .../hadoop/hive/metastore/HiveMetaStore.java | 42 +- .../hive/metastore/HmsThriftHttpServlet.java | 2 +- .../hive/metastore/PropertyServlet.java | 27 +- .../hive/metastore/SecureServletCaller.java | 60 ++ .../hive/metastore/ServletSecurity.java | 44 +- standalone-metastore/pom.xml | 1 + 20 files changed, 2672 insertions(+), 33 deletions(-) create mode 100644 standalone-metastore/metastore-catalog/data/conf/README.txt create mode 100644 standalone-metastore/metastore-catalog/pom.xml create mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java create mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java create mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java create mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java create mode 100644 standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java create mode 100644 standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java create mode 100644 standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java create mode 100644 standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json create mode 100644 standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json create mode 100644 standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json create mode 100644 standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java diff --git a/standalone-metastore/metastore-catalog/data/conf/README.txt b/standalone-metastore/metastore-catalog/data/conf/README.txt new file mode 100644 index 000000000000..0b2f0f032f2f --- /dev/null +++ b/standalone-metastore/metastore-catalog/data/conf/README.txt @@ -0,0 +1 @@ +Need to force creation of a directory. diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml new file mode 100644 index 000000000000..69522c7adb3c --- /dev/null +++ b/standalone-metastore/metastore-catalog/pom.xml @@ -0,0 +1,340 @@ + + + + + hive-standalone-metastore + org.apache.hive + 4.1.0-SNAPSHOT + + 4.0.0 + hive-metastore-icecat + Hive Metastore Iceberg Catalog + + .. + 8 + 8 + UTF-8 + false + ${project.parent.version} + ${hive.version} + 1.6.1 + + + + org.apache.hive + hive-standalone-metastore-server + ${revision} + + + org.apache.hive + hive-standalone-metastore-common + ${revision} + + + org.apache.hive + hive-iceberg-shading + ${revision} + + + org.apache.hive + hive-iceberg-handler + ${revision} + + + org.apache.hive + hive-iceberg-catalog + ${revision} + + + org.apache.iceberg + iceberg-bundled-guava + ${iceberg.version} + + + + + org.apache.httpcomponents.core5 + httpcore5 + 5.2 + + + junit + junit + test + + + com.github.tomakehurst + wiremock-jre8-standalone + 2.32.0 + test + + + org.assertj + assertj-core + 3.19.0 + test + + + org.junit.jupiter + junit-jupiter-api + 5.10.0 + test + + + org.apache.hadoop + hadoop-auth + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + commons-beanutils + commons-beanutils + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + commons-logging + commons-logging + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + commons-logging + commons-logging + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + assemble + none + + single + + + + + + org.apache.rat + apache-rat-plugin + + + process-resources + + check + + + + + + *.patch + DEV-README + **/src/main/sql/** + **/README.md + **/*.iml + **/*.txt + **/*.log + **/package-info.java + **/*.properties + **/*.q + **/*.q.out + **/*.xml + **/gen/** + **/patchprocess/** + **/metastore_db/** + **/test/resources/**/*.ldif + **/test/resources/**/*.sql + **/test/resources/**/*.json + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + TestHMSCatalog.java + TestHiveCatalog.java + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + test + + + + log4j2.debug + false + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + generate-test-sources + + + + + + + + + + + + + + + + + + + + copy + + run + + + + + + + diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java new file mode 100644 index 000000000000..ff55cd943ad1 --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; +import com.github.benmanes.caffeine.cache.Ticker; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + + +/** + * Class that wraps an Iceberg Catalog to cache tables. + * Initial code in: + * https://github.com/apache/iceberg/blob/1.3.x/core/src/main/java/org/apache/iceberg/CachingCatalog.java + * Main difference is the SupportsNamespace and the fact that loadTable performs a metadata refresh. + * + *

See {@link CatalogProperties#CACHE_EXPIRATION_INTERVAL_MS} for more details regarding special + * values for {@code expirationIntervalMillis}. + */ +public class HiveCachingCatalog implements Catalog, SupportsNamespaces { + private static final Logger LOG = LoggerFactory.getLogger(HiveCachingCatalog.class); + @SuppressWarnings("checkstyle:VisibilityModifier") + protected final long expirationIntervalMillis; + @SuppressWarnings("checkstyle:VisibilityModifier") + protected final Cache tableCache; + private final CATALOG catalog; + private final boolean caseSensitive; + + @SuppressWarnings("checkstyle:VisibilityModifier") + protected HiveCachingCatalog(CATALOG catalog, long expirationIntervalMillis) { + Preconditions.checkArgument( + expirationIntervalMillis != 0, + "When %s is set to 0, the catalog cache should be disabled. This indicates a bug.", + CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS); + this.catalog = catalog; + this.caseSensitive = true; + this.expirationIntervalMillis = expirationIntervalMillis; + this.tableCache = createTableCache(Ticker.systemTicker()); + } + + public CATALOG unwrap() { + return catalog; + } + + public static + HiveCachingCatalog wrap(C catalog, long expirationIntervalMillis) { + return new HiveCachingCatalog(catalog, expirationIntervalMillis); + } + + + private Cache createTableCache(Ticker ticker) { + Caffeine cacheBuilder = Caffeine.newBuilder().softValues(); + if (expirationIntervalMillis > 0) { + return cacheBuilder + .removalListener(new MetadataTableInvalidatingRemovalListener()) + .executor(Runnable::run) // Makes the callbacks to removal listener synchronous + .expireAfterAccess(Duration.ofMillis(expirationIntervalMillis)) + .ticker(ticker) + .build(); + } + return cacheBuilder.build(); + } + + private TableIdentifier canonicalizeIdentifier(TableIdentifier tableIdentifier) { + return caseSensitive ? tableIdentifier : tableIdentifier.toLowerCase(); + } + + @Override + public String name() { + return catalog.name(); + } + + @Override + public List listTables(Namespace namespace) { + return catalog.listTables(namespace); + } + + private Table loadTableHive(TableIdentifier ident) { + return catalog.loadTable(ident); + } + + @Override + public Table loadTable(TableIdentifier ident) { + TableIdentifier canonicalized = canonicalizeIdentifier(ident); + Table cached = tableCache.getIfPresent(canonicalized); + if (cached != null) { + return cached; + } + + if (MetadataTableUtils.hasMetadataTableName(canonicalized)) { + TableIdentifier originTableIdentifier = + TableIdentifier.of(canonicalized.namespace().levels()); + Table originTable = tableCache.get(originTableIdentifier, this::loadTableHive); + + // share TableOperations instance of origin table for all metadata tables, so that metadata + // table instances are + // also refreshed as well when origin table instance is refreshed. + if (originTable instanceof HasTableOperations) { + TableOperations ops = ((HasTableOperations) originTable).operations(); + MetadataTableType type = MetadataTableType.from(canonicalized.name()); + Table metadataTable = + MetadataTableUtils.createMetadataTableInstance( + ops, catalog.name(), originTableIdentifier, canonicalized, type); + tableCache.put(canonicalized, metadataTable); + return metadataTable; + } + } + return tableCache.get(canonicalized, this::loadTableHive); + } + + @Override + public boolean dropTable(TableIdentifier ident, boolean purge) { + boolean dropped = catalog.dropTable(ident, purge); + invalidateTable(ident); + return dropped; + } + + @Override + public void renameTable(TableIdentifier from, TableIdentifier to) { + catalog.renameTable(from, to); + invalidateTable(from); + } + + @Override + public void invalidateTable(TableIdentifier ident) { + catalog.invalidateTable(ident); + TableIdentifier canonicalized = canonicalizeIdentifier(ident); + tableCache.invalidate(canonicalized); + tableCache.invalidateAll(metadataTableIdentifiers(canonicalized)); + } + + @Override + public Table registerTable(TableIdentifier identifier, String metadataFileLocation) { + Table table = catalog.registerTable(identifier, metadataFileLocation); + invalidateTable(identifier); + return table; + } + + private Iterable metadataTableIdentifiers(TableIdentifier ident) { + ImmutableList.Builder builder = ImmutableList.builder(); + + for (MetadataTableType type : MetadataTableType.values()) { + // metadata table resolution is case insensitive right now + builder.add(TableIdentifier.parse(ident + "." + type.name())); + builder.add(TableIdentifier.parse(ident + "." + type.name().toLowerCase(Locale.ROOT))); + } + + return builder.build(); + } + + @Override + public TableBuilder buildTable(TableIdentifier identifier, Schema schema) { + return new CachingTableBuilder(identifier, schema); + } + + @Override + public void createNamespace(Namespace nmspc, Map map) { + catalog.createNamespace(nmspc, map); + } + + @Override + public List listNamespaces(Namespace nmspc) throws NoSuchNamespaceException { + return catalog.listNamespaces(nmspc); + } + + @Override + public Map loadNamespaceMetadata(Namespace nmspc) throws NoSuchNamespaceException { + return catalog.loadNamespaceMetadata(nmspc); + } + + @Override + public boolean dropNamespace(Namespace nmspc) throws NamespaceNotEmptyException { + List tables = listTables(nmspc); + for (TableIdentifier ident : tables) { + TableIdentifier canonicalized = canonicalizeIdentifier(ident); + tableCache.invalidate(canonicalized); + tableCache.invalidateAll(metadataTableIdentifiers(canonicalized)); + } + return catalog.dropNamespace(nmspc); + } + + @Override + public boolean setProperties(Namespace nmspc, Map map) throws NoSuchNamespaceException { + return catalog.setProperties(nmspc, map); + } + + @Override + public boolean removeProperties(Namespace nmspc, Set set) throws NoSuchNamespaceException { + return catalog.removeProperties(nmspc, set); + } + + /** + * RemovalListener class for removing metadata tables when their associated data table is expired + * via cache expiration. + */ + class MetadataTableInvalidatingRemovalListener + implements RemovalListener { + @Override + public void onRemoval(TableIdentifier tableIdentifier, Table table, RemovalCause cause) { + LOG.debug("Evicted {} from the table cache ({})", tableIdentifier, cause); + if (RemovalCause.EXPIRED.equals(cause)) { + if (!MetadataTableUtils.hasMetadataTableName(tableIdentifier)) { + tableCache.invalidateAll(metadataTableIdentifiers(tableIdentifier)); + } + } + } + } + + private class CachingTableBuilder implements TableBuilder { + private final TableIdentifier ident; + private final TableBuilder innerBuilder; + + private CachingTableBuilder(TableIdentifier identifier, Schema schema) { + this.innerBuilder = catalog.buildTable(identifier, schema); + this.ident = identifier; + } + + @Override + public TableBuilder withPartitionSpec(PartitionSpec spec) { + innerBuilder.withPartitionSpec(spec); + return this; + } + + @Override + public TableBuilder withSortOrder(SortOrder sortOrder) { + innerBuilder.withSortOrder(sortOrder); + return this; + } + + @Override + public TableBuilder withLocation(String location) { + innerBuilder.withLocation(location); + return this; + } + + @Override + public TableBuilder withProperties(Map properties) { + innerBuilder.withProperties(properties); + return this; + } + + @Override + public TableBuilder withProperty(String key, String value) { + innerBuilder.withProperty(key, value); + return this; + } + + @Override + public Table create() { + AtomicBoolean created = new AtomicBoolean(false); + Table table = + tableCache.get( + canonicalizeIdentifier(ident), + identifier -> { + created.set(true); + return innerBuilder.create(); + }); + if (!created.get()) { + throw new AlreadyExistsException("Table already exists: %s", ident); + } + return table; + } + + @Override + public Transaction createTransaction() { + // create a new transaction without altering the cache. the table doesn't exist until the + // transaction is + // committed. if the table is created before the transaction commits, any cached version is + // correct and the + // transaction create will fail. if the transaction commits before another create, then the + // cache will be empty. + return innerBuilder.createTransaction(); + } + + @Override + public Transaction replaceTransaction() { + // create a new transaction without altering the cache. the table doesn't change until the + // transaction is + // committed. when the transaction commits, invalidate the table in the cache if it is + // present. + return CommitCallbackTransaction.addCallback( + innerBuilder.replaceTransaction(), () -> invalidateTable(ident)); + } + + @Override + public Transaction createOrReplaceTransaction() { + // create a new transaction without altering the cache. the table doesn't change until the + // transaction is + // committed. when the transaction commits, invalidate the table in the cache if it is + // present. + return CommitCallbackTransaction.addCallback( + innerBuilder.createOrReplaceTransaction(), () -> invalidateTable(ident)); + } + } +} diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java new file mode 100644 index 000000000000..c598f7b64d5c --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -0,0 +1,672 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import com.codahale.metrics.Counter; +import org.apache.hadoop.hive.metastore.metrics.Metrics; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.BaseTransaction; +import org.apache.iceberg.Table; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.Transactions; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.exceptions.RESTException; +import org.apache.iceberg.exceptions.UnprocessableEntityException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.requests.CommitTransactionRequest; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.RegisterTableRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; +import org.apache.iceberg.rest.requests.ReportMetricsRequest; +import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.CreateNamespaceResponse; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.apache.iceberg.util.Pair; +import org.apache.iceberg.util.PropertyUtil; + +/** + * Original @ https://github.com/apache/iceberg/blob/main/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java + * Adaptor class to translate REST requests into {@link Catalog} API calls. + */ +public class HMSCatalogAdapter implements RESTClient { + /** The metric names prefix. */ + static final String HMS_METRIC_PREFIX = "hmscatalog."; + private static final Splitter SLASH = Splitter.on('/'); + + private static final Map, Integer> EXCEPTION_ERROR_CODES = + ImmutableMap., Integer>builder() + .put(NamespaceNotSupported.class, 400) + .put(IllegalArgumentException.class, 400) + .put(ValidationException.class, 400) + .put(NamespaceNotEmptyException.class, 400) // TODO: should this be more specific? + .put(NotAuthorizedException.class, 401) + .put(ForbiddenException.class, 403) + .put(NoSuchNamespaceException.class, 404) + .put(NoSuchTableException.class, 404) + .put(NoSuchIcebergTableException.class, 404) + .put(UnsupportedOperationException.class, 406) + .put(AlreadyExistsException.class, 409) + .put(CommitFailedException.class, 409) + .put(UnprocessableEntityException.class, 422) + .put(CommitStateUnknownException.class, 500) + .buildOrThrow(); + + private final Catalog catalog; + private final SupportsNamespaces asNamespaceCatalog; + + public HMSCatalogAdapter(Catalog catalog) { + this.catalog = catalog; + this.asNamespaceCatalog = + catalog instanceof SupportsNamespaces ? (SupportsNamespaces) catalog : null; + } + + enum HTTPMethod { + GET, + HEAD, + POST, + DELETE + } + + enum Route { + TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null, OAuthTokenResponse.class), + CONFIG(HTTPMethod.GET, "v1/config", null, ConfigResponse.class), + LIST_NAMESPACES(HTTPMethod.GET, "v1/namespaces", null, ListNamespacesResponse.class), + CREATE_NAMESPACE( + HTTPMethod.POST, + "v1/namespaces", + CreateNamespaceRequest.class, + CreateNamespaceResponse.class), + LOAD_NAMESPACE(HTTPMethod.GET, "v1/namespaces/{namespace}", null, GetNamespaceResponse.class), + DROP_NAMESPACE(HTTPMethod.DELETE, "v1/namespaces/{namespace}"), + UPDATE_NAMESPACE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/properties", + UpdateNamespacePropertiesRequest.class, + UpdateNamespacePropertiesResponse.class), + LIST_TABLES(HTTPMethod.GET, "v1/namespaces/{namespace}/tables", null, ListTablesResponse.class), + CREATE_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables", + CreateTableRequest.class, + LoadTableResponse.class), + LOAD_TABLE( + HTTPMethod.GET, "v1/namespaces/{namespace}/tables/{table}", null, LoadTableResponse.class), + REGISTER_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/register", + RegisterTableRequest.class, + LoadTableResponse.class), + UPDATE_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables/{table}", + UpdateTableRequest.class, + LoadTableResponse.class), + DROP_TABLE(HTTPMethod.DELETE, "v1/namespaces/{namespace}/tables/{table}"), + RENAME_TABLE(HTTPMethod.POST, "v1/tables/rename", RenameTableRequest.class, null), + REPORT_METRICS( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables/{table}/metrics", + ReportMetricsRequest.class, + null), + COMMIT_TRANSACTION( + HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null); + + private final HTTPMethod method; + private final int requiredLength; + private final Map requirements; + private final Map variables; + private final Class requestClass; + private final Class responseClass; + + /** + * An exception safe way of getting a route by name. + * + * @param name the route name + * @return the route instance or null if it could not be found + */ + static Route byName(String name) { + try { + return valueOf(name.toUpperCase()); + } catch (IllegalArgumentException xill) { + return null; + } + } + + Route(HTTPMethod method, String pattern) { + this(method, pattern, null, null); + } + + Route( + HTTPMethod method, + String pattern, + Class requestClass, + Class responseClass + ) { + this.method = method; + // parse the pattern into requirements and variables + List parts = SLASH.splitToList(pattern); + ImmutableMap.Builder requirementsBuilder = ImmutableMap.builder(); + ImmutableMap.Builder variablesBuilder = ImmutableMap.builder(); + for (int pos = 0; pos < parts.size(); pos += 1) { + String part = parts.get(pos); + if (part.startsWith("{") && part.endsWith("}")) { + variablesBuilder.put(pos, part.substring(1, part.length() - 1)); + } else { + requirementsBuilder.put(pos, part); + } + } + + this.requestClass = requestClass; + this.responseClass = responseClass; + + this.requiredLength = parts.size(); + this.requirements = requirementsBuilder.build(); + this.variables = variablesBuilder.build(); + } + + private boolean matches(HTTPMethod requestMethod, List requestPath) { + return method == requestMethod && + requiredLength == requestPath.size() && + requirements.entrySet().stream() + .allMatch( + requirement -> + requirement + .getValue() + .equalsIgnoreCase(requestPath.get(requirement.getKey()))); + } + + private Map variables(List requestPath) { + ImmutableMap.Builder vars = ImmutableMap.builder(); + variables.forEach((key, value) -> vars.put(value, requestPath.get(key))); + return vars.build(); + } + + public static Pair> from(HTTPMethod method, String path) { + List parts = SLASH.splitToList(path); + for (Route candidate : Route.values()) { + if (candidate.matches(method, parts)) { + return Pair.of(candidate, candidate.variables(parts)); + } + } + + return null; + } + + public Class requestClass() { + return requestClass; + } + + public Class responseClass() { + return responseClass; + } + } + + /** + * @param route a route/api-call name + * @return the metric counter name for the api-call + */ + static String hmsCatalogMetricCount(String route) { + return HMS_METRIC_PREFIX + route.toLowerCase() + ".count"; + } + + /** + * @param apis an optional list of known api call names + * @return the list of metric names for the HMSCatalog class + */ + public static List getMetricNames(String... apis) { + final List routes; + if (apis != null && apis.length > 0) { + routes = Arrays.stream(apis) + .map(HMSCatalogAdapter.Route::byName) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } else { + routes = Arrays.asList(HMSCatalogAdapter.Route.values()); + } + final List metricNames = new ArrayList<>(routes.size()); + for (HMSCatalogAdapter.Route route : routes) { + metricNames.add(hmsCatalogMetricCount(route.name())); + } + return metricNames; + } + + private ConfigResponse config(Map vars, Object body) { + return castResponse(ConfigResponse.class, ConfigResponse.builder().build()); + } + + private OAuthTokenResponse tokens(Map vars, Object body) { + Class responseType = OAuthTokenResponse.class; + @SuppressWarnings("unchecked") + Map request = (Map) castRequest(Map.class, body); + String grantType = request.get("grant_type"); + switch (grantType) { + case "client_credentials": + return castResponse( + responseType, + OAuthTokenResponse.builder() + .withToken("client-credentials-token:sub=" + request.get("client_id")) + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .withTokenType("Bearer") + .build()); + + case "urn:ietf:params:oauth:grant-type:token-exchange": + String actor = request.get("actor_token"); + String token = + String.format( + "token-exchange-token:sub=%s%s", + request.get("subject_token"), actor != null ? ",act=" + actor : ""); + return castResponse( + responseType, + OAuthTokenResponse.builder() + .withToken(token) + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .withTokenType("Bearer") + .build()); + + default: + throw new UnsupportedOperationException("Unsupported grant_type: " + grantType); + } + } + + private ListNamespacesResponse listNamespaces(Map vars, Object body) { + if (asNamespaceCatalog != null) { + Namespace ns; + if (vars.containsKey("parent")) { + ns = Namespace.of(RESTUtil.NAMESPACE_SPLITTER.splitToStream(vars.get("parent")).toArray(String[]::new)); + } else { + ns = Namespace.empty(); + } + return castResponse(ListNamespacesResponse.class, CatalogHandlers.listNamespaces(asNamespaceCatalog, ns)); + } + throw new NamespaceNotSupported(catalog.toString()); + } + + private CreateNamespaceResponse createNamespace(Map vars, Object body) { + if (asNamespaceCatalog != null) { + CreateNamespaceRequest request = castRequest(CreateNamespaceRequest.class, body); + return castResponse( + CreateNamespaceResponse.class, CatalogHandlers.createNamespace(asNamespaceCatalog, request)); + } + throw new NamespaceNotSupported(catalog.toString()); + } + + private GetNamespaceResponse loadNamespace(Map vars, Object body) { + if (asNamespaceCatalog != null) { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse( + GetNamespaceResponse.class, CatalogHandlers.loadNamespace(asNamespaceCatalog, namespace)); + } + throw new NamespaceNotSupported(catalog.toString()); + } + + private RESTResponse dropNamespace(Map vars, Object body) { + if (asNamespaceCatalog != null) { + CatalogHandlers.dropNamespace(asNamespaceCatalog, namespaceFromPathVars(vars)); + } + throw new NamespaceNotSupported(catalog.toString()); + } + + private UpdateNamespacePropertiesResponse updateNamespace(Map vars, Object body) { + if (asNamespaceCatalog != null) { + Namespace namespace = namespaceFromPathVars(vars); + UpdateNamespacePropertiesRequest request = + castRequest(UpdateNamespacePropertiesRequest.class, body); + return castResponse( + UpdateNamespacePropertiesResponse.class, + CatalogHandlers.updateNamespaceProperties(asNamespaceCatalog, namespace, request)); + } + throw new NamespaceNotSupported(catalog.toString()); + } + + private ListTablesResponse listTables(Map vars, Object body) { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse(ListTablesResponse.class, CatalogHandlers.listTables(catalog, namespace)); + } + + private LoadTableResponse createTable(Map vars, Object body) { + final Class responseType = LoadTableResponse.class; + Namespace namespace = namespaceFromPathVars(vars); + CreateTableRequest request = castRequest(CreateTableRequest.class, body); + request.validate(); + if (request.stageCreate()) { + return castResponse( + responseType, CatalogHandlers.stageTableCreate(catalog, namespace, request)); + } else { + return castResponse( + responseType, CatalogHandlers.createTable(catalog, namespace, request)); + } + } + + private RESTResponse dropTable(Map vars, Object body) { + final Class responseType = RESTResponse.class; + if (PropertyUtil.propertyAsBoolean(vars, "purgeRequested", false)) { + CatalogHandlers.purgeTable(catalog, identFromPathVars(vars)); + } else { + CatalogHandlers.dropTable(catalog, identFromPathVars(vars)); + } + return null; + } + + private LoadTableResponse loadTable(Map vars, Object body) { + TableIdentifier ident = identFromPathVars(vars); + return castResponse(LoadTableResponse.class, CatalogHandlers.loadTable(catalog, ident)); + } + + private LoadTableResponse registerTable(Map vars, Object body) { + Namespace namespace = namespaceFromPathVars(vars); + RegisterTableRequest request = castRequest(RegisterTableRequest.class, body); + return castResponse(LoadTableResponse.class, CatalogHandlers.registerTable(catalog, namespace, request)); + } + + private LoadTableResponse updateTable(Map vars, Object body) { + TableIdentifier ident = identFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse(LoadTableResponse.class, CatalogHandlers.updateTable(catalog, ident, request)); + } + + private RESTResponse renameTable(Map vars, Object body) { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameTable(catalog, request); + return null; + } + + private RESTResponse reportMetrics(Map vars, Object body) { + // nothing to do here other than checking that we're getting the correct request + castRequest(ReportMetricsRequest.class, body); + return null; + } + + private RESTResponse commitTransaction(Map vars, Object body) { + CommitTransactionRequest request = castRequest(CommitTransactionRequest.class, body); + commitTransaction(catalog, request); + return null; + } + + + @SuppressWarnings("MethodLength") + private T handleRequest( + Route route, Map vars, Object body, Class responseType) { + // update HMS catalog route counter metric + final String metricName = hmsCatalogMetricCount(route.name()); + Counter counter = Metrics.getOrCreateCounter(metricName); + if (counter != null) { + counter.inc(); + } + switch (route) { + case TOKENS: + return (T) tokens(vars, body); + + case CONFIG: + return (T) config(vars, body); + + case LIST_NAMESPACES: + return (T) listNamespaces(vars, body); + + case CREATE_NAMESPACE: + return (T) createNamespace(vars, body); + + case LOAD_NAMESPACE: + return (T) loadNamespace(vars, body); + + case DROP_NAMESPACE: + return (T) dropNamespace(vars, body); + + case UPDATE_NAMESPACE: + return (T) updateNamespace(vars, body); + + case LIST_TABLES: + return (T) listTables(vars, body); + + case CREATE_TABLE: + return (T) createTable(vars, body); + + case DROP_TABLE: + return (T) dropTable(vars, body); + + case LOAD_TABLE: + return (T) loadTable(vars, body); + + case REGISTER_TABLE: + return (T) registerTable(vars, body); + + case UPDATE_TABLE: + return (T) updateTable(vars, body); + + case RENAME_TABLE: + return (T) renameTable(vars, body); + + case REPORT_METRICS: + return (T) reportMetrics(vars, body); + + case COMMIT_TRANSACTION: + return (T) commitTransaction(vars, body); + + default: + } + return null; + } + + /** + * This is a very simplistic approach that only validates the requirements for each table and does + * not do any other conflict detection. Therefore, it does not guarantee true transactional + * atomicity, which is left to the implementation details of a REST server. + */ + private static void commitTransaction(Catalog catalog, CommitTransactionRequest request) { + List transactions = Lists.newArrayList(); + + for (UpdateTableRequest tableChange : request.tableChanges()) { + Table table = catalog.loadTable(tableChange.identifier()); + if (table instanceof BaseTable) { + Transaction transaction = + Transactions.newTransaction( + tableChange.identifier().toString(), ((BaseTable) table).operations()); + transactions.add(transaction); + + BaseTransaction.TransactionTable txTable = + (BaseTransaction.TransactionTable) transaction.table(); + + // this performs validations and makes temporary commits that are in-memory + CatalogHandlers.commit(txTable.operations(), tableChange); + } else { + throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); + } + } + // only commit if validations passed previously + transactions.forEach(Transaction::commitTransaction); + } + + T execute( + HTTPMethod method, + String path, + Map queryParams, + Object body, + Class responseType, + Map headers, + Consumer errorHandler) { + ErrorResponse.Builder errorBuilder = ErrorResponse.builder(); + Pair> routeAndVars = Route.from(method, path); + if (routeAndVars != null) { + try { + ImmutableMap.Builder vars = ImmutableMap.builder(); + if (queryParams != null) { + vars.putAll(queryParams); + } + vars.putAll(routeAndVars.second()); + return handleRequest(routeAndVars.first(), vars.build(), body, responseType); + } catch (RuntimeException e) { + configureResponseFromException(e, errorBuilder); + } + + } else { + errorBuilder + .responseCode(400) + .withType("BadRequestException") + .withMessage(String.format("No route for request: %s %s", method, path)); + } + ErrorResponse error = errorBuilder.build(); + errorHandler.accept(error); + // if the error handler doesn't throw an exception, throw a generic one + throw new RESTException("Unhandled error: %s", error); + } + + @Override + public T delete( + String path, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.DELETE, path, null, null, responseType, headers, errorHandler); + } + + @Override + public T delete( + String path, + Map queryParams, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.DELETE, path, queryParams, null, responseType, headers, errorHandler); + } + + @Override + public T post( + String path, + RESTRequest body, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.POST, path, null, body, responseType, headers, errorHandler); + } + + @Override + public T get( + String path, + Map queryParams, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.GET, path, queryParams, null, responseType, headers, errorHandler); + } + + @Override + public void head(String path, Map headers, Consumer errorHandler) { + execute(HTTPMethod.HEAD, path, null, null, null, headers, errorHandler); + } + + @Override + public T postForm( + String path, + Map formData, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.POST, path, null, formData, responseType, headers, errorHandler); + } + + @Override + public void close() throws IOException { + // The calling test is responsible for closing the underlying catalog backing this REST catalog + // so that the underlying backend catalog is not closed and reopened during the REST catalog's + // initialize method when fetching the server configuration. + } + + private static class NamespaceNotSupported extends RuntimeException { + NamespaceNotSupported(String catalog) { + super("catalog " + catalog + " does not support namespace"); + } + } + + private static class BadResponseType extends RuntimeException { + private BadResponseType(Class responseType, Object response) { + super( + String.format("Invalid response object, not a %s: %s", responseType.getName(), response)); + } + } + + private static class BadRequestType extends RuntimeException { + private BadRequestType(Class requestType, Object request) { + super(String.format("Invalid request object, not a %s: %s", requestType.getName(), request)); + } + } + + public static T castRequest(Class requestType, Object request) { + if (requestType.isInstance(request)) { + return requestType.cast(request); + } + throw new BadRequestType(requestType, request); + } + + public static T castResponse(Class responseType, Object response) { + if (responseType.isInstance(response)) { + return responseType.cast(response); + } + throw new BadResponseType(responseType, response); + } + + public static void configureResponseFromException( + Exception exc, ErrorResponse.Builder errorBuilder) { + errorBuilder + .responseCode(EXCEPTION_ERROR_CODES.getOrDefault(exc.getClass(), 500)) + .withType(exc.getClass().getSimpleName()) + .withMessage(exc.getMessage()) + .withStackTrace(exc); + } + + private static Namespace namespaceFromPathVars(Map pathVars) { + return RESTUtil.decodeNamespace(pathVars.get("namespace")); + } + + private static TableIdentifier identFromPathVars(Map pathVars) { + return TableIdentifier.of( + namespaceFromPathVars(pathVars), RESTUtil.decodeString(pathVars.get("table"))); + } +} diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java new file mode 100644 index 000000000000..6ddaf2331d1f --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import java.io.IOException; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.util.Collections; +import java.util.Map; +import java.util.TreeMap; +import javax.servlet.http.HttpServlet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.SecureServletCaller; +import org.apache.hadoop.hive.metastore.ServletSecurity; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.HiveCachingCatalog; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.hive.HiveCatalog; +import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.gzip.GzipHandler; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class HMSCatalogServer { + private static final String CACHE_EXPIRY = "hive.metastore.catalog.cache.expiry"; + private static final String JETTY_THREADPOOL_MIN = "hive.metastore.catalog.jetty.threadpool.min"; + private static final String JETTY_THREADPOOL_MAX = "hive.metastore.catalog.jetty.threadpool.max"; + private static final String JETTY_THREADPOOL_IDLE = "hive.metastore.catalog.jetty.threadpool.idle"; + private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class); + private static Reference catalogRef; + + static Catalog getLastCatalog() { + return catalogRef != null ? catalogRef.get() : null; + } + + private HMSCatalogServer() { + // nothing + } + + public static HttpServlet createServlet(SecureServletCaller security, Catalog catalog) throws IOException { + return new HMSCatalogServlet(security, new HMSCatalogAdapter(catalog)); + } + + public static Catalog createCatalog(Configuration configuration) { + final String curi = configuration.get(MetastoreConf.ConfVars.THRIFT_URIS.getVarname()); + final String cwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE.getVarname()); + final String cextwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); + MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); + MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.HMS_HANDLER_CREATE, "newHMSRetryingLocalHandler"); + final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog(); + catalog.setConf(configuration); + Map properties = new TreeMap<>(); + if (curi != null) { + properties.put("uri", curi); + } + if (cwarehouse != null) { + properties.put("warehouse", cwarehouse); + } + if (cextwarehouse != null) { + properties.put("external-warehouse", cextwarehouse); + } + catalog.initialize("hive", properties); + long expiry = configuration.getLong(CACHE_EXPIRY, 60_000L); + return expiry > 0? HiveCachingCatalog.wrap(catalog, expiry) : catalog; + } + + public static HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException { + String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH); + boolean jwt = "jwt".equalsIgnoreCase(auth); + SecureServletCaller security = new ServletSecurity(configuration, jwt); + Catalog actualCatalog = catalog; + if (actualCatalog == null) { + actualCatalog = createCatalog(configuration); + actualCatalog.initialize("hive", Collections.emptyMap()); + } + catalogRef = new SoftReference<>(actualCatalog); + return createServlet(security, actualCatalog); + } + + /** + * Convenience method to start a http server that only serves this servlet. + * @param conf the configuration + * @param catalog the catalog instance to serve + * @return the server instance + * @throws Exception if servlet initialization fails + */ + public static Server startServer(Configuration conf, HiveCatalog catalog) throws Exception { + int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT); + if (port < 0) { + return null; + } + final HttpServlet servlet = createServlet(conf, catalog); + ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.setContextPath("/"); + ServletHolder servletHolder = new ServletHolder(servlet); + servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic"); + final String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH); + context.addServlet(servletHolder, "/" + cli + "/*"); + context.setVirtualHosts(null); + context.setGzipHandler(new GzipHandler()); + + final Server httpServer = createHttpServer(conf, port); + httpServer.setHandler(context); + LOG.info("Starting HMS REST Catalog Server with context path:/{}/ on port:{}", cli, port); + httpServer.start(); + return httpServer; + } + + private static Server createHttpServer(Configuration conf, int port) throws IOException { + final int maxThreads = conf.getInt(JETTY_THREADPOOL_MAX, 256); + final int minThreads = conf.getInt(JETTY_THREADPOOL_MIN, 8); + final int idleTimeout = conf.getInt(JETTY_THREADPOOL_IDLE, 60_000); + final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout); + final Server httpServer = new Server(threadPool); + final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf); + final ServerConnector connector = new ServerConnector(httpServer, sslContextFactory); + connector.setPort(port); + connector.setReuseAddress(true); + httpServer.setConnectors(new Connector[] {connector}); + for (ConnectionFactory factory : connector.getConnectionFactories()) { + if (factory instanceof HttpConnectionFactory) { + HttpConnectionFactory httpFactory = (HttpConnectionFactory) factory; + HttpConfiguration httpConf = httpFactory.getHttpConfiguration(); + httpConf.setSendServerVersion(false); + httpConf.setSendXPoweredBy(false); + } + } + return httpServer; + } + + + /** + * Convenience method to start a http server that only serves this servlet. + *

This one is looked up through reflection to start from HMS.

+ * @param conf the configuration + * @return the server instance + * @throws Exception if servlet initialization fails + */ + public static Server startServer(Configuration conf) throws Exception { + return startServer(conf, null); + } +} diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java new file mode 100644 index 000000000000..d5be0a92522f --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.hive.metastore.SecureServletCaller; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.io.CharStreams; +import org.apache.iceberg.rest.HMSCatalogAdapter.HTTPMethod; +import org.apache.iceberg.rest.HMSCatalogAdapter.Route; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Original @ https://github.com/apache/iceberg/blob/main/core/src/test/java/org/apache/iceberg/rest/RESTCatalogServlet.java + * The RESTCatalogServlet provides a servlet implementation used in combination with a + * RESTCatalogAdaptor to proxy the REST Spec to any Catalog implementation. + */ +public class HMSCatalogServlet extends HttpServlet { + private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServlet.class); + /** + * The security. + */ + private final SecureServletCaller security; + + private final HMSCatalogAdapter restCatalogAdapter; + private final Map responseHeaders = + ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + + public HMSCatalogServlet(SecureServletCaller security, HMSCatalogAdapter restCatalogAdapter) { + this.security = security; + this.restCatalogAdapter = restCatalogAdapter; + } + + @Override + public void init() throws ServletException { + super.init(); + security.init(); + } + @Override + protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String method = req.getMethod(); + if (!"PATCH".equals(method)) { + super.service(req, resp); + } else { + this.doPatch(req, resp); + } + } + + protected void doPatch(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("PATCH failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("GET failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + @Override + protected void doPut(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("PUT failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + @Override + protected void doHead(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("HEAD failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("POST failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + @Override + protected void doDelete(HttpServletRequest request, HttpServletResponse response) { + try { + security.execute(request, response, this::execute); + } catch (IOException e) { + LOG.error("DELETE failed", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + private void execute(HttpServletRequest request, HttpServletResponse response) { + try { + ServletRequestContext context = ServletRequestContext.from(request); + response.setStatus(HttpServletResponse.SC_OK); + responseHeaders.forEach(response::setHeader); + + final Optional error = context.error(); + if (error.isPresent()) { + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + RESTObjectMapper.mapper().writeValue(response.getWriter(), error.get()); + return; + } + Object responseBody = + restCatalogAdapter.execute( + context.method(), + context.path(), + context.queryParams(), + context.body(), + context.route().responseClass(), + context.headers(), + handle(response)); + + if (responseBody != null) { + RESTObjectMapper.mapper().writeValue(response.getWriter(), responseBody); + } + } catch (RuntimeException e) { + // should be a RESTException but not able to see them through dependencies + LOG.error("Error processing REST request", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } catch (Exception e) { + LOG.error("Unexpected exception when processing REST request", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + + protected Consumer handle(HttpServletResponse response) { + return errorResponse -> { + response.setStatus(errorResponse.code()); + try { + RESTObjectMapper.mapper().writeValue(response.getWriter(), errorResponse); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } + + public static class ServletRequestContext { + private HTTPMethod method; + private Route route; + private String path; + private Map headers; + private Map queryParams; + private Object body; + + private ErrorResponse errorResponse; + + private ServletRequestContext(ErrorResponse errorResponse) { + this.errorResponse = errorResponse; + } + + private ServletRequestContext( + HTTPMethod method, + Route route, + String path, + Map headers, + Map queryParams, + Object body) { + this.method = method; + this.route = route; + this.path = path; + this.headers = headers; + this.queryParams = queryParams; + this.body = body; + } + + static ServletRequestContext from(HttpServletRequest request) throws IOException { + HTTPMethod method = HTTPMethod.valueOf(request.getMethod()); + // path = uri - context-path + servlet-path + / + String path = request.getPathInfo(); + if (path == null) { + path = request.getRequestURI().substring( + request.getContextPath().length() + request.getServletPath().length()); + } + // remove leading / + path = path.substring(1); + Pair> routeContext = Route.from(method, path); + + if (routeContext == null) { + return new ServletRequestContext( + ErrorResponse.builder() + .responseCode(400) + .withType("BadRequestException") + .withMessage(String.format("No route for request: %s %s", method, path)) + .build()); + } + + Route route = routeContext.first(); + Object requestBody = null; + if (route.requestClass() != null) { + requestBody = + RESTObjectMapper.mapper().readValue(request.getReader(), route.requestClass()); + } else if (route == Route.TOKENS) { + try (Reader reader = new InputStreamReader(request.getInputStream())) { + requestBody = RESTUtil.decodeFormData(CharStreams.toString(reader)); + } + } + + Map queryParams = + request.getParameterMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue()[0])); + Map headers = + Collections.list(request.getHeaderNames()).stream() + .collect(Collectors.toMap(Function.identity(), request::getHeader)); + + return new ServletRequestContext(method, route, path, headers, queryParams, requestBody); + } + + public HTTPMethod method() { + return method; + } + + public Route route() { + return route; + } + + public String path() { + return path; + } + + public Map headers() { + return headers; + } + + public Map queryParams() { + return queryParams; + } + + public Object body() { + return body; + } + + public Optional error() { + return Optional.ofNullable(errorResponse); + } + } +} diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java new file mode 100644 index 000000000000..fb31799b5d6d --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iceberg.hive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.hive.HiveCatalog; +import org.apache.iceberg.hive.HiveTableOperations; +import org.apache.iceberg.io.FileIO; + +import java.util.Map; + +/** + * A Friend bridge to Iceberg. + */ +public class HiveUtil { + + public static final HiveTableOperations newTableOperations(Configuration conf, String catalogName, String database, String table) { + return new HiveTableOperations(conf, null, null, catalogName, database, table); + } + + public static final HiveTableOperations newTableOperations(Configuration conf, ClientPool metaClients, FileIO fileIO, String catalogName, String database, String table) { + return new HiveTableOperations(conf, null, null, catalogName, database, table); + } + + public static Database convertToDatabase(HiveCatalog catalog, Namespace ns, Map meta) { + return catalog.convertToDatabase(ns, meta); + } + + public static void setSnapshotSummary(HiveTableOperations ops, Map parameters, Snapshot snapshot) { + ops.setSnapshotSummary(parameters, snapshot); + } + + public static void setSnapshotStats(HiveTableOperations ops, TableMetadata metadata, Map parameters) { + ops.setSnapshotStats(metadata, parameters); + } + + public static void setSchema(HiveTableOperations ops, TableMetadata metadata, Map parameters) { + ops.setSchema(metadata.schema(), parameters); + } + + public static void setPartitionSpec(HiveTableOperations ops, TableMetadata metadata, Map parameters) { + ops.setPartitionSpec(metadata, parameters); + } + + public static void setSortOrder(HiveTableOperations ops, TableMetadata metadata, Map parameters) { + ops.setSortOrder(metadata, parameters); + } +} diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java new file mode 100644 index 000000000000..8e1fb3691a53 --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.MetricRegistry; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; +import com.google.gson.ToNumberStrategy; +import com.nimbusds.jose.JWSAlgorithm; +import com.nimbusds.jose.JWSHeader; +import com.nimbusds.jose.JWSSigner; +import com.nimbusds.jose.crypto.RSASSASigner; +import com.nimbusds.jose.jwk.RSAKey; +import com.nimbusds.jwt.JWTClaimsSet; +import com.nimbusds.jwt.SignedJWT; +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import javax.servlet.http.HttpServletResponse; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.metrics.Metrics; +import org.apache.hadoop.hive.metastore.properties.HMSPropertyManager; +import org.apache.hadoop.hive.metastore.properties.PropertyManager; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.hive.HiveCatalog; +import org.eclipse.jetty.server.Server; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class HMSTestBase { + protected static final Logger LOG = LoggerFactory.getLogger(HMSTestBase.class.getName()); + protected static final String BASE_DIR = System.getProperty("basedir"); + protected static Random RND = new Random(20230922); + protected static final String USER_1 = "USER_1"; + protected static final String DB_NAME = "hivedb"; + + protected static final long EVICTION_INTERVAL = TimeUnit.SECONDS.toMillis(10); + private static final File JWT_AUTHKEY_FILE = + new File(BASE_DIR,"src/test/resources/auth/jwt/jwt-authorized-key.json"); + protected static final File JWT_NOAUTHKEY_FILE = + new File(BASE_DIR,"src/test/resources/auth/jwt/jwt-unauthorized-key.json"); + protected static final File JWT_JWKS_FILE = + new File(BASE_DIR,"src/test/resources/auth/jwt/jwt-verification-jwks.json"); + protected static final int MOCK_JWKS_SERVER_PORT = 8089; + @ClassRule + public static final WireMockRule MOCK_JWKS_SERVER = new WireMockRule(MOCK_JWKS_SERVER_PORT); + + + public static class TestSchemaInfo extends MetaStoreSchemaInfo { + public TestSchemaInfo(String metastoreHome, String dbType) throws HiveMetaException { + super(metastoreHome, dbType); + } + @Override + public String getMetaStoreScriptDir() { + return new File(BASE_DIR,"src/test/resources").getAbsolutePath() + File.separatorChar + + "scripts" + File.separatorChar + "metastore" + + File.separatorChar + "upgrade" + File.separatorChar + dbType; + } + } + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + protected Configuration conf = null; + protected String NS = "hms" + RND.nextInt(100); + + protected int port = -1; + protected int catalogPort = -1; + protected final String catalogPath = "hmscatalog"; + // for direct calls + protected Catalog catalog; + protected SupportsNamespaces nsCatalog; + + protected int createMetastoreServer(Configuration conf) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); + } + + protected void stopMetastoreServer(int port) { + MetaStoreTestUtils.close(port); + } + + @Before + public void setUp() throws Exception { + NS = "hms" + RND.nextInt(100); + conf = MetastoreConf.newMetastoreConf(); + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.CAPABILITY_CHECK, false); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); + // new 2024-10-02 + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.SCHEMA_VERIFICATION, false); + + conf.setBoolean(MetastoreConf.ConfVars.METRICS_ENABLED.getVarname(), true); + // "hive.metastore.warehouse.dir" + String whpath = new File(BASE_DIR,"target/tmp/warehouse/managed").toURI()/*.getAbsolutePath()*/.toString(); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.WAREHOUSE, whpath); + HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_WAREHOUSE, whpath); + // "hive.metastore.warehouse.external.dir" + String extwhpath = new File(BASE_DIR,"target/tmp/warehouse/external").toURI()/*.getAbsolutePath()*/.toString(); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL, extwhpath); + conf.set(HiveConf.ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL.varname, extwhpath); + + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SCHEMA_INFO_CLASS, "org.apache.iceberg.rest.HMSTestBase$TestSchemaInfo"); + // Events that get cleaned happen in batches of 1 to exercise batching code + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L); + MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT, 0); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH, "jwt"); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH, catalogPath); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION_JWT_JWKS_URL, + "http://localhost:" + MOCK_JWKS_SERVER_PORT + "/jwks"); + MOCK_JWKS_SERVER.stubFor(get("/jwks") + .willReturn(ok() + .withBody(Files.readAllBytes(JWT_JWKS_FILE.toPath())))); + Metrics.initialize(conf); + // The server + port = createMetastoreServer(conf); + System.out.println("Starting MetaStore Server on port " + port); + // The manager decl + PropertyManager.declare(NS, HMSPropertyManager.class); + // The client + HiveMetaStoreClient client = createClient(conf, port); + Assert.assertNotNull("Unable to connect to the MetaStore server", client); + + // create a managed root + Warehouse wh = new Warehouse(conf); + String location = temp.newFolder("hivedb2023").getAbsolutePath(); + Database db = new Database(DB_NAME, "catalog test", location, Collections.emptyMap()); + client.createDatabase(db); + + Server iceServer = HiveMetaStore.getIcebergServer(); + int tries = 5; + while(iceServer == null && tries-- > 0) { + Thread.sleep(100); + iceServer = HiveMetaStore.getIcebergServer(); + } + Catalog ice = HMSCatalogServer.getLastCatalog(); + if (iceServer != null) { + while (iceServer.isStarting()) { + Thread.sleep(100); + } + catalog = ice != null? ice : HMSCatalogServer.getLastCatalog(); + nsCatalog = catalog instanceof SupportsNamespaces? (SupportsNamespaces) catalog : null; + catalogPort = iceServer.getURI().getPort(); + } else { + throw new NullPointerException("no server"); + } + } + + protected HiveMetaStoreClient createClient(Configuration conf, int port) throws Exception { + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, ""); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI, false); + return new HiveMetaStoreClient(conf); + } + + /** + * @param apis a list of api calls + * @return the map of HMSCatalog route counter metrics keyed by their names + */ + static Map reportMetricCounters(String... apis) { + Map map = new LinkedHashMap<>(); + MetricRegistry registry = Metrics.getRegistry(); + List names = HMSCatalogAdapter.getMetricNames(apis); + for(String name : names) { + Counter counter = registry.counter(name); + if (counter != null) { + long count = counter.getCount(); + map.put(name, count); + } + } + return map; + } + + @After + public synchronized void tearDown() throws Exception { + try { + if (port >= 0) { + System.out.println("Stopping MetaStore Server on port " + port); + stopMetastoreServer(port); + port = -1; + } + // Clear the SSL system properties before each test. + System.clearProperty(ObjectStore.TRUSTSTORE_PATH_KEY); + System.clearProperty(ObjectStore.TRUSTSTORE_PASSWORD_KEY); + System.clearProperty(ObjectStore.TRUSTSTORE_TYPE_KEY); + // + } finally { + catalog = null; + nsCatalog = null; + catalogPort = -1; + conf = null; + } + } + + protected String generateJWT() throws Exception { + return generateJWT(JWT_AUTHKEY_FILE.toPath()); + } + protected String generateJWT(Path path) throws Exception { + return generateJWT(USER_1, path, TimeUnit.MINUTES.toMillis(5)); + } + + private static String generateJWT(String user, Path keyFile, long lifeTimeMillis) throws Exception { + RSAKey rsaKeyPair = RSAKey.parse(new String(java.nio.file.Files.readAllBytes(keyFile), StandardCharsets.UTF_8)); + // Create RSA-signer with the private key + JWSSigner signer = new RSASSASigner(rsaKeyPair); + JWSHeader header = new JWSHeader + .Builder(JWSAlgorithm.RS256) + .keyID(rsaKeyPair.getKeyID()) + .build(); + Date now = new Date(); + Date expirationTime = new Date(now.getTime() + lifeTimeMillis); + JWTClaimsSet claimsSet = new JWTClaimsSet.Builder() + .jwtID(UUID.randomUUID().toString()) + .issueTime(now) + .issuer("auth-server") + .subject(user) + .expirationTime(expirationTime) + .claim("custom-claim-or-payload", "custom-claim-or-payload") + .build(); + SignedJWT signedJWT = new SignedJWT(header, claimsSet); + // Compute the RSA signature + signedJWT.sign(signer); + return signedJWT.serialize(); + } + + /** + * Performs a Json client call. + * @param jwt the jwt token + * @param url the url + * @param method the http method + * @param arg the argument that will be transported as JSon + * @return the result the was returned through Json + * @throws IOException if marshalling the request/response fail + */ + public static Object clientCall(String jwt, URL url, String method, Object arg) throws IOException { + return clientCall(jwt, url, method, true, arg); + } + + public static class ServerResponse { + private final int code; + private final String content; + public ServerResponse(int code, String content) { + this.code = code; + this.content = content; + } + } + + /** + * Performs an http client call. + * @param jwt a JWT bearer token (can be null) + * @param url the url to call + * @param method the http method to use + * @param json whether the call is application/json (true) or application/x-www-form-urlencoded (false) + * @param arg the query argument + * @return the (JSON) response + * @throws IOException + */ + public static Object clientCall(String jwt, URL url, String method, boolean json, Object arg) throws IOException { + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + try { + if ("PATCH".equals(method)) { + con.setRequestMethod("POST"); + con.setRequestProperty("X-HTTP-Method-Override", "PATCH"); + } else { + con.setRequestMethod(method); + } + con.setRequestProperty(MetaStoreUtils.USER_NAME_HTTP_HEADER, url.getUserInfo()); + if (json) { + con.setRequestProperty("Content-Type", "application/json"); + } else { + con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); + } + con.setRequestProperty("Accept", "application/json"); + if (jwt != null) { + con.setRequestProperty("Authorization", "Bearer " + jwt); + } + con.setDoInput(true); + if (arg != null) { + con.setDoOutput(true); + DataOutputStream wr = new DataOutputStream(con.getOutputStream()); + if (json) { + String outjson = GSON.toJson(arg); + wr.writeBytes(outjson); + } else { + wr.writeBytes(arg.toString()); + } + wr.flush(); + wr.close(); + } + // perform http method + return httpResponse(con); + } finally { + con.disconnect(); + } + } + + private static Object httpResponse(HttpURLConnection con) throws IOException { + int responseCode = con.getResponseCode(); + InputStream responseStream = con.getErrorStream(); + if (responseStream == null) { + try { + responseStream = con.getInputStream(); + } catch (IOException e) { + return new ServerResponse(responseCode, e.getMessage()); + } + } + if (responseStream != null) { + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(responseStream, StandardCharsets.UTF_8))) { + // if not strictly ok, check we are still receiving a JSON + if (responseCode != HttpServletResponse.SC_OK) { + String contentType = con.getContentType(); + if (contentType == null || !contentType.contains("application/json")) { + String line; + StringBuilder response = new StringBuilder("error " + responseCode + ":"); + while ((line = reader.readLine()) != null) response.append(line); + return new ServerResponse(responseCode, response.toString()); + } + } + Object r = GSON.fromJson(reader, Object.class); + if (r instanceof Map) { + ((Map) r).put("status", responseCode); + } + return r; + } + } + return responseCode; +} + + /** + * Making integer more pervasive when converting JSON. + */ + private static final ToNumberStrategy NARROW_NUMBER = jsonReader -> { + Number number = ToNumberPolicy.LONG_OR_DOUBLE.readNumber(jsonReader); + if (number instanceof Long) { + long n = number.longValue(); + int i = (int) n; + if (i == n) { + return i; + } + } + return number; + }; + + public static final Gson GSON = new GsonBuilder() + .setNumberToNumberStrategy(NARROW_NUMBER) + .setObjectToNumberStrategy(NARROW_NUMBER) + .setPrettyPrinting() + .create(); +} \ No newline at end of file diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java new file mode 100644 index 000000000000..456398909f54 --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import com.google.gson.Gson; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Types; +import static org.apache.iceberg.types.Types.NestedField.required; +import org.junit.Assert; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; + +public class TestHMSCatalog extends HMSTestBase { + public TestHMSCatalog() { + super(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Test + public void testCreateNamespaceHttp() throws Exception { + String ns = "nstesthttp"; + // list namespaces + URL url = new URL("http://hive@localhost:" + catalogPort + "/"+catalogPath+"/v1/namespaces"); + String jwt = generateJWT(); + // check namespaces list (ie 0) + Object response = clientCall(jwt, url, "GET", null); + Assert.assertTrue(response instanceof Map); + Map nsrep = (Map) response; + List nslist = (List) nsrep.get("namespaces"); + Assert.assertEquals(2, nslist.size()); + Assert.assertTrue((nslist.contains(Arrays.asList("default")))); + Assert.assertTrue((nslist.contains(Arrays.asList("hivedb")))); + // succeed + response = clientCall(jwt, url, "POST", false, "{ \"namespace\" : [ \""+ns+"\" ], "+ + "\"properties\":{ \"owner\": \"apache\", \"group\" : \"iceberg\" }" + +"}"); + Assert.assertNotNull(response); + HiveMetaStoreClient client = createClient(conf, port); + Database database1 = client.getDatabase(ns); + Assert.assertEquals("apache", database1.getParameters().get("owner")); + Assert.assertEquals("iceberg", database1.getParameters().get("group")); + + List tis = catalog.listTables(Namespace.of(ns)); + Assert.assertTrue(tis.isEmpty()); + + // list tables in hivedb + url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + ns + "/tables"); + // succeed + response = clientCall(jwt, url, "GET", null); + Assert.assertNotNull(response); + + // quick check on metrics + Map counters = reportMetricCounters("list_namespaces", "list_tables"); + counters.entrySet().forEach(m->{ + Assert.assertTrue(m.getKey(), m.getValue() > 0); + }); + } + + private Schema getTestSchema() { + return new Schema( + required(1, "id", Types.IntegerType.get(), "unique ID"), + required(2, "data", Types.StringType.get())); + } + + + @Test + public void testCreateTableTxnBuilder() throws Exception { + Schema schema = getTestSchema(); + final String tblName = "tbl_" + Integer.toHexString(RND.nextInt(65536)); + final TableIdentifier tableIdent = TableIdentifier.of(DB_NAME, tblName); + String location = temp.newFolder(tableIdent.toString()).toString(); + + try { + Transaction txn = catalog.buildTable(tableIdent, schema) + .withLocation(location) + .createTransaction(); + txn.commitTransaction(); + Table table = catalog.loadTable(tableIdent); + + Assert.assertEquals(location, table.location()); + Assert.assertEquals(2, table.schema().columns().size()); + Assert.assertTrue(table.spec().isUnpartitioned()); + List tis = catalog.listTables(Namespace.of(DB_NAME)); + Assert.assertFalse(tis.isEmpty()); + + // list namespaces + URL url = new URL("http://hive@localhost:" + catalogPort + "/"+catalogPath+"/v1/namespaces"); + String jwt = generateJWT(); + // succeed + Object response = clientCall(jwt, url, "GET", null); + Assert.assertNotNull(response); + + // list tables in hivedb + url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + DB_NAME + "/tables"); + // succeed + response = clientCall(jwt, url, "GET", null); + Assert.assertNotNull(response); + + // load table + url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + DB_NAME + "/tables/" + tblName); + // succeed + response = clientCall(jwt, url, "GET", null); + Assert.assertNotNull(response); + String str = new Gson().toJson(response); + + // quick check on metrics + Map counters = reportMetricCounters("list_namespaces", "list_tables", "load_table"); + counters.forEach((key, value) -> Assert.assertTrue(key, value > 0)); + table = catalog.loadTable(tableIdent); + Assert.assertNotNull(table); + } catch (Exception xany) { + String str = xany.getMessage(); + } finally { + //metastoreClient.dropTable(DB_NAME, tblName); + catalog.dropTable(tableIdent, false); + } + } + +} diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json new file mode 100644 index 000000000000..b5b4fb40e7c9 --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json @@ -0,0 +1,12 @@ +{ + "p": "-8lxjB9JZA44XBLLVGnY20x28uT8NQ1BlbqI0Tlr96An4B_PzgPL5_bFFB7SWs8ehSWn9z2SJfClhQpBLfy-2mXvJek_xgibESIlPXqY9Qrg7-PhRmPs3whyiIsnn8tpPMm2XJ_4n0Y-Yfx4nwErGdy84LiKFMDXPEk2a7ndYWs", + "kty": "RSA", + "q": "0YAcTLBnTrSUiciE0lliIkAidW0TnHP48v-vJitLEz0d8mlTZ_aeOQJm6CUOqF7BqQv3Z8OK_HYKXfOr7xzUlfROONybUXRFE0LvT5Fjvrq-56QGB6GeFq5i6HKlRcC_8TD6WwUJWIzeYuPqhp_FYIpT4ds131d5VYPKDCdY_dM", + "d": "VsxW72idEAtoZQDphvxJ0t54EyRfcIJVB9BZuqnyNTfH-VsaUO3st86w_PMU_i0lmyIc8dkCmwOb8R2pRXDo6UxEYUe5YfBnvn9iYF3Ll2QfPOKfZhDBOfqSjEb1po20is7mXTQORBv3bhSo664pasHItTwDz-KKI-FiIu_PYq0lYihuaedUUMp3MQTvDFulpFWEKzqseBDat07BholvxjzlnBK-Ez3KI9qGH8VIIk5TGW5pVu3cQe1WC8NJOe3xR9vu7XX6xvhVLPP7fvKiXJWJ_I_SagAhR1JW0uDJl_b0CrYYeVUnt_pzvW1BeJGz7ysCXcHlLBUh72XrpW-O7Q", + "e": "AQAB", + "kid": "123", + "qi": "9yk0mg4LY48YS8cvG51wMVfKfEjSbt2ygKxqabdsP-qSVpz-KVJtCmbKa57jm2BaMV_mRBQFodxu4XN58VGsj5MzXC5Jb_CkLeQfkp6ZKvehZhiJn3HF0Kb19u9xPvKDclHpKl-UMM1Pcu8Ww52DOyOYcHa1_SLZ05CcOWvMkS8", + "dp": "HYtToYeCSxVIE7W42hzZb1IXmwS3e1ok2fbbWwGL47CNPUU-UwQrBvrzwRqkwDcRc7opbV9yKLWGFohPgZ_onSPc3evyqcAUwfvptr8N96LhJgTtSB8tijYpilAZxCxQGuvoVBIJUFcjtsezN6Uhc5VtLEk7GphOKSrGEfnrOiU", + "dq": "tF2uf5v0JT-1DnazW4IWydQblqtlEfKKp3LX8W2egh7BNJ3XcA9UI1LdFAord2u1IXwq8YvZkgdyX3bVVNSmdb_SxIOxuMv4WF_tNry-eku-5iFCC7nqKC7U-rkRb19GIToAoPJSHImTQOJmXKcbQEV3eGDJHdLqpGQFRLdvl38", + "n": "zg12QaFTsez1EijOYRFzNZdowOt79ePqxCMQ-EEHynUhEZ6TIDnXfjWfuWocS1qRRglUUbHerEtmACUKPQShaG8uL0ZXiLqDr2QSuqrTtr2VUGesxZc6GiqkZlnWFNu5kSUvtemcKxWl8OLFf-5kNnGW4_4xM6BIwosYZnddfFqQT5IP6iTMZIUIKXxY4s1dadYRIiMteNutro67fhOLKabHkyC6ILE6f6VZsYbb_NXC5yC--7DiC2GYKzy7TKmaczuDfQZVgVY-nL9kTPIdhf334EYHQfYmLdvLc56g8-cxY3xh2GnwAj1JcT2u3hsS4KS05bUFHFnveO5uxIYKMQ" +} \ No newline at end of file diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json new file mode 100644 index 000000000000..f4845de7459d --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json @@ -0,0 +1,12 @@ +{ + "p": "wvzuDSY6dIsIJB0UM5BIncN6ui5ee-KHpCmBhh_ia2iX3DluQODEgITw7gDATTDdQsBD-nJLjrqUs5g5Gmt0UgZucXQ5PCt1CK6dLEZCaLivw2fsHYvOKeTkdA49wqLkTc8pkfQs09N-b6NspDDqVJPFffBvFpR_IBFay-xKa5k", + "kty": "RSA", + "q": "sQzza69VkEmgUm50pEGjgu-OxugOrjcHrjQ42A23YVwAAJ90qPNQa62O7dv5oWmSX2PJ7TgjkzbvtTycLfT_vUeapwfCcJe4WoDg54xF3E35yBvBIwReRiavxf5nWsHEtd5kBg6wRIndGwGUBE91xaLg21spjH7nQKtG9vKeNM8", + "d": "UbiPIpr7agQqpM3ERfaXsKNMETyBrIYr3yoggHQ7XQkSPepCgRhE86puRmjf76FtZ3RwpJwjLfO6Ap0fIE9LXXE8otTF9sMnC9fe7odHkEu61Wr3aQM-53dgZoJL7XU53LOo0cNO44SBbw11d2cYlAR3KuCEK7bCLMBOkK1gdxVpgDC7DgxVgnP39bUlf4fA5gQeT5nNGnCWTV4jMVWCyEb0Ck5CvGJp1cCKaMSEvV4j6AM72EkAn8PogTSOJpurRJaTky0De7-ncT2Sv5DCuOIkMhsHqayLbm7a84ORHqsnWpZV85WVW-xxiivkVpqtSDRKCI94pMa9DWszjNJW8Q", + "e": "AQAB", + "kid": "sig-1642039368", + "qi": "CXP_tewCHyXk6PNDcbI0wtXsaWJryOJfMsc7roBCoOwDbTekUFXhOfRmFX5ZTNetRNDpw9nNiQDXt8pyw7UZ-0EhD1cLst1slS__hBi5QEAGo9cUxl3RGeMAFtY9O8B1gjFyKkG5BzdddGBKGQT3Tg23Eyzn6EA_NCw4XAKnkwQ", + "dp": "aAdzphZQN595n3LYNU50P59sWeqlRCkuvvnZ_coDDdUGuFr3pKuGix7iP8is0EISuitD2VmjUCnhbhP3202bCKwfvm4Inz58OT6X4mg1xBNMys8mHPla6-UPsY9rie1IKu8suY7xX65FlaA2NT9XtfoE8tUVH5HoZR59N7EAX3k", + "dq": "mTkZDO-fgBCH4-7dmS2JIY7KpI897T2IsxVUwH4WXvastd1Jq9FuntGEKYu_HRbtawpEPbzg5M2dY97BVvB5xshKKhWIC8Lx87knapw19XOyIKEMY46rO9DNO-9waNXatH5zV96sY5RgOrgB7j0KMnFEYfIiIgnNfmT8NElB63c", + "n": "htq92ltGQrZv19TlhluoqmXjjRXw_NWEd0nPZsWrbLnr8lZ-gOxsjIsDMjb5HNDNmuAS7pg2d_o5ZZAY1sSjKf_EuUPZN-MOej8ZBOtrMxEH7e_t37kYIbbJSuzt55poZdRli6BE8CVDesS4W-wsFZ0MvUazAUADh3onARN7Arf3jwknm5CLafE_JzKrNKZadBElEFEAEu5y9n_SuTlemw3P81lOVmZmjGjfqtPx01O5aV_truMjrQa3NUivu1ihrjvJl0xc3rwJe7qDrfEqgvpBQ-vrAsvg3Jiz5Idj6cU3J0hNtV4ixYxcDQecNlgR7gBeIp3E8BXL1kGOOHYUtw" +} \ No newline at end of file diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json new file mode 100644 index 000000000000..a6fd935a0a3b --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json @@ -0,0 +1,20 @@ +{ + "keys": [ + { + "kty": "RSA", + "e": "AQAB", + "alg": "RS256", + "kid": "819d1e61429dd3d3caef129c0ac2bae8c6d46fbc", + "use": "sig", + "n": "qfR12Bcs_hSL0Y1fN5TYZeUQIFmuVRYa210na81BFj91xxwtICJY6ckZCI3Jf0v2tPLOT_iKVk4WBCZ7AVJVvZqHuttkyrdFROMVTe6DwmcjbbkgACMVildTnHy9xy2KuX-OZsEYzgHuRgfe_Y-JN6LoxBYZx6VoBLpgK-F0Q-0O_bRgZhHifVG4ZzARjhgz0PvBb700GtOTHS6mQIfToPErbgqcowKN9k-mJqJr8xpXSHils-Yw97LHjICZmvA5B8EPNW28DwFOE5JrsPcyrFKOAYl4NcSYQgjl-17TWE5_tFdZ8Lz-srjiPMoHlBjZD1C7aO03LI-_9u8lVsktMw" + }, + { + "kty": "RSA", + "e": "AQAB", + "alg": "RS256", + "kid": "123", + "use": "sig", + "n": "zg12QaFTsez1EijOYRFzNZdowOt79ePqxCMQ-EEHynUhEZ6TIDnXfjWfuWocS1qRRglUUbHerEtmACUKPQShaG8uL0ZXiLqDr2QSuqrTtr2VUGesxZc6GiqkZlnWFNu5kSUvtemcKxWl8OLFf-5kNnGW4_4xM6BIwosYZnddfFqQT5IP6iTMZIUIKXxY4s1dadYRIiMteNutro67fhOLKabHkyC6ILE6f6VZsYbb_NXC5yC--7DiC2GYKzy7TKmaczuDfQZVgVY-nL9kTPIdhf334EYHQfYmLdvLc56g8-cxY3xh2GnwAj1JcT2u3hsS4KS05bUFHFnveO5uxIYKMQ" + } + ] +} \ No newline at end of file diff --git a/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties b/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties new file mode 100644 index 000000000000..7243144ed6ff --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name=PropertiesConfig +property.filename = logs +appenders = console,captured + +appender.console.type = Console +appender.console.name = STDOUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n + +appender.captured.type = CapturingLogAppender +appender.captured.name = CAPTURED + +loggers=file +logger.file.name=guru.springframework.blog.log4j2properties +logger.file.level = debug +logger.file.appenderRefs = file +logger.file.appenderRef.file.ref = LOGFILE + +rootLogger.level = debug +rootLogger.appenderRefs = stdout,captured +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.captured.ref = CAPTURED diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 544f26dd9d88..9f9e33742009 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1827,6 +1827,20 @@ public enum ConfVars { "hive.metastore.properties.servlet.auth", "jwt", "Property-maps servlet authentication method (simple or jwt)." ), + ICEBERG_CATALOG_SERVLET_PATH("hive.metastore.catalog.servlet.path", + "hive.metastore.catalog.servlet.path", "icecli", + "HMS Iceberg Catalog servlet path component of URL endpoint." + ), + ICEBERG_CATALOG_SERVLET_PORT("hive.metastore.catalog.servlet.port", + "hive.metastore.catalog.servlet.port", -1, + "HMS Iceberg Catalog servlet server port. Negative value disables the servlet," + + " 0 will let the system determine the catalog server port," + + " positive value will be used as-is." + ), + ICEBERG_CATALOG_SERVLET_AUTH("hive.metastore.catalog.servlet.auth", + "hive.metastore.catalog.servlet.auth", "jwt", + "HMS Iceberg Catalog servlet authentication method (simple or jwt)." + ), // Deprecated Hive values that we are keeping for backwards compatibility. @Deprecated diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 0fd998b2df57..3ce67f8f23ff 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.metastore; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import org.apache.commons.cli.OptionBuilder; @@ -120,12 +122,17 @@ public class HiveMetaStore extends ThriftHiveMetastore { private static String msHost = null; private static ThriftServer thriftServer; private static Server propertyServer = null; + private static Server icebergServer = null; public static Server getPropertyServer() { return propertyServer; } + public static Server getIcebergServer() { + return icebergServer; + } + public static boolean isRenameAllowed(Database srcDB, Database destDB) { if (!srcDB.getName().equalsIgnoreCase(destDB.getName())) { if (ReplChangeManager.isSourceOfReplication(srcDB) || ReplChangeManager.isSourceOfReplication(destDB)) { @@ -319,6 +326,23 @@ public static void main(String[] args) throws Throwable { if (isCliVerbose) { System.err.println(shutdownMsg); } + // property server + if (propertyServer != null) { + try { + propertyServer.stop(); + } catch (Exception e) { + LOG.error("Error stopping Property Map server.", e); + } + } + // iceberg server + if (icebergServer != null) { + try { + icebergServer.stop(); + } catch (Exception e) { + LOG.error("Error stopping Iceberg API server.", e); + } + } + // metrics if (MetastoreConf.getBoolVar(conf, ConfVars.METRICS_ENABLED)) { try { Metrics.shutdown(); @@ -389,7 +413,7 @@ private static ThriftServer startHttpMetastore(int port, Configuration conf) throws Exception { LOG.info("Attempting to start http metastore server on port: {}", port); // login principal if security is enabled - ServletSecurity.loginServerPincipal(conf); + ServletSecurity.loginServerPrincipal(conf); long maxMessageSize = MetastoreConf.getLongVar(conf, ConfVars.SERVER_MAX_MESSAGE_SIZE); int minWorkerThreads = MetastoreConf.getIntVar(conf, ConfVars.SERVER_MIN_THREADS); @@ -742,10 +766,26 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, } // optionally create and start the property server and servlet propertyServer = PropertyServlet.startServer(conf); + // optionally create and start the Iceberg REST server and servlet + icebergServer = startIcebergCatalog(conf); thriftServer.start(); } + static Server startIcebergCatalog(Configuration configuration) { + try { + Class iceClazz = Class.forName("org.apache.iceberg.rest.HMSCatalogServer"); + Method iceStart = iceClazz.getMethod("startServer", Configuration.class); + return (Server) iceStart.invoke(null, configuration); + } catch (ClassNotFoundException xnf) { + LOG.warn("unable to start Iceberg REST Catalog server {}, missing jar?", xnf); + return null; + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + LOG.error("unable to start Iceberg REST Catalog server {}", e); + return null; + } + } + /** * @param port where metastore server is running * @return metastore server instance URL. If the metastore server was bound to a configured diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java index 4572f86e0247..2ecd00810484 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java @@ -35,7 +35,7 @@ JWTs sent in the Authorization header in HTTP request. */ public class HmsThriftHttpServlet extends TServlet { - private final ServletSecurity security; + private final SecureServletCaller security; public HmsThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory, Configuration conf) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java index 1d6cc9d6ade1..0f82dce30b9b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java @@ -37,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.servlet.Servlet; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServlet; @@ -65,14 +66,22 @@ public class PropertyServlet extends HttpServlet { /** The configuration. */ private final Configuration configuration; /** The security. */ - private final ServletSecurity security; + private final SecureServletCaller security; - PropertyServlet(Configuration configuration) { + static boolean isAuthJwt(Configuration configuration) { String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH); - boolean jwt = auth != null && "jwt".equals(auth.toLowerCase()); - this.security = new ServletSecurity(configuration, jwt); + return "jwt".equalsIgnoreCase(auth); + } + + PropertyServlet(Configuration configuration) { + this(configuration, new ServletSecurity(configuration, isAuthJwt(configuration))); + } + + PropertyServlet(Configuration configuration, SecureServletCaller security) { + this.security = security; this.configuration = configuration; } + private String strError(String msg, Object...args) { return String.format(PTYERROR + msg, args); } @@ -344,6 +353,10 @@ public static Server startServer(Configuration conf) throws Exception { return null; } String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH); + return startServer(conf, port, cli, new PropertyServlet(conf)); + } + + public static Server startServer(Configuration conf, int port, String path, Servlet servlet) throws Exception { // HTTP Server Server server = new Server(); server.setStopAtShutdown(true); @@ -359,11 +372,11 @@ public static Server startServer(Configuration conf) throws Exception { ServletHandler handler = new ServletHandler(); server.setHandler(handler); ServletHolder holder = handler.newServletHolder(Source.EMBEDDED); - holder.setServlet(new PropertyServlet(conf)); // - handler.addServletWithMapping(holder, "/"+cli+"/*"); + holder.setServlet(servlet); // + handler.addServletWithMapping(holder, "/"+path+"/*"); server.start(); if (!server.isStarted()) { - LOGGER.error("unable to start property-maps servlet server, path {}, port {}", cli, port); + LOGGER.error("unable to start property-maps servlet server, path {}, port {}", path, port); } else { LOGGER.info("started property-maps servlet server on {}", server.getURI()); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java new file mode 100644 index 000000000000..206684bf1b2e --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java @@ -0,0 +1,60 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * Secures servlet processing. + */ +public interface SecureServletCaller { + /** + * Should be called in Servlet.init() + * @throws ServletException if the jwt validator creation throws an exception + */ + public void init() throws ServletException; + + /** + * Any http method executor. + */ + @FunctionalInterface + interface MethodExecutor { + /** + * The method to call to secure the execution of a (http) method. + * @param request the request + * @param response the response + * @throws ServletException if the method executor fails + * @throws IOException if the Json in/out fail + */ + void execute(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException; + } + + /** + * The method to call to secure the execution of a (http) method. + * @param request the request + * @param response the response + * @param executor the method executor + */ + void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) + throws IOException; + + +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index 76181722ca85..2f2c7cda7830 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -44,7 +44,7 @@ /** * Secures servlet processing. */ -public class ServletSecurity { +public class ServletSecurity implements SecureServletCaller { private static final Logger LOG = LoggerFactory.getLogger(ServletSecurity.class); static final String X_USER = MetaStoreUtils.USER_NAME_HTTP_HEADER; private final boolean isSecurityEnabled; @@ -52,7 +52,7 @@ public class ServletSecurity { private JWTValidator jwtValidator = null; private final Configuration conf; - ServletSecurity(Configuration conf, boolean jwt) { + public ServletSecurity(Configuration conf, boolean jwt) { this.conf = conf; this.isSecurityEnabled = UserGroupInformation.isSecurityEnabled(); this.jwtAuthEnabled = jwt; @@ -73,24 +73,15 @@ public void init() throws ServletException { } } - /** - * Any http method executor. - */ - @FunctionalInterface - interface MethodExecutor { - void execute(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException; - } - /** * The method to call to secure the execution of a (http) method. * @param request the request * @param response the response * @param executor the method executor - * @throws ServletException if the method executor fails * @throws IOException if the Json in/out fail */ public void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) - throws ServletException, IOException { + throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Logging headers in "+request.getMethod()+" request"); Enumeration headerNames = request.getHeaderNames(); @@ -124,7 +115,7 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me } catch (RuntimeException e) { LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(), e); - throw new ServletException(e); + throw new IOException(e); } } catch (HttpAuthenticationException e) { response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); @@ -178,7 +169,7 @@ private String extractBearerToken(HttpServletRequest request, * @param conf the configuration * @throws IOException if getting the server principal fails */ - static void loginServerPincipal(Configuration conf) throws IOException { + static void loginServerPrincipal(Configuration conf) throws IOException { // This check is likely pointless, especially with the current state of the http // servlet which respects whatever comes in. Putting this in place for the moment // only to enable testing on an otherwise secure cluster. @@ -193,35 +184,40 @@ static void loginServerPincipal(Configuration conf) throws IOException { LOG.info("Security is not enabled. Not logging in via keytab"); } } + /** * Creates an SSL context factory if configuration states so. * @param conf the configuration * @return null if no ssl in config, an instance otherwise * @throws IOException if getting password fails */ - static SslContextFactory createSslContextFactory(Configuration conf) throws IOException { + public static SslContextFactory createSslContextFactory(Configuration conf) throws IOException { final boolean useSsl = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL); if (!useSsl) { return null; } - String keyStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH).trim(); + final String keyStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH).trim(); if (keyStorePath.isEmpty()) { throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_KEYSTORE_PATH.toString() + " Not configured for SSL connection"); } - String keyStorePassword = + final String keyStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD); - String keyStoreType = + final String keyStoreType = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_TYPE).trim(); - String keyStoreAlgorithm = + final String keyStoreAlgorithm = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim(); - + final String[] excludedProtocols = + MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_PROTOCOL_BLACKLIST).split(","); + if (LOG.isInfoEnabled()) { + LOG.info("HTTP Server SSL: adding excluded protocols: {}", Arrays.toString(excludedProtocols)); + } SslContextFactory factory = new SslContextFactory.Server(); - String[] excludedProtocols = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_PROTOCOL_BLACKLIST).split(","); - LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols)); factory.addExcludeProtocols(excludedProtocols); - LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = " - + Arrays.toString(factory.getExcludeProtocols())); + if (LOG.isInfoEnabled()) { + LOG.info("HTTP Server SSL: SslContextFactory.getExcludeProtocols = {}", + Arrays.toString(factory.getExcludeProtocols())); + } factory.setKeyStorePath(keyStorePath); factory.setKeyStorePassword(keyStorePassword); factory.setKeyStoreType(keyStoreType); diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 81e80833a8bf..5262d5d7b8c5 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -29,6 +29,7 @@ metastore-common metastore-server metastore-tools + metastore-catalog 4.1.0-SNAPSHOT From e90e1d89749fbaf912de2fb72af6c76b070921e0 Mon Sep 17 00:00:00 2001 From: Henrib Date: Mon, 3 Feb 2025 16:13:03 +0100 Subject: [PATCH 03/40] HIVE-28059 : rollback changes that are unnecessary; --- .../hive/TestHiveIcebergBranchOperation.java | 4 +- .../TestHiveIcebergStorageHandlerNoScan.java | 2 +- ...n_partition_evolution_w_id_spec_w_filter.q | 4 +- .../src/test/queries/positive/iceberg_stats.q | 22 +++ .../test/results/positive/iceberg_stats.q.out | 159 ++++++++++++++++++ 5 files changed, 186 insertions(+), 5 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java index c5eb59987fb0..d62f83eb6fcc 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergBranchOperation.java @@ -231,12 +231,12 @@ public void testCreateBranchFromTag() throws IOException, InterruptedException { // Create a branch based on a tag which doesn't exist will fail. Assertions.assertThatThrownBy(() -> shell.executeStatement(String.format( "ALTER TABLE customers CREATE BRANCH %s FOR TAG AS OF %s", branchName2, nonExistTag))) - .isInstanceOf(IllegalArgumentException.class).hasMessageEndingWith("does not exist"); + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("does not exist"); // Create a branch based on a branch will fail. Assertions.assertThatThrownBy(() -> shell.executeStatement(String.format( "ALTER TABLE customers CREATE BRANCH %s FOR TAG AS OF %s", branchName2, branchName1))) - .isInstanceOf(IllegalArgumentException.class).hasMessageEndingWith("does not exist"); + .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("does not exist"); } @Test diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index 70c5f6880418..e9bad07f9aa7 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -700,7 +700,7 @@ public void testCreateTableError() { "')")) .isInstanceOf(IllegalArgumentException.class) .hasMessageStartingWith("Failed to execute Hive query") - .hasMessageEndingWith("Table location not set"); + .hasMessageContaining("Table location not set"); } } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q index 38bf51fbb613..19202ae4407a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_partition_evolution_w_id_spec_w_filter.q @@ -69,8 +69,8 @@ delete from ice_orc where last_name in ('ln5', 'ln13'); select * from ice_orc; describe formatted ice_orc; -explain alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15'); -alter table ice_orc COMPACT 'major' and wait where team_id=10 or first_name in ('fn3', 'fn11') or last_name in ('ln7', 'ln15'); +explain alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2); +alter table ice_orc COMPACT 'major' and wait where company_id=100 or dept_id in (1,2); select * from ice_orc; describe formatted ice_orc; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q index de88018f32e0..6fc965e17456 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q @@ -28,5 +28,27 @@ select count(*) from ice01; insert overwrite table ice01 select * from ice01; explain select count(*) from ice01; +-- false means that count(*) query won't use row count stored in HMS +set iceberg.hive.keep.stats=false; + +create external table ice03 (id int, key int) Stored by Iceberg stored as ORC + TBLPROPERTIES('format-version'='2'); + +insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1); +-- Iceberg table can utilize fetch task to directly retrieve the row count from iceberg SnapshotSummary +explain select count(*) from ice03; +select count(*) from ice03; + +-- delete some values +delete from ice03 where id in (2,4); + +explain select count(*) from ice03; +select count(*) from ice03; + +-- iow +insert overwrite table ice03 select * from ice03; +explain select count(*) from ice03; + drop table ice01; drop table ice02; +drop table ice03; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out index 33c60b54608d..43bd6e42513e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out @@ -192,6 +192,155 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: create external table ice03 (id int, key int) Stored by Iceberg stored as ORC + TBLPROPERTIES('format-version'='2') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice03 +POSTHOOK: query: create external table ice03 (id int, key int) Stored by Iceberg stored as ORC + TBLPROPERTIES('format-version'='2') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice03 +PREHOOK: query: insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice03 +POSTHOOK: query: insert into ice03 values (1,1),(2,1),(3,1),(4,1),(5,1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice03 +PREHOOK: query: explain select count(*) from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select count(*) from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: hdfs://### HDFS PATH ### +5 +PREHOOK: query: delete from ice03 where id in (2,4) +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: default@ice03 +POSTHOOK: query: delete from ice03 where id in (2,4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: default@ice03 +PREHOOK: query: explain select count(*) from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select count(*) from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: ice03 + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 3 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + minReductionHashAggr: 0.6666666 + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + null sort order: + sort order: + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select count(*) from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: hdfs://### HDFS PATH ### +3 +PREHOOK: query: insert overwrite table ice03 select * from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: default@ice03 +POSTHOOK: query: insert overwrite table ice03 select * from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: default@ice03 +PREHOOK: query: explain select count(*) from ice03 +PREHOOK: type: QUERY +PREHOOK: Input: default@ice03 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: explain select count(*) from ice03 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: hdfs://### HDFS PATH ### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 1 + Processor Tree: + ListSink + PREHOOK: query: drop table ice01 PREHOOK: type: DROPTABLE PREHOOK: Input: default@ice01 @@ -212,3 +361,13 @@ POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@ice02 POSTHOOK: Output: database:default POSTHOOK: Output: default@ice02 +PREHOOK: query: drop table ice03 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@ice03 +PREHOOK: Output: database:default +PREHOOK: Output: default@ice03 +POSTHOOK: query: drop table ice03 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@ice03 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice03 From f7da91b5d0112d0a6bc278792b84e5a5b9bfce88 Mon Sep 17 00:00:00 2001 From: Henrib Date: Mon, 3 Feb 2025 16:16:14 +0100 Subject: [PATCH 04/40] HIVE-28059 : rollback changes that are unnecessary; --- .../iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java index e9bad07f9aa7..a8072f45afe6 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java @@ -775,7 +775,7 @@ public void testCreatePartitionedTableWithPropertiesAndWithColumnSpecification() "')")) .isInstanceOf(IllegalArgumentException.class) .hasMessageStartingWith("Failed to execute Hive query") - .hasMessageEndingWith( + .hasMessageContaining( "Provide only one of the following: Hive partition transform specification, " + "or the iceberg.mr.table.partition.spec property"); } From ef8b220265b0b77bfffe8c5d9d7527355e230ce7 Mon Sep 17 00:00:00 2001 From: Henrib Date: Mon, 3 Feb 2025 16:28:36 +0100 Subject: [PATCH 05/40] Update iceberg_stats.q.out --- .../src/test/results/positive/iceberg_stats.q.out | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out index 43bd6e42513e..4e5b70945016 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_stats.q.out @@ -265,7 +265,7 @@ STAGE PLANS: Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: - Map 1 + Map 1 Map Operator Tree: TableScan alias: ice03 @@ -279,12 +279,12 @@ STAGE PLANS: outputColumnNames: _col0 Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator - null sort order: - sort order: + null sort order: + sort order: Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE value expressions: _col0 (type: bigint) Execution mode: vectorized - Reducer 2 + Reducer 2 Execution mode: vectorized Reduce Operator Tree: Group By Operator From 4ee172071f61f95782eb8eb09c7228dfda10edea Mon Sep 17 00:00:00 2001 From: Henrib Date: Tue, 4 Feb 2025 09:36:48 +0100 Subject: [PATCH 06/40] HIVE-28059 : increase test server startup wait time; --- .../org/apache/iceberg/rest/HMSTestBase.java | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index 8e1fb3691a53..eab55c19a1e7 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -124,6 +124,7 @@ public String getMetaStoreScriptDir() { protected int port = -1; protected int catalogPort = -1; protected final String catalogPath = "hmscatalog"; + protected static final int WAIT_FOR_SERVER = 5000; // for direct calls protected Catalog catalog; protected SupportsNamespaces nsCatalog; @@ -183,22 +184,42 @@ public void setUp() throws Exception { Database db = new Database(DB_NAME, "catalog test", location, Collections.emptyMap()); client.createDatabase(db); + int[] aport = { -1 }; + Catalog ice = acquireServer(aport); + catalog = ice; + nsCatalog = catalog instanceof SupportsNamespaces? (SupportsNamespaces) catalog : null; + catalogPort = aport[0]; + } + + private static String format(String format, Object... params) { + return org.slf4j.helpers.MessageFormatter.arrayFormat(format, params).getMessage(); + } + + private static Catalog acquireServer(int[] port) throws InterruptedException { + final int wait = 200; Server iceServer = HiveMetaStore.getIcebergServer(); - int tries = 5; + int tries = WAIT_FOR_SERVER / wait; while(iceServer == null && tries-- > 0) { - Thread.sleep(100); + Thread.sleep(wait); iceServer = HiveMetaStore.getIcebergServer(); } - Catalog ice = HMSCatalogServer.getLastCatalog(); if (iceServer != null) { - while (iceServer.isStarting()) { - Thread.sleep(100); + port[0] = iceServer.getURI().getPort(); + boolean starting; + tries = WAIT_FOR_SERVER / wait; + while((starting = iceServer.isStarting()) && tries-- > 0) { + Thread.sleep(wait); } - catalog = ice != null? ice : HMSCatalogServer.getLastCatalog(); - nsCatalog = catalog instanceof SupportsNamespaces? (SupportsNamespaces) catalog : null; - catalogPort = iceServer.getURI().getPort(); + if (starting) { + LOG.warn("server still starting after {}ms", WAIT_FOR_SERVER); + } + Catalog ice = HMSCatalogServer.getLastCatalog(); + if (ice == null) { + throw new NullPointerException(format("unable to acquire catalog after {}ms", WAIT_FOR_SERVER)); + } + return ice; } else { - throw new NullPointerException("no server"); + throw new NullPointerException(format("unable to acquire server after {}ms", WAIT_FOR_SERVER)); } } From 8a092459641c8849effcdd10db2b3a91b9fdc904 Mon Sep 17 00:00:00 2001 From: Henrib Date: Tue, 4 Feb 2025 19:03:45 +0100 Subject: [PATCH 07/40] HIVE-28059 : attempting a rename on artefactId to blindfix build issue; --- standalone-metastore/metastore-catalog/pom.xml | 2 +- .../src/test/java/org/apache/iceberg/rest/HMSTestBase.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml index 69522c7adb3c..66b1c7c06b6f 100644 --- a/standalone-metastore/metastore-catalog/pom.xml +++ b/standalone-metastore/metastore-catalog/pom.xml @@ -17,7 +17,7 @@ 4.1.0-SNAPSHOT 4.0.0 - hive-metastore-icecat + hive-standalone-metastore-icecat Hive Metastore Iceberg Catalog .. diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index eab55c19a1e7..0125a42652e8 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -73,7 +73,6 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.hive.HiveCatalog; import org.eclipse.jetty.server.Server; import org.junit.After; import org.junit.Assert; From 92b988335e25a02594d46b64d290d06ea84dbce4 Mon Sep 17 00:00:00 2001 From: Henrib Date: Thu, 6 Feb 2025 08:10:45 +0100 Subject: [PATCH 08/40] HIVE-28059 : fixing src packaging; --- packaging/src/main/assembly/src.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml index 18b80a6d64a2..654682d89926 100644 --- a/packaging/src/main/assembly/src.xml +++ b/packaging/src/main/assembly/src.xml @@ -105,6 +105,7 @@ standalone-metastore/metastore-common/**/* standalone-metastore/metastore-server/**/* standalone-metastore/metastore-tools/**/* + standalone-metastore/metastore-catalog/**/* standalone-metastore/src/assembly/src.xml standalone-metastore/pom.xml streaming/**/* From bc36af2b4c0c89dcc521ddc62e60cff78767e918 Mon Sep 17 00:00:00 2001 From: Henrib Date: Tue, 11 Feb 2025 13:13:58 +0100 Subject: [PATCH 09/40] HIVE-28059 : fixing PR comments; - improve tests; --- .../metastore-catalog/data/conf/README.txt | 1 - .../metastore-catalog/pom.xml | 21 +-- .../apache/iceberg/HiveCachingCatalog.java | 32 ++-- .../iceberg/rest/HMSCatalogAdapter.java | 146 +++++++++++++++--- .../org/apache/iceberg/rest/HMSTestBase.java | 92 +++++++---- .../apache/iceberg/rest/TestHMSCatalog.java | 77 +++++++-- standalone-metastore/metastore-server/pom.xml | 2 +- .../hadoop/hive/metastore/HiveMetaStore.java | 20 ++- .../hive/metastore/SecureServletCaller.java | 2 +- .../hive/metastore/ServletSecurity.java | 1 + standalone-metastore/pom.xml | 1 + 11 files changed, 303 insertions(+), 92 deletions(-) delete mode 100644 standalone-metastore/metastore-catalog/data/conf/README.txt diff --git a/standalone-metastore/metastore-catalog/data/conf/README.txt b/standalone-metastore/metastore-catalog/data/conf/README.txt deleted file mode 100644 index 0b2f0f032f2f..000000000000 --- a/standalone-metastore/metastore-catalog/data/conf/README.txt +++ /dev/null @@ -1 +0,0 @@ -Need to force creation of a directory. diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml index 66b1c7c06b6f..365cc60d2d23 100644 --- a/standalone-metastore/metastore-catalog/pom.xml +++ b/standalone-metastore/metastore-catalog/pom.xml @@ -26,34 +26,33 @@ UTF-8 false ${project.parent.version} - ${hive.version} 1.6.1 org.apache.hive hive-standalone-metastore-server - ${revision} + ${hive.version} org.apache.hive hive-standalone-metastore-common - ${revision} + ${hive.version} org.apache.hive hive-iceberg-shading - ${revision} + ${hive.version} org.apache.hive hive-iceberg-handler - ${revision} + ${hive.version} org.apache.hive hive-iceberg-catalog - ${revision} + ${hive.version} org.apache.iceberg @@ -88,19 +87,13 @@ com.github.tomakehurst wiremock-jre8-standalone - 2.32.0 - test - - - org.assertj - assertj-core - 3.19.0 + ${wiremock.jre8.standalone.version} test org.junit.jupiter junit-jupiter-api - 5.10.0 + ${junit.jupiter.version} test diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java index ff55cd943ad1..d72f02840bf7 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java @@ -48,29 +48,27 @@ * Class that wraps an Iceberg Catalog to cache tables. * Initial code in: * https://github.com/apache/iceberg/blob/1.3.x/core/src/main/java/org/apache/iceberg/CachingCatalog.java - * Main difference is the SupportsNamespace and the fact that loadTable performs a metadata refresh. + * Main difference is the SupportsNamespace. * *

See {@link CatalogProperties#CACHE_EXPIRATION_INTERVAL_MS} for more details regarding special * values for {@code expirationIntervalMillis}. + * @param the catalog class */ public class HiveCachingCatalog implements Catalog, SupportsNamespaces { private static final Logger LOG = LoggerFactory.getLogger(HiveCachingCatalog.class); @SuppressWarnings("checkstyle:VisibilityModifier") - protected final long expirationIntervalMillis; + protected final long expirationMs; @SuppressWarnings("checkstyle:VisibilityModifier") protected final Cache tableCache; private final CATALOG catalog; - private final boolean caseSensitive; @SuppressWarnings("checkstyle:VisibilityModifier") - protected HiveCachingCatalog(CATALOG catalog, long expirationIntervalMillis) { - Preconditions.checkArgument( - expirationIntervalMillis != 0, - "When %s is set to 0, the catalog cache should be disabled. This indicates a bug.", + protected HiveCachingCatalog(CATALOG catalog, long expiration) { + Preconditions.checkArgument(expiration > 0, + "When %s is <= 0, the catalog cache should be disabled.", CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS); this.catalog = catalog; - this.caseSensitive = true; - this.expirationIntervalMillis = expirationIntervalMillis; + this.expirationMs = expiration; this.tableCache = createTableCache(Ticker.systemTicker()); } @@ -86,19 +84,24 @@ HiveCachingCatalog wrap(C catalog, long expirationIntervalMillis) { private Cache createTableCache(Ticker ticker) { Caffeine cacheBuilder = Caffeine.newBuilder().softValues(); - if (expirationIntervalMillis > 0) { + if (expirationMs > 0) { return cacheBuilder .removalListener(new MetadataTableInvalidatingRemovalListener()) .executor(Runnable::run) // Makes the callbacks to removal listener synchronous - .expireAfterAccess(Duration.ofMillis(expirationIntervalMillis)) + .expireAfterAccess(Duration.ofMillis(expirationMs)) .ticker(ticker) .build(); } return cacheBuilder.build(); } - private TableIdentifier canonicalizeIdentifier(TableIdentifier tableIdentifier) { - return caseSensitive ? tableIdentifier : tableIdentifier.toLowerCase(); + /** + * Eventual canonical transformation. + * @param tableIdentifier the identifier + * @return the canonical representation + */ + protected TableIdentifier canonicalizeIdentifier(TableIdentifier tableIdentifier) { + return tableIdentifier; } @Override @@ -129,8 +132,7 @@ public Table loadTable(TableIdentifier ident) { Table originTable = tableCache.get(originTableIdentifier, this::loadTableHive); // share TableOperations instance of origin table for all metadata tables, so that metadata - // table instances are - // also refreshed as well when origin table instance is refreshed. + // table instances are also refreshed as well when origin table instance is refreshed. if (originTable instanceof HasTableOperations) { TableOperations ops = ((HasTableOperations) originTable).operations(); MetadataTableType type = MetadataTableType.from(canonicalized.name()); diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java index c598f7b64d5c..550df6e70109 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -38,6 +38,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.exceptions.CommitStateUnknownException; @@ -56,6 +57,7 @@ import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; import org.apache.iceberg.rest.requests.RegisterTableRequest; import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.rest.requests.ReportMetricsRequest; @@ -68,6 +70,7 @@ import org.apache.iceberg.rest.responses.ListNamespacesResponse; import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.apache.iceberg.rest.responses.OAuthTokenResponse; import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; import org.apache.iceberg.util.Pair; @@ -102,11 +105,14 @@ public class HMSCatalogAdapter implements RESTClient { private final Catalog catalog; private final SupportsNamespaces asNamespaceCatalog; + private final ViewCatalog asViewCatalog; + public HMSCatalogAdapter(Catalog catalog) { this.catalog = catalog; this.asNamespaceCatalog = catalog instanceof SupportsNamespaces ? (SupportsNamespaces) catalog : null; + this.asViewCatalog = catalog instanceof ViewCatalog ? (ViewCatalog) catalog : null; } enum HTTPMethod { @@ -115,9 +121,16 @@ enum HTTPMethod { POST, DELETE } + + @FunctionalInterface + interface Operation { + T exec(Map vars, Object body); + } enum Route { TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null, OAuthTokenResponse.class), + SEPARATE_AUTH_TOKENS_URI( + HTTPMethod.POST, "https://auth-server.com/token", null, OAuthTokenResponse.class), CONFIG(HTTPMethod.GET, "v1/config", null, ConfigResponse.class), LIST_NAMESPACES(HTTPMethod.GET, "v1/namespaces", null, ListNamespacesResponse.class), CREATE_NAMESPACE( @@ -158,7 +171,22 @@ enum Route { ReportMetricsRequest.class, null), COMMIT_TRANSACTION( - HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null); + HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null), + LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views", null, ListTablesResponse.class), + LOAD_VIEW( + HTTPMethod.GET, "v1/namespaces/{namespace}/views/{name}", null, LoadViewResponse.class), + CREATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views", + CreateViewRequest.class, + LoadViewResponse.class), + UPDATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views/{name}", + UpdateTableRequest.class, + LoadViewResponse.class), + RENAME_VIEW(HTTPMethod.POST, "v1/views/rename", RenameTableRequest.class, null), + DROP_VIEW(HTTPMethod.DELETE, "v1/namespaces/{namespace}/views/{name}"); private final HTTPMethod method; private final int requiredLength; @@ -284,33 +312,26 @@ private ConfigResponse config(Map vars, Object body) { } private OAuthTokenResponse tokens(Map vars, Object body) { - Class responseType = OAuthTokenResponse.class; - @SuppressWarnings("unchecked") Map request = (Map) castRequest(Map.class, body); String grantType = request.get("grant_type"); switch (grantType) { case "client_credentials": - return castResponse( - responseType, - OAuthTokenResponse.builder() - .withToken("client-credentials-token:sub=" + request.get("client_id")) - .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") - .withTokenType("Bearer") - .build()); + return OAuthTokenResponse.builder() + .withToken("client-credentials-token:sub=" + request.get("client_id")) + .withTokenType("Bearer") + .build(); case "urn:ietf:params:oauth:grant-type:token-exchange": String actor = request.get("actor_token"); String token = - String.format( - "token-exchange-token:sub=%s%s", - request.get("subject_token"), actor != null ? ",act=" + actor : ""); - return castResponse( - responseType, - OAuthTokenResponse.builder() - .withToken(token) - .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") - .withTokenType("Bearer") - .build()); + String.format( + "token-exchange-token:sub=%s%s", + request.get("subject_token"), actor != null ? ",act=" + actor : ""); + return OAuthTokenResponse.builder() + .withToken(token) + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .withTokenType("Bearer") + .build(); default: throw new UnsupportedOperationException("Unsupported grant_type: " + grantType); @@ -431,6 +452,67 @@ private RESTResponse commitTransaction(Map vars, Object body) { return null; } + private ListTablesResponse listViews(Map vars, Object body) { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + String pageToken = PropertyUtil.propertyAsString(vars, "pageToken", null); + String pageSize = PropertyUtil.propertyAsString(vars, "pageSize", null); + if (pageSize != null) { + return castResponse( + ListTablesResponse.class, + CatalogHandlers.listViews(asViewCatalog, namespace, pageToken, pageSize)); + } else { + return castResponse( + ListTablesResponse.class, CatalogHandlers.listViews(asViewCatalog, namespace)); + } + } + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse createView(Map vars, Object body) { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + CreateViewRequest request = castRequest(CreateViewRequest.class, body); + return castResponse( + LoadViewResponse.class, CatalogHandlers.createView(asViewCatalog, namespace, request)); + } + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse loadView(Map vars, Object body) { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + return castResponse(LoadViewResponse.class, CatalogHandlers.loadView(asViewCatalog, ident)); + } + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse updateView(Map vars, Object body) { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse( + LoadViewResponse.class, CatalogHandlers.updateView(asViewCatalog, ident, request)); + } + throw new ViewNotSupported(catalog.toString()); + } + + private RESTResponse renameView(Map vars, Object body) { + if (null != asViewCatalog) { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameView(asViewCatalog, request); + return null; + } + throw new ViewNotSupported(catalog.toString()); + } + + private RESTResponse dropView(Map vars, Object body) { + if (null != asViewCatalog) { + CatalogHandlers.dropView(asViewCatalog, identFromPathVars(vars)); + return null; + } + throw new ViewNotSupported(catalog.toString()); + } @SuppressWarnings("MethodLength") private T handleRequest( @@ -489,6 +571,24 @@ private T handleRequest( case COMMIT_TRANSACTION: return (T) commitTransaction(vars, body); + + case LIST_VIEWS: + return (T) listViews(vars, body); + + case CREATE_VIEW: + return (T) createView(vars, body); + + case LOAD_VIEW: + return (T) loadView(vars, body); + + case UPDATE_VIEW: + return (T) updateView(vars, body); + + case RENAME_VIEW: + return (T) renameView(vars, body); + + case DROP_VIEW: + return (T) dropView(vars, body); default: } @@ -624,6 +724,12 @@ private static class NamespaceNotSupported extends RuntimeException { super("catalog " + catalog + " does not support namespace"); } } + + private static class ViewNotSupported extends RuntimeException { + ViewNotSupported(String catalog) { + super("catalog " + catalog + " does not support views"); + } + } private static class BadResponseType extends RuntimeException { private BadResponseType(Class responseType, Object response) { diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index 0125a42652e8..27605fa2a8a5 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -24,10 +24,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.ok; import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.ToNumberPolicy; -import com.google.gson.ToNumberStrategy; import com.nimbusds.jose.JWSAlgorithm; import com.nimbusds.jose.JWSHeader; import com.nimbusds.jose.JWSSigner; @@ -48,6 +44,7 @@ import java.nio.file.Path; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -55,6 +52,13 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.jexl3.JexlBuilder; +import org.apache.commons.jexl3.JexlContext; +import org.apache.commons.jexl3.JexlEngine; +import org.apache.commons.jexl3.JexlException; +import org.apache.commons.jexl3.JexlFeatures; +import org.apache.commons.jexl3.MapContext; +import org.apache.commons.jexl3.introspection.JexlPermissions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaException; @@ -71,6 +75,9 @@ import org.apache.hadoop.hive.metastore.properties.PropertyManager; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hive.iceberg.com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.hive.iceberg.com.fasterxml.jackson.core.type.TypeReference; +import org.apache.hive.iceberg.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; import org.eclipse.jetty.server.Server; @@ -89,6 +96,19 @@ public abstract class HMSTestBase { protected static Random RND = new Random(20230922); protected static final String USER_1 = "USER_1"; protected static final String DB_NAME = "hivedb"; + /** A Jexl engine for convenience. */ + static final JexlEngine JEXL; + static { + JexlFeatures features = new JexlFeatures() + .sideEffect(false) + .sideEffectGlobal(false); + JexlPermissions p = JexlPermissions.RESTRICTED + .compose("org.apache.hadoop.hive.metastore.*", "org.apache.iceberg.*"); + JEXL = new JexlBuilder() + .features(features) + .permissions(p) + .create(); + } protected static final long EVICTION_INTERVAL = TimeUnit.SECONDS.toMillis(10); private static final File JWT_AUTHKEY_FILE = @@ -354,7 +374,7 @@ public static Object clientCall(String jwt, URL url, String method, boolean json con.setDoOutput(true); DataOutputStream wr = new DataOutputStream(con.getOutputStream()); if (json) { - String outjson = GSON.toJson(arg); + String outjson = serialize(arg); wr.writeBytes(outjson); } else { wr.writeBytes(arg.toString()); @@ -368,7 +388,7 @@ public static Object clientCall(String jwt, URL url, String method, boolean json con.disconnect(); } } - + private static Object httpResponse(HttpURLConnection con) throws IOException { int responseCode = con.getResponseCode(); InputStream responseStream = con.getErrorStream(); @@ -392,7 +412,8 @@ private static Object httpResponse(HttpURLConnection con) throws IOException { return new ServerResponse(responseCode, response.toString()); } } - Object r = GSON.fromJson(reader, Object.class); + // there might be no answer which is still ok + Object r = reader.ready() ? deserialize(reader) : new HashMap<>(1); if (r instanceof Map) { ((Map) r).put("status", responseCode); } @@ -401,25 +422,42 @@ private static Object httpResponse(HttpURLConnection con) throws IOException { } return responseCode; } - - /** - * Making integer more pervasive when converting JSON. - */ - private static final ToNumberStrategy NARROW_NUMBER = jsonReader -> { - Number number = ToNumberPolicy.LONG_OR_DOUBLE.readNumber(jsonReader); - if (number instanceof Long) { - long n = number.longValue(); - int i = (int) n; - if (i == n) { - return i; - } + + private static final ObjectMapper MAPPER = RESTObjectMapper.mapper(); + + static String serialize(T object) { + try { + return MAPPER.writeValueAsString(object); + } catch (JsonProcessingException xany) { + throw new RuntimeException(xany); + } } - return number; - }; - - public static final Gson GSON = new GsonBuilder() - .setNumberToNumberStrategy(NARROW_NUMBER) - .setObjectToNumberStrategy(NARROW_NUMBER) - .setPrettyPrinting() - .create(); + + static T deserialize(String s) { + try { + return MAPPER.readValue(s, new TypeReference() {}); + } catch (JsonProcessingException xany) { + throw new RuntimeException(xany); + } + } + + static T deserialize(BufferedReader s) { + try { + return MAPPER.readValue(s, new TypeReference() {}); + } catch (IOException xany) { + throw new RuntimeException(xany); + } + } + + static Object eval(Object properties, String expr) { + try { + JexlContext context = properties instanceof Map + ? new MapContext((Map) properties) + : JexlEngine.EMPTY_CONTEXT; + Object result = JEXL.createScript(expr).execute(context, properties); + return result; + } catch (JexlException xany) { + throw xany; + } + } } \ No newline at end of file diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java index 456398909f54..7a50b1ea5d8d 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java @@ -20,6 +20,8 @@ package org.apache.iceberg.rest; import com.google.gson.Gson; +import java.io.IOException; +import java.net.URI; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -31,11 +33,14 @@ import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; import org.apache.iceberg.types.Types; import static org.apache.iceberg.types.Types.NestedField.required; +import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.After; import org.junit.Test; public class TestHMSCatalog extends HMSTestBase { @@ -65,7 +70,7 @@ public void testCreateNamespaceHttp() throws Exception { Object response = clientCall(jwt, url, "GET", null); Assert.assertTrue(response instanceof Map); Map nsrep = (Map) response; - List nslist = (List) nsrep.get("namespaces"); + List> nslist = (List>) nsrep.get("namespaces"); Assert.assertEquals(2, nslist.size()); Assert.assertTrue((nslist.contains(Arrays.asList("default")))); Assert.assertTrue((nslist.contains(Arrays.asList("hivedb")))); @@ -104,6 +109,8 @@ private Schema getTestSchema() { @Test public void testCreateTableTxnBuilder() throws Exception { + URI iceUri = URI.create("http://hive@localhost:" + catalogPort + "/"+catalogPath+"/v1/"); + String jwt = generateJWT(); Schema schema = getTestSchema(); final String tblName = "tbl_" + Integer.toHexString(RND.nextInt(65536)); final TableIdentifier tableIdent = TableIdentifier.of(DB_NAME, tblName); @@ -123,36 +130,88 @@ public void testCreateTableTxnBuilder() throws Exception { Assert.assertFalse(tis.isEmpty()); // list namespaces - URL url = new URL("http://hive@localhost:" + catalogPort + "/"+catalogPath+"/v1/namespaces"); - String jwt = generateJWT(); + URL url = iceUri.resolve("namespaces").toURL(); // succeed Object response = clientCall(jwt, url, "GET", null); Assert.assertNotNull(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + List> nslist = (List>) eval(response, "json -> json.namespaces"); + Assert.assertEquals(2, nslist.size()); + Assert.assertTrue((nslist.contains(Arrays.asList("default")))); + Assert.assertTrue((nslist.contains(Arrays.asList("hivedb")))); // list tables in hivedb - url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + DB_NAME + "/tables"); + url = iceUri.resolve("namespaces/" + DB_NAME + "/tables").toURL(); // succeed response = clientCall(jwt, url, "GET", null); Assert.assertNotNull(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + Assert.assertEquals(1, (int) eval(response, "json -> size(json.identifiers)")); + Assert.assertEquals(tblName, eval(response, "json -> json.identifiers[0].name")); // load table - url = new URL("http://hive@localhost:" + catalogPort + "/" + catalogPath+"/v1/namespaces/" + DB_NAME + "/tables/" + tblName); + url = iceUri.resolve("namespaces/" + DB_NAME + "/tables/" + tblName).toURL(); // succeed response = clientCall(jwt, url, "GET", null); Assert.assertNotNull(response); - String str = new Gson().toJson(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + Assert.assertEquals(location, eval(response, "json -> json.metadata.location")); // quick check on metrics Map counters = reportMetricCounters("list_namespaces", "list_tables", "load_table"); counters.forEach((key, value) -> Assert.assertTrue(key, value > 0)); table = catalog.loadTable(tableIdent); Assert.assertNotNull(table); - } catch (Exception xany) { - String str = xany.getMessage(); + } catch (IOException xany) { + Assert.fail(xany.getMessage()); } finally { //metastoreClient.dropTable(DB_NAME, tblName); catalog.dropTable(tableIdent, false); } } + + @Test + public void testTableAPI() throws Exception { + URI iceUri = URI.create("http://hive@localhost:" + catalogPort + "/"+catalogPath+"/v1/"); + String jwt = generateJWT(); + Schema schema = getTestSchema(); + final String tblName = "tbl_" + Integer.toHexString(RND.nextInt(65536)); + final TableIdentifier tableIdent = TableIdentifier.of(DB_NAME, tblName); + String location = temp.newFolder(tableIdent.toString()).toString(); + // create table + CreateTableRequest create = CreateTableRequest.builder(). + withName(tblName). + withLocation(location). + withSchema(schema).build(); + URL url = iceUri.resolve("namespaces/" + DB_NAME + "/tables").toURL(); + Object response = clientCall(jwt, url, "POST", create); + Assert.assertNotNull(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + Assert.assertEquals(location, eval(response, "json -> json.metadata.location")); + Table table = catalog.loadTable(tableIdent); + Assert.assertEquals(location, table.location()); + + // rename table + final String rtblName = "TBL_" + Integer.toHexString(RND.nextInt(65536)); + final TableIdentifier rtableIdent = TableIdentifier.of(DB_NAME, rtblName); + RenameTableRequest rename = RenameTableRequest.builder(). + withSource(tableIdent). + withDestination(rtableIdent). + build(); + url = iceUri.resolve("tables/rename").toURL(); + response = clientCall(jwt, url, "POST", rename); + Assert.assertNotNull(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + table = catalog.loadTable(rtableIdent); + Assert.assertEquals(location, table.location()); + + // delete table + url = iceUri.resolve("namespaces/" + DB_NAME + "/tables/" + rtblName).toURL(); + response = clientCall(jwt, url, "DELETE", null); + Assert.assertNotNull(response); + Assert.assertEquals(200, (int) eval(response, "json -> json.status")); + Assert.assertThrows(NoSuchTableException.class, () -> catalog.loadTable(rtableIdent)); + } + } diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 883c55a48392..a133641d0cb6 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -413,7 +413,7 @@ com.github.tomakehurst wiremock-jre8-standalone - 2.32.0 + ${wiremock.jre8.standalone.version} test diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 3ce67f8f23ff..076fbbcd275c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -159,7 +159,19 @@ static Iface newHMSHandler(Configuration conf) HMSHandler baseHandler = new HMSHandler("hive client", conf); return HMSHandlerProxyFactory.getProxy(conf, baseHandler, true); } - + + /** + * Create HMS handler for embedded metastore. + * + *

IMPORTANT

+ * + * This method is called indirectly by HiveMetastoreClient using reflection when using an embedded + * Iceberg Catalog. It can not be removed and its arguments can't be changed without matching + * change in HiveMetastoreClient . + * + * @param conf configuration to use + * @throws MetaException + */ static Iface newHMSRetryingLocalHandler(Configuration conf) throws MetaException { HMSHandler baseHandler = new HMSHandler("hive client", conf); @@ -778,10 +790,10 @@ static Server startIcebergCatalog(Configuration configuration) { Method iceStart = iceClazz.getMethod("startServer", Configuration.class); return (Server) iceStart.invoke(null, configuration); } catch (ClassNotFoundException xnf) { - LOG.warn("unable to start Iceberg REST Catalog server {}, missing jar?", xnf); + LOG.warn("unable to start Iceberg REST Catalog server, missing jar?", xnf); return null; - } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - LOG.error("unable to start Iceberg REST Catalog server {}", e); + } catch (Exception e) { + LOG.error("unable to start Iceberg REST Catalog server", e); return null; } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java index 206684bf1b2e..0cffc3da5d1a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java @@ -30,7 +30,7 @@ public interface SecureServletCaller { * Should be called in Servlet.init() * @throws ServletException if the jwt validator creation throws an exception */ - public void init() throws ServletException; + void init() throws ServletException; /** * Any http method executor. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index 2f2c7cda7830..57db75d325c7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -80,6 +80,7 @@ public void init() throws ServletException { * @param executor the method executor * @throws IOException if the Json in/out fail */ + @Override public void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) throws IOException { if (LOG.isDebugEnabled()) { diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 1f4855d39fee..4152fa505399 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -121,6 +121,7 @@ ${basedir}/src/gen/thrift -I ${thrift.home} -strict --gen java:beans,generated_annotations=undated --gen cpp --gen php --gen py --gen rb + 2.32.0 2024-01-01T00:00:00Z From 399e25bb69bec3b824abccd2228d1fc7dd0d84d3 Mon Sep 17 00:00:00 2001 From: Henrib Date: Tue, 11 Feb 2025 17:10:29 +0100 Subject: [PATCH 10/40] HIVE-28059 : fixing more PR comments (conf, pom); --- .../metastore-catalog/pom.xml | 75 ++++--------------- .../apache/iceberg/rest/HMSCatalogServer.java | 1 - .../org/apache/iceberg/rest/HMSTestBase.java | 4 +- .../apache/iceberg/rest/TestHMSCatalog.java | 1 - .../src/test/resources/hive-log4j2.properties | 2 +- .../hive/metastore/HiveMetaStoreClient.java | 3 +- .../hive/metastore/conf/MetastoreConf.java | 2 - 7 files changed, 17 insertions(+), 71 deletions(-) diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml index 365cc60d2d23..3988823d7b22 100644 --- a/standalone-metastore/metastore-catalog/pom.xml +++ b/standalone-metastore/metastore-catalog/pom.xml @@ -74,6 +74,20 @@ true --> + + org.apache.hive + hive-standalone-metastore-common + ${hive.version} + tests + test + + + org.apache.hive + hive-standalone-metastore-server + ${hive.version} + tests + test + org.apache.httpcomponents.core5 httpcore5 @@ -243,39 +257,11 @@ - - - *.patch - DEV-README - **/src/main/sql/** - **/README.md - **/*.iml - **/*.txt - **/*.log - **/package-info.java - **/*.properties - **/*.q - **/*.q.out - **/*.xml - **/gen/** - **/patchprocess/** - **/metastore_db/** - **/test/resources/**/*.ldif - **/test/resources/**/*.sql - **/test/resources/**/*.json - - org.apache.maven.plugins maven-surefire-plugin ${surefire.version} - - - TestHMSCatalog.java - TestHiveCatalog.java - - org.codehaus.mojo @@ -295,39 +281,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - 3.1.0 - - - generate-test-sources - - - - - - - - - - - - - - - - - - - - copy - - run - - - - diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java index 6ddaf2331d1f..ced0207a823b 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -73,7 +73,6 @@ public static Catalog createCatalog(Configuration configuration) { final String cwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE.getVarname()); final String cextwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); - MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.HMS_HANDLER_CREATE, "newHMSRetryingLocalHandler"); final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog(); catalog.setConf(configuration); Map properties = new TreeMap<>(); diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index 27605fa2a8a5..8b40b068b408 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -128,9 +128,7 @@ public TestSchemaInfo(String metastoreHome, String dbType) throws HiveMetaExcept } @Override public String getMetaStoreScriptDir() { - return new File(BASE_DIR,"src/test/resources").getAbsolutePath() + File.separatorChar + - "scripts" + File.separatorChar + "metastore" + - File.separatorChar + "upgrade" + File.separatorChar + dbType; + return new File(BASE_DIR, "../metastore-server/src/main/sql/derby").getAbsolutePath(); } } diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java index 7a50b1ea5d8d..d3b50fa928f3 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java @@ -19,7 +19,6 @@ package org.apache.iceberg.rest; -import com.google.gson.Gson; import java.io.IOException; import java.net.URI; import java.net.URL; diff --git a/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties b/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties index 7243144ed6ff..36aaa3ef9ccf 100644 --- a/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties +++ b/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties @@ -33,7 +33,7 @@ logger.file.level = debug logger.file.appenderRefs = file logger.file.appenderRef.file.ref = LOGFILE -rootLogger.level = debug +rootLogger.level = info rootLogger.appenderRefs = stdout,captured rootLogger.appenderRef.stdout.ref = STDOUT rootLogger.appenderRef.captured.ref = CAPTURED diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index e0b64f2c4927..df072e03b325 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -304,8 +304,7 @@ static ThriftHiveMetastore.Iface callEmbeddedMetastore(Configuration conf) throw try { Class clazz = Class.forName(HIVE_METASTORE_CLASS); //noinspection JavaReflectionMemberAccess - String methodName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_CREATE); - Method method = clazz.getDeclaredMethod(methodName,Configuration.class); + Method method = clazz.getDeclaredMethod(HIVE_METASTORE_CREATE_HANDLER_METHOD, Configuration.class); method.setAccessible(true); return (ThriftHiveMetastore.Iface) method.invoke(null, conf); } catch (InvocationTargetException e) { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 9f9e33742009..a1634ead255d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -911,8 +911,6 @@ public enum ConfVars { HMS_HANDLER_PROXY_CLASS("metastore.hmshandler.proxy", "hive.metastore.hmshandler.proxy", METASTORE_RETRYING_HANDLER_CLASS, "The proxy class name of HMSHandler, default is RetryingHMSHandler."), - HMS_HANDLER_CREATE("metastore.hmshandler.create", "metastore.hmshandler.create","newHMSHandler", - "The method name to create new HMSHandler"), IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus.identifierFactory", "datanucleus1", "Name of the identifier factory to use when generating table/column names etc. \n" + From cfe85d2c922c00146f20219ceeba507ca2447a94 Mon Sep 17 00:00:00 2001 From: Henrib Date: Wed, 12 Feb 2025 01:02:20 +0100 Subject: [PATCH 11/40] HIVE-28059 : refactored cache; - cleaner init & server; - one issue/FIXME (no Thrift conf does not work correctly); --- .../apache/iceberg/HiveCachingCatalog.java | 333 ------------------ .../iceberg/rest/HMSCachingCatalog.java | 85 +++++ .../iceberg/rest/HMSCatalogAdapter.java | 96 +++-- .../apache/iceberg/rest/HMSCatalogServer.java | 44 +-- .../hive/metastore/conf/MetastoreConf.java | 2 + 5 files changed, 150 insertions(+), 410 deletions(-) delete mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java create mode 100644 standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java deleted file mode 100644 index d72f02840bf7..000000000000 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/HiveCachingCatalog.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import com.github.benmanes.caffeine.cache.RemovalCause; -import com.github.benmanes.caffeine.cache.RemovalListener; -import com.github.benmanes.caffeine.cache.Ticker; -import java.time.Duration; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.iceberg.catalog.Catalog; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NamespaceNotEmptyException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - - -/** - * Class that wraps an Iceberg Catalog to cache tables. - * Initial code in: - * https://github.com/apache/iceberg/blob/1.3.x/core/src/main/java/org/apache/iceberg/CachingCatalog.java - * Main difference is the SupportsNamespace. - * - *

See {@link CatalogProperties#CACHE_EXPIRATION_INTERVAL_MS} for more details regarding special - * values for {@code expirationIntervalMillis}. - * @param the catalog class - */ -public class HiveCachingCatalog implements Catalog, SupportsNamespaces { - private static final Logger LOG = LoggerFactory.getLogger(HiveCachingCatalog.class); - @SuppressWarnings("checkstyle:VisibilityModifier") - protected final long expirationMs; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected final Cache tableCache; - private final CATALOG catalog; - - @SuppressWarnings("checkstyle:VisibilityModifier") - protected HiveCachingCatalog(CATALOG catalog, long expiration) { - Preconditions.checkArgument(expiration > 0, - "When %s is <= 0, the catalog cache should be disabled.", - CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS); - this.catalog = catalog; - this.expirationMs = expiration; - this.tableCache = createTableCache(Ticker.systemTicker()); - } - - public CATALOG unwrap() { - return catalog; - } - - public static - HiveCachingCatalog wrap(C catalog, long expirationIntervalMillis) { - return new HiveCachingCatalog(catalog, expirationIntervalMillis); - } - - - private Cache createTableCache(Ticker ticker) { - Caffeine cacheBuilder = Caffeine.newBuilder().softValues(); - if (expirationMs > 0) { - return cacheBuilder - .removalListener(new MetadataTableInvalidatingRemovalListener()) - .executor(Runnable::run) // Makes the callbacks to removal listener synchronous - .expireAfterAccess(Duration.ofMillis(expirationMs)) - .ticker(ticker) - .build(); - } - return cacheBuilder.build(); - } - - /** - * Eventual canonical transformation. - * @param tableIdentifier the identifier - * @return the canonical representation - */ - protected TableIdentifier canonicalizeIdentifier(TableIdentifier tableIdentifier) { - return tableIdentifier; - } - - @Override - public String name() { - return catalog.name(); - } - - @Override - public List listTables(Namespace namespace) { - return catalog.listTables(namespace); - } - - private Table loadTableHive(TableIdentifier ident) { - return catalog.loadTable(ident); - } - - @Override - public Table loadTable(TableIdentifier ident) { - TableIdentifier canonicalized = canonicalizeIdentifier(ident); - Table cached = tableCache.getIfPresent(canonicalized); - if (cached != null) { - return cached; - } - - if (MetadataTableUtils.hasMetadataTableName(canonicalized)) { - TableIdentifier originTableIdentifier = - TableIdentifier.of(canonicalized.namespace().levels()); - Table originTable = tableCache.get(originTableIdentifier, this::loadTableHive); - - // share TableOperations instance of origin table for all metadata tables, so that metadata - // table instances are also refreshed as well when origin table instance is refreshed. - if (originTable instanceof HasTableOperations) { - TableOperations ops = ((HasTableOperations) originTable).operations(); - MetadataTableType type = MetadataTableType.from(canonicalized.name()); - Table metadataTable = - MetadataTableUtils.createMetadataTableInstance( - ops, catalog.name(), originTableIdentifier, canonicalized, type); - tableCache.put(canonicalized, metadataTable); - return metadataTable; - } - } - return tableCache.get(canonicalized, this::loadTableHive); - } - - @Override - public boolean dropTable(TableIdentifier ident, boolean purge) { - boolean dropped = catalog.dropTable(ident, purge); - invalidateTable(ident); - return dropped; - } - - @Override - public void renameTable(TableIdentifier from, TableIdentifier to) { - catalog.renameTable(from, to); - invalidateTable(from); - } - - @Override - public void invalidateTable(TableIdentifier ident) { - catalog.invalidateTable(ident); - TableIdentifier canonicalized = canonicalizeIdentifier(ident); - tableCache.invalidate(canonicalized); - tableCache.invalidateAll(metadataTableIdentifiers(canonicalized)); - } - - @Override - public Table registerTable(TableIdentifier identifier, String metadataFileLocation) { - Table table = catalog.registerTable(identifier, metadataFileLocation); - invalidateTable(identifier); - return table; - } - - private Iterable metadataTableIdentifiers(TableIdentifier ident) { - ImmutableList.Builder builder = ImmutableList.builder(); - - for (MetadataTableType type : MetadataTableType.values()) { - // metadata table resolution is case insensitive right now - builder.add(TableIdentifier.parse(ident + "." + type.name())); - builder.add(TableIdentifier.parse(ident + "." + type.name().toLowerCase(Locale.ROOT))); - } - - return builder.build(); - } - - @Override - public TableBuilder buildTable(TableIdentifier identifier, Schema schema) { - return new CachingTableBuilder(identifier, schema); - } - - @Override - public void createNamespace(Namespace nmspc, Map map) { - catalog.createNamespace(nmspc, map); - } - - @Override - public List listNamespaces(Namespace nmspc) throws NoSuchNamespaceException { - return catalog.listNamespaces(nmspc); - } - - @Override - public Map loadNamespaceMetadata(Namespace nmspc) throws NoSuchNamespaceException { - return catalog.loadNamespaceMetadata(nmspc); - } - - @Override - public boolean dropNamespace(Namespace nmspc) throws NamespaceNotEmptyException { - List tables = listTables(nmspc); - for (TableIdentifier ident : tables) { - TableIdentifier canonicalized = canonicalizeIdentifier(ident); - tableCache.invalidate(canonicalized); - tableCache.invalidateAll(metadataTableIdentifiers(canonicalized)); - } - return catalog.dropNamespace(nmspc); - } - - @Override - public boolean setProperties(Namespace nmspc, Map map) throws NoSuchNamespaceException { - return catalog.setProperties(nmspc, map); - } - - @Override - public boolean removeProperties(Namespace nmspc, Set set) throws NoSuchNamespaceException { - return catalog.removeProperties(nmspc, set); - } - - /** - * RemovalListener class for removing metadata tables when their associated data table is expired - * via cache expiration. - */ - class MetadataTableInvalidatingRemovalListener - implements RemovalListener { - @Override - public void onRemoval(TableIdentifier tableIdentifier, Table table, RemovalCause cause) { - LOG.debug("Evicted {} from the table cache ({})", tableIdentifier, cause); - if (RemovalCause.EXPIRED.equals(cause)) { - if (!MetadataTableUtils.hasMetadataTableName(tableIdentifier)) { - tableCache.invalidateAll(metadataTableIdentifiers(tableIdentifier)); - } - } - } - } - - private class CachingTableBuilder implements TableBuilder { - private final TableIdentifier ident; - private final TableBuilder innerBuilder; - - private CachingTableBuilder(TableIdentifier identifier, Schema schema) { - this.innerBuilder = catalog.buildTable(identifier, schema); - this.ident = identifier; - } - - @Override - public TableBuilder withPartitionSpec(PartitionSpec spec) { - innerBuilder.withPartitionSpec(spec); - return this; - } - - @Override - public TableBuilder withSortOrder(SortOrder sortOrder) { - innerBuilder.withSortOrder(sortOrder); - return this; - } - - @Override - public TableBuilder withLocation(String location) { - innerBuilder.withLocation(location); - return this; - } - - @Override - public TableBuilder withProperties(Map properties) { - innerBuilder.withProperties(properties); - return this; - } - - @Override - public TableBuilder withProperty(String key, String value) { - innerBuilder.withProperty(key, value); - return this; - } - - @Override - public Table create() { - AtomicBoolean created = new AtomicBoolean(false); - Table table = - tableCache.get( - canonicalizeIdentifier(ident), - identifier -> { - created.set(true); - return innerBuilder.create(); - }); - if (!created.get()) { - throw new AlreadyExistsException("Table already exists: %s", ident); - } - return table; - } - - @Override - public Transaction createTransaction() { - // create a new transaction without altering the cache. the table doesn't exist until the - // transaction is - // committed. if the table is created before the transaction commits, any cached version is - // correct and the - // transaction create will fail. if the transaction commits before another create, then the - // cache will be empty. - return innerBuilder.createTransaction(); - } - - @Override - public Transaction replaceTransaction() { - // create a new transaction without altering the cache. the table doesn't change until the - // transaction is - // committed. when the transaction commits, invalidate the table in the cache if it is - // present. - return CommitCallbackTransaction.addCallback( - innerBuilder.replaceTransaction(), () -> invalidateTable(ident)); - } - - @Override - public Transaction createOrReplaceTransaction() { - // create a new transaction without altering the cache. the table doesn't change until the - // transaction is - // committed. when the transaction commits, invalidate the table in the cache if it is - // present. - return CommitCallbackTransaction.addCallback( - innerBuilder.createOrReplaceTransaction(), () -> invalidateTable(ident)); - } - } -} diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java new file mode 100644 index 000000000000..6b5d76f818a8 --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.rest; + +import com.github.benmanes.caffeine.cache.Ticker; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.CachingCatalog; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; + + +/** + * Class that wraps an Iceberg Catalog to cache tables. + * @param the catalog class + */ +public class HMSCachingCatalog extends CachingCatalog implements SupportsNamespaces { + protected final CATALOG nsCatalog; + + public HMSCachingCatalog(CATALOG catalog, long expiration) { + super(catalog, true, expiration, Ticker.systemTicker()); + nsCatalog = catalog; + } + + public CATALOG hmsUnwrap() { + return nsCatalog; + } + + @Override + public void createNamespace(Namespace nmspc, Map map) { + nsCatalog.createNamespace(nmspc, map); + } + + @Override + public List listNamespaces(Namespace nmspc) throws NoSuchNamespaceException { + return nsCatalog.listNamespaces(nmspc); + } + + @Override + public Map loadNamespaceMetadata(Namespace nmspc) throws NoSuchNamespaceException { + return nsCatalog.loadNamespaceMetadata(nmspc); + } + + @Override + public boolean dropNamespace(Namespace nmspc) throws NamespaceNotEmptyException { + List tables = listTables(nmspc); + for (TableIdentifier ident : tables) { + invalidateTable(ident); + } + return nsCatalog.dropNamespace(nmspc); + } + + @Override + public boolean setProperties(Namespace nmspc, Map map) throws NoSuchNamespaceException { + return nsCatalog.setProperties(nmspc, map); + } + + @Override + public boolean removeProperties(Namespace nmspc, Set set) throws NoSuchNamespaceException { + return nsCatalog.removeProperties(nmspc, set); + } + +} diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java index 550df6e70109..8d694d15ac04 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -128,64 +128,48 @@ interface Operation { } enum Route { - TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null, OAuthTokenResponse.class), - SEPARATE_AUTH_TOKENS_URI( - HTTPMethod.POST, "https://auth-server.com/token", null, OAuthTokenResponse.class), - CONFIG(HTTPMethod.GET, "v1/config", null, ConfigResponse.class), - LIST_NAMESPACES(HTTPMethod.GET, "v1/namespaces", null, ListNamespacesResponse.class), - CREATE_NAMESPACE( - HTTPMethod.POST, - "v1/namespaces", - CreateNamespaceRequest.class, - CreateNamespaceResponse.class), - LOAD_NAMESPACE(HTTPMethod.GET, "v1/namespaces/{namespace}", null, GetNamespaceResponse.class), + TOKENS(HTTPMethod.POST, "v1/oauth/tokens", + null, OAuthTokenResponse.class), + SEPARATE_AUTH_TOKENS_URI(HTTPMethod.POST, "https://auth-server.com/token", + null, OAuthTokenResponse.class), + CONFIG(HTTPMethod.GET, "v1/config", + null, ConfigResponse.class), + LIST_NAMESPACES(HTTPMethod.GET, "v1/namespaces", + null, ListNamespacesResponse.class), + CREATE_NAMESPACE(HTTPMethod.POST, "v1/namespaces", + CreateNamespaceRequest.class, CreateNamespaceResponse.class), + LOAD_NAMESPACE(HTTPMethod.GET, "v1/namespaces/{namespace}", + null, GetNamespaceResponse.class), DROP_NAMESPACE(HTTPMethod.DELETE, "v1/namespaces/{namespace}"), - UPDATE_NAMESPACE( - HTTPMethod.POST, - "v1/namespaces/{namespace}/properties", - UpdateNamespacePropertiesRequest.class, - UpdateNamespacePropertiesResponse.class), - LIST_TABLES(HTTPMethod.GET, "v1/namespaces/{namespace}/tables", null, ListTablesResponse.class), - CREATE_TABLE( - HTTPMethod.POST, - "v1/namespaces/{namespace}/tables", - CreateTableRequest.class, - LoadTableResponse.class), - LOAD_TABLE( - HTTPMethod.GET, "v1/namespaces/{namespace}/tables/{table}", null, LoadTableResponse.class), - REGISTER_TABLE( - HTTPMethod.POST, - "v1/namespaces/{namespace}/register", - RegisterTableRequest.class, - LoadTableResponse.class), - UPDATE_TABLE( - HTTPMethod.POST, - "v1/namespaces/{namespace}/tables/{table}", - UpdateTableRequest.class, - LoadTableResponse.class), + UPDATE_NAMESPACE(HTTPMethod.POST, "v1/namespaces/{namespace}/properties", + UpdateNamespacePropertiesRequest.class, UpdateNamespacePropertiesResponse.class), + LIST_TABLES(HTTPMethod.GET, "v1/namespaces/{namespace}/tables", + null, ListTablesResponse.class), + CREATE_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/tables", + CreateTableRequest.class, LoadTableResponse.class), + LOAD_TABLE(HTTPMethod.GET, "v1/namespaces/{namespace}/tables/{table}", + null, LoadTableResponse.class), + REGISTER_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/register", + RegisterTableRequest.class, LoadTableResponse.class), + UPDATE_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/tables/{table}", + UpdateTableRequest.class, LoadTableResponse.class), DROP_TABLE(HTTPMethod.DELETE, "v1/namespaces/{namespace}/tables/{table}"), - RENAME_TABLE(HTTPMethod.POST, "v1/tables/rename", RenameTableRequest.class, null), - REPORT_METRICS( - HTTPMethod.POST, - "v1/namespaces/{namespace}/tables/{table}/metrics", - ReportMetricsRequest.class, - null), - COMMIT_TRANSACTION( - HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null), - LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views", null, ListTablesResponse.class), - LOAD_VIEW( - HTTPMethod.GET, "v1/namespaces/{namespace}/views/{name}", null, LoadViewResponse.class), - CREATE_VIEW( - HTTPMethod.POST, - "v1/namespaces/{namespace}/views", - CreateViewRequest.class, - LoadViewResponse.class), - UPDATE_VIEW( - HTTPMethod.POST, - "v1/namespaces/{namespace}/views/{name}", - UpdateTableRequest.class, - LoadViewResponse.class), - RENAME_VIEW(HTTPMethod.POST, "v1/views/rename", RenameTableRequest.class, null), + RENAME_TABLE(HTTPMethod.POST, "v1/tables/rename", + RenameTableRequest.class, null), + REPORT_METRICS(HTTPMethod.POST, "v1/namespaces/{namespace}/tables/{table}/metrics", + ReportMetricsRequest.class, null), + COMMIT_TRANSACTION(HTTPMethod.POST, "v1/transactions/commit", + CommitTransactionRequest.class, null), + LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views", + null, ListTablesResponse.class), + LOAD_VIEW(HTTPMethod.GET, "v1/namespaces/{namespace}/views/{name}", + null, LoadViewResponse.class), + CREATE_VIEW(HTTPMethod.POST, "v1/namespaces/{namespace}/views", + CreateViewRequest.class, LoadViewResponse.class), + UPDATE_VIEW(HTTPMethod.POST, "v1/namespaces/{namespace}/views/{name}", + UpdateTableRequest.class, LoadViewResponse.class), + RENAME_VIEW(HTTPMethod.POST, "v1/views/rename", + RenameTableRequest.class, null), DROP_VIEW(HTTPMethod.DELETE, "v1/namespaces/{namespace}/views/{name}"); private final HTTPMethod method; diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java index ced0207a823b..4174e7f1b438 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -30,7 +30,6 @@ import org.apache.hadoop.hive.metastore.SecureServletCaller; import org.apache.hadoop.hive.metastore.ServletSecurity; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.iceberg.HiveCachingCatalog; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.hive.HiveCatalog; import org.eclipse.jetty.server.ConnectionFactory; @@ -48,6 +47,9 @@ import org.slf4j.LoggerFactory; +/** + * Iceberg Catalog server. + */ public class HMSCatalogServer { private static final String CACHE_EXPIRY = "hive.metastore.catalog.cache.expiry"; private static final String JETTY_THREADPOOL_MIN = "hive.metastore.catalog.jetty.threadpool.min"; @@ -60,44 +62,44 @@ static Catalog getLastCatalog() { return catalogRef != null ? catalogRef.get() : null; } - private HMSCatalogServer() { + protected HMSCatalogServer() { // nothing } - public static HttpServlet createServlet(SecureServletCaller security, Catalog catalog) throws IOException { + protected HttpServlet createServlet(SecureServletCaller security, Catalog catalog) throws IOException { return new HMSCatalogServlet(security, new HMSCatalogAdapter(catalog)); } - public static Catalog createCatalog(Configuration configuration) { - final String curi = configuration.get(MetastoreConf.ConfVars.THRIFT_URIS.getVarname()); - final String cwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE.getVarname()); - final String cextwarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); - MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); - final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog(); - catalog.setConf(configuration); + protected Catalog createCatalog(Configuration configuration) { + final String configUri = configuration.get(MetastoreConf.ConfVars.THRIFT_URIS.getVarname()); + final String configWarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE.getVarname()); + final String configExtWarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); Map properties = new TreeMap<>(); - if (curi != null) { - properties.put("uri", curi); + if (configUri != null) { + properties.put("uri", configUri); } - if (cwarehouse != null) { - properties.put("warehouse", cwarehouse); + if (configWarehouse != null) { + properties.put("warehouse", configWarehouse); } - if (cextwarehouse != null) { - properties.put("external-warehouse", cextwarehouse); + if (configExtWarehouse != null) { + properties.put("external-warehouse", configExtWarehouse); } + final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog(); + catalog.setConf(configuration); catalog.initialize("hive", properties); long expiry = configuration.getLong(CACHE_EXPIRY, 60_000L); - return expiry > 0? HiveCachingCatalog.wrap(catalog, expiry) : catalog; + return expiry > 0? new HMSCachingCatalog(catalog, expiry) : catalog; } - public static HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException { + protected HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException { String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH); boolean jwt = "jwt".equalsIgnoreCase(auth); SecureServletCaller security = new ServletSecurity(configuration, jwt); Catalog actualCatalog = catalog; if (actualCatalog == null) { + // FIXME: 20250212 - the no-Thrift requires this but then test fail with closed Peristence Manager + // MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); actualCatalog = createCatalog(configuration); - actualCatalog.initialize("hive", Collections.emptyMap()); } catalogRef = new SoftReference<>(actualCatalog); return createServlet(security, actualCatalog); @@ -110,7 +112,7 @@ public static HttpServlet createServlet(Configuration configuration, Catalog cat * @return the server instance * @throws Exception if servlet initialization fails */ - public static Server startServer(Configuration conf, HiveCatalog catalog) throws Exception { + protected Server startServer(Configuration conf, HiveCatalog catalog) throws Exception { int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT); if (port < 0) { return null; @@ -163,6 +165,6 @@ private static Server createHttpServer(Configuration conf, int port) throws IOEx * @throws Exception if servlet initialization fails */ public static Server startServer(Configuration conf) throws Exception { - return startServer(conf, null); + return new HMSCatalogServer().startServer(conf, null); } } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index a1634ead255d..9f9e33742009 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -911,6 +911,8 @@ public enum ConfVars { HMS_HANDLER_PROXY_CLASS("metastore.hmshandler.proxy", "hive.metastore.hmshandler.proxy", METASTORE_RETRYING_HANDLER_CLASS, "The proxy class name of HMSHandler, default is RetryingHMSHandler."), + HMS_HANDLER_CREATE("metastore.hmshandler.create", "metastore.hmshandler.create","newHMSHandler", + "The method name to create new HMSHandler"), IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus.identifierFactory", "datanucleus1", "Name of the identifier factory to use when generating table/column names etc. \n" + From 81db7e46b352c6e3db033fc6b0987d261284a6ab Mon Sep 17 00:00:00 2001 From: Henrib Date: Wed, 12 Feb 2025 01:32:01 +0100 Subject: [PATCH 12/40] HIVE-28059 : remove dead class & code; - simplify servlet; --- .../iceberg/rest/HMSCatalogServlet.java | 54 ++++----------- .../org/apache/iceberg/hive/HiveUtil.java | 68 ------------------- .../apache/iceberg/rest/TestHMSCatalog.java | 1 - 3 files changed, 15 insertions(+), 108 deletions(-) delete mode 100644 standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java index d5be0a92522f..2843b5003e5d 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -80,64 +80,43 @@ protected void service(HttpServletRequest req, HttpServletResponse resp) throws this.doPatch(req, resp); } } - - protected void doPatch(HttpServletRequest request, HttpServletResponse response) { + + private void doExecute(String method, HttpServletRequest request, HttpServletResponse response) { try { security.execute(request, response, this::execute); } catch (IOException e) { - LOG.error("PATCH failed", e); + LOG.error(method + " failed", e); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } } + protected void doPatch(HttpServletRequest request, HttpServletResponse response) { + doExecute("PATCH", request, response); + } + @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error("GET failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + doExecute("GET", request, response); } @Override protected void doPut(HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error("PUT failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + doExecute("PUT", request, response); } @Override protected void doHead(HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error("HEAD failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + doExecute("HEAD", request, response); } @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error("POST failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + doExecute("POST", request, response); } @Override protected void doDelete(HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error("DELETE failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + doExecute("DELETE", request, response); } private void execute(HttpServletRequest request, HttpServletResponse response) { @@ -165,13 +144,10 @@ private void execute(HttpServletRequest request, HttpServletResponse response) { if (responseBody != null) { RESTObjectMapper.mapper().writeValue(response.getWriter(), responseBody); } - } catch (RuntimeException e) { + } catch (RuntimeException | IOException e) { // should be a RESTException but not able to see them through dependencies LOG.error("Error processing REST request", e); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } catch (Exception e) { - LOG.error("Unexpected exception when processing REST request", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } } @@ -257,11 +233,11 @@ static ServletRequestContext from(HttpServletRequest request) throws IOException return new ServletRequestContext(method, route, path, headers, queryParams, requestBody); } - public HTTPMethod method() { + HTTPMethod method() { return method; } - public Route route() { + Route route() { return route; } diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java deleted file mode 100644 index fb31799b5d6d..000000000000 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/HiveUtil.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.iceberg.hive; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.iceberg.ClientPool; -import org.apache.iceberg.Snapshot; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.hive.HiveCatalog; -import org.apache.iceberg.hive.HiveTableOperations; -import org.apache.iceberg.io.FileIO; - -import java.util.Map; - -/** - * A Friend bridge to Iceberg. - */ -public class HiveUtil { - - public static final HiveTableOperations newTableOperations(Configuration conf, String catalogName, String database, String table) { - return new HiveTableOperations(conf, null, null, catalogName, database, table); - } - - public static final HiveTableOperations newTableOperations(Configuration conf, ClientPool metaClients, FileIO fileIO, String catalogName, String database, String table) { - return new HiveTableOperations(conf, null, null, catalogName, database, table); - } - - public static Database convertToDatabase(HiveCatalog catalog, Namespace ns, Map meta) { - return catalog.convertToDatabase(ns, meta); - } - - public static void setSnapshotSummary(HiveTableOperations ops, Map parameters, Snapshot snapshot) { - ops.setSnapshotSummary(parameters, snapshot); - } - - public static void setSnapshotStats(HiveTableOperations ops, TableMetadata metadata, Map parameters) { - ops.setSnapshotStats(metadata, parameters); - } - - public static void setSchema(HiveTableOperations ops, TableMetadata metadata, Map parameters) { - ops.setSchema(metadata.schema(), parameters); - } - - public static void setPartitionSpec(HiveTableOperations ops, TableMetadata metadata, Map parameters) { - ops.setPartitionSpec(metadata, parameters); - } - - public static void setSortOrder(HiveTableOperations ops, TableMetadata metadata, Map parameters) { - ops.setSortOrder(metadata, parameters); - } -} diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java index d3b50fa928f3..1dd2034cb417 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java @@ -164,7 +164,6 @@ public void testCreateTableTxnBuilder() throws Exception { } catch (IOException xany) { Assert.fail(xany.getMessage()); } finally { - //metastoreClient.dropTable(DB_NAME, tblName); catalog.dropTable(tableIdent, false); } } From 931eff51ed80dfcbd69e5a6a1829418213b5ed65 Mon Sep 17 00:00:00 2001 From: Henrib Date: Wed, 12 Feb 2025 11:37:54 +0100 Subject: [PATCH 13/40] HIVE-28059 : remove dead code; - fix test issue (thanks Zhihua !); --- .../apache/iceberg/rest/HMSCatalogServer.java | 3 +- .../java/org/apache/iceberg/hive/Friend.java | 29 +++++++++++++++++++ .../org/apache/iceberg/rest/HMSTestBase.java | 6 ++-- .../hive/metastore/HiveMetaStoreClient.java | 2 +- .../hive/metastore/conf/MetastoreConf.java | 2 -- .../hadoop/hive/metastore/HiveMetaStore.java | 22 -------------- 6 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java index 4174e7f1b438..e238fe145cba 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -97,8 +97,7 @@ protected HttpServlet createServlet(Configuration configuration, Catalog catalog SecureServletCaller security = new ServletSecurity(configuration, jwt); Catalog actualCatalog = catalog; if (actualCatalog == null) { - // FIXME: 20250212 - the no-Thrift requires this but then test fail with closed Peristence Manager - // MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); + MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); actualCatalog = createCatalog(configuration); } catalogRef = new SoftReference<>(actualCatalog); diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java new file mode 100644 index 000000000000..a60b9637637e --- /dev/null +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.hive; + +/** + * Friend of CachedClientPool. + */ +public class Friend { + public static void cleanPoolCache() { + CachedClientPool.clientPoolCache().invalidateAll(); + } + +} diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index 8b40b068b408..cc49cdf7d4c1 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -80,6 +80,7 @@ import org.apache.hive.iceberg.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.hive.Friend; import org.eclipse.jetty.server.Server; import org.junit.After; import org.junit.Assert; @@ -166,11 +167,11 @@ public void setUp() throws Exception { conf.setBoolean(MetastoreConf.ConfVars.METRICS_ENABLED.getVarname(), true); // "hive.metastore.warehouse.dir" - String whpath = new File(BASE_DIR,"target/tmp/warehouse/managed").toURI()/*.getAbsolutePath()*/.toString(); + String whpath = new File(BASE_DIR,"target/tmp/warehouse/managed").toURI().toString(); MetastoreConf.setVar(conf, MetastoreConf.ConfVars.WAREHOUSE, whpath); HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_WAREHOUSE, whpath); // "hive.metastore.warehouse.external.dir" - String extwhpath = new File(BASE_DIR,"target/tmp/warehouse/external").toURI()/*.getAbsolutePath()*/.toString(); + String extwhpath = new File(BASE_DIR,"target/tmp/warehouse/external").toURI().toString(); MetastoreConf.setVar(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL, extwhpath); conf.set(HiveConf.ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL.varname, extwhpath); @@ -277,6 +278,7 @@ public synchronized void tearDown() throws Exception { System.clearProperty(ObjectStore.TRUSTSTORE_PASSWORD_KEY); System.clearProperty(ObjectStore.TRUSTSTORE_TYPE_KEY); // + Friend.cleanPoolCache(); } finally { catalog = null; nsCatalog = null; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index df072e03b325..0b6a2e2b8868 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -282,7 +282,7 @@ public Void run() throws Exception { open(); } - /** + /** * Instantiate the metastore server handler directly instead of connecting * through the network * diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 9f9e33742009..a1634ead255d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -911,8 +911,6 @@ public enum ConfVars { HMS_HANDLER_PROXY_CLASS("metastore.hmshandler.proxy", "hive.metastore.hmshandler.proxy", METASTORE_RETRYING_HANDLER_CLASS, "The proxy class name of HMSHandler, default is RetryingHMSHandler."), - HMS_HANDLER_CREATE("metastore.hmshandler.create", "metastore.hmshandler.create","newHMSHandler", - "The method name to create new HMSHandler"), IDENTIFIER_FACTORY("datanucleus.identifierFactory", "datanucleus.identifierFactory", "datanucleus1", "Name of the identifier factory to use when generating table/column names etc. \n" + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 076fbbcd275c..03729bc588c4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -160,28 +160,6 @@ static Iface newHMSHandler(Configuration conf) return HMSHandlerProxyFactory.getProxy(conf, baseHandler, true); } - /** - * Create HMS handler for embedded metastore. - * - *

IMPORTANT

- * - * This method is called indirectly by HiveMetastoreClient using reflection when using an embedded - * Iceberg Catalog. It can not be removed and its arguments can't be changed without matching - * change in HiveMetastoreClient . - * - * @param conf configuration to use - * @throws MetaException - */ - static Iface newHMSRetryingLocalHandler(Configuration conf) - throws MetaException { - HMSHandler baseHandler = new HMSHandler("hive client", conf); - RetryingHMSHandler handler = new RetryingHMSHandler(conf, baseHandler, true); - return (IHMSHandler) java.lang.reflect.Proxy.newProxyInstance( - RetryingHMSHandler.class.getClassLoader(), - new Class[] { IHMSHandler.class }, handler); - } - - /** * Discard a current delegation token. * From 37ae8f3b8a56b31202459f359a4110051ee82f1c Mon Sep 17 00:00:00 2001 From: Henrib Date: Wed, 12 Feb 2025 17:15:58 +0100 Subject: [PATCH 14/40] HIVE-28059 : moving configuration variables to the common place; - remove first set of warnings, remove unused variables/arguments; --- .../iceberg/rest/HMSCatalogAdapter.java | 242 +++++++++--------- .../apache/iceberg/rest/HMSCatalogServer.java | 41 +-- .../hive/metastore/conf/MetastoreConf.java | 16 ++ 3 files changed, 154 insertions(+), 145 deletions(-) diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java index 8d694d15ac04..2662b9f2d1be 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java @@ -121,11 +121,6 @@ enum HTTPMethod { POST, DELETE } - - @FunctionalInterface - interface Operation { - T exec(Map vars, Object body); - } enum Route { TOKENS(HTTPMethod.POST, "v1/oauth/tokens", @@ -197,9 +192,7 @@ static Route byName(String name) { this(method, pattern, null, null); } - Route( - HTTPMethod method, - String pattern, + Route(HTTPMethod method, String pattern, Class requestClass, Class responseClass ) { @@ -216,10 +209,8 @@ static Route byName(String name) { requirementsBuilder.put(pos, part); } } - this.requestClass = requestClass; this.responseClass = responseClass; - this.requiredLength = parts.size(); this.requirements = requirementsBuilder.build(); this.variables = variablesBuilder.build(); @@ -291,11 +282,11 @@ public static List getMetricNames(String... apis) { return metricNames; } - private ConfigResponse config(Map vars, Object body) { + private ConfigResponse config() { return castResponse(ConfigResponse.class, ConfigResponse.builder().build()); } - private OAuthTokenResponse tokens(Map vars, Object body) { + private OAuthTokenResponse tokens(Object body) { Map request = (Map) castRequest(Map.class, body); String grantType = request.get("grant_type"); switch (grantType) { @@ -322,7 +313,7 @@ private OAuthTokenResponse tokens(Map vars, Object body) { } } - private ListNamespacesResponse listNamespaces(Map vars, Object body) { + private ListNamespacesResponse listNamespaces(Map vars) { if (asNamespaceCatalog != null) { Namespace ns; if (vars.containsKey("parent")) { @@ -335,7 +326,7 @@ private ListNamespacesResponse listNamespaces(Map vars, Object b throw new NamespaceNotSupported(catalog.toString()); } - private CreateNamespaceResponse createNamespace(Map vars, Object body) { + private CreateNamespaceResponse createNamespace(Object body) { if (asNamespaceCatalog != null) { CreateNamespaceRequest request = castRequest(CreateNamespaceRequest.class, body); return castResponse( @@ -344,7 +335,7 @@ private CreateNamespaceResponse createNamespace(Map vars, Object throw new NamespaceNotSupported(catalog.toString()); } - private GetNamespaceResponse loadNamespace(Map vars, Object body) { + private GetNamespaceResponse loadNamespace(Map vars) { if (asNamespaceCatalog != null) { Namespace namespace = namespaceFromPathVars(vars); return castResponse( @@ -353,7 +344,7 @@ private GetNamespaceResponse loadNamespace(Map vars, Object body throw new NamespaceNotSupported(catalog.toString()); } - private RESTResponse dropNamespace(Map vars, Object body) { + private RESTResponse dropNamespace(Map vars) { if (asNamespaceCatalog != null) { CatalogHandlers.dropNamespace(asNamespaceCatalog, namespaceFromPathVars(vars)); } @@ -372,7 +363,7 @@ private UpdateNamespacePropertiesResponse updateNamespace(Map va throw new NamespaceNotSupported(catalog.toString()); } - private ListTablesResponse listTables(Map vars, Object body) { + private ListTablesResponse listTables(Map vars) { Namespace namespace = namespaceFromPathVars(vars); return castResponse(ListTablesResponse.class, CatalogHandlers.listTables(catalog, namespace)); } @@ -391,8 +382,7 @@ private LoadTableResponse createTable(Map vars, Object body) { } } - private RESTResponse dropTable(Map vars, Object body) { - final Class responseType = RESTResponse.class; + private RESTResponse dropTable(Map vars) { if (PropertyUtil.propertyAsBoolean(vars, "purgeRequested", false)) { CatalogHandlers.purgeTable(catalog, identFromPathVars(vars)); } else { @@ -401,7 +391,7 @@ private RESTResponse dropTable(Map vars, Object body) { return null; } - private LoadTableResponse loadTable(Map vars, Object body) { + private LoadTableResponse loadTable(Map vars) { TableIdentifier ident = identFromPathVars(vars); return castResponse(LoadTableResponse.class, CatalogHandlers.loadTable(catalog, ident)); } @@ -418,89 +408,118 @@ private LoadTableResponse updateTable(Map vars, Object body) { return castResponse(LoadTableResponse.class, CatalogHandlers.updateTable(catalog, ident, request)); } - private RESTResponse renameTable(Map vars, Object body) { + private RESTResponse renameTable(Object body) { RenameTableRequest request = castRequest(RenameTableRequest.class, body); CatalogHandlers.renameTable(catalog, request); return null; } - private RESTResponse reportMetrics(Map vars, Object body) { + private RESTResponse reportMetrics(Object body) { // nothing to do here other than checking that we're getting the correct request castRequest(ReportMetricsRequest.class, body); return null; } - private RESTResponse commitTransaction(Map vars, Object body) { + private RESTResponse commitTransaction(Object body) { CommitTransactionRequest request = castRequest(CommitTransactionRequest.class, body); commitTransaction(catalog, request); return null; } - private ListTablesResponse listViews(Map vars, Object body) { - if (null != asViewCatalog) { - Namespace namespace = namespaceFromPathVars(vars); - String pageToken = PropertyUtil.propertyAsString(vars, "pageToken", null); - String pageSize = PropertyUtil.propertyAsString(vars, "pageSize", null); - if (pageSize != null) { - return castResponse( - ListTablesResponse.class, - CatalogHandlers.listViews(asViewCatalog, namespace, pageToken, pageSize)); - } else { - return castResponse( - ListTablesResponse.class, CatalogHandlers.listViews(asViewCatalog, namespace)); - } - } - throw new ViewNotSupported(catalog.toString()); - } - - private LoadViewResponse createView(Map vars, Object body) { - if (null != asViewCatalog) { - Namespace namespace = namespaceFromPathVars(vars); - CreateViewRequest request = castRequest(CreateViewRequest.class, body); - return castResponse( - LoadViewResponse.class, CatalogHandlers.createView(asViewCatalog, namespace, request)); + private ListTablesResponse listViews(Map vars) { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + String pageToken = PropertyUtil.propertyAsString(vars, "pageToken", null); + String pageSize = PropertyUtil.propertyAsString(vars, "pageSize", null); + if (pageSize != null) { + return castResponse( + ListTablesResponse.class, + CatalogHandlers.listViews(asViewCatalog, namespace, pageToken, pageSize)); + } else { + return castResponse( + ListTablesResponse.class, CatalogHandlers.listViews(asViewCatalog, namespace)); } - throw new ViewNotSupported(catalog.toString()); } - - private LoadViewResponse loadView(Map vars, Object body) { - if (null != asViewCatalog) { - TableIdentifier ident = identFromPathVars(vars); - return castResponse(LoadViewResponse.class, CatalogHandlers.loadView(asViewCatalog, ident)); - } - throw new ViewNotSupported(catalog.toString()); - } - - private LoadViewResponse updateView(Map vars, Object body) { - if (null != asViewCatalog) { - TableIdentifier ident = identFromPathVars(vars); - UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); - return castResponse( - LoadViewResponse.class, CatalogHandlers.updateView(asViewCatalog, ident, request)); - } - throw new ViewNotSupported(catalog.toString()); + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse createView(Map vars, Object body) { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + CreateViewRequest request = castRequest(CreateViewRequest.class, body); + return castResponse( + LoadViewResponse.class, CatalogHandlers.createView(asViewCatalog, namespace, request)); } - - private RESTResponse renameView(Map vars, Object body) { - if (null != asViewCatalog) { - RenameTableRequest request = castRequest(RenameTableRequest.class, body); - CatalogHandlers.renameView(asViewCatalog, request); - return null; - } - throw new ViewNotSupported(catalog.toString()); + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse loadView(Map vars) { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + return castResponse(LoadViewResponse.class, CatalogHandlers.loadView(asViewCatalog, ident)); } - - private RESTResponse dropView(Map vars, Object body) { - if (null != asViewCatalog) { - CatalogHandlers.dropView(asViewCatalog, identFromPathVars(vars)); - return null; - } - throw new ViewNotSupported(catalog.toString()); + throw new ViewNotSupported(catalog.toString()); + } + + private LoadViewResponse updateView(Map vars, Object body) { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse( + LoadViewResponse.class, CatalogHandlers.updateView(asViewCatalog, ident, request)); + } + throw new ViewNotSupported(catalog.toString()); + } + + private RESTResponse renameView(Object body) { + if (null != asViewCatalog) { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameView(asViewCatalog, request); + return null; + } + throw new ViewNotSupported(catalog.toString()); + } + + private RESTResponse dropView(Map vars) { + if (null != asViewCatalog) { + CatalogHandlers.dropView(asViewCatalog, identFromPathVars(vars)); + return null; } + throw new ViewNotSupported(catalog.toString()); + } + + /** + * This is a very simplistic approach that only validates the requirements for each table and does + * not do any other conflict detection. Therefore, it does not guarantee true transactional + * atomicity, which is left to the implementation details of a REST server. + */ + private static void commitTransaction(Catalog catalog, CommitTransactionRequest request) { + List transactions = Lists.newArrayList(); + for (UpdateTableRequest tableChange : request.tableChanges()) { + Table table = catalog.loadTable(tableChange.identifier()); + if (table instanceof BaseTable) { + Transaction transaction = + Transactions.newTransaction( + tableChange.identifier().toString(), ((BaseTable) table).operations()); + transactions.add(transaction); + + BaseTransaction.TransactionTable txTable = + (BaseTransaction.TransactionTable) transaction.table(); + + // this performs validations and makes temporary commits that are in-memory + CatalogHandlers.commit(txTable.operations(), tableChange); + } else { + throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); + } + } + // only commit if validations passed previously + transactions.forEach(Transaction::commitTransaction); + } + @SuppressWarnings("MethodLength") private T handleRequest( - Route route, Map vars, Object body, Class responseType) { + Route route, Map vars, Object body) { // update HMS catalog route counter metric final String metricName = hmsCatalogMetricCount(route.name()); Counter counter = Metrics.getOrCreateCounter(metricName); @@ -509,37 +528,37 @@ private T handleRequest( } switch (route) { case TOKENS: - return (T) tokens(vars, body); + return (T) tokens(body); case CONFIG: - return (T) config(vars, body); + return (T) config(); case LIST_NAMESPACES: - return (T) listNamespaces(vars, body); + return (T) listNamespaces(vars); case CREATE_NAMESPACE: - return (T) createNamespace(vars, body); + return (T) createNamespace(body); case LOAD_NAMESPACE: - return (T) loadNamespace(vars, body); + return (T) loadNamespace(vars); case DROP_NAMESPACE: - return (T) dropNamespace(vars, body); + return (T) dropNamespace(vars); case UPDATE_NAMESPACE: return (T) updateNamespace(vars, body); case LIST_TABLES: - return (T) listTables(vars, body); + return (T) listTables(vars); case CREATE_TABLE: return (T) createTable(vars, body); case DROP_TABLE: - return (T) dropTable(vars, body); + return (T) dropTable(vars); case LOAD_TABLE: - return (T) loadTable(vars, body); + return (T) loadTable(vars); case REGISTER_TABLE: return (T) registerTable(vars, body); @@ -548,65 +567,37 @@ private T handleRequest( return (T) updateTable(vars, body); case RENAME_TABLE: - return (T) renameTable(vars, body); + return (T) renameTable(body); case REPORT_METRICS: - return (T) reportMetrics(vars, body); + return (T) reportMetrics(body); case COMMIT_TRANSACTION: - return (T) commitTransaction(vars, body); + return (T) commitTransaction(body); case LIST_VIEWS: - return (T) listViews(vars, body); + return (T) listViews(vars); case CREATE_VIEW: return (T) createView(vars, body); case LOAD_VIEW: - return (T) loadView(vars, body); + return (T) loadView(vars); case UPDATE_VIEW: return (T) updateView(vars, body); case RENAME_VIEW: - return (T) renameView(vars, body); + return (T) renameView(vars); case DROP_VIEW: - return (T) dropView(vars, body); + return (T) dropView(vars); default: } return null; } - /** - * This is a very simplistic approach that only validates the requirements for each table and does - * not do any other conflict detection. Therefore, it does not guarantee true transactional - * atomicity, which is left to the implementation details of a REST server. - */ - private static void commitTransaction(Catalog catalog, CommitTransactionRequest request) { - List transactions = Lists.newArrayList(); - - for (UpdateTableRequest tableChange : request.tableChanges()) { - Table table = catalog.loadTable(tableChange.identifier()); - if (table instanceof BaseTable) { - Transaction transaction = - Transactions.newTransaction( - tableChange.identifier().toString(), ((BaseTable) table).operations()); - transactions.add(transaction); - - BaseTransaction.TransactionTable txTable = - (BaseTransaction.TransactionTable) transaction.table(); - - // this performs validations and makes temporary commits that are in-memory - CatalogHandlers.commit(txTable.operations(), tableChange); - } else { - throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); - } - } - // only commit if validations passed previously - transactions.forEach(Transaction::commitTransaction); - } T execute( HTTPMethod method, @@ -625,11 +616,10 @@ T execute( vars.putAll(queryParams); } vars.putAll(routeAndVars.second()); - return handleRequest(routeAndVars.first(), vars.build(), body, responseType); + return handleRequest(routeAndVars.first(), vars.build(), body); } catch (RuntimeException e) { configureResponseFromException(e, errorBuilder); } - } else { errorBuilder .responseCode(400) diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java index e238fe145cba..0ebb8c92772c 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -22,9 +22,9 @@ import java.io.IOException; import java.lang.ref.Reference; import java.lang.ref.SoftReference; -import java.util.Collections; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; import javax.servlet.http.HttpServlet; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.SecureServletCaller; @@ -51,18 +51,19 @@ * Iceberg Catalog server. */ public class HMSCatalogServer { - private static final String CACHE_EXPIRY = "hive.metastore.catalog.cache.expiry"; - private static final String JETTY_THREADPOOL_MIN = "hive.metastore.catalog.jetty.threadpool.min"; - private static final String JETTY_THREADPOOL_MAX = "hive.metastore.catalog.jetty.threadpool.max"; - private static final String JETTY_THREADPOOL_IDLE = "hive.metastore.catalog.jetty.threadpool.idle"; private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class); - private static Reference catalogRef; + private static final AtomicReference> catalogRef = new AtomicReference<>(); - static Catalog getLastCatalog() { - return catalogRef != null ? catalogRef.get() : null; + public static Catalog getLastCatalog() { + Reference soft = catalogRef.get(); + return soft != null ? soft.get() : null; + } + + protected static void setLastCatalog(Catalog catalog) { + catalogRef.set(new SoftReference<>(catalog)); } - protected HMSCatalogServer() { + private HMSCatalogServer() { // nothing } @@ -71,23 +72,24 @@ protected HttpServlet createServlet(SecureServletCaller security, Catalog catalo } protected Catalog createCatalog(Configuration configuration) { - final String configUri = configuration.get(MetastoreConf.ConfVars.THRIFT_URIS.getVarname()); - final String configWarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE.getVarname()); - final String configExtWarehouse = configuration.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); - Map properties = new TreeMap<>(); + final Map properties = new TreeMap<>(); + final String configUri = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS); if (configUri != null) { properties.put("uri", configUri); } + final String configWarehouse = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.WAREHOUSE); if (configWarehouse != null) { properties.put("warehouse", configWarehouse); } + final String configExtWarehouse = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL); if (configExtWarehouse != null) { properties.put("external-warehouse", configExtWarehouse); } final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog(); catalog.setConf(configuration); - catalog.initialize("hive", properties); - long expiry = configuration.getLong(CACHE_EXPIRY, 60_000L); + final String catalogName = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.CATALOG_DEFAULT); + catalog.initialize(catalogName, properties); + long expiry = MetastoreConf.getLongVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_CACHE_EXPIRY); return expiry > 0? new HMSCachingCatalog(catalog, expiry) : catalog; } @@ -100,9 +102,10 @@ protected HttpServlet createServlet(Configuration configuration, Catalog catalog MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); actualCatalog = createCatalog(configuration); } - catalogRef = new SoftReference<>(actualCatalog); + setLastCatalog(actualCatalog); return createServlet(security, actualCatalog); } + /** * Convenience method to start a http server that only serves this servlet. @@ -134,9 +137,9 @@ protected Server startServer(Configuration conf, HiveCatalog catalog) throws Exc } private static Server createHttpServer(Configuration conf, int port) throws IOException { - final int maxThreads = conf.getInt(JETTY_THREADPOOL_MAX, 256); - final int minThreads = conf.getInt(JETTY_THREADPOOL_MIN, 8); - final int idleTimeout = conf.getInt(JETTY_THREADPOOL_IDLE, 60_000); + final int maxThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MAX); + final int minThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MIN); + final int idleTimeout = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE); final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout); final Server httpServer = new Server(threadPool); final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index a1634ead255d..2bd60ecffa26 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1839,6 +1839,22 @@ public enum ConfVars { "hive.metastore.catalog.servlet.auth", "jwt", "HMS Iceberg Catalog servlet authentication method (simple or jwt)." ), + ICEBERG_CATALOG_JETTY_THREADPOOL_MIN("hive.metastore.catalog.jetty.threadpool.min", + "hive.metastore.catalog.jetty.threadpool.min", 8, + "HMS Iceberg Catalog embedded Jetty minimum number of threads." + ), + ICEBERG_CATALOG_JETTY_THREADPOOL_MAX("hive.metastore.catalog.jetty.threadpool.max", + "hive.metastore.catalog.jetty.threadpool.max", 256, + "HMS Iceberg Catalog embedded Jetty maximum number of threads." + ), + ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE("hive.metastore.catalog.jetty.threadpool.idle", + "hive.metastore.catalog.jetty.threadpool.idle", 60_000L, + "HMS Iceberg Catalog embedded Jetty thread idle time." + ), + ICEBERG_CATALOG_CACHE_EXPIRY("hive.metastore.catalog.cache.expiry", + "hive.metastore.catalog.cache.expiry", 60_000L, + "HMS Iceberg Catalog cache expiry." + ), // Deprecated Hive values that we are keeping for backwards compatibility. @Deprecated From 1ab4d94f4da82e4aa92a7817527584242e50de31 Mon Sep 17 00:00:00 2001 From: Henrib Date: Thu, 13 Feb 2025 15:20:02 +0100 Subject: [PATCH 15/40] HIVE-28059 : default servlet path update (iceberg); - control authentication type values (jwt, simple); --- .../org/apache/hadoop/hive/metastore/conf/MetastoreConf.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 2bd60ecffa26..1745e8e1a3d2 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1823,10 +1823,11 @@ public enum ConfVars { ), PROPERTIES_SERVLET_AUTH("hive.metastore.properties.servlet.auth", "hive.metastore.properties.servlet.auth", "jwt", + new StringSetValidator("simple", "jwt"), "Property-maps servlet authentication method (simple or jwt)." ), ICEBERG_CATALOG_SERVLET_PATH("hive.metastore.catalog.servlet.path", - "hive.metastore.catalog.servlet.path", "icecli", + "hive.metastore.catalog.servlet.path", "iceberg", "HMS Iceberg Catalog servlet path component of URL endpoint." ), ICEBERG_CATALOG_SERVLET_PORT("hive.metastore.catalog.servlet.port", From 7cb3ce2795bb44015e75a8acf039f6d08cabe16a Mon Sep 17 00:00:00 2001 From: Henrib Date: Thu, 13 Feb 2025 15:27:40 +0100 Subject: [PATCH 16/40] HIVE-28059 : pom cleanup; --- standalone-metastore/metastore-catalog/pom.xml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml index 3988823d7b22..dcf552755bd7 100644 --- a/standalone-metastore/metastore-catalog/pom.xml +++ b/standalone-metastore/metastore-catalog/pom.xml @@ -59,21 +59,6 @@ iceberg-bundled-guava ${iceberg.version} - - org.apache.hive hive-standalone-metastore-common From cae7f2ff3e43e46d83be1fe825925b6d8a2f0ec5 Mon Sep 17 00:00:00 2001 From: Henrib Date: Thu, 13 Feb 2025 18:38:08 +0100 Subject: [PATCH 17/40] HIVE-28059 : renamed & commented test helper class --- .../hive/{Friend.java => IcebergTestHelper.java} | 12 +++++++++--- .../java/org/apache/iceberg/rest/HMSTestBase.java | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) rename standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/{Friend.java => IcebergTestHelper.java} (71%) diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java similarity index 71% rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java rename to standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java index a60b9637637e..8ae4faf51400 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/Friend.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java @@ -19,10 +19,16 @@ package org.apache.iceberg.hive; /** - * Friend of CachedClientPool. + * Test helper utility. */ -public class Friend { - public static void cleanPoolCache() { +public class IcebergTestHelper { + /** + * Invalidates all clients remaining in the cached client pool held by the + * Catalog instance(s). + *

This is necessary when a new catalog is instantiated to avoid reusing + * old clients that may point to a (now) defunct catalog.

+ */ + public static void invalidatePoolCache() { CachedClientPool.clientPoolCache().invalidateAll(); } diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index cc49cdf7d4c1..b87c147e7eb4 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -80,7 +80,7 @@ import org.apache.hive.iceberg.com.fasterxml.jackson.databind.ObjectMapper; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.SupportsNamespaces; -import org.apache.iceberg.hive.Friend; +import org.apache.iceberg.hive.IcebergTestHelper; import org.eclipse.jetty.server.Server; import org.junit.After; import org.junit.Assert; @@ -278,7 +278,7 @@ public synchronized void tearDown() throws Exception { System.clearProperty(ObjectStore.TRUSTSTORE_PASSWORD_KEY); System.clearProperty(ObjectStore.TRUSTSTORE_TYPE_KEY); // - Friend.cleanPoolCache(); + IcebergTestHelper.invalidatePoolCache(); } finally { catalog = null; nsCatalog = null; From a4d66fd11738e0b82db14d1706cdea6a2b7839c1 Mon Sep 17 00:00:00 2001 From: Henrib Date: Thu, 13 Feb 2025 19:49:20 +0100 Subject: [PATCH 18/40] HIVE-28059 : removed useless synchronized; --- .../src/test/java/org/apache/iceberg/rest/HMSTestBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java index b87c147e7eb4..045964bfef64 100644 --- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java +++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java @@ -266,7 +266,7 @@ static Map reportMetricCounters(String... apis) { } @After - public synchronized void tearDown() throws Exception { + public void tearDown() throws Exception { try { if (port >= 0) { System.out.println("Stopping MetaStore Server on port " + port); From b83cfafa29c841a3be2459c60ab389739a8985d3 Mon Sep 17 00:00:00 2001 From: Henrib Date: Fri, 14 Feb 2025 16:42:11 +0100 Subject: [PATCH 19/40] HIVE-28059 : removed SecureServletCaller; - removed HmsThriftHttpServlet; - add servlet proxy to execute calls through ServletSecurity; - simplify PropertyServlet/HMSCatalogServlet and wrap them in proxy at instantiation time; - javadoc; --- .../apache/iceberg/rest/HMSCatalogServer.java | 13 ++- .../iceberg/rest/HMSCatalogServlet.java | 63 +------------- .../hadoop/hive/metastore/HiveMetaStore.java | 4 +- .../hive/metastore/HmsThriftHttpServlet.java | 57 ------------ .../hive/metastore/PropertyServlet.java | 30 +------ .../hive/metastore/SecureServletCaller.java | 60 ------------- .../hive/metastore/ServletSecurity.java | 86 ++++++++++++++++++- 7 files changed, 95 insertions(+), 218 deletions(-) delete mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java delete mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java index 0ebb8c92772c..076dcbc5d278 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java @@ -27,7 +27,6 @@ import java.util.concurrent.atomic.AtomicReference; import javax.servlet.http.HttpServlet; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.SecureServletCaller; import org.apache.hadoop.hive.metastore.ServletSecurity; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.iceberg.catalog.Catalog; @@ -67,8 +66,8 @@ private HMSCatalogServer() { // nothing } - protected HttpServlet createServlet(SecureServletCaller security, Catalog catalog) throws IOException { - return new HMSCatalogServlet(security, new HMSCatalogAdapter(catalog)); + protected HttpServlet createServlet(ServletSecurity security, Catalog catalog) throws IOException { + return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog))); } protected Catalog createCatalog(Configuration configuration) { @@ -94,9 +93,7 @@ protected Catalog createCatalog(Configuration configuration) { } protected HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException { - String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_AUTH); - boolean jwt = "jwt".equalsIgnoreCase(auth); - SecureServletCaller security = new ServletSecurity(configuration, jwt); + ServletSecurity security = new ServletSecurity(configuration); Catalog actualCatalog = catalog; if (actualCatalog == null) { MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, ""); @@ -106,7 +103,6 @@ protected HttpServlet createServlet(Configuration configuration, Catalog catalog return createServlet(security, actualCatalog); } - /** * Convenience method to start a http server that only serves this servlet. * @param conf the configuration @@ -135,13 +131,14 @@ protected Server startServer(Configuration conf, HiveCatalog catalog) throws Exc httpServer.start(); return httpServer; } - + private static Server createHttpServer(Configuration conf, int port) throws IOException { final int maxThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MAX); final int minThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MIN); final int idleTimeout = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE); final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout); final Server httpServer = new Server(threadPool); + httpServer.setStopAtShutdown(true); final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf); final ServerConnector connector = new ServerConnector(httpServer, sslContextFactory); connector.setPort(port); diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java index 2843b5003e5d..22acfd190214 100644 --- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java +++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java @@ -33,7 +33,6 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.hive.metastore.SecureServletCaller; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpHeaders; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -52,74 +51,16 @@ */ public class HMSCatalogServlet extends HttpServlet { private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServlet.class); - /** - * The security. - */ - private final SecureServletCaller security; - private final HMSCatalogAdapter restCatalogAdapter; private final Map responseHeaders = ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); - public HMSCatalogServlet(SecureServletCaller security, HMSCatalogAdapter restCatalogAdapter) { - this.security = security; + public HMSCatalogServlet(HMSCatalogAdapter restCatalogAdapter) { this.restCatalogAdapter = restCatalogAdapter; } - - @Override - public void init() throws ServletException { - super.init(); - security.init(); - } - @Override - protected void service(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - String method = req.getMethod(); - if (!"PATCH".equals(method)) { - super.service(req, resp); - } else { - this.doPatch(req, resp); - } - } - private void doExecute(String method, HttpServletRequest request, HttpServletResponse response) { - try { - security.execute(request, response, this::execute); - } catch (IOException e) { - LOG.error(method + " failed", e); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } - } - - protected void doPatch(HttpServletRequest request, HttpServletResponse response) { - doExecute("PATCH", request, response); - } - @Override - protected void doGet(HttpServletRequest request, HttpServletResponse response) { - doExecute("GET", request, response); - } - - @Override - protected void doPut(HttpServletRequest request, HttpServletResponse response) { - doExecute("PUT", request, response); - } - - @Override - protected void doHead(HttpServletRequest request, HttpServletResponse response) { - doExecute("HEAD", request, response); - } - - @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) { - doExecute("POST", request, response); - } - - @Override - protected void doDelete(HttpServletRequest request, HttpServletResponse response) { - doExecute("DELETE", request, response); - } - - private void execute(HttpServletRequest request, HttpServletResponse response) { + protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { try { ServletRequestContext context = ServletRequestContext.from(request); response.setStatus(HttpServletResponse.SC_OK); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 03729bc588c4..13463355a284 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -96,6 +96,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import javax.servlet.Servlet; import javax.servlet.ServletRequestEvent; import javax.servlet.ServletRequestListener; /** @@ -469,7 +470,8 @@ public void setThreadFactory(ThreadFactory threadFactory) { IHMSHandler handler = HMSHandlerProxyFactory.getProxy(conf, baseHandler, false); processor = new ThriftHiveMetastore.Processor<>(handler); LOG.info("Starting DB backed MetaStore Server with generic processor"); - TServlet thriftHttpServlet = new HmsThriftHttpServlet(processor, protocolFactory, conf); + ServletSecurity security = new ServletSecurity(conf); + Servlet thriftHttpServlet = security.proxy(new TServlet(processor, protocolFactory)); boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL); HMSHandler.LOG.info("Direct SQL optimization = {}", directSqlEnabled); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java deleted file mode 100644 index 2ecd00810484..000000000000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HmsThriftHttpServlet.java +++ /dev/null @@ -1,57 +0,0 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.io.IOException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; - -import org.apache.thrift.TProcessor; -import org.apache.thrift.protocol.TProtocolFactory; -import org.apache.thrift.server.TServlet; - -/* -Servlet class used by HiveMetastore server when running in HTTP mode. -If JWT auth is enabled, then the servlet is also responsible for validating -JWTs sent in the Authorization header in HTTP request. - */ -public class HmsThriftHttpServlet extends TServlet { - private final SecureServletCaller security; - - public HmsThriftHttpServlet(TProcessor processor, - TProtocolFactory protocolFactory, Configuration conf) { - super(processor, protocolFactory); - boolean jwt = MetastoreConf.getVar(conf,ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt"); - security = new ServletSecurity(conf, jwt); - } - - public void init() throws ServletException { - super.init(); - security.init(); - } - - @Override - protected void doPost(HttpServletRequest request, - HttpServletResponse response) throws ServletException, IOException { - security.execute(request, response, (q,a)->super.doPost(q,a)); - } -} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java index 0f82dce30b9b..9633c2065d4b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java @@ -65,8 +65,6 @@ public class PropertyServlet extends HttpServlet { public static final Logger LOGGER = LoggerFactory.getLogger(PropertyServlet.class); /** The configuration. */ private final Configuration configuration; - /** The security. */ - private final SecureServletCaller security; static boolean isAuthJwt(Configuration configuration) { String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH); @@ -74,11 +72,6 @@ static boolean isAuthJwt(Configuration configuration) { } PropertyServlet(Configuration configuration) { - this(configuration, new ServletSecurity(configuration, isAuthJwt(configuration))); - } - - PropertyServlet(Configuration configuration, SecureServletCaller security) { - this.security = security; this.configuration = configuration; } @@ -147,16 +140,10 @@ private void writeJson(HttpServletResponse response, Object value) throws IOExce public void init() throws ServletException { super.init(); - security.init(); } @Override protected void doPost(HttpServletRequest request, - HttpServletResponse response) throws ServletException, IOException { - security.execute(request, response, PropertyServlet.this::runPost); - } - - private void runPost(HttpServletRequest request, HttpServletResponse response) throws ServletException { final RawStore ms = getMS(); final String ns = getNamespace(request.getRequestURI()); @@ -267,10 +254,6 @@ private void runPost(HttpServletRequest request, @Override protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - security.execute(request, response, PropertyServlet.this::runPut); - } - private void runPut(HttpServletRequest request, - HttpServletResponse response) throws ServletException { final String ns = getNamespace(request.getRequestURI()); final RawStore ms = getMS(); try { @@ -303,11 +286,6 @@ private void runPut(HttpServletRequest request, @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - security.execute(request, response, PropertyServlet.this::runGet); - } - - private void runGet(HttpServletRequest request, - HttpServletResponse response) throws ServletException { final String ns = getNamespace(request.getRequestURI()); final RawStore ms = getMS(); try { @@ -352,11 +330,9 @@ public static Server startServer(Configuration conf) throws Exception { if (port < 0) { return null; } - String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH); - return startServer(conf, port, cli, new PropertyServlet(conf)); - } - - public static Server startServer(Configuration conf, int port, String path, Servlet servlet) throws Exception { + String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH); + ServletSecurity security = new ServletSecurity(conf); + Servlet servlet = security.proxy(new PropertyServlet(conf)); // HTTP Server Server server = new Server(); server.setStopAtShutdown(true); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java deleted file mode 100644 index 0cffc3da5d1a..000000000000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/SecureServletCaller.java +++ /dev/null @@ -1,60 +0,0 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; - -/** - * Secures servlet processing. - */ -public interface SecureServletCaller { - /** - * Should be called in Servlet.init() - * @throws ServletException if the jwt validator creation throws an exception - */ - void init() throws ServletException; - - /** - * Any http method executor. - */ - @FunctionalInterface - interface MethodExecutor { - /** - * The method to call to secure the execution of a (http) method. - * @param request the request - * @param response the response - * @throws ServletException if the method executor fails - * @throws IOException if the Json in/out fail - */ - void execute(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException; - } - - /** - * The method to call to secure the execution of a (http) method. - * @param request the request - * @param response the response - * @param executor the method executor - */ - void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) - throws IOException; - - -} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index 57db75d325c7..a40f410843aa 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; @@ -43,14 +44,44 @@ /** * Secures servlet processing. + *

This is to be used by servlets that require impersonation through UserGroupInformation.doAs mechanism when + * providing service. The servlet request header provides user identification + * that Hadoop"e;s security uses to perform actions, the + * {@link ServletSecurity#execute(HttpServletRequest, HttpServletResponse, ServletSecurity.MethodExecutor)} + * method takes care of running code in the expected UserGroupInformation context. + *

+ * A typical usage in a servlet is the following: + * 
+ * {@code
+ * SecureServletCaller security; // ...
+ *
+ * @Override protected void doPost(HttpServletRequest request,
+ * HttpServletResponse response) throws ServletException, IOException {
+ * security.execute(request, response, this::runPost);
+ * }
+ *
+ * private void runPost(HttpServletRequest request,
+ * HttpServletResponse response) throws ServletException {
+ * ...
+ * }
+ * }
+ *
+ * + *

This implementation performs user extraction and eventual JWT validation to + * execute (servlet service) methods within the context of the retrieved UserGroupInformation.

*/ -public class ServletSecurity implements SecureServletCaller { +public class ServletSecurity { private static final Logger LOG = LoggerFactory.getLogger(ServletSecurity.class); static final String X_USER = MetaStoreUtils.USER_NAME_HTTP_HEADER; private final boolean isSecurityEnabled; private final boolean jwtAuthEnabled; - private JWTValidator jwtValidator = null; private final Configuration conf; + private JWTValidator jwtValidator = null; + + public ServletSecurity(Configuration conf) { + this(conf, MetastoreConf.getVar(conf, + MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt")); + } public ServletSecurity(Configuration conf, boolean jwt) { this.conf = conf; @@ -63,7 +94,7 @@ public ServletSecurity(Configuration conf, boolean jwt) { * @throws ServletException if the jwt validator creation throws an exception */ public void init() throws ServletException { - if (jwtAuthEnabled) { + if (jwtAuthEnabled && jwtValidator == null) { try { jwtValidator = new JWTValidator(this.conf); } catch (Exception e) { @@ -73,6 +104,54 @@ public void init() throws ServletException { } } + /** + * Proxy a servlet instance service through this security executor. + */ + public class ProxyServlet extends HttpServlet { + private final HttpServlet delegate; + + ProxyServlet(HttpServlet delegate) { + this.delegate = delegate; + } + + public void init() throws ServletException { + ServletSecurity.this.init(); + delegate.init(); + } + + @Override public void service(HttpServletRequest request, HttpServletResponse response) throws IOException { + execute(request, response, delegate::service); + } + } + + /** + * Creates a proxy servlet. + * @param servlet the servlet to serve within this security context + * @return a servlet instance + */ + public HttpServlet proxy(HttpServlet servlet) { + return new ProxyServlet(servlet); + } + + /** + * Any http method executor. + *

A method whose signature is similar to + * {@link HttpServlet#doPost(HttpServletRequest, HttpServletResponse)}, + * {@link HttpServlet#doGet(HttpServletRequest, HttpServletResponse)}, + * etc.

+ */ + @FunctionalInterface + public interface MethodExecutor { + /** + * The method to call to secure the execution of a (http) method. + * @param request the request + * @param response the response + * @throws ServletException if the method executor fails + * @throws IOException if the Json in/out fail + */ + void execute(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException; + } + /** * The method to call to secure the execution of a (http) method. * @param request the request @@ -80,7 +159,6 @@ public void init() throws ServletException { * @param executor the method executor * @throws IOException if the Json in/out fail */ - @Override public void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) throws IOException { if (LOG.isDebugEnabled()) { From 107e42395ec5ec649f26384609f50c4f0570e3b0 Mon Sep 17 00:00:00 2001 From: Henrib Date: Sat, 15 Feb 2025 16:07:50 +0100 Subject: [PATCH 20/40] HIVE-28059 : nit, javadoc; --- .../hive/metastore/ServletSecurity.java | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index a40f410843aa..c851b09125f7 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -37,6 +37,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Enumeration; @@ -44,28 +45,35 @@ /** * Secures servlet processing. - *

This is to be used by servlets that require impersonation through UserGroupInformation.doAs mechanism when - * providing service. The servlet request header provides user identification - * that Hadoop"e;s security uses to perform actions, the + *

This is to be used by servlets that require impersonation through {@link UserGroupInformation#doAs(PrivilegedAction)} + * method when providing service. The servlet request header provides user identification + * that Hadoop{@literal '}s security uses to perform actions, the * {@link ServletSecurity#execute(HttpServletRequest, HttpServletResponse, ServletSecurity.MethodExecutor)} - * method takes care of running code in the expected UserGroupInformation context. + * method invokes the executor through a {@link PrivilegedAction} in the expected {@link UserGroupInformation} context. *

* A typical usage in a servlet is the following: * 
- * {@code
+ *{@code
  * SecureServletCaller security; // ...
- *
- * @Override protected void doPost(HttpServletRequest request,
- * HttpServletResponse response) throws ServletException, IOException {
- * security.execute(request, response, this::runPost);
+ * @Override protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ *    throws ServletException, IOException {
+ *  security.execute(request, response, this::runPost);
  * }
- *
- * private void runPost(HttpServletRequest request,
- * HttpServletResponse response) throws ServletException {
+ * private void runPost(HttpServletRequest request, HttpServletResponse response) throws ServletException {
  * ...
  * }
- * }
+ *}
  *
+ *

+ * As a convenience, instead of embedding the security instance, one can wrap an existing servlet in a proxy that + * will ensure all its service methods are called with the expected {@link UserGroupInformation} . + * 

+ * {@code
+ *  HttpServlet myServlet = ...;
+ *  ServletSecurity security = ...: ;
+ *  Servlet ugiServlet = security.proxy(mySerlvet);
+ *  }
+ *

* *

This implementation performs user extraction and eventual JWT validation to * execute (servlet service) methods within the context of the retrieved UserGroupInformation.

@@ -114,7 +122,7 @@ public class ProxyServlet extends HttpServlet { this.delegate = delegate; } - public void init() throws ServletException { + @Override public void init() throws ServletException { ServletSecurity.this.init(); delegate.init(); } @@ -162,7 +170,7 @@ public interface MethodExecutor { public void execute(HttpServletRequest request, HttpServletResponse response, MethodExecutor executor) throws IOException { if (LOG.isDebugEnabled()) { - LOG.debug("Logging headers in "+request.getMethod()+" request"); + LOG.debug("Logging headers in {} request", request.getMethod()); Enumeration headerNames = request.getHeaderNames(); while (headerNames.hasMoreElements()) { String headerName = headerNames.nextElement(); @@ -170,9 +178,9 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me request.getHeader(headerName)); } } + final UserGroupInformation clientUgi; try { String userFromHeader = extractUserName(request, response); - UserGroupInformation clientUgi; // Temporary, and useless for now. Here only to allow this to work on an otherwise kerberized // server. if (isSecurityEnabled || jwtAuthEnabled) { @@ -182,25 +190,25 @@ public void execute(HttpServletRequest request, HttpServletResponse response, Me LOG.info("Creating remote user for: {}", userFromHeader); clientUgi = UserGroupInformation.createRemoteUser(userFromHeader); } - PrivilegedExceptionAction action = () -> { - executor.execute(request, response); - return null; - }; - try { - clientUgi.doAs(action); - } catch (InterruptedException e) { - LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(), e); - Thread.currentThread().interrupt(); - } catch (RuntimeException e) { - LOG.error("Exception when executing http request as user: " + clientUgi.getUserName(), - e); - throw new IOException(e); - } } catch (HttpAuthenticationException e) { response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); response.getWriter().println("Authentication error: " + e.getMessage()); // Also log the error message on server side LOG.error("Authentication error: ", e); + // no need to go further + return; + } + final PrivilegedExceptionAction action = () -> { + executor.execute(request, response); + return null; + }; + try { + clientUgi.doAs(action); + } catch (InterruptedException e) { + LOG.info("Interrupted when executing http request as user: {}", clientUgi.getUserName(), e); + Thread.currentThread().interrupt(); + } catch (RuntimeException e) { + throw new IOException("Exception when executing http request as user: "+ clientUgi.getUserName(), e); } } From 4921457b56b9301fd125429a37c96ef817a53875 Mon Sep 17 00:00:00 2001 From: Henrib Date: Sat, 15 Feb 2025 19:52:25 +0100 Subject: [PATCH 21/40] HIVE-28059 : nit, javadoc (again!); --- .../hadoop/hive/metastore/PropertyServlet.java | 1 + .../hadoop/hive/metastore/ServletSecurity.java | 14 ++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java index 9633c2065d4b..9965b52dda6c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java @@ -138,6 +138,7 @@ private void writeJson(HttpServletResponse response, Object value) throws IOExce writer.flush(); } + @Override public void init() throws ServletException { super.init(); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index c851b09125f7..b0a226e116c1 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -52,27 +52,25 @@ * method invokes the executor through a {@link PrivilegedAction} in the expected {@link UserGroupInformation} context. *

* A typical usage in a servlet is the following: - * 
- *{@code
+ * 

  * SecureServletCaller security; // ...
- * @Override protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ * {@literal @}Override protected void doPost(HttpServletRequest request, HttpServletResponse response)
  *    throws ServletException, IOException {
  *  security.execute(request, response, this::runPost);
  * }
  * private void runPost(HttpServletRequest request, HttpServletResponse response) throws ServletException {
  * ...
  * }
- *}
- * 

+ *
*

* As a convenience, instead of embedding the security instance, one can wrap an existing servlet in a proxy that * will ensure all its service methods are called with the expected {@link UserGroupInformation} . - * 

- * {@code
+ *  

  *  HttpServlet myServlet = ...;
  *  ServletSecurity security = ...: ;
  *  Servlet ugiServlet = security.proxy(mySerlvet);
- *  }

+ *  }
+ *
*

* *

This implementation performs user extraction and eventual JWT validation to From 9b7eb152021c0bd22eebf95c4d805a48e456e92d Mon Sep 17 00:00:00 2001 From: Henrib Date: Sun, 16 Feb 2025 10:28:29 +0100 Subject: [PATCH 22/40] HIVE-28059 : javadoc (again!!); --- .../hadoop/hive/metastore/ServletSecurity.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java index b0a226e116c1..21f39eb9f932 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java @@ -63,18 +63,19 @@ * } * *

- * As a convenience, instead of embedding the security instance, one can wrap an existing servlet in a proxy that - * will ensure all its service methods are called with the expected {@link UserGroupInformation} . - * 


+ * As a convenience, instead of embedding the security instance, one can wrap an existing servlet in a proxy that
+ * will ensure all its service methods are called with the expected {@link UserGroupInformation} .
+ * 

+ * 

  *  HttpServlet myServlet = ...;
  *  ServletSecurity security = ...: ;
  *  Servlet ugiServlet = security.proxy(mySerlvet);
  *  }
  *  

+ * 

+ * This implementation performs user extraction and eventual JWT validation to
+ * execute (servlet service) methods within the context of the retrieved UserGroupInformation.
  * 

- *
- * 
This implementation performs user extraction and eventual JWT validation to
- * execute (servlet service) methods within the context of the retrieved UserGroupInformation.

  */
 public class ServletSecurity {
   private static final Logger LOG = LoggerFactory.getLogger(ServletSecurity.class);

From 4c41e9ce749852b1a3b1511b4713b984f495f2b8 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Sun, 16 Feb 2025 17:39:38 +0100
Subject: [PATCH 23/40] HIVE-28059 : fix regression on PropertyServlet & test;

---
 .../java/org/apache/hadoop/hive/metastore/PropertyServlet.java  | 2 +-
 .../apache/hadoop/hive/metastore/properties/HMSTestBase.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index 9965b52dda6c..0fecc92f2514 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -332,7 +332,7 @@ public static Server startServer(Configuration conf) throws Exception {
       return null;
     }
     String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
-    ServletSecurity security = new ServletSecurity(conf);
+    ServletSecurity security = new ServletSecurity(conf, PropertyServlet.isAuthJwt(conf));
     Servlet servlet =  security.proxy(new PropertyServlet(conf));
     // HTTP Server
     Server server = new Server();
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
index 30635e8bbeee..66fefbe0faf3 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
@@ -111,7 +111,7 @@ public void setUp() throws Exception {
     NS = "hms" + RND.nextInt(100);
     conf = MetastoreConf.newMetastoreConf();
     MetaStoreTestUtils.setConfForStandloneMode(conf);
-
+    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION, "jwt");
     MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
     // Events that get cleaned happen in batches of 1 to exercise batching code
     MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);

From 507dd6e8e937bd967691a6e64bd8ab5c39366481 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Mon, 17 Feb 2025 09:19:42 +0100
Subject: [PATCH 24/40] HIVE-28059 : fix regression on PropertyServlet test;

---
 .../org/apache/hadoop/hive/metastore/properties/HMSTestBase.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
index 66fefbe0faf3..4023076c04da 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
@@ -111,7 +111,6 @@ public void setUp() throws Exception {
     NS = "hms" + RND.nextInt(100);
     conf = MetastoreConf.newMetastoreConf();
     MetaStoreTestUtils.setConfForStandloneMode(conf);
-    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION, "jwt");
     MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
     // Events that get cleaned happen in batches of 1 to exercise batching code
     MetastoreConf.setLongVar(conf, MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);

From edd71486f1b639f97f5904686571c2044f01e35c Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Tue, 18 Feb 2025 18:59:54 +0100
Subject: [PATCH 25/40] HIVE-28059 : fix nits; - Created helper class to
 abstract creating embedded Jetty serving one servlet on one port; - Added
 embedded jetty configuration generic properties (threadpool); - Use
 ServletServerBuilder to create PropertyServlet and HMSCatalogServlet servers;

---
 .../iceberg/rest/HMSCatalogAdapter.java       |  37 +++--
 .../apache/iceberg/rest/HMSCatalogServer.java | 123 ++++++--------
 .../hive/metastore/conf/MetastoreConf.java    |  12 ++
 .../hive/metastore/PropertyServlet.java       |  72 ++++-----
 .../hive/metastore/ServletServerBuilder.java  | 151 ++++++++++++++++++
 5 files changed, 267 insertions(+), 128 deletions(-)
 create mode 100644 standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java

diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
index 2662b9f2d1be..776d53dbdb11 100644
--- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
+++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
@@ -90,7 +90,7 @@ public class HMSCatalogAdapter implements RESTClient {
           .put(NamespaceNotSupported.class, 400)
           .put(IllegalArgumentException.class, 400)
           .put(ValidationException.class, 400)
-          .put(NamespaceNotEmptyException.class, 400) // TODO: should this be more specific?
+          .put(NamespaceNotEmptyException.class, 400)
           .put(NotAuthorizedException.class, 401)
           .put(ForbiddenException.class, 403)
           .put(NoSuchNamespaceException.class, 404)
@@ -103,6 +103,15 @@ public class HMSCatalogAdapter implements RESTClient {
           .put(CommitStateUnknownException.class, 500)
           .buildOrThrow();
 
+  private static final String URN_OAUTH_TOKEN_EXCHANGE = "urn:ietf:params:oauth:grant-type:token-exchange";
+  private static final String URN_OAUTH_ACCESS_TOKEN = "urn:ietf:params:oauth:token-type:access_token";
+  private static final String GRANT_TYPE = "grant_type";
+  private static final String CLIENT_CREDENTIALS = "client_credentials";
+  private static final String BEARER = "Bearer";
+  private static final String CLIENT_ID = "client_id";
+  private static final String ACTOR_TOKEN = "actor_token";
+  private static final String SUBJECT_TOKEN = "subject_token";
+
   private final Catalog catalog;
   private final SupportsNamespaces asNamespaceCatalog;
   private final ViewCatalog asViewCatalog;
@@ -288,24 +297,24 @@ private ConfigResponse config() {
 
   private OAuthTokenResponse tokens(Object body) {
     Map request = (Map) castRequest(Map.class, body);
-    String grantType = request.get("grant_type");
+    String grantType = request.get(GRANT_TYPE);
     switch (grantType) {
-      case "client_credentials":
+      case CLIENT_CREDENTIALS:
         return OAuthTokenResponse.builder()
-            .withToken("client-credentials-token:sub=" + request.get("client_id"))
-            .withTokenType("Bearer")
+            .withToken("client-credentials-token:sub=" + request.get(CLIENT_ID))
+            .withTokenType(BEARER)
             .build();
 
-      case "urn:ietf:params:oauth:grant-type:token-exchange":
-        String actor = request.get("actor_token");
+      case URN_OAUTH_TOKEN_EXCHANGE:
+        String actor = request.get(ACTOR_TOKEN);
         String token =
             String.format(
                 "token-exchange-token:sub=%s%s",
-                request.get("subject_token"), actor != null ? ",act=" + actor : "");
+                request.get(SUBJECT_TOKEN), actor != null ? ",act=" + actor : "");
         return OAuthTokenResponse.builder()
             .withToken(token)
-            .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token")
-            .withTokenType("Bearer")
+            .withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN)
+            .withTokenType(BEARER)
             .build();
 
       default:
@@ -315,13 +324,13 @@ private OAuthTokenResponse tokens(Object body) {
 
   private ListNamespacesResponse listNamespaces(Map vars) {
     if (asNamespaceCatalog != null) {
-      Namespace ns;
+      Namespace namespace;
       if (vars.containsKey("parent")) {
-        ns = Namespace.of(RESTUtil.NAMESPACE_SPLITTER.splitToStream(vars.get("parent")).toArray(String[]::new));
+        namespace = Namespace.of(RESTUtil.NAMESPACE_SPLITTER.splitToStream(vars.get("parent")).toArray(String[]::new));
       } else {
-        ns = Namespace.empty();
+        namespace = Namespace.empty();
       }
-      return castResponse(ListNamespacesResponse.class, CatalogHandlers.listNamespaces(asNamespaceCatalog, ns));
+      return castResponse(ListNamespacesResponse.class, CatalogHandlers.listNamespaces(asNamespaceCatalog, namespace));
     }
     throw new NamespaceNotSupported(catalog.toString());
   }
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
index 076dcbc5d278..76394861765c 100644
--- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
+++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.apache.iceberg.rest;
 
 import java.io.IOException;
@@ -28,49 +27,53 @@
 import javax.servlet.http.HttpServlet;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.hadoop.hive.metastore.ServletServerBuilder;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.hive.HiveCatalog;
-import org.eclipse.jetty.server.ConnectionFactory;
-import org.eclipse.jetty.server.Connector;
-import org.eclipse.jetty.server.HttpConfiguration;
-import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 /**
- * Iceberg Catalog server.
+ * Iceberg Catalog server creator.
  */
-public class HMSCatalogServer {
+public class HMSCatalogServer extends ServletServerBuilder {
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class);
-  private static final AtomicReference> catalogRef = new AtomicReference<>();
+  protected static final AtomicReference> catalogRef = new AtomicReference<>();
 
   public static Catalog getLastCatalog() {
     Reference soft = catalogRef.get();
-    return soft != null ? soft.get() :  null;
+    return soft != null ? soft.get() : null;
   }
-  
+
   protected static void setLastCatalog(Catalog catalog) {
     catalogRef.set(new SoftReference<>(catalog));
   }
 
-  private HMSCatalogServer() {
-    // nothing
+  protected final int port;
+  protected final String path;
+  protected Catalog catalog;
+
+  protected HMSCatalogServer(Configuration conf, Catalog catalog) {
+    super(conf);
+    port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
+    path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
+    this.catalog = catalog;
   }
 
-  protected HttpServlet createServlet(ServletSecurity security, Catalog catalog) throws IOException {
-    return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
+  @Override
+  protected String getServletPath() {
+    return path;
   }
 
-  protected Catalog createCatalog(Configuration configuration) {
+  @Override
+  protected int getServerPort() {
+    return port;
+  }
+
+  protected Catalog createCatalog() {
     final Map properties = new TreeMap<>();
     final String configUri = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS);
     if (configUri != null) {
@@ -89,81 +92,45 @@ protected Catalog createCatalog(Configuration configuration) {
     final String catalogName = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.CATALOG_DEFAULT);
     catalog.initialize(catalogName, properties);
     long expiry = MetastoreConf.getLongVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_CACHE_EXPIRY);
-    return expiry > 0? new HMSCachingCatalog(catalog, expiry) : catalog;
+    return expiry > 0 ? new HMSCachingCatalog(catalog, expiry) : catalog;
   }
 
-  protected HttpServlet createServlet(Configuration configuration, Catalog catalog) throws IOException {
+  protected HttpServlet createServlet(Catalog catalog) throws IOException {
     ServletSecurity security = new ServletSecurity(configuration);
+    return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
+  }
+
+  @Override
+  protected HttpServlet createServlet() throws IOException {
     Catalog actualCatalog = catalog;
     if (actualCatalog == null) {
       MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, "");
-      actualCatalog = createCatalog(configuration);
+      actualCatalog = catalog = createCatalog();
     }
     setLastCatalog(actualCatalog);
-    return createServlet(security, actualCatalog);
+    return createServlet(actualCatalog);
   }
-  
-  /**
-   * Convenience method to start a http server that only serves this servlet.
-   * @param conf the configuration
-   * @param catalog the catalog instance to serve
-   * @return the server instance
-   * @throws Exception if servlet initialization fails
-   */
-  protected Server startServer(Configuration conf, HiveCatalog catalog) throws Exception {
-    int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
-    if (port < 0) {
-      return null;
-    }
-    final HttpServlet servlet = createServlet(conf, catalog);
-    ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
-    context.setContextPath("/");
-    ServletHolder servletHolder = new ServletHolder(servlet);
-    servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
-    final String cli = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
-    context.addServlet(servletHolder, "/" + cli + "/*");
-    context.setVirtualHosts(null);
-    context.setGzipHandler(new GzipHandler());
 
-    final Server httpServer = createHttpServer(conf, port);
-    httpServer.setHandler(context);
-    LOG.info("Starting HMS REST Catalog Server with context path:/{}/ on port:{}", cli, port);
-    httpServer.start();
-    return httpServer;
-  }
-  
-  private static Server createHttpServer(Configuration conf, int port) throws IOException {
-    final int maxThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MAX);
-    final int minThreads = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MIN);
-    final int idleTimeout = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE);
+  @Override
+  protected Server createServer() {
+    final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MAX);
+    final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MIN);
+    final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE);
     final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
-    final Server httpServer = new Server(threadPool);
-    httpServer.setStopAtShutdown(true);
-    final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf);
-    final ServerConnector connector = new ServerConnector(httpServer, sslContextFactory);
-    connector.setPort(port);
-    connector.setReuseAddress(true);
-    httpServer.setConnectors(new Connector[] {connector});
-    for (ConnectionFactory factory : connector.getConnectionFactories()) {
-      if (factory instanceof HttpConnectionFactory) {
-        HttpConnectionFactory httpFactory = (HttpConnectionFactory) factory;
-        HttpConfiguration httpConf = httpFactory.getHttpConfiguration();
-        httpConf.setSendServerVersion(false);
-        httpConf.setSendXPoweredBy(false);
-      }
-    }
-    return httpServer;
+    return new Server(threadPool);
   }
 
-
   /**
-   * Convenience method to start a http server that only serves this servlet.
-   * 
This one is looked up through reflection to start from HMS.

+   * Convenience method to start a http server that only serves the Iceberg
+   * catalog servlet.
+   * 

+   * This one is looked up through reflection to start from HMS.

+   *
    * @param conf the configuration
    * @return the server instance
    * @throws Exception if servlet initialization fails
    */
   public static Server startServer(Configuration conf) throws Exception {
-    return new HMSCatalogServer().startServer(conf, null);
+    return new HMSCatalogServer(conf, null).startServer();
   }
 }
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 1745e8e1a3d2..eb0c6cce90d5 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1856,6 +1856,18 @@ public enum ConfVars {
         "hive.metastore.catalog.cache.expiry", 60_000L,
         "HMS Iceberg Catalog cache expiry."
     ),
+    EMBEDDED_JETTY_THREADPOOL_MIN("hive.metastore.embedded.jetty.threadpool.min",
+            "hive.metastore.embedded.jetty.threadpool.min", 2,
+            "HMS embedded Jetty server(s) minimum number of threads."
+    ),
+    EMBEDDED_JETTY_THREADPOOL_MAX("hive.metastore.embedded.jetty.threadpool.max",
+            "hive.metastore.embedded.jetty.threadpool.max", 256,
+            "HMS embedded Jetty server(s) maximum number of threads."
+    ),
+    EMBEDDED_JETTY_THREADPOOL_IDLE("hive.metastore.embedded.jetty.threadpool.idle",
+            "hive.metastore.embedded.jetty.threadpool.idle", 60_000L,
+            "HMS embedded Jetty server(s) thread idle time."
+    ),
 
     // Deprecated Hive values that we are keeping for backwards compatibility.
     @Deprecated
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index 0fecc92f2514..8a2a7fe14aa8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -29,15 +29,9 @@
 import org.apache.hadoop.hive.metastore.properties.PropertyMap;
 import org.apache.hadoop.hive.metastore.properties.PropertyStore;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.servlet.ServletHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.servlet.Source;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.servlet.Servlet;
 import javax.servlet.ServletException;
 import javax.servlet.ServletOutputStream;
 import javax.servlet.http.HttpServlet;
@@ -319,6 +313,35 @@ protected void doGet(HttpServletRequest request,
     }
   }
 
+  /**
+   * Single servlet creation helper.
+   */
+  private static class ServerBuilder extends ServletServerBuilder {
+    final int port;
+    final String path;
+    ServerBuilder(Configuration conf) {
+      super(conf);
+      port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
+      path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+    }
+
+    @Override
+    protected String getServletPath() {
+      return path;
+    }
+
+    @Override
+    protected int getServerPort() {
+      return port;
+    }
+
+    @Override
+    protected HttpServlet createServlet() throws IOException {
+      ServletSecurity security = new ServletSecurity(configuration, PropertyServlet.isAuthJwt(configuration));
+      return security.proxy(new PropertyServlet(configuration));
+    }
+  }
+
   /**
    * Convenience method to start a http server that only serves this servlet.
    * @param conf the configuration
@@ -326,36 +349,13 @@ protected void doGet(HttpServletRequest request,
    * @throws Exception if servlet initialization fails
    */
   public static Server startServer(Configuration conf) throws Exception {
-    // no port, no server
-    int port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
-    if (port < 0) {
-      return null;
-    }
-    String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
-    ServletSecurity security = new ServletSecurity(conf, PropertyServlet.isAuthJwt(conf));
-    Servlet servlet =  security.proxy(new PropertyServlet(conf));
-    // HTTP Server
-    Server server = new Server();
-    server.setStopAtShutdown(true);
-
-    // Optional SSL
-    final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(conf);
-    final ServerConnector connector = new ServerConnector(server, sslContextFactory);
-    connector.setPort(port);
-    connector.setReuseAddress(true);
-    server.addConnector(connector);
-
-    // Hook the servlet
-    ServletHandler handler = new ServletHandler();
-    server.setHandler(handler);
-    ServletHolder holder = handler.newServletHolder(Source.EMBEDDED);
-    holder.setServlet(servlet); //
-    handler.addServletWithMapping(holder, "/"+path+"/*");
-    server.start();
-    if (!server.isStarted()) {
-      LOGGER.error("unable to start property-maps servlet server, path {}, port {}", path, port);
-    } else {
-      LOGGER.info("started property-maps servlet server on {}", server.getURI());
+    Server server = new ServerBuilder(conf).startServer();
+    if (server != null) {
+      if (!server.isStarted()) {
+        LOGGER.error("Unable to start property-maps servlet server on {}", server.getURI());
+      } else {
+        LOGGER.info("Started property-maps servlet server on {}", server.getURI());
+      }
     }
     return server;
   }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
new file mode 100644
index 000000000000..dd2f355cfc32
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import java.io.IOException;
+import javax.servlet.http.HttpServlet;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.eclipse.jetty.server.HttpConfiguration;
+import org.eclipse.jetty.server.HttpConnectionFactory;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.gzip.GzipHandler;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+
+/**
+ * Helper class to ease creation of embedded Jetty serving one servlet on a given port.
+ * 
When using Jetty, the easiest way - and may be only - to serve different servlets
+ * on different ports is to create 2 separate Jetty instances; this helper eases creation
+ * of such a dedicated server.

+ */
+public abstract class ServletServerBuilder {
+  /**
+   * The configuration instance.
+   */
+  protected final Configuration configuration;
+
+  /**
+   * Creates a builder instance.
+   * @param conf the configuration
+   */
+  protected ServletServerBuilder(Configuration conf) {
+    this.configuration = conf;
+  }
+  /**
+   * Gets the servlet path.
+   * @return the path
+   */
+  protected abstract String getServletPath();
+
+  /**
+   * Gets the server port.
+   * @return the port
+   */
+  protected abstract int getServerPort();
+
+  /**
+   * Creates the servlet instance.
+   * 
It is often advisable to use {@link ServletSecurity} to proxy the actual servlet instance.

+   * @return the servlet instance
+   * @throws IOException if servlet creation fails
+   */
+  protected abstract HttpServlet createServlet() throws IOException;
+
+  /**
+   * Creates the servlet context.
+   * @param servlet the servlet
+   * @return a context instance
+   */
+  protected ServletContextHandler createContext(HttpServlet servlet) {
+    // hook the servlet
+    ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+    context.setContextPath("/");
+    ServletHolder servletHolder = new ServletHolder(servlet);
+    servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
+    final String path = getServletPath();
+    context.addServlet(servletHolder, "/" + path + "/*");
+    context.setVirtualHosts(null);
+    context.setGzipHandler(new GzipHandler());
+    return context;
+  }
+
+  /**
+   * Creates a server instance.
+   * 
Default use configuration to determine threadpool constants?

+   * @return the server instance
+   * @throws IOException if server creation fails
+   */
+  protected Server createServer() throws IOException {
+    final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_MAX);
+    final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_MIN);
+    final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_IDLE);
+    final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
+    return new Server(threadPool);
+  }
+
+  /**
+   * Creates a server instance and a connector on a given port.
+   * @param port the port
+   * @return the server instance listening to the port
+   * @throws IOException if server creation fails
+   */
+  protected Server createServer(int port) throws IOException {
+    final Server server = createServer();
+    server.setStopAtShutdown(true);
+    final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(configuration);
+    final ServerConnector connector = new ServerConnector(server, sslContextFactory);
+    connector.setPort(port);
+    connector.setReuseAddress(true);
+    server.addConnector(connector);
+    HttpConnectionFactory httpFactory = connector.getConnectionFactory(HttpConnectionFactory.class);
+    // do not leak information
+    if (httpFactory != null) {
+      HttpConfiguration httpConf = httpFactory.getHttpConfiguration();
+      httpConf.setSendServerVersion(false);
+      httpConf.setSendXPoweredBy(false);
+    }
+    return server;
+  }
+
+  /**
+   * Convenience method to start a http server that only serves this servlet.
+   * @return the server instance or null if port < 0
+   * @throws Exception if servlet initialization fails
+   */
+  public Server startServer() throws Exception {
+    int port = getServerPort();
+    if (port < 0) {
+      return null;
+    }
+    // create the servlet
+    final HttpServlet servlet = createServlet();
+    // hook the servlet
+    ServletContextHandler context = createContext(servlet);
+    // Http server
+    final Server httpServer = createServer(port);
+    httpServer.setHandler(context);
+    httpServer.start();
+    return httpServer;
+  }
+
+}

From 0a6381cfcde34d9206e54f86c3c7437712c1f5b9 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Wed, 19 Feb 2025 12:38:11 +0100
Subject: [PATCH 26/40] HIVE-28059 : fixing more nits; - moving to
 metastore-iceberg-catalog; - removing never thrown exceptions from
 signatures; - fixing javadoc;

---
 .../pom.xml                                     | 17 ++++++++++++++++-
 .../apache/iceberg/rest/HMSCachingCatalog.java  |  0
 .../apache/iceberg/rest/HMSCatalogAdapter.java  |  8 +++-----
 .../apache/iceberg/rest/HMSCatalogServer.java   | 10 +++++++++-
 .../apache/iceberg/rest/HMSCatalogServlet.java  |  5 ++---
 .../apache/iceberg/hive/IcebergTestHelper.java  |  0
 .../org/apache/iceberg/rest/HMSTestBase.java    | 11 ++---------
 .../org/apache/iceberg/rest/TestHMSCatalog.java |  0
 .../resources/auth/jwt/jwt-authorized-key.json  |  0
 .../auth/jwt/jwt-unauthorized-key.json          |  0
 .../auth/jwt/jwt-verification-jwks.json         |  0
 .../src/test/resources/hive-log4j2.properties   |  0
 .../hadoop/hive/metastore/PropertyServlet.java  |  2 +-
 .../hadoop/hive/metastore/ServletSecurity.java  |  2 +-
 standalone-metastore/pom.xml                    |  2 +-
 15 files changed, 35 insertions(+), 22 deletions(-)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/pom.xml (93%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java (98%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java (93%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java (96%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/java/org/apache/iceberg/rest/HMSTestBase.java (98%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/resources/auth/jwt/jwt-authorized-key.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/resources/auth/jwt/jwt-unauthorized-key.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/resources/auth/jwt/jwt-verification-jwks.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-iceberg-catalog}/src/test/resources/hive-log4j2.properties (100%)

diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-iceberg-catalog/pom.xml
similarity index 93%
rename from standalone-metastore/metastore-catalog/pom.xml
rename to standalone-metastore/metastore-iceberg-catalog/pom.xml
index dcf552755bd7..812d0b36bdfb 100644
--- a/standalone-metastore/metastore-catalog/pom.xml
+++ b/standalone-metastore/metastore-iceberg-catalog/pom.xml
@@ -17,7 +17,7 @@
     4.1.0-SNAPSHOT
   
   4.0.0
-  hive-standalone-metastore-icecat
+  hive-standalone-metastore-iceberg-catalog
   Hive Metastore Iceberg Catalog
   
     ..
@@ -59,6 +59,21 @@
       iceberg-bundled-guava
       ${iceberg.version}
     
+    
+    
     
       org.apache.hive
       hive-standalone-metastore-common
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
rename to standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
similarity index 98%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
rename to standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
index 776d53dbdb11..80652a2a7ec6 100644
--- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
+++ b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
@@ -77,7 +77,7 @@
 import org.apache.iceberg.util.PropertyUtil;
 
 /**
- * Original @ https://github.com/apache/iceberg/blob/main/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
+ * Original @ https://github.com/apache/iceberg/blob/1.6.x/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java
  * Adaptor class to translate REST requests into {@link Catalog} API calls.
  */
 public class HMSCatalogAdapter implements RESTClient {
@@ -696,10 +696,8 @@ public  T postForm(
   }
 
   @Override
-  public void close() throws IOException {
-    // The calling test is responsible for closing the underlying catalog backing this REST catalog
-    // so that the underlying backend catalog is not closed and reopened during the REST catalog's
-    // initialize method when fetching the server configuration.
+  public void close() {
+    // The caller is responsible for closing the underlying catalog backing this REST catalog.
   }
 
   private static class NamespaceNotSupported extends RuntimeException {
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
similarity index 93%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
rename to standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
index 76394861765c..6e24ffbddc72 100644
--- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
+++ b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
@@ -131,6 +131,14 @@ protected Server createServer() {
    * @throws Exception if servlet initialization fails
    */
   public static Server startServer(Configuration conf) throws Exception {
-    return new HMSCatalogServer(conf, null).startServer();
+    Server server = new HMSCatalogServer(conf, null).startServer();
+    if (server != null) {
+      if (!server.isStarted()) {
+        LOG.error("Unable to start property-maps servlet server on {}", server.getURI());
+      } else {
+        LOG.info("Started property-maps servlet server on {}", server.getURI());
+      }
+    }
+    return server;
   }
 }
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
similarity index 96%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
rename to standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
index 22acfd190214..bc581fc26b9c 100644
--- a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
+++ b/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
@@ -29,7 +29,6 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -45,7 +44,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Original @ https://github.com/apache/iceberg/blob/main/core/src/test/java/org/apache/iceberg/rest/RESTCatalogServlet.java
+ * Original @ https://github.com/apache/iceberg/blob/1.6.x/core/src/test/java/org/apache/iceberg/rest/RESTCatalogServlet.java
  * The RESTCatalogServlet provides a servlet implementation used in combination with a
  * RESTCatalogAdaptor to proxy the REST Spec to any Catalog implementation.
  */
@@ -60,7 +59,7 @@ public HMSCatalogServlet(HMSCatalogAdapter restCatalogAdapter) {
   }
   
   @Override
-  protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+  protected void service(HttpServletRequest request, HttpServletResponse response) {
     try {
       ServletRequestContext context = ServletRequestContext.from(request);
       response.setStatus(HttpServletResponse.SC_OK);
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java b/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
rename to standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
similarity index 98%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
rename to standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
index 045964bfef64..989aeaea5682 100644
--- a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
+++ b/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
@@ -37,6 +37,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.Reader;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -433,15 +434,7 @@ static  String serialize(T object) {
         }
     }
 
-    static  T deserialize(String s) {
-        try {
-            return MAPPER.readValue(s, new TypeReference() {});
-        } catch (JsonProcessingException xany) {
-            throw new RuntimeException(xany);
-        }
-    }
-
-    static  T deserialize(BufferedReader s) {
+    static  T deserialize(Reader s) {
         try {
             return MAPPER.readValue(s, new TypeReference() {});
         } catch (IOException xany) {
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
rename to standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json b/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
rename to standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json b/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
rename to standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json b/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
rename to standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties b/standalone-metastore/metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/hive-log4j2.properties
rename to standalone-metastore/metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index 8a2a7fe14aa8..f30f7b6563a9 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -336,7 +336,7 @@ protected int getServerPort() {
     }
 
     @Override
-    protected HttpServlet createServlet() throws IOException {
+    protected HttpServlet createServlet() {
       ServletSecurity security = new ServletSecurity(configuration, PropertyServlet.isAuthJwt(configuration));
       return security.proxy(new PropertyServlet(configuration));
     }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
index 21f39eb9f932..71c9b20fb80e 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
@@ -53,7 +53,7 @@
  * 

  * A typical usage in a servlet is the following:
  * 

- * SecureServletCaller security; // ...
+ * ServletSecurity security; // ...
  * {@literal @}Override protected void doPost(HttpServletRequest request, HttpServletResponse response)
  *    throws ServletException, IOException {
  *  security.execute(request, response, this::runPost);
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 1dcfd8127654..6e55634f2fdd 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -29,7 +29,7 @@
     metastore-common
     metastore-server
     metastore-tools
-    metastore-catalog
+    metastore-iceberg-catalog
   
   
     4.1.0-SNAPSHOT

From 2be93277d0b4768c4fa25e60d635bf263c6d760b Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Wed, 19 Feb 2025 21:18:52 +0100
Subject: [PATCH 27/40] Javadoc nit;

---
 .../org/apache/hadoop/hive/metastore/ServletServerBuilder.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index dd2f355cfc32..e2cb5c971900 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -129,7 +129,7 @@ protected Server createServer(int port) throws IOException {
 
   /**
    * Convenience method to start a http server that only serves this servlet.
-   * @return the server instance or null if port < 0
+   * @return the server instance or null if port < 0
    * @throws Exception if servlet initialization fails
    */
   public Server startServer() throws Exception {

From 6c2b9308344c26ea829b3eca6845c643a2937d47 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Fri, 21 Feb 2025 12:39:12 +0100
Subject: [PATCH 28/40] HIVE-28059 : create only one Jetty instance for HMS
 property & Iceberg Catalog servlets; - tests servlet builder including port
 sharing or not; - moving module to metastore-catalog; - fixing nits;

---
 .../pom.xml                                   |   2 +-
 .../iceberg/rest/HMSCachingCatalog.java       |   0
 .../iceberg/rest/HMSCatalogAdapter.java       |   1 -
 .../apache/iceberg/rest/HMSCatalogServer.java |  78 +++--
 .../iceberg/rest/HMSCatalogServlet.java       |   8 +-
 .../iceberg/hive/IcebergTestHelper.java       |   0
 .../org/apache/iceberg/rest/HMSTestBase.java  |   6 +-
 .../apache/iceberg/rest/TestHMSCatalog.java   |   2 +-
 .../auth/jwt/jwt-authorized-key.json          |   0
 .../auth/jwt/jwt-unauthorized-key.json        |   0
 .../auth/jwt/jwt-verification-jwks.json       |   0
 .../src/test/resources/log4j2.properties}     |  25 +-
 .../hive/metastore/conf/MetastoreConf.java    |  19 +-
 .../hadoop/hive/metastore/HiveMetaStore.java  |  44 ++-
 .../hive/metastore/PropertyServlet.java       |  58 ++--
 .../hive/metastore/ServletServerBuilder.java  | 270 ++++++++++++++----
 .../metastore/TestServletServerBuilder.java   | 233 +++++++++++++++
 standalone-metastore/pom.xml                  |   2 +-
 18 files changed, 558 insertions(+), 190 deletions(-)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/pom.xml (99%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java (100%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java (99%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java (69%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java (97%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java (100%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/java/org/apache/iceberg/rest/HMSTestBase.java (98%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java (99%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/resources/auth/jwt/jwt-authorized-key.json (100%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/resources/auth/jwt/jwt-unauthorized-key.json (100%)
 rename standalone-metastore/{metastore-iceberg-catalog => metastore-catalog}/src/test/resources/auth/jwt/jwt-verification-jwks.json (100%)
 rename standalone-metastore/{metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties => metastore-catalog/src/test/resources/log4j2.properties} (71%)
 create mode 100644 standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java

diff --git a/standalone-metastore/metastore-iceberg-catalog/pom.xml b/standalone-metastore/metastore-catalog/pom.xml
similarity index 99%
rename from standalone-metastore/metastore-iceberg-catalog/pom.xml
rename to standalone-metastore/metastore-catalog/pom.xml
index 812d0b36bdfb..865c5e8b6f02 100644
--- a/standalone-metastore/metastore-iceberg-catalog/pom.xml
+++ b/standalone-metastore/metastore-catalog/pom.xml
@@ -17,7 +17,7 @@
     4.1.0-SNAPSHOT
   
   4.0.0
-  hive-standalone-metastore-iceberg-catalog
+  hive-standalone-metastore-catalog
   Hive Metastore Iceberg Catalog
   
     ..
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
similarity index 100%
rename from standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
rename to standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
similarity index 99%
rename from standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
rename to standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
index 80652a2a7ec6..063dbb59e311 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
+++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
@@ -21,7 +21,6 @@
 
 import com.codahale.metrics.Counter;
 import org.apache.hadoop.hive.metastore.metrics.Metrics;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
similarity index 69%
rename from standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
rename to standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
index 6e24ffbddc72..eb5886972eaf 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
+++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
@@ -32,14 +32,13 @@
 import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.hive.HiveCatalog;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Iceberg Catalog server creator.
  */
-public class HMSCatalogServer extends ServletServerBuilder {
+public class HMSCatalogServer {
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class);
   protected static final AtomicReference> catalogRef = new AtomicReference<>();
 
@@ -52,29 +51,33 @@ protected static void setLastCatalog(Catalog catalog) {
     catalogRef.set(new SoftReference<>(catalog));
   }
 
+  protected final Configuration configuration;
   protected final int port;
   protected final String path;
   protected Catalog catalog;
 
   protected HMSCatalogServer(Configuration conf, Catalog catalog) {
-    super(conf);
     port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
     path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
+    this.configuration = conf;
     this.catalog = catalog;
   }
-
-  @Override
-  protected String getServletPath() {
+  
+  public int getPort() {
+    return port;
+  }
+  
+  public String getPath() {
     return path;
   }
-
-  @Override
-  protected int getServerPort() {
-    return port;
+  
+  public Catalog getCatalog() {
+    return catalog;
   }
 
   protected Catalog createCatalog() {
     final Map properties = new TreeMap<>();
+    MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, "");
     final String configUri = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS);
     if (configUri != null) {
       properties.put("uri", configUri);
@@ -100,45 +103,40 @@ protected HttpServlet createServlet(Catalog catalog) throws IOException {
     return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
   }
 
-  @Override
   protected HttpServlet createServlet() throws IOException {
-    Catalog actualCatalog = catalog;
-    if (actualCatalog == null) {
-      MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, "");
-      actualCatalog = catalog = createCatalog();
+    if (port >= 0 && path != null && !path.isEmpty()) {
+      Catalog actualCatalog = catalog;
+      if (actualCatalog == null) {
+        actualCatalog = catalog = createCatalog();
+      }
+      setLastCatalog(actualCatalog);
+      return createServlet(actualCatalog);
     }
-    setLastCatalog(actualCatalog);
-    return createServlet(actualCatalog);
-  }
-
-  @Override
-  protected Server createServer() {
-    final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MAX);
-    final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_MIN);
-    final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE);
-    final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
-    return new Server(threadPool);
+    return null;
   }
 
+  
   /**
-   * Convenience method to start a http server that only serves the Iceberg
-   * catalog servlet.
-   * 

-   * This one is looked up through reflection to start from HMS.

+   * Factory method to describe Iceberg servlet.
+   * 
This one is looked up through reflection to start from HMS.

    *
    * @param conf the configuration
-   * @return the server instance
-   * @throws Exception if servlet initialization fails
+   * @return the servlet descriptor instance
    */
-  public static Server startServer(Configuration conf) throws Exception {
-    Server server = new HMSCatalogServer(conf, null).startServer();
-    if (server != null) {
-      if (!server.isStarted()) {
-        LOG.error("Unable to start property-maps servlet server on {}", server.getURI());
-      } else {
-        LOG.info("Started property-maps servlet server on {}", server.getURI());
+  public static ServletServerBuilder.Descriptor createServlet(Configuration configuration) {
+    try {
+      HMSCatalogServer hms = new HMSCatalogServer(configuration, null);
+      HttpServlet servlet = hms.createServlet();
+      if (servlet != null) {
+        return new ServletServerBuilder.Descriptor(hms.getPort(), hms.getPath(), servlet) {
+          @Override public String toString() {
+            return "Iceberg REST Catalog";
+          }
+        };
       }
+    } catch (Exception exception) {
+      LOG.error("failed to create servlet ", exception);
     }
-    return server;
+    return null;
   }
 }
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
similarity index 97%
rename from standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
rename to standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
index bc581fc26b9c..4b0e6a47080c 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
+++ b/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
@@ -32,8 +32,6 @@
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.apache.hc.core5.http.ContentType;
-import org.apache.hc.core5.http.HttpHeaders;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.io.CharStreams;
 import org.apache.iceberg.rest.HMSCatalogAdapter.HTTPMethod;
@@ -50,9 +48,12 @@
  */
 public class HMSCatalogServlet extends HttpServlet {
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServlet.class);
+  private static final String CONTENT_TYPE = "Content-Type";
+  private static final String APPLICATION_JSON = "application/json";
+  
   private final HMSCatalogAdapter restCatalogAdapter;
   private final Map responseHeaders =
-      ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType());
+      ImmutableMap.of(CONTENT_TYPE, APPLICATION_JSON);
 
   public HMSCatalogServlet(HMSCatalogAdapter restCatalogAdapter) {
     this.restCatalogAdapter = restCatalogAdapter;
@@ -64,7 +65,6 @@ protected void service(HttpServletRequest request, HttpServletResponse response)
       ServletRequestContext context = ServletRequestContext.from(request);
       response.setStatus(HttpServletResponse.SC_OK);
       responseHeaders.forEach(response::setHeader);
-
       final Optional error = context.error();
       if (error.isPresent()) {
         response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
similarity index 100%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
rename to standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
similarity index 98%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
rename to standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
index 989aeaea5682..abcce301ca22 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
+++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
@@ -68,7 +68,6 @@
 import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
 import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.Warehouse;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.metrics.Metrics;
@@ -194,11 +193,10 @@ public void setUp() throws Exception {
     // The manager decl
     PropertyManager.declare(NS, HMSPropertyManager.class);
     // The client
-    HiveMetaStoreClient client = createClient(conf, port);
+    HiveMetaStoreClient client = createClient(conf);
     Assert.assertNotNull("Unable to connect to the MetaStore server", client);
 
     // create a managed root
-    Warehouse wh = new Warehouse(conf);
     String location = temp.newFolder("hivedb2023").getAbsolutePath();
     Database db = new Database(DB_NAME, "catalog test", location, Collections.emptyMap());
     client.createDatabase(db);
@@ -242,7 +240,7 @@ private static Catalog acquireServer(int[] port) throws InterruptedException {
     }
   }
 
-  protected HiveMetaStoreClient createClient(Configuration conf, int port) throws Exception {
+  protected HiveMetaStoreClient createClient(Configuration conf) throws Exception {
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "");
     MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI, false);
     return new HiveMetaStoreClient(conf);
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
similarity index 99%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
rename to standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
index 1dd2034cb417..8b5a795e2c2d 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
+++ b/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
@@ -78,7 +78,7 @@ public void testCreateNamespaceHttp() throws Exception {
             "\"properties\":{ \"owner\": \"apache\", \"group\" : \"iceberg\" }"
             +"}");
     Assert.assertNotNull(response);
-    HiveMetaStoreClient client = createClient(conf, port);
+    HiveMetaStoreClient client = createClient(conf);
     Database database1 = client.getDatabase(ns);
     Assert.assertEquals("apache", database1.getParameters().get("owner"));
     Assert.assertEquals("iceberg", database1.getParameters().get("group"));
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
rename to standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
rename to standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json b/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
similarity index 100%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
rename to standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
diff --git a/standalone-metastore/metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties b/standalone-metastore/metastore-catalog/src/test/resources/log4j2.properties
similarity index 71%
rename from standalone-metastore/metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties
rename to standalone-metastore/metastore-catalog/src/test/resources/log4j2.properties
index 36aaa3ef9ccf..7d592ef2df94 100644
--- a/standalone-metastore/metastore-iceberg-catalog/src/test/resources/hive-log4j2.properties
+++ b/standalone-metastore/metastore-catalog/src/test/resources/log4j2.properties
@@ -17,23 +17,22 @@
 
 name=PropertiesConfig
 property.filename = logs
-appenders = console,captured
+appenders = console
 
 appender.console.type = Console
 appender.console.name = STDOUT
 appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n
+appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{5} - %msg%n
 
-appender.captured.type = CapturingLogAppender
-appender.captured.name = CAPTURED
+rootLogger.level = INFO
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = STDOUT
 
-loggers=file
-logger.file.name=guru.springframework.blog.log4j2properties
-logger.file.level = debug
-logger.file.appenderRefs = file
-logger.file.appenderRef.file.ref = LOGFILE
+loggers = HttpClient, JettyHttpServer
+
+logger.HttpClient.name = org.apache.http.client
+logger.HttpClient.level = INFO
+
+logger.JettyHttpServer.name = org.eclipse.jetty.server
+logger.JettyHttpServer.level = INFO
 
-rootLogger.level = info
-rootLogger.appenderRefs = stdout,captured
-rootLogger.appenderRef.stdout.ref = STDOUT
-rootLogger.appenderRef.captured.ref = CAPTURED
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index eb0c6cce90d5..ce91867922fe 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1826,6 +1826,13 @@ public enum ConfVars {
             new StringSetValidator("simple", "jwt"),
         "Property-maps servlet authentication method (simple or jwt)."
     ),
+    ICEBERG_CATALOG_SERVLET_FACTORY("hive.metastore.catalog.servlet.factory",
+            "hive.metastore.catalog.servlet.factory",
+            "org.apache.iceberg.rest.HMSCatalogServer",
+            "HMS Iceberg Catalog servlet factory class name."
+            + "The factory needs to expose a method: "
+            + "public static HttpServlet createServlet(Configuration configuration);"
+    ),
     ICEBERG_CATALOG_SERVLET_PATH("hive.metastore.catalog.servlet.path",
         "hive.metastore.catalog.servlet.path", "iceberg",
         "HMS Iceberg Catalog servlet path component of URL endpoint."
@@ -1840,18 +1847,6 @@ public enum ConfVars {
         "hive.metastore.catalog.servlet.auth", "jwt",
         "HMS Iceberg Catalog servlet authentication method (simple or jwt)."
     ),
-    ICEBERG_CATALOG_JETTY_THREADPOOL_MIN("hive.metastore.catalog.jetty.threadpool.min",
-        "hive.metastore.catalog.jetty.threadpool.min", 8,
-        "HMS Iceberg Catalog embedded Jetty minimum number of threads."
-    ),
-    ICEBERG_CATALOG_JETTY_THREADPOOL_MAX("hive.metastore.catalog.jetty.threadpool.max",
-        "hive.metastore.catalog.jetty.threadpool.max", 256,
-        "HMS Iceberg Catalog embedded Jetty maximum number of threads."
-    ),
-    ICEBERG_CATALOG_JETTY_THREADPOOL_IDLE("hive.metastore.catalog.jetty.threadpool.idle",
-        "hive.metastore.catalog.jetty.threadpool.idle", 60_000L,
-        "HMS Iceberg Catalog embedded Jetty thread idle time."
-    ),
     ICEBERG_CATALOG_CACHE_EXPIRY("hive.metastore.catalog.cache.expiry",
         "hive.metastore.catalog.cache.expiry", 60_000L,
         "HMS Iceberg Catalog cache expiry."
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 13463355a284..1576cb6a2933 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -122,16 +122,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
   private static ZooKeeperHiveHelper zooKeeperHelper = null;
   private static String msHost = null;
   private static ThriftServer thriftServer;
-  private static Server propertyServer = null;
-  private static Server icebergServer = null;
+  private static Server servletServer = null;
 
 
   public static Server getPropertyServer() {
-    return propertyServer;
+    return servletServer;
   }
 
   public static Server getIcebergServer() {
-    return icebergServer;
+    return servletServer;
   }
 
   public static boolean isRenameAllowed(Database srcDB, Database destDB) {
@@ -317,22 +316,14 @@ public static void main(String[] args) throws Throwable {
         if (isCliVerbose) {
           System.err.println(shutdownMsg);
         }
-        // property server
-        if (propertyServer != null) {
+        // servlet server
+        if (servletServer != null) {
           try {
-            propertyServer.stop();
+            servletServer.stop();
           } catch (Exception e) {
             LOG.error("Error stopping Property Map server.", e);
           }
         }
-        // iceberg server
-        if (icebergServer != null) {
-          try {
-            icebergServer.stop();
-          } catch (Exception e) {
-            LOG.error("Error stopping Iceberg API server.", e);
-          }
-        }
         // metrics
         if (MetastoreConf.getBoolVar(conf, ConfVars.METRICS_ENABLED)) {
           try {
@@ -756,19 +747,24 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
         throw e;
       }
     }
-    // optionally create and start the property server and servlet
-    propertyServer = PropertyServlet.startServer(conf);
-    // optionally create and start the Iceberg REST server and servlet
-    icebergServer = startIcebergCatalog(conf);
-
+    // optionally create and start the property and Iceberg REST server
+    servletServer = ServletServerBuilder.startServer(LOG, conf,
+            PropertyServlet::createServlet,
+            HiveMetaStore::createIcebergServlet);
     thriftServer.start();
   }
 
-  static Server startIcebergCatalog(Configuration configuration) {
+  /**
+   * Creates the Iceberg REST catalog servlet descriptor.
+   * @param configuration the configuration
+   * @return the servlet descriptor (can be null)
+   */
+  static ServletServerBuilder.Descriptor createIcebergServlet(Configuration configuration) {
     try {
-      Class iceClazz = Class.forName("org.apache.iceberg.rest.HMSCatalogServer");
-      Method iceStart = iceClazz.getMethod("startServer", Configuration.class);
-      return (Server) iceStart.invoke(null, configuration);
+      String className = MetastoreConf.getVar(configuration, ConfVars.ICEBERG_CATALOG_SERVLET_FACTORY);
+      Class iceClazz = Class.forName(className);
+      Method iceStart = iceClazz.getMethod("createServlet", Configuration.class);
+      return (ServletServerBuilder.Descriptor) iceStart.invoke(null, configuration);
     } catch (ClassNotFoundException xnf) {
       LOG.warn("unable to start Iceberg REST Catalog server, missing jar?", xnf);
       return null;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index f30f7b6563a9..c5893ca4f1f8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -69,6 +69,10 @@ static boolean isAuthJwt(Configuration configuration) {
     this.configuration = configuration;
   }
 
+  @Override public String getServletName() {
+    return "HMS property";
+  }
+  
   private String strError(String msg, Object...args) {
     return String.format(PTYERROR + msg, args);
   }
@@ -313,50 +317,34 @@ protected void doGet(HttpServletRequest request,
     }
   }
 
-  /**
-   * Single servlet creation helper.
-   */
-  private static class ServerBuilder extends ServletServerBuilder {
-    final int port;
-    final String path;
-    ServerBuilder(Configuration conf) {
-      super(conf);
-      port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
-      path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
-    }
-
-    @Override
-    protected String getServletPath() {
-      return path;
-    }
-
-    @Override
-    protected int getServerPort() {
-      return port;
-    }
-
-    @Override
-    protected HttpServlet createServlet() {
-      ServletSecurity security = new ServletSecurity(configuration, PropertyServlet.isAuthJwt(configuration));
-      return security.proxy(new PropertyServlet(configuration));
+  public static ServletServerBuilder.Descriptor createServlet(Configuration configuration) {
+    try {
+      int port = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
+      String path = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+      if (port >= 0 && path != null && !path.isEmpty()) {
+        ServletSecurity security = new ServletSecurity(configuration, PropertyServlet.isAuthJwt(configuration));
+        HttpServlet servlet = security.proxy(new PropertyServlet(configuration));
+        return new ServletServerBuilder.Descriptor(port, path, servlet) {
+          @Override public String toString() {
+            return "HMS property";
+          }
+        };
+      }
+    } catch (Exception io) {
+      LOGGER.error("failed to create servlet ", io);
     }
+    return null;
   }
 
   /**
    * Convenience method to start a http server that only serves this servlet.
+   *
    * @param conf the configuration
    * @return the server instance
    * @throws Exception if servlet initialization fails
    */
   public static Server startServer(Configuration conf) throws Exception {
-    Server server = new ServerBuilder(conf).startServer();
-    if (server != null) {
-      if (!server.isStarted()) {
-        LOGGER.error("Unable to start property-maps servlet server on {}", server.getURI());
-      } else {
-        LOGGER.info("Started property-maps servlet server on {}", server.getURI());
-      }
-    }
-    return server;
+   return ServletServerBuilder.startServer(LOGGER, conf, PropertyServlet::createServlet);
   }
+
 }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index e2cb5c971900..54d77ba46e3f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -19,6 +19,15 @@
 package org.apache.hadoop.hive.metastore;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import javax.servlet.Servlet;
 import javax.servlet.http.HttpServlet;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
@@ -26,19 +35,24 @@
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.ContextHandlerCollection;
+import org.eclipse.jetty.server.handler.HandlerCollection;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.slf4j.Logger;
 
 /**
- * Helper class to ease creation of embedded Jetty serving one servlet on a given port.
- * 
When using Jetty, the easiest way - and may be only - to serve different servlets
- * on different ports is to create 2 separate Jetty instances; this helper eases creation
- * of such a dedicated server.

+ * Helper class to ease creation of embedded Jetty serving servlets on
+ * different ports.
  */
-public abstract class ServletServerBuilder {
+public class ServletServerBuilder {
+  /**
+   * Keeping track of descriptors.
+   */
+  private Map descriptorsMap = new IdentityHashMap<>();
   /**
    * The configuration instance.
    */
@@ -46,52 +60,86 @@ public abstract class ServletServerBuilder {
 
   /**
    * Creates a builder instance.
+   *
    * @param conf the configuration
    */
   protected ServletServerBuilder(Configuration conf) {
     this.configuration = conf;
   }
-  /**
-   * Gets the servlet path.
-   * @return the path
-   */
-  protected abstract String getServletPath();
+
+  public Configuration getConfiguration() {
+    return configuration;
+  }
 
   /**
-   * Gets the server port.
-   * @return the port
+   * A descriptor of a servlet.
+   * 
After server is started, unspecified port will be updated to reflect
+   * what the system allocated.

    */
-  protected abstract int getServerPort();
+  public static class Descriptor {
+    private int port;
+    private final String path;
+    private final HttpServlet servlet;
+
+    /**
+     * Create a servlet descriptor.
+     * @param port the servlet port (or 0 if system allocated)
+     * @param path the servlet path
+     * @param servlet the servlet instance
+     */
+    public Descriptor(int port, String path, HttpServlet servlet) {
+      this.port = port;
+      this.path = path;
+      this.servlet = servlet;
+    }
+    
+    public String toString() {
+      return servlet.getClass().getSimpleName() + ":" + port+ "/"+ path ;
+    }
+
+    public int getPort() {
+      return port;
+    }
+
+    public String getPath() {
+      return path;
+    }
+
+    public HttpServlet getServlet() {
+      return servlet;
+    }
+  }
 
   /**
-   * Creates the servlet instance.
-   * 
It is often advisable to use {@link ServletSecurity} to proxy the actual servlet instance.

-   * @return the servlet instance
-   * @throws IOException if servlet creation fails
+   * Adds a servlet instance.
+   * 
The servlet port can be shared between servlets; if 0, the system will provide
+   * a port. If the port is < 0, the system will provide a port dedicated (ie non-shared)
+   * to the servlet.

+   * @param port the servlet port
+   * @param path the servlet path
+   * @param servlet a servlet instance
+   * @return a descriptor
    */
-  protected abstract HttpServlet createServlet() throws IOException;
+  public Descriptor addServlet(int port, String path, HttpServlet servlet){
+    Descriptor descriptor  = new Descriptor(port, path, servlet);
+    return addServlet(descriptor);
+  }
 
   /**
-   * Creates the servlet context.
-   * @param servlet the servlet
-   * @return a context instance
+   * Adds a servlet instance.
+   *
+   * @param descriptor a descriptor
+   * @return the descriptor
    */
-  protected ServletContextHandler createContext(HttpServlet servlet) {
-    // hook the servlet
-    ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
-    context.setContextPath("/");
-    ServletHolder servletHolder = new ServletHolder(servlet);
-    servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
-    final String path = getServletPath();
-    context.addServlet(servletHolder, "/" + path + "/*");
-    context.setVirtualHosts(null);
-    context.setGzipHandler(new GzipHandler());
-    return context;
+  public Descriptor addServlet(Descriptor descriptor){
+    descriptorsMap.put(descriptor.getServlet(), descriptor);
+    return descriptor;
   }
 
   /**
    * Creates a server instance.
-   * 
Default use configuration to determine threadpool constants?

+   * 
Default use configuration to determine thread-pool constants?

+   *
    * @return the server instance
    * @throws IOException if server creation fails
    */
@@ -100,23 +148,24 @@ protected Server createServer() throws IOException {
     final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_MIN);
     final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_IDLE);
     final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
-    return new Server(threadPool);
+    Server server = new Server(threadPool);
+    server.setStopAtShutdown(true);
+    return server;
   }
 
   /**
    * Creates a server instance and a connector on a given port.
+   *
+   * @param server the server instance
+   * @param sslContextFactory the ssl factory
    * @param port the port
-   * @return the server instance listening to the port
+   * @return the server connector listening to the port
    * @throws IOException if server creation fails
    */
-  protected Server createServer(int port) throws IOException {
-    final Server server = createServer();
-    server.setStopAtShutdown(true);
-    final SslContextFactory sslContextFactory = ServletSecurity.createSslContextFactory(configuration);
+  protected ServerConnector createConnector(Server server, SslContextFactory sslContextFactory, int port) throws IOException {
     final ServerConnector connector = new ServerConnector(server, sslContextFactory);
     connector.setPort(port);
     connector.setReuseAddress(true);
-    server.addConnector(connector);
     HttpConnectionFactory httpFactory = connector.getConnectionFactory(HttpConnectionFactory.class);
     // do not leak information
     if (httpFactory != null) {
@@ -124,28 +173,141 @@ protected Server createServer(int port) throws IOException {
       httpConf.setSendServerVersion(false);
       httpConf.setSendXPoweredBy(false);
     }
-    return server;
+    return connector;
+  }
+
+  /**
+   * Adds a servlet to its intended servlet context handler.
+   * @param handlersMap the map of port to handlers
+   * @param descriptor the servlet descriptor
+   * @throws IOException
+   */
+  protected void addServlet(Map handlersMap, Descriptor descriptor) throws IOException {
+    final int port = descriptor.getPort();
+    final String path = descriptor.getPath();
+    final HttpServlet servlet = descriptor.getServlet();
+    // if port is < 0, use one for this servlet only
+    int key = port < 0 ? -1 - handlersMap.size() : port;
+    ServletContextHandler handler = handlersMap.computeIfAbsent(key, p -> {
+      ServletContextHandler servletHandler = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+      servletHandler.setContextPath("/");
+      servletHandler.setGzipHandler(new GzipHandler());
+      return servletHandler;
+    });
+    ServletHolder servletHolder = new ServletHolder(servlet);
+    servletHolder.setInitParameter("javax.ws.rs.Application", "ServiceListPublic");
+    handler.addServlet(servletHolder, "/" + path + "/*");
   }
 
   /**
-   * Convenience method to start a http server that only serves this servlet.
-   * @return the server instance or null if port < 0
+   * Convenience method to start a http server that serves all configured
+   * servlets.
+   *
+   * @return the server instance or null if no servlet was configured
    * @throws Exception if servlet initialization fails
    */
   public Server startServer() throws Exception {
-    int port = getServerPort();
-    if (port < 0) {
+    // add all servlets
+    Map handlersMap = new HashMap<>();
+    for(Descriptor descriptor : descriptorsMap.values()) {
+      addServlet(handlersMap, descriptor);
+    }
+    final int size = handlersMap.size();
+    if (size == 0) {
       return null;
     }
-    // create the servlet
-    final HttpServlet servlet = createServlet();
-    // hook the servlet
-    ServletContextHandler context = createContext(servlet);
-    // Http server
-    final Server httpServer = createServer(port);
-    httpServer.setHandler(context);
-    httpServer.start();
-    return httpServer;
+    final Server server = createServer();
+    // create the connectors
+    final SslContextFactory sslFactory = ServletSecurity.createSslContextFactory(configuration);
+    final int[] keys = new int[size];
+    final ServerConnector[] connectors = new ServerConnector[size];
+    final ServletContextHandler[] handlers = new ServletContextHandler[size];
+    Iterator> it = handlersMap.entrySet().iterator();
+    for (int c = 0; it.hasNext(); ++c) {
+      Map.Entry entry = it.next();
+      int key = entry.getKey();
+      keys[c] = key;
+      int port = key < 0? 0 : key;
+      ServerConnector connector = createConnector(server, sslFactory, port);
+      connectors[c] = connector;
+      ServletContextHandler handler = entry.getValue();
+      handlers[c] = handler;
+      // make each servlet context be served only by its dedicated connector
+      String host = "hms" + Integer.toString(c);
+      connector.setName(host);
+      handler.setVirtualHosts(new String[]{"@"+host});
+    }
+    // hook the connectors and the handlers
+    server.setConnectors(connectors);
+    HandlerCollection portHandler = new ContextHandlerCollection();
+    portHandler.setHandlers(handlers);
+    server.setHandler(portHandler);
+    // start the server
+    server.start();
+    // collect auto ports
+    for (int i = 0; i < connectors.length; ++i) {
+      int port = connectors[i].getLocalPort();
+      ServletContextHandler handler = handlers[i];
+      ServletHolder[] holders = handler.getServletHandler().getServlets();
+      for(ServletHolder holder : holders) {
+        Servlet servlet = holder.getServletInstance();
+        if (servlet != null) {
+          Descriptor descriptor = descriptorsMap.get(servlet);
+          if (descriptor != null) {
+            descriptor.port = port;
+          }
+        }
+      }
+    }
+    return server;
   }
 
+  /**
+   * Helper for generic use case.
+   * @param logger the logger
+   * @param conf the configuration
+   * @param describe the functions to create descriptors
+   * @return a server instance
+   */
+  @SafeVarargs
+  public static Server startServer(
+          Logger logger,
+          Configuration conf,
+          Function... describe) {
+    List descriptors = new ArrayList();
+    Arrays.asList(describe).forEach(functor -> {
+      ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
+      if (descriptor != null) {
+        descriptors.add(descriptor);
+      };
+    });
+    if (!descriptors.isEmpty()) {
+      ServletServerBuilder builder = new ServletServerBuilder(conf);
+      descriptors.forEach(d -> builder.addServlet(d));
+      try {
+        Server server = builder.startServer();
+        if (server != null) {
+          if (!server.isStarted()) {
+            logger.error("Unable to start property-maps servlet server on {}", server.getURI());
+          } else {
+            descriptors.forEach(descriptor -> {
+            logger.info("Started {} servlet on {}:{}",
+                    descriptor.toString(),
+                   descriptor.getPort(),
+                   descriptor.getPath());
+            });
+          }
+        }
+        return server;
+      } catch(Exception exception) {
+        logger.error("Unable to start servlet server", exception);
+        return null;
+      } catch(Throwable throwable) {
+        logger.error("Unable to start servlet server", throwable);
+        return null;
+      }
+    }
+    return null;
+  }
 }
+
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java
new file mode 100644
index 000000000000..e752ef592e26
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestServletServerBuilder.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import com.google.gson.Gson;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Map;
+import java.util.function.Function;
+import javax.servlet.Servlet;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.eclipse.jetty.server.Server;
+import org.junit.experimental.categories.Category;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.hive.metastore.ServletServerBuilder.Descriptor;
+
+@Category(MetastoreUnitTest.class)
+public class TestServletServerBuilder {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestServletServerBuilder.class);
+
+  private static Function describeServlet(final Map descriptors, int port, String greeting) {
+    return configuration -> {
+      String name = greeting.toLowerCase();
+      HttpServlet s1 = new HelloServlet(greeting) {
+        @Override
+        public String getServletName() {
+          return name + "()";
+        }
+      };
+      Descriptor descriptor = new Descriptor(port, name, s1);
+      descriptors.put(s1.getServletName(), descriptor);
+      return descriptor;
+    };
+  }
+
+  @Test
+  public void testOne() throws Exception {
+    Configuration conf = new Configuration();
+    // keeping track of what is built
+    final Map descriptors = new HashMap();
+    Function fd1 = describeServlet(descriptors, 0, "ONE");
+    Function fd2 = describeServlet(descriptors, 0, "TWO");
+    // the 'conventional' way of starting the server
+    Server server = ServletServerBuilder.startServer(LOG, conf, fd1, fd2);
+
+    Descriptor d1 = descriptors.get("one()");
+    Descriptor d2 = descriptors.get("two()");
+    // same port for both servlets
+    Assert.assertTrue(d1.getPort() > 0);
+    Assert.assertEquals(d1.getPort(), d2.getPort());
+    // check 
+    URI uri = URI.create("http://localhost:" + d1.getPort());
+    Object one = clientCall(uri.resolve("/one").toURL());
+    Assert.assertEquals("ONE", one);
+    uri = URI.create("http://localhost:" + d2.getPort());
+    Object two = clientCall(uri.resolve("/two").toURL());
+    Assert.assertEquals("TWO", two);
+    server.stop();
+  }
+
+  @Test
+  public void testOnePort() throws Exception {
+    int port;
+    try (ServerSocket server0 = new ServerSocket(0)) {
+      port = server0.getLocalPort();
+    } catch (IOException xio) {
+      // cant run test if can not get free port
+      return;
+    }
+    onePort(port);
+  }
+
+  @Test
+  public void testOnePortAuto() throws Exception {
+    onePort(0);
+  }
+
+  void onePort(int port) throws Exception {
+    Configuration conf = new Configuration();
+    ServletServerBuilder ssb = new ServletServerBuilder(conf);
+    HttpServlet s1 = new HelloServlet("ONE");
+    HttpServlet s2 = new HelloServlet("TWO");
+    Descriptor d1 = ssb.addServlet(port, "one", s1);
+    Descriptor d2 = ssb.addServlet(port, "two", s2);
+    Server server = ssb.startServer();
+    // same port for both servlets
+    Assert.assertTrue(d1.getPort() > 0);
+    Assert.assertEquals(d1.getPort(), d2.getPort());
+    // check 
+    URI uri = URI.create("http://localhost:" + d1.getPort());
+    Object one = clientCall(uri.resolve("/one").toURL());
+    Assert.assertEquals("ONE", one);
+    uri = URI.create("http://localhost:" + d2.getPort());
+    Object two = clientCall(uri.resolve("/two").toURL());
+    Assert.assertEquals("TWO", two);
+    server.stop();
+  }
+
+  @Test
+  public void testTwoPorts() throws Exception {
+    runTwoPorts(-1, -2);
+  }
+
+  @Test
+  public void testTwoPortsAuto() throws Exception {
+    int p0, p1;
+    try (ServerSocket server0 = new ServerSocket(0); ServerSocket server1 = new ServerSocket(0)) {
+      p0 = server0.getLocalPort();
+      p1 = server1.getLocalPort();
+    } catch (IOException xio) {
+      // cant do test if can not get port
+      return;
+    }
+    runTwoPorts(p0, p1);
+  }
+
+  void runTwoPorts(int p1, int p2) throws Exception {
+    Configuration conf = new Configuration();
+    ServletServerBuilder ssb = new ServletServerBuilder(conf);
+    HttpServlet s1 = new HelloServlet("ONE");
+    HttpServlet s2 = new HelloServlet("TWO");
+    Descriptor d1 = ssb.addServlet(p1, "one", s1);
+    Descriptor d2 = ssb.addServlet(p2, "two", s2);
+    Map mappings = new IdentityHashMap<>();
+    Server server = ssb.startServer();
+    // different port for both servlets
+    Assert.assertNotEquals(d1.getPort(), d2.getPort());
+
+    URI uri = URI.create("http://localhost:" + d1.getPort());
+    Object one = clientCall(uri.resolve("/one").toURL());
+    Assert.assertEquals("ONE", one);
+    // fail, not found
+    Object o404 = clientCall(uri.resolve("/two").toURL());
+    Assert.assertEquals(404, o404);
+    uri = URI.create("http://localhost:" + d2.getPort());
+    Object two = clientCall(uri.resolve("/two").toURL());
+    Assert.assertEquals("TWO", two);
+    // fail, not found
+    o404 = clientCall(uri.resolve("/one").toURL());
+    Assert.assertEquals(404, o404);
+    server.stop();
+  }
+
+  static int findFreePort() throws IOException {
+    try (ServerSocket server0 = new ServerSocket(0)) {
+      return server0.getLocalPort();
+    }
+  }
+
+  static int find2FreePort() throws IOException {
+    try (ServerSocket socket0 = new ServerSocket(0)) {
+      return socket0.getLocalPort();
+    }
+  }
+
+  /**
+   * Performs a Json client call.
+   *
+   * @param url the url
+   * @return the result the was returned through Json
+   * @throws IOException if marshalling the request/response fail
+   */
+  static Object clientCall(URL url) throws IOException {
+    HttpURLConnection con = (HttpURLConnection) url.openConnection();
+    con.setRequestMethod("GET");
+    con.setRequestProperty("Content-Type", "application/json");
+    con.setRequestProperty("Accept", "application/json");
+    con.setDoOutput(true);
+    int responseCode = con.getResponseCode();
+    if (responseCode == HttpServletResponse.SC_OK) {
+      try (Reader reader = new BufferedReader(
+              new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8))) {
+        return new Gson().fromJson(reader, Object.class);
+      }
+    }
+    return responseCode;
+  }
+
+}
+
+class HelloServlet extends HttpServlet {
+
+  final String greeting;
+
+  public HelloServlet() {
+    this("Hello");
+  }
+
+  public HelloServlet(String greeting) {
+    this.greeting = greeting;
+  }
+
+  @Override
+  protected void doGet(HttpServletRequest request,
+          HttpServletResponse response) throws ServletException, IOException {
+    response.setContentType("application/json");
+    response.setStatus(HttpServletResponse.SC_OK);
+    response.getWriter().println(greeting);
+  }
+}
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 6e55634f2fdd..1dcfd8127654 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -29,7 +29,7 @@
     metastore-common
     metastore-server
     metastore-tools
-    metastore-iceberg-catalog
+    metastore-catalog
   
   
     4.1.0-SNAPSHOT

From 11c97181ce76151d08a1f44be1d6b2351e8e3d45 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Fri, 21 Feb 2025 15:01:55 +0100
Subject: [PATCH 29/40] HIVE-28059 : - moving module to metastore-rest-catalog;

---
 .../{metastore-catalog => metastore-rest-catalog}/pom.xml     | 4 ++--
 .../main/java/org/apache/iceberg/rest/HMSCachingCatalog.java  | 0
 .../main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java  | 0
 .../main/java/org/apache/iceberg/rest/HMSCatalogServer.java   | 0
 .../main/java/org/apache/iceberg/rest/HMSCatalogServlet.java  | 0
 .../test/java/org/apache/iceberg/hive/IcebergTestHelper.java  | 0
 .../src/test/java/org/apache/iceberg/rest/HMSTestBase.java    | 0
 .../src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java | 0
 .../src/test/resources/auth/jwt/jwt-authorized-key.json       | 0
 .../src/test/resources/auth/jwt/jwt-unauthorized-key.json     | 0
 .../src/test/resources/auth/jwt/jwt-verification-jwks.json    | 0
 .../src/test/resources/log4j2.properties                      | 0
 standalone-metastore/pom.xml                                  | 2 +-
 13 files changed, 3 insertions(+), 3 deletions(-)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/pom.xml (98%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/java/org/apache/iceberg/rest/HMSTestBase.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/resources/auth/jwt/jwt-authorized-key.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/resources/auth/jwt/jwt-unauthorized-key.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/resources/auth/jwt/jwt-verification-jwks.json (100%)
 rename standalone-metastore/{metastore-catalog => metastore-rest-catalog}/src/test/resources/log4j2.properties (100%)

diff --git a/standalone-metastore/metastore-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml
similarity index 98%
rename from standalone-metastore/metastore-catalog/pom.xml
rename to standalone-metastore/metastore-rest-catalog/pom.xml
index 865c5e8b6f02..8202741fdded 100644
--- a/standalone-metastore/metastore-catalog/pom.xml
+++ b/standalone-metastore/metastore-rest-catalog/pom.xml
@@ -17,8 +17,8 @@
     4.1.0-SNAPSHOT
   
   4.0.0
-  hive-standalone-metastore-catalog
-  Hive Metastore Iceberg Catalog
+  hive-standalone-metastore-rest-catalog
+  Hive Metastore Iceberg REST Catalog
   
     ..
     8
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
rename to standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCachingCatalog.java
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
rename to standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
rename to standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
diff --git a/standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
rename to standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
rename to standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/hive/IcebergTestHelper.java
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
rename to standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
diff --git a/standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
rename to standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json b/standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
rename to standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-authorized-key.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json b/standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
rename to standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-unauthorized-key.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json b/standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
rename to standalone-metastore/metastore-rest-catalog/src/test/resources/auth/jwt/jwt-verification-jwks.json
diff --git a/standalone-metastore/metastore-catalog/src/test/resources/log4j2.properties b/standalone-metastore/metastore-rest-catalog/src/test/resources/log4j2.properties
similarity index 100%
rename from standalone-metastore/metastore-catalog/src/test/resources/log4j2.properties
rename to standalone-metastore/metastore-rest-catalog/src/test/resources/log4j2.properties
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 1dcfd8127654..d712cf0fd495 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -29,7 +29,7 @@
     metastore-common
     metastore-server
     metastore-tools
-    metastore-catalog
+    metastore-rest-catalog
   
   
     4.1.0-SNAPSHOT

From 0b1add3dc31291efc9ee5b1d6802a500a5349842 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Fri, 21 Feb 2025 15:04:00 +0100
Subject: [PATCH 30/40] HIVE-28059: Update src.xml

---
 packaging/src/main/assembly/src.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/src/main/assembly/src.xml b/packaging/src/main/assembly/src.xml
index 654682d89926..9cdbed13776e 100644
--- a/packaging/src/main/assembly/src.xml
+++ b/packaging/src/main/assembly/src.xml
@@ -105,7 +105,7 @@
         standalone-metastore/metastore-common/**/*
         standalone-metastore/metastore-server/**/*
         standalone-metastore/metastore-tools/**/*
-        standalone-metastore/metastore-catalog/**/*
+        standalone-metastore/metastore-rest-catalog/**/*
         standalone-metastore/src/assembly/src.xml
         standalone-metastore/pom.xml
         streaming/**/*

From 670ab10677569e480be38edf1719336ee3158194 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Fri, 21 Feb 2025 21:04:03 +0100
Subject: [PATCH 31/40] HIVE-28059 : - latest remarks (naming, etc)

---
 .../hive/metastore/conf/MetastoreConf.java    | 20 ++--
 .../metastore-rest-catalog/pom.xml            |  2 +-
 ...alogServer.java => HMSCatalogFactory.java} | 13 ++-
 .../org/apache/iceberg/rest/HMSTestBase.java  | 14 ++-
 .../hadoop/hive/metastore/HiveMetaStore.java  | 53 +++++++++--
 .../hive/metastore/ServletServerBuilder.java  | 92 ++++++++++++-------
 .../metastore/properties/HMSServletTest.java  |  3 +-
 .../properties/HMSServletTest1A.java          |  4 +-
 .../metastore/properties/HMSServletTestA.java |  4 +-
 9 files changed, 129 insertions(+), 76 deletions(-)
 rename standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/{HMSCatalogServer.java => HMSCatalogFactory.java} (94%)

diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index ce91867922fe..e0571664193c 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1828,7 +1828,7 @@ public enum ConfVars {
     ),
     ICEBERG_CATALOG_SERVLET_FACTORY("hive.metastore.catalog.servlet.factory",
             "hive.metastore.catalog.servlet.factory",
-            "org.apache.iceberg.rest.HMSCatalogServer",
+            "org.apache.iceberg.rest.HMSCatalogFactory",
             "HMS Iceberg Catalog servlet factory class name."
             + "The factory needs to expose a method: "
             + "public static HttpServlet createServlet(Configuration configuration);"
@@ -1851,17 +1851,17 @@ public enum ConfVars {
         "hive.metastore.catalog.cache.expiry", 60_000L,
         "HMS Iceberg Catalog cache expiry."
     ),
-    EMBEDDED_JETTY_THREADPOOL_MIN("hive.metastore.embedded.jetty.threadpool.min",
-            "hive.metastore.embedded.jetty.threadpool.min", 2,
-            "HMS embedded Jetty server(s) minimum number of threads."
+    HTTPSERVER_THREADPOOL_MIN("hive.metastore.httpserver.threadpool.min",
+            "hive.metastore.httpserver.threadpool.min", 8,
+            "HMS embedded HTTP server minimum number of threads."
     ),
-    EMBEDDED_JETTY_THREADPOOL_MAX("hive.metastore.embedded.jetty.threadpool.max",
-            "hive.metastore.embedded.jetty.threadpool.max", 256,
-            "HMS embedded Jetty server(s) maximum number of threads."
+    HTTPSERVER_THREADPOOL_MAX("hive.metastore.httpserver.threadpool.max",
+            "hive.metastore.httpserver.threadpool.max", 256,
+            "HMS embedded HTTP server maximum number of threads."
     ),
-    EMBEDDED_JETTY_THREADPOOL_IDLE("hive.metastore.embedded.jetty.threadpool.idle",
-            "hive.metastore.embedded.jetty.threadpool.idle", 60_000L,
-            "HMS embedded Jetty server(s) thread idle time."
+    HTTPSERVER_THREADPOOL_IDLE("hive.metastore.httpserver.threadpool.idle",
+            "hive.metastore.httpserver.threadpool.idle", 60_000L,
+            "HMS embedded HTTP server thread idle time."
     ),
 
     // Deprecated Hive values that we are keeping for backwards compatibility.
diff --git a/standalone-metastore/metastore-rest-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml
index 8202741fdded..43871e6f456b 100644
--- a/standalone-metastore/metastore-rest-catalog/pom.xml
+++ b/standalone-metastore/metastore-rest-catalog/pom.xml
@@ -18,7 +18,7 @@
   
   4.0.0
   hive-standalone-metastore-rest-catalog
-  Hive Metastore Iceberg REST Catalog
+  Hive Standalone Metastore REST Catalog
   
     ..
     8
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
similarity index 94%
rename from standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
rename to standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
index eb5886972eaf..6c214f2a13f6 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServer.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
@@ -31,15 +31,14 @@
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.hive.HiveCatalog;
-import org.eclipse.jetty.server.Server;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Iceberg Catalog server creator.
  */
-public class HMSCatalogServer {
-  private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogServer.class);
+public class HMSCatalogFactory {
+  private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogFactory.class);
   protected static final AtomicReference> catalogRef = new AtomicReference<>();
 
   public static Catalog getLastCatalog() {
@@ -56,7 +55,7 @@ protected static void setLastCatalog(Catalog catalog) {
   protected final String path;
   protected Catalog catalog;
 
-  protected HMSCatalogServer(Configuration conf, Catalog catalog) {
+  protected HMSCatalogFactory(Configuration conf, Catalog catalog) {
     port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
     path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
     this.configuration = conf;
@@ -120,12 +119,12 @@ protected HttpServlet createServlet() throws IOException {
    * Factory method to describe Iceberg servlet.
    * 
This one is looked up through reflection to start from HMS.

    *
-   * @param conf the configuration
+   * @param configuration the configuration
    * @return the servlet descriptor instance
    */
   public static ServletServerBuilder.Descriptor createServlet(Configuration configuration) {
     try {
-      HMSCatalogServer hms = new HMSCatalogServer(configuration, null);
+      HMSCatalogFactory hms = new HMSCatalogFactory(configuration, null);
       HttpServlet servlet = hms.createServlet();
       if (servlet != null) {
         return new ServletServerBuilder.Descriptor(hms.getPort(), hms.getPath(), servlet) {
@@ -134,7 +133,7 @@ public static ServletServerBuilder.Descriptor createServlet(Configuration config
           }
         };
       }
-    } catch (Exception exception) {
+    } catch (IOException exception) {
       LOG.error("failed to create servlet ", exception);
     }
     return null;
diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
index abcce301ca22..8fbaae049706 100644
--- a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
+++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
@@ -201,27 +201,25 @@ public void setUp() throws Exception {
     Database db = new Database(DB_NAME, "catalog test", location, Collections.emptyMap());
     client.createDatabase(db);
 
-    int[] aport = { -1 };
-    Catalog ice = acquireServer(aport);
+    Catalog ice = acquireServer();
       catalog =  ice;
       nsCatalog = catalog instanceof SupportsNamespaces? (SupportsNamespaces) catalog : null;
-      catalogPort = aport[0];
+      catalogPort = HiveMetaStore.getCatalogServletPort();
   }
   
   private static String format(String format, Object... params) {
     return org.slf4j.helpers.MessageFormatter.arrayFormat(format, params).getMessage();
   }
 
-  private static Catalog acquireServer(int[] port) throws InterruptedException {
+  private static Catalog acquireServer() throws InterruptedException {
     final int wait = 200;
-    Server iceServer = HiveMetaStore.getIcebergServer();
+    Server iceServer = HiveMetaStore.getServletServer();
     int tries = WAIT_FOR_SERVER / wait;
     while(iceServer == null && tries-- > 0) {
       Thread.sleep(wait);
-      iceServer = HiveMetaStore.getIcebergServer();
+      iceServer = HiveMetaStore.getServletServer();
     }
     if (iceServer != null) {
-      port[0] = iceServer.getURI().getPort();
       boolean starting;
       tries = WAIT_FOR_SERVER / wait;
       while((starting = iceServer.isStarting()) && tries-- > 0) {
@@ -230,7 +228,7 @@ private static Catalog acquireServer(int[] port) throws InterruptedException {
       if (starting) {
         LOG.warn("server still starting after {}ms", WAIT_FOR_SERVER);
       }
-      Catalog ice = HMSCatalogServer.getLastCatalog();
+      Catalog ice = HMSCatalogFactory.getLastCatalog();
       if (ice == null) {
         throw new NullPointerException(format("unable to acquire catalog after {}ms", WAIT_FOR_SERVER));
       }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 1576cb6a2933..adb9c78fb698 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hive.metastore;
 
-import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
@@ -122,17 +121,39 @@ public class HiveMetaStore extends ThriftHiveMetastore {
   private static ZooKeeperHiveHelper zooKeeperHelper = null;
   private static String msHost = null;
   private static ThriftServer thriftServer;
+  /** the servlet server. */
   private static Server servletServer = null;
+  /** the port and path of the property servlet. */
+  private static int propertyServletPort = -1;
+  /** the port and path of the catalog servlet. */
+  private static int catalogServletPort = -1;
 
-
-  public static Server getPropertyServer() {
+  /**
+   * Gets the embedded servlet server.
+   * @return the server instance or null
+   */
+  public static Server getServletServer() {
     return servletServer;
   }
 
-  public static Server getIcebergServer() {
-    return servletServer;
+  /**
+   * Gets the property servlet connector port.
+   * 
If configuration is 0, this port is allocated by the system.

+   * @return the connector port or -1 if not configured
+   */
+  public static int getPropertyServletPort() {
+    return propertyServletPort;
   }
-
+  
+  /**
+   * Gets the catalog servlet connector port.
+   * 
If configuration is 0, this port is allocated by the system.

+   * @return the connector port or -1 if not configured
+   */
+  public static int getCatalogServletPort() {
+    return catalogServletPort;
+  }
+  
   public static boolean isRenameAllowed(Database srcDB, Database destDB) {
     if (!srcDB.getName().equalsIgnoreCase(destDB.getName())) {
       if (ReplChangeManager.isSourceOfReplication(srcDB) || ReplChangeManager.isSourceOfReplication(destDB)) {
@@ -748,12 +769,24 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
       }
     }
     // optionally create and start the property and Iceberg REST server
-    servletServer = ServletServerBuilder.startServer(LOG, conf,
-            PropertyServlet::createServlet,
-            HiveMetaStore::createIcebergServlet);
+    ServletServerBuilder.Descriptor properties = PropertyServlet.createServlet(conf);
+    ServletServerBuilder.Descriptor catalog = createIcebergServlet(conf);
+    ServletServerBuilder builder = new ServletServerBuilder(conf);
+    builder.addServlet(properties);
+    builder.addServlet(catalog);
+    servletServer = builder.start(LOG);
+    if (servletServer != null) {
+      if (properties != null) {
+          propertyServletPort = properties.getPort();
+      }
+      if (catalog != null) {
+        catalogServletPort = catalog.getPort();
+      }
+    }
+    // main server
     thriftServer.start();
   }
-
+  
   /**
    * Creates the Iceberg REST catalog servlet descriptor.
    * @param configuration the configuration
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index 54d77ba46e3f..80354a20678e 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -52,7 +52,7 @@ public class ServletServerBuilder {
   /**
    * Keeping track of descriptors.
    */
-  private Map descriptorsMap = new IdentityHashMap<>();
+  private final Map descriptorsMap = new IdentityHashMap<>();
   /**
    * The configuration instance.
    */
@@ -93,6 +93,7 @@ public Descriptor(int port, String path, HttpServlet servlet) {
       this.servlet = servlet;
     }
     
+    @Override
     public String toString() {
       return servlet.getClass().getSimpleName() + ":" + port+ "/"+ path ;
     }
@@ -131,8 +132,10 @@ public Descriptor addServlet(int port, String path, HttpServlet servlet){
    * @param descriptor a descriptor
    * @return the descriptor
    */
-  public Descriptor addServlet(Descriptor descriptor){
-    descriptorsMap.put(descriptor.getServlet(), descriptor);
+  public Descriptor addServlet(Descriptor descriptor) {
+    if (descriptor != null) {
+      descriptorsMap.put(descriptor.getServlet(), descriptor);
+    }
     return descriptor;
   }
 
@@ -144,9 +147,9 @@ public Descriptor addServlet(Descriptor descriptor){
    * @throws IOException if server creation fails
    */
   protected Server createServer() throws IOException {
-    final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_MAX);
-    final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_MIN);
-    final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.EMBEDDED_JETTY_THREADPOOL_IDLE);
+    final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MAX);
+    final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MIN);
+    final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_IDLE);
     final QueuedThreadPool threadPool = new QueuedThreadPool(maxThreads, minThreads, idleTimeout);
     Server server = new Server(threadPool);
     server.setStopAtShutdown(true);
@@ -219,14 +222,12 @@ public Server startServer() throws Exception {
     final Server server = createServer();
     // create the connectors
     final SslContextFactory sslFactory = ServletSecurity.createSslContextFactory(configuration);
-    final int[] keys = new int[size];
     final ServerConnector[] connectors = new ServerConnector[size];
     final ServletContextHandler[] handlers = new ServletContextHandler[size];
     Iterator> it = handlersMap.entrySet().iterator();
     for (int c = 0; it.hasNext(); ++c) {
       Map.Entry entry = it.next();
       int key = entry.getKey();
-      keys[c] = key;
       int port = key < 0? 0 : key;
       ServerConnector connector = createConnector(server, sslFactory, port);
       connectors[c] = connector;
@@ -263,51 +264,72 @@ public Server startServer() throws Exception {
   }
 
   /**
-   * Helper for generic use case.
-   * @param logger the logger
+   * Creates a builder.
    * @param conf the configuration
-   * @param describe the functions to create descriptors
-   * @return a server instance
+   * @param describe the functions to call that create servlet descriptors
+   * @return the builder or null if no descriptors
    */
   @SafeVarargs
-  public static Server startServer(
-          Logger logger,
-          Configuration conf,
+  public static ServletServerBuilder builder(Configuration conf,
           Function... describe) {
     List descriptors = new ArrayList();
     Arrays.asList(describe).forEach(functor -> {
       ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
       if (descriptor != null) {
         descriptors.add(descriptor);
-      };
+      }
     });
     if (!descriptors.isEmpty()) {
       ServletServerBuilder builder = new ServletServerBuilder(conf);
       descriptors.forEach(d -> builder.addServlet(d));
-      try {
-        Server server = builder.startServer();
-        if (server != null) {
-          if (!server.isStarted()) {
-            logger.error("Unable to start property-maps servlet server on {}", server.getURI());
-          } else {
-            descriptors.forEach(descriptor -> {
+      return builder;
+    }
+    return null;
+  }
+
+  /**
+   * Creates and starts the server.
+   * @param logger a logger to output info
+   * @return the server instance (or null if error)
+   */
+  public Server start(Logger logger) {
+    try {
+      Server server = startServer();
+      if (server != null) {
+        if (!server.isStarted()) {
+          logger.error("Unable to start servlet server on {}", server.getURI());
+        } else {
+          descriptorsMap.values().forEach(descriptor -> {
             logger.info("Started {} servlet on {}:{}",
                     descriptor.toString(),
-                   descriptor.getPort(),
-                   descriptor.getPath());
-            });
-          }
+                    descriptor.getPort(),
+                    descriptor.getPath());
+          });
         }
-        return server;
-      } catch(Exception exception) {
-        logger.error("Unable to start servlet server", exception);
-        return null;
-      } catch(Throwable throwable) {
-        logger.error("Unable to start servlet server", throwable);
-        return null;
       }
+      return server;
+    } catch (Exception exception) {
+      logger.error("Unable to start servlet server", exception);
+      return null;
+    } catch (Throwable throwable) {
+      logger.error("Unable to start servlet server", throwable);
+      return null;
     }
-    return null;
+  }
+  
+   /**
+   * Helper for generic use case.
+   * @param logger the logger
+   * @param conf the configuration
+   * @param describe the functions to create descriptors
+   * @return a server instance
+   */
+  @SafeVarargs
+  public static Server startServer(
+          Logger logger,
+          Configuration conf,
+          Function... describe) {
+    return builder(conf, describe).start(logger);
   }
 }
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
index 75f670409225..b7728a3385ab 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
@@ -51,6 +51,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
 
 public class HMSServletTest extends HMSTestBase {
   protected static final String CLI = "hmscli";
@@ -64,7 +65,7 @@ public class HMSServletTest extends HMSTestBase {
       if (servletServer == null || !servletServer.isStarted()) {
         Assert.fail("http server did not start");
       }
-      sport = servletServer.getURI().getPort();
+      sport = HiveMetaStore.getPropertyServletPort();
     }
     return sport;
   }
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
index 5ff45d90dd82..88f55f82ac23 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
@@ -45,11 +45,11 @@ protected int createServer(Configuration conf) throws Exception {
         .willReturn(ok()
             .withBody(Files.readAllBytes(jwtVerificationJWKSFile.toPath()))));
     thriftPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf);
-    servletServer = HiveMetaStore.getPropertyServer();
+    servletServer = HiveMetaStore.getServletServer();
     if (servletServer == null || !servletServer.isStarted()) {
       Assert.fail("http server did not start");
     }
-    sport = servletServer.getURI().getPort();
+    sport = HiveMetaStore.getPropertyServletPort();
     return sport;
   }
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
index 10a54457ab12..22f245330b55 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
@@ -43,11 +43,11 @@ protected int createServer(Configuration conf) throws Exception {
         .willReturn(ok()
             .withBody(Files.readAllBytes(jwtVerificationJWKSFile.toPath()))));
     thriftPort = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf);
-    servletServer = HiveMetaStore.getPropertyServer();
+    servletServer = HiveMetaStore.getServletServer();
     if (servletServer == null || !servletServer.isStarted()) {
       Assert.fail("http server did not start");
     }
-    sport = servletServer.getURI().getPort();
+    sport = HiveMetaStore.getPropertyServletPort();
     return sport;
   }
 

From 77fb8672821b1d4525aeb0bdfbe15dcf56ca8258 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Sun, 23 Feb 2025 19:37:23 +0100
Subject: [PATCH 32/40] HIVE-28059 : fixing property servlet tests; - catalog &
 property servlet ports shall not be confused; - some property tests were not
 referring to the proper port; - no tests were executed since they lacked the
 proper annotation (MetastoreUnitTest); - nits on ServletServerBuilder;

---
 .../hive/metastore/ServletServerBuilder.java  | 10 +++-
 .../metastore/properties/HMSDirectTest.java   | 21 +++----
 .../metastore/properties/HMSServletTest.java  | 58 +++++++++++--------
 .../metastore/properties/HMSServletTest1.java | 26 +++++----
 .../properties/HMSServletTest1A.java          | 11 ++--
 .../metastore/properties/HMSServletTestA.java | 10 ++--
 .../metastore/properties/HMSTestBase.java     | 45 +++++++-------
 .../metastore/properties/HMSThriftTest.java   | 17 +++---
 .../properties/PropertyStoreTest.java         |  6 +-
 9 files changed, 111 insertions(+), 93 deletions(-)

diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index 80354a20678e..46baad72fdc1 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -109,6 +109,10 @@ public String getPath() {
     public HttpServlet getServlet() {
       return servlet;
     }
+
+    void setPort(int port) {
+      this.port = port;
+    }
   }
 
   /**
@@ -180,7 +184,7 @@ protected ServerConnector createConnector(Server server, SslContextFactory sslCo
   }
 
   /**
-   * Adds a servlet to its intended servlet context handler.
+   * Adds a servlet to its intended servlet context context.
    * @param handlersMap the map of port to handlers
    * @param descriptor the servlet descriptor
    * @throws IOException
@@ -245,7 +249,7 @@ public Server startServer() throws Exception {
     server.setHandler(portHandler);
     // start the server
     server.start();
-    // collect auto ports
+    // collect automatically assigned connector ports
     for (int i = 0; i < connectors.length; ++i) {
       int port = connectors[i].getLocalPort();
       ServletContextHandler handler = handlers[i];
@@ -255,7 +259,7 @@ public Server startServer() throws Exception {
         if (servlet != null) {
           Descriptor descriptor = descriptorsMap.get(servlet);
           if (descriptor != null) {
-            descriptor.port = port;
+            descriptor.setPort(port);
           }
         }
       }
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
index 7c3c77451649..3e7c0cd2e600 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSDirectTest.java
@@ -17,32 +17,33 @@
  */
 package org.apache.hadoop.hive.metastore.properties;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.HMSHandler;
 import org.apache.hadoop.hive.metastore.ObjectStore;
 import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
 import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
 /**
  * In-process property manager test.
  */
+@Category(MetastoreUnitTest.class)
 public class HMSDirectTest extends HMSTestBase {
   protected ObjectStore objectStore = null;
   static Random RND = new Random(20230424);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
index b7728a3385ab..51c4e26727d2 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
@@ -18,8 +18,25 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import javax.servlet.http.HttpServletResponse;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.PropertyServlet;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -34,30 +51,21 @@
 import org.apache.http.message.BasicNameValuePair;
 import org.eclipse.jetty.server.Server;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-import org.apache.hadoop.hive.metastore.HiveMetaStore;
-
+@Category(MetastoreUnitTest.class)
 public class HMSServletTest extends HMSTestBase {
-  protected static final String CLI = "hmscli";
+  String path = null;
   Server servletServer = null;
   int sport = -1;
-
+  
+  @Before
+  public void setUp() throws Exception {
+    super.setUp();
+    path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+  }
 
   @Override protected int createServer(Configuration conf) throws Exception {
     if (servletServer == null) {
@@ -65,7 +73,7 @@ public class HMSServletTest extends HMSTestBase {
       if (servletServer == null || !servletServer.isStarted()) {
         Assert.fail("http server did not start");
       }
-      sport = HiveMetaStore.getPropertyServletPort();
+      sport = servletServer.getURI().getPort();
     }
     return sport;
   }
@@ -82,9 +90,11 @@ public class HMSServletTest extends HMSTestBase {
     }
   }
 
+
   @Override
   protected PropertyClient createClient(Configuration conf, int sport) throws Exception {
-    URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+    String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
+    URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
     String jwt = generateJWT();
     return new JSonClient(jwt, url);
   }
@@ -144,7 +154,7 @@ public Map getProperties(List selection) {
 
   @Test
   public void testServletEchoA() throws Exception {
-    URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+    URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
     Map json = Collections.singletonMap("method", "echo");
     String jwt = generateJWT();
     // succeed
@@ -177,7 +187,7 @@ public void testProperties0() throws Exception {
         .setUserInfo("hive")
         .setHost("localhost")
         .setPort(sport)
-        .setPath("/" + CLI + "/" + NS)
+        .setPath("/" + path + "/" + NS)
         .setParameters(nvp)
         .build();
     HttpGet get = new HttpGet(uri);
@@ -293,7 +303,7 @@ public static Object clientCall(String jwt, URL url, String method, Object arg)
    * @throws Exception
    */
   private HttpPost createPost(String jwt, String msgBody) {
-    HttpPost method = new HttpPost("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+    HttpPost method = new HttpPost("http://hive@localhost:" + sport + "/" + path + "/" + NS);
     method.addHeader("Authorization", "Bearer " + jwt);
     method.addHeader("Content-Type", "application/json");
     method.addHeader("Accept", "application/json");
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
index db15d52e12d4..1a096e38f30c 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
@@ -18,7 +18,18 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import javax.servlet.http.HttpServletResponse;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -29,18 +40,9 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.HttpClients;
+import org.junit.experimental.categories.Category;
 
-import javax.servlet.http.HttpServletResponse;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
+@Category(MetastoreUnitTest.class)
 public class HMSServletTest1 extends HMSServletTest {
   @Override
   public void tearDown() throws Exception {
@@ -52,7 +54,7 @@ public void tearDown() throws Exception {
 
   @Override
   protected PropertyClient createClient(Configuration conf, int sport) throws Exception {
-    URL url = new URL("http://hive@localhost:" + sport + "/" + CLI + "/" + NS);
+    URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
     String jwt = generateJWT();
     return new JSonHttpClient(jwt, url.toString());
   }
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
index 88f55f82ac23..1cf4b3e4e26e 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
@@ -16,22 +16,23 @@
  */
 package org.apache.hadoop.hive.metastore.properties;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import java.nio.file.Files;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
 import org.junit.Assert;
-
-import java.nio.file.Files;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import org.junit.experimental.categories.Category;
 
 /**
  * Test using the servlet server created by the MetaStore and
  * the client based on Apache HttpClient.
  */
+@Category(MetastoreUnitTest.class)
 public class HMSServletTest1A extends HMSServletTest1 {
   protected int thriftPort;
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
index 22f245330b55..41a2ba06233d 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
@@ -16,20 +16,22 @@
  */
 package org.apache.hadoop.hive.metastore.properties;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import java.nio.file.Files;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.HiveMetaStore;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
 import org.junit.Assert;
-import java.nio.file.Files;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
+import org.junit.experimental.categories.Category;
 
 /**
  * Test using the servlet server created by the MetaStore.
  */
+@Category(MetastoreUnitTest.class)
 public class HMSServletTestA extends HMSServletTest {
   protected int thriftPort;
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
index 4023076c04da..60d239bcbcbf 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hive.metastore.properties;
 
+import static com.github.tomakehurst.wiremock.client.WireMock.get;
+import static com.github.tomakehurst.wiremock.client.WireMock.ok;
 import com.github.tomakehurst.wiremock.junit.WireMockRule;
 import com.nimbusds.jose.JWSAlgorithm;
 import com.nimbusds.jose.JWSHeader;
@@ -25,25 +27,6 @@
 import com.nimbusds.jose.jwk.RSAKey;
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.jexl3.JxltEngine;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
-import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.TestObjectStore;
-
-import static com.github.tomakehurst.wiremock.client.WireMock.get;
-import static com.github.tomakehurst.wiremock.client.WireMock.ok;
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpStatus;
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpType;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
@@ -58,16 +41,30 @@
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.JEXL;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.jexl3.JxltEngine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.TestObjectStore;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_OPERATION;
 import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_STATUS;
+import org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpStatus;
+import org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MaintenanceOpType;
+import static org.apache.hadoop.hive.metastore.properties.PropertyManager.JEXL;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.BOOLEAN;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.JSON;
 import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public abstract class HMSTestBase {
   protected static final String baseDir = System.getProperty("basedir");
@@ -87,12 +84,12 @@ public abstract class HMSTestBase {
   /**
    * Abstract the property client access on a given namespace.
    */
-  interface PropertyClient {
+  protected interface PropertyClient {
     boolean setProperties(Map properties);
     Map> getProperties(String mapPrefix, String mapPredicate, String... selection) throws IOException;
   }
 
-  interface HttpPropertyClient extends PropertyClient {
+  protected interface HttpPropertyClient extends PropertyClient {
     default Map getProperties(List selection) throws IOException {
       throw new UnsupportedOperationException("not implemented in " + this.getClass());
     }
@@ -202,6 +199,8 @@ protected void stopServer(int port) throws Exception {
 
   /**
    * Creates a client.
+   * @param conf the configuration
+   * @param port the servlet port
    * @return the client instance
    * @throws Exception
    */
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
index 33354ad17b54..b7fa65d6d771 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSThriftTest.java
@@ -17,17 +17,19 @@
  */
 package org.apache.hadoop.hive.metastore.properties;
 
+import java.io.IOException;
+import java.util.Map;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
 import org.apache.thrift.TException;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 
-import java.io.IOException;
-import java.util.Map;
-
+@Category(MetastoreUnitTest.class)
 public class HMSThriftTest extends HMSTestBase {
   /**
    * A Thrift based property client.
@@ -67,16 +69,11 @@ public Map> getProperties(String mapPrefix, String m
     MetaStoreTestUtils.close(port);
   }
 
-  /**
-   * Creates a client.
-   * @return the client instance
-   * @throws Exception
-   */
   @Override protected PropertyClient createClient(Configuration conf, int port) throws Exception {
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "http://localhost:" + port);
     MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI, false);
-    HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
-    return new ThriftPropertyClient(NS, client);
+    HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(conf);
+    return new ThriftPropertyClient(NS, hiveClient);
   }
 
   @Test
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
index 50ab770aaabf..1ef1c2119194 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/PropertyStoreTest.java
@@ -18,12 +18,14 @@
 package org.apache.hadoop.hive.metastore.properties;
 
 import com.google.common.base.Supplier;
+import java.nio.charset.StandardCharsets;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.HMSHandler;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
 import org.apache.hadoop.hive.metastore.ObjectStore;
 import org.apache.hadoop.hive.metastore.TestObjectStore;
 import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
 import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties;
@@ -31,11 +33,11 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.nio.charset.StandardCharsets;
-
+@Category(MetastoreUnitTest.class)
 public class PropertyStoreTest {
   private ObjectStore objectStore = null;
   private Configuration conf;

From 447f48d5b58575aacb5adfa8d03d4869595650ac Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Mon, 24 Feb 2025 08:36:18 +0100
Subject: [PATCH 33/40] HIVE-28059 : adding REST catalog to packaging;

---
 packaging/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/packaging/pom.xml b/packaging/pom.xml
index cfc37f087f90..f7c6ac7aee96 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -424,6 +424,11 @@
       hive-webhcat-java-client
       ${project.version}
     
+    
+      org.apache.hive
+      hive-standalone-metastore-rest-catalog
+      ${project.version}
+    
     
       org.apache.hadoop
       hadoop-hdfs-client

From b99d91ae76706ccf302d329d69b3f0b7f145b579 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Wed, 26 Feb 2025 21:06:36 +0100
Subject: [PATCH 34/40] HIVE-28059 : addressing easiest pr comments;

---
 .../metastore-rest-catalog/pom.xml            |  14 +-
 .../iceberg/rest/HMSCatalogAdapter.java       |  16 +-
 .../iceberg/rest/HMSCatalogFactory.java       |   2 +-
 .../org/apache/iceberg/rest/HMSTestBase.java  |  15 +-
 .../hadoop/hive/metastore/HiveMetaStore.java  |   4 +-
 .../hive/metastore/PropertyServlet.java       |  97 ++++----
 .../hive/metastore/ServletSecurity.java       |  10 +-
 .../hive/metastore/ServletServerBuilder.java  | 220 +++++++++---------
 .../metastore/properties/HMSTestBase.java     |  24 +-
 9 files changed, 195 insertions(+), 207 deletions(-)

diff --git a/standalone-metastore/metastore-rest-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml
index 43871e6f456b..7b09557ef878 100644
--- a/standalone-metastore/metastore-rest-catalog/pom.xml
+++ b/standalone-metastore/metastore-rest-catalog/pom.xml
@@ -17,8 +17,8 @@
     4.1.0-SNAPSHOT
   
   4.0.0
-  hive-standalone-metastore-rest-catalog
-  Hive Standalone Metastore REST Catalog
+  hive-metastore-rest-catalog
+  Hive Metastore REST Catalog
   
     ..
     8
@@ -39,16 +39,6 @@
       hive-standalone-metastore-common
       ${hive.version}
     
-    
-      org.apache.hive
-      hive-iceberg-shading
-      ${hive.version}
-    
-    
-      org.apache.hive
-      hive-iceberg-handler
-      ${hive.version}
-    
     
       org.apache.hive
       hive-iceberg-catalog
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
index 063dbb59e311..cc2738008f75 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
@@ -110,6 +110,8 @@ public class HMSCatalogAdapter implements RESTClient {
   private static final String CLIENT_ID = "client_id";
   private static final String ACTOR_TOKEN = "actor_token";
   private static final String SUBJECT_TOKEN = "subject_token";
+  private static final String VIEWS_PATH = "v1/namespaces/{namespace}/views/{name}";
+  private static final String TABLES_PATH = "v1/namespaces/{namespace}/tables/{table}";
 
   private final Catalog catalog;
   private final SupportsNamespaces asNamespaceCatalog;
@@ -150,13 +152,13 @@ enum Route {
             null, ListTablesResponse.class),
     CREATE_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/tables",
             CreateTableRequest.class, LoadTableResponse.class),
-    LOAD_TABLE(HTTPMethod.GET, "v1/namespaces/{namespace}/tables/{table}",
+    LOAD_TABLE(HTTPMethod.GET, TABLES_PATH,
             null, LoadTableResponse.class),
     REGISTER_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/register",
             RegisterTableRequest.class, LoadTableResponse.class),
-    UPDATE_TABLE(HTTPMethod.POST, "v1/namespaces/{namespace}/tables/{table}",
+    UPDATE_TABLE(HTTPMethod.POST, TABLES_PATH,
             UpdateTableRequest.class, LoadTableResponse.class),
-    DROP_TABLE(HTTPMethod.DELETE, "v1/namespaces/{namespace}/tables/{table}"),
+    DROP_TABLE(HTTPMethod.DELETE, TABLES_PATH),
     RENAME_TABLE(HTTPMethod.POST, "v1/tables/rename",
             RenameTableRequest.class, null),
     REPORT_METRICS(HTTPMethod.POST, "v1/namespaces/{namespace}/tables/{table}/metrics",
@@ -165,15 +167,15 @@ enum Route {
             CommitTransactionRequest.class, null),
     LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views",
             null, ListTablesResponse.class),
-    LOAD_VIEW(HTTPMethod.GET, "v1/namespaces/{namespace}/views/{name}",
+    LOAD_VIEW(HTTPMethod.GET, VIEWS_PATH,
             null, LoadViewResponse.class),
     CREATE_VIEW(HTTPMethod.POST, "v1/namespaces/{namespace}/views",
             CreateViewRequest.class, LoadViewResponse.class),
-    UPDATE_VIEW(HTTPMethod.POST, "v1/namespaces/{namespace}/views/{name}",
+    UPDATE_VIEW(HTTPMethod.POST, VIEWS_PATH,
             UpdateTableRequest.class, LoadViewResponse.class),
     RENAME_VIEW(HTTPMethod.POST, "v1/views/rename",
             RenameTableRequest.class, null),
-    DROP_VIEW(HTTPMethod.DELETE, "v1/namespaces/{namespace}/views/{name}");
+    DROP_VIEW(HTTPMethod.DELETE, VIEWS_PATH);
 
     private final HTTPMethod method;
     private final int requiredLength;
@@ -525,7 +527,7 @@ private static void commitTransaction(Catalog catalog, CommitTransactionRequest
     transactions.forEach(Transaction::commitTransaction);
   }
   
-  @SuppressWarnings("MethodLength")
+  @SuppressWarnings({"MethodLength", "unchecked"})
   private  T handleRequest(
       Route route, Map vars, Object body) {
     // update HMS catalog route counter metric
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
index 6c214f2a13f6..1d6d575ac6f2 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
@@ -94,7 +94,7 @@ protected Catalog createCatalog() {
     final String catalogName = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.CATALOG_DEFAULT);
     catalog.initialize(catalogName, properties);
     long expiry = MetastoreConf.getLongVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_CACHE_EXPIRY);
-    return expiry > 0 ? new HMSCachingCatalog(catalog, expiry) : catalog;
+    return expiry > 0 ? new HMSCachingCatalog<>(catalog, expiry) : catalog;
   }
 
   protected HttpServlet createServlet(Catalog catalog) throws IOException {
diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
index 8fbaae049706..d6b48a84dec4 100644
--- a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
+++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/HMSTestBase.java
@@ -369,15 +369,14 @@ public static Object clientCall(String jwt, URL url, String method, boolean json
       con.setDoInput(true);
       if (arg != null) {
         con.setDoOutput(true);
-        DataOutputStream wr = new DataOutputStream(con.getOutputStream());
-        if (json) {
-          String outjson = serialize(arg);
-          wr.writeBytes(outjson);
-        } else {
-          wr.writeBytes(arg.toString());
+        try (DataOutputStream wr = new DataOutputStream(con.getOutputStream())) {
+          if (json) {
+            wr.writeBytes(serialize(arg));
+          } else {
+            wr.writeBytes(arg.toString());
+          }
+          wr.flush();
         }
-        wr.flush();
-        wr.close();
       }
       // perform http method
       return httpResponse(con);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index adb9c78fb698..503f15702c42 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -769,10 +769,10 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
       }
     }
     // optionally create and start the property and Iceberg REST server
-    ServletServerBuilder.Descriptor properties = PropertyServlet.createServlet(conf);
-    ServletServerBuilder.Descriptor catalog = createIcebergServlet(conf);
     ServletServerBuilder builder = new ServletServerBuilder(conf);
+    ServletServerBuilder.Descriptor properties = PropertyServlet.createServlet(conf);
     builder.addServlet(properties);
+    ServletServerBuilder.Descriptor catalog = createIcebergServlet(conf);
     builder.addServlet(catalog);
     servletServer = builder.start(LOG);
     if (servletServer != null) {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index c5893ca4f1f8..fab395df577a 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -60,16 +60,12 @@ public class PropertyServlet extends HttpServlet {
   /** The configuration. */
   private final Configuration configuration;
 
-  static boolean isAuthJwt(Configuration configuration) {
-    String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH);
-    return "jwt".equalsIgnoreCase(auth);
-  }
-
   PropertyServlet(Configuration configuration) {
     this.configuration = configuration;
   }
 
-  @Override public String getServletName() {
+  @Override
+  public String getServletName() {
     return "HMS property";
   }
   
@@ -166,44 +162,12 @@ protected void doPost(HttpServletRequest request,
             switch (method) {
               // fetch a list of qualified keys by name
               case "fetchProperties": {
-                // one or many keys
-                Object jsonKeys = call.get("keys");
-                if (jsonKeys == null) {
-                  throw new IllegalArgumentException("null keys");
-                }
-                Iterable keys = jsonKeys instanceof List
-                    ? (List) jsonKeys
-                    : Collections.singletonList(jsonKeys);
-                Map properties = new TreeMap<>();
-                for (Object okey : keys) {
-                  String key = okey.toString();
-                  String value = mgr.exportPropertyValue(key);
-                  if (value != null) {
-                    properties.put(key, value);
-                  }
-                }
-                reactions.add(properties);
+                fetchProperties( mgr, call, reactions);
                 break;
               }
               // select a list of qualified keys by prefix/predicate/selection
               case "selectProperties": {
-                String prefix = (String) call.get("prefix");
-                if (prefix == null) {
-                  throw new IllegalArgumentException("null prefix");
-                }
-                String predicate = (String) call.get("predicate");
-                // selection may be null, a sole property or a list
-                Object selection = call.get("selection");
-                @SuppressWarnings("unchecked") List project =
-                    selection == null
-                        ? null
-                        : selection instanceof List
-                        ? (List) selection
-                        : Collections.singletonList(selection.toString());
-                Map selected = mgr.selectProperties(prefix, predicate, project);
-                Map> returned = new TreeMap<>();
-                selected.forEach((k, v) -> returned.put(k, v.export(project == null)));
-                reactions.add(returned);
+                selectProperties(mgr, call, reactions);
                 break;
               }
               case "script": {
@@ -237,18 +201,45 @@ protected void doPost(HttpServletRequest request,
     }
   }
 
-//  A way to import values using files sent over http
-//  private void importProperties(HttpServletRequest request) throws ServletException, IOException {
-//    List fileParts = request.getParts().stream()
-//        .filter(part -> "files".equals(part.getName()) && part.getSize() > 0)
-//        .collect(Collectors.toList()); // Retrieves 
-//
-//    for (Part filePart : fileParts) {
-//      String fileName = Paths.get(filePart.getSubmittedFileName()).getFileName().toString(); // MSIE fix.
-//      InputStream fileContent = filePart.getInputStream();
-//      // ... (do your job here)
-//    }
-//  }
+  private static void fetchProperties(PropertyManager mgr, Map call, List reactions) {
+    // one or many keys
+    Object jsonKeys = call.get("keys");
+    if (jsonKeys == null) {
+      throw new IllegalArgumentException("null keys");
+    }
+    Iterable keys = jsonKeys instanceof List
+        ? (List) jsonKeys
+        : Collections.singletonList(jsonKeys);
+    Map properties = new TreeMap<>();
+    for (Object okey : keys) {
+      String key = okey.toString();
+      String value = mgr.exportPropertyValue(key);
+      if (value != null) {
+        properties.put(key, value);
+      }
+    }
+    reactions.add(properties);
+  }
+
+  private static void selectProperties(PropertyManager mgr, Map call, List reactions) {
+    String prefix = (String) call.get("prefix");
+    if (prefix == null) {
+      throw new IllegalArgumentException("null prefix");
+    }
+    String predicate = (String) call.get("predicate");
+    // selection may be null, a sole property or a list
+    Object selection = call.get("selection");
+    @SuppressWarnings("unchecked") List project =
+        selection == null
+            ? null
+            : selection instanceof List
+            ? (List) selection
+            : Collections.singletonList(selection.toString());
+    Map selected = mgr.selectProperties(prefix, predicate, project);
+    Map> returned = new TreeMap<>();
+    selected.forEach((k, v) -> returned.put(k, v.export(project == null)));
+    reactions.add(returned);
+  }
 
   @Override
   protected void doPut(HttpServletRequest request,
@@ -322,7 +313,7 @@ public static ServletServerBuilder.Descriptor createServlet(Configuration config
       int port = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PORT);
       String path = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
       if (port >= 0 && path != null && !path.isEmpty()) {
-        ServletSecurity security = new ServletSecurity(configuration, PropertyServlet.isAuthJwt(configuration));
+        ServletSecurity security = new ServletSecurity(configuration);
         HttpServlet servlet = security.proxy(new PropertyServlet(configuration));
         return new ServletServerBuilder.Descriptor(port, path, servlet) {
           @Override public String toString() {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
index 71c9b20fb80e..197c56e057c2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
@@ -86,8 +86,7 @@ public class ServletSecurity {
   private JWTValidator jwtValidator = null;
 
   public ServletSecurity(Configuration conf) {
-    this(conf, MetastoreConf.getVar(conf,
-            MetastoreConf.ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt"));
+    this(conf, isAuthJwt(conf));
   }
 
   public ServletSecurity(Configuration conf, boolean jwt) {
@@ -96,6 +95,11 @@ public ServletSecurity(Configuration conf, boolean jwt) {
     this.jwtAuthEnabled = jwt;
   }
 
+  public static boolean isAuthJwt(Configuration configuration) {
+    String auth = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.PROPERTIES_SERVLET_AUTH);
+    return "jwt".equalsIgnoreCase(auth);
+  }
+
   /**
    * Should be called in Servlet.init()
    * @throws ServletException if the jwt validator creation throws an exception
@@ -284,7 +288,7 @@ public static SslContextFactory createSslContextFactory(Configuration conf) thro
     }
     final String keyStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH).trim();
     if (keyStorePath.isEmpty()) {
-      throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_KEYSTORE_PATH.toString()
+      throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_KEYSTORE_PATH
           + " Not configured for SSL connection");
     }
     final String keyStorePassword =
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index 46baad72fdc1..9f6ac2b76661 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -18,17 +18,6 @@
  */
 package org.apache.hadoop.hive.metastore;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import javax.servlet.Servlet;
-import javax.servlet.http.HttpServlet;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.eclipse.jetty.server.HttpConfiguration;
@@ -44,19 +33,31 @@
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.slf4j.Logger;
 
+import javax.servlet.Servlet;
+import javax.servlet.http.HttpServlet;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
 /**
  * Helper class to ease creation of embedded Jetty serving servlets on
  * different ports.
  */
 public class ServletServerBuilder {
-  /**
-   * Keeping track of descriptors.
-   */
-  private final Map descriptorsMap = new IdentityHashMap<>();
   /**
    * The configuration instance.
    */
   protected final Configuration configuration;
+  /**
+   * Keeping track of descriptors.
+   */
+  private final Map descriptorsMap = new IdentityHashMap<>();
 
   /**
    * Creates a builder instance.
@@ -67,52 +68,49 @@ protected ServletServerBuilder(Configuration conf) {
     this.configuration = conf;
   }
 
-  public Configuration getConfiguration() {
-    return configuration;
-  }
-
   /**
-   * A descriptor of a servlet.
-   * 
After server is started, unspecified port will be updated to reflect
-   * what the system allocated.

+   * Creates a builder.
+   *
+   * @param conf     the configuration
+   * @param describe the functions to call that create servlet descriptors
+   * @return the builder or null if no descriptors
    */
-  public static class Descriptor {
-    private int port;
-    private final String path;
-    private final HttpServlet servlet;
-
-    /**
-     * Create a servlet descriptor.
-     * @param port the servlet port (or 0 if system allocated)
-     * @param path the servlet path
-     * @param servlet the servlet instance
-     */
-    public Descriptor(int port, String path, HttpServlet servlet) {
-      this.port = port;
-      this.path = path;
-      this.servlet = servlet;
-    }
-    
-    @Override
-    public String toString() {
-      return servlet.getClass().getSimpleName() + ":" + port+ "/"+ path ;
-    }
-
-    public int getPort() {
-      return port;
-    }
-
-    public String getPath() {
-      return path;
+  @SafeVarargs
+  public static ServletServerBuilder builder(Configuration conf,
+                                             Function... describe) {
+    List descriptors = new ArrayList();
+    Arrays.asList(describe).forEach(functor -> {
+      ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
+      if (descriptor != null) {
+        descriptors.add(descriptor);
+      }
+    });
+    if (!descriptors.isEmpty()) {
+      ServletServerBuilder builder = new ServletServerBuilder(conf);
+      descriptors.forEach(d -> builder.addServlet(d));
+      return builder;
     }
+    return null;
+  }
 
-    public HttpServlet getServlet() {
-      return servlet;
-    }
+  /**
+   * Helper for generic use case.
+   *
+   * @param logger   the logger
+   * @param conf     the configuration
+   * @param describe the functions to create descriptors
+   * @return a server instance
+   */
+  @SafeVarargs
+  public static Server startServer(
+          Logger logger,
+          Configuration conf,
+          Function... describe) {
+    return builder(conf, describe).start(logger);
+  }
 
-    void setPort(int port) {
-      this.port = port;
-    }
+  public Configuration getConfiguration() {
+    return configuration;
   }
 
   /**
@@ -120,13 +118,14 @@ void setPort(int port) {
    * 
The servlet port can be shared between servlets; if 0, the system will provide
    * a port. If the port is < 0, the system will provide a port dedicated (ie non-shared)
    * to the servlet.

-   * @param port the servlet port
-   * @param path the servlet path
+   *
+   * @param port    the servlet port
+   * @param path    the servlet path
    * @param servlet a servlet instance
    * @return a descriptor
    */
-  public Descriptor addServlet(int port, String path, HttpServlet servlet){
-    Descriptor descriptor  = new Descriptor(port, path, servlet);
+  public Descriptor addServlet(int port, String path, HttpServlet servlet) {
+    Descriptor descriptor = new Descriptor(port, path, servlet);
     return addServlet(descriptor);
   }
 
@@ -163,9 +162,9 @@ protected Server createServer() throws IOException {
   /**
    * Creates a server instance and a connector on a given port.
    *
-   * @param server the server instance
+   * @param server            the server instance
    * @param sslContextFactory the ssl factory
-   * @param port the port
+   * @param port              the port
    * @return the server connector listening to the port
    * @throws IOException if server creation fails
    */
@@ -185,8 +184,9 @@ protected ServerConnector createConnector(Server server, SslContextFactory sslCo
 
   /**
    * Adds a servlet to its intended servlet context context.
+   *
    * @param handlersMap the map of port to handlers
-   * @param descriptor the servlet descriptor
+   * @param descriptor  the servlet descriptor
    * @throws IOException
    */
   protected void addServlet(Map handlersMap, Descriptor descriptor) throws IOException {
@@ -216,7 +216,7 @@ protected void addServlet(Map handlersMap, Descr
   public Server startServer() throws Exception {
     // add all servlets
     Map handlersMap = new HashMap<>();
-    for(Descriptor descriptor : descriptorsMap.values()) {
+    for (Descriptor descriptor : descriptorsMap.values()) {
       addServlet(handlersMap, descriptor);
     }
     final int size = handlersMap.size();
@@ -232,15 +232,15 @@ public Server startServer() throws Exception {
     for (int c = 0; it.hasNext(); ++c) {
       Map.Entry entry = it.next();
       int key = entry.getKey();
-      int port = key < 0? 0 : key;
+      int port = Math.max(key, 0);
       ServerConnector connector = createConnector(server, sslFactory, port);
       connectors[c] = connector;
       ServletContextHandler handler = entry.getValue();
       handlers[c] = handler;
       // make each servlet context be served only by its dedicated connector
-      String host = "hms" + Integer.toString(c);
+      String host = "hms" + c;
       connector.setName(host);
-      handler.setVirtualHosts(new String[]{"@"+host});
+      handler.setVirtualHosts(new String[]{"@" + host});
     }
     // hook the connectors and the handlers
     server.setConnectors(connectors);
@@ -254,7 +254,7 @@ public Server startServer() throws Exception {
       int port = connectors[i].getLocalPort();
       ServletContextHandler handler = handlers[i];
       ServletHolder[] holders = handler.getServletHandler().getServlets();
-      for(ServletHolder holder : holders) {
+      for (ServletHolder holder : holders) {
         Servlet servlet = holder.getServletInstance();
         if (servlet != null) {
           Descriptor descriptor = descriptorsMap.get(servlet);
@@ -267,32 +267,9 @@ public Server startServer() throws Exception {
     return server;
   }
 
-  /**
-   * Creates a builder.
-   * @param conf the configuration
-   * @param describe the functions to call that create servlet descriptors
-   * @return the builder or null if no descriptors
-   */
-  @SafeVarargs
-  public static ServletServerBuilder builder(Configuration conf,
-          Function... describe) {
-    List descriptors = new ArrayList();
-    Arrays.asList(describe).forEach(functor -> {
-      ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
-      if (descriptor != null) {
-        descriptors.add(descriptor);
-      }
-    });
-    if (!descriptors.isEmpty()) {
-      ServletServerBuilder builder = new ServletServerBuilder(conf);
-      descriptors.forEach(d -> builder.addServlet(d));
-      return builder;
-    }
-    return null;
-  }
-
   /**
    * Creates and starts the server.
+   *
    * @param logger a logger to output info
    * @return the server instance (or null if error)
    */
@@ -312,28 +289,55 @@ public Server start(Logger logger) {
         }
       }
       return server;
-    } catch (Exception exception) {
-      logger.error("Unable to start servlet server", exception);
-      return null;
     } catch (Throwable throwable) {
       logger.error("Unable to start servlet server", throwable);
       return null;
     }
   }
-  
-   /**
-   * Helper for generic use case.
-   * @param logger the logger
-   * @param conf the configuration
-   * @param describe the functions to create descriptors
-   * @return a server instance
+
+  /**
+   * A descriptor of a servlet.
+   * 
After server is started, unspecified port will be updated to reflect
+   * what the system allocated.

    */
-  @SafeVarargs
-  public static Server startServer(
-          Logger logger,
-          Configuration conf,
-          Function... describe) {
-    return builder(conf, describe).start(logger);
+  public static class Descriptor {
+    private final String path;
+    private final HttpServlet servlet;
+    private int port;
+
+    /**
+     * Create a servlet descriptor.
+     *
+     * @param port    the servlet port (or 0 if system allocated)
+     * @param path    the servlet path
+     * @param servlet the servlet instance
+     */
+    public Descriptor(int port, String path, HttpServlet servlet) {
+      this.port = port;
+      this.path = path;
+      this.servlet = servlet;
+    }
+
+    @Override
+    public String toString() {
+      return servlet.getClass().getSimpleName() + ":" + port + "/" + path;
+    }
+
+    public int getPort() {
+      return port;
+    }
+
+    void setPort(int port) {
+      this.port = port;
+    }
+
+    public String getPath() {
+      return path;
+    }
+
+    public HttpServlet getServlet() {
+      return servlet;
+    }
   }
 }
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
index 60d239bcbcbf..cded98bd07cc 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSTestBase.java
@@ -46,7 +46,6 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
 import org.apache.hadoop.hive.metastore.ObjectStore;
-import org.apache.hadoop.hive.metastore.TestObjectStore;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_OPERATION;
 import static org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_STATUS;
@@ -97,7 +96,7 @@ default Map getProperties(List selection) throws IOExcep
 
   protected Configuration conf = null;
 
-  protected static final Logger LOG = LoggerFactory.getLogger(TestObjectStore.class.getName());
+  protected static final Logger LOG = LoggerFactory.getLogger(HMSTestBase.class);
   static Random RND = new Random(20230424);
   protected String NS;// = "hms" + RND.nextInt(100);
   protected PropertyClient client;
@@ -180,9 +179,9 @@ private static String generateJWT(String user, Path keyFile, long lifeTimeMillis
 
   /**
    * Creates and starts the server.
-   * @param conf
+   * @param conf the configuration
    * @return the server port
-   * @throws Exception
+   * @throws Exception if creation fails
    */
   protected int createServer(Configuration conf) throws Exception {
     return 0;
@@ -191,7 +190,7 @@ protected int createServer(Configuration conf) throws Exception {
   /**
    * Stops the server.
    * @param port the server port
-   * @throws Exception
+   * @throws Exception if stopping the server fails
    */
   protected void stopServer(int port) throws Exception {
     // nothing
@@ -202,12 +201,12 @@ protected void stopServer(int port) throws Exception {
    * @param conf the configuration
    * @param port the servlet port
    * @return the client instance
-   * @throws Exception
+   * @throws Exception if client creation fails
    */
   protected abstract PropertyClient createClient(Configuration conf, int port) throws Exception;
 
 
-  public void runOtherProperties0(PropertyClient client) throws Exception {
+  void runOtherProperties0(PropertyClient client) throws Exception {
     Map ptyMap = createProperties0();
     boolean commit = client.setProperties(ptyMap);
     Assert.assertTrue(commit);
@@ -234,7 +233,7 @@ static Map createProperties0() {
     try {
       String json = IOUtils.toString(
           HMSDirectTest.class.getResourceAsStream("payload.json"),
-          "UTF-8"
+              StandardCharsets.UTF_8
       );
       JxltEngine JXLT = JEXL.createJxltEngine();
       JxltEngine.Template jsonjexl = JXLT.createTemplate(json, "table", "delta", "g");
@@ -263,7 +262,7 @@ static Map createProperties0() {
     }
   }
 
-  public void runOtherProperties1(PropertyClient client) throws Exception {
+  void runOtherProperties1(PropertyClient client) throws Exception {
     Map ptyMap = createProperties1();
     boolean commit = client.setProperties(ptyMap);
     Assert.assertTrue(commit);
@@ -276,12 +275,11 @@ public void runOtherProperties1(PropertyClient client) throws Exception {
       HttpPropertyClient httpClient = (HttpPropertyClient) client;
       // get fillfactors using getProperties, create args array from previous result
       List keys = new ArrayList<>(maps.keySet());
-      for (int k = 0; k < keys.size(); ++k) {
-        keys.set(k, keys.get(k) + ".fillFactor");
-      }
+      keys.replaceAll(s -> s + ".fillFactor");
       Object values = httpClient.getProperties(keys);
       Assert.assertTrue(values instanceof Map);
-      Map getm = (Map) values;
+      @SuppressWarnings("unchecked")
+      final Map getm = (Map) values;
       for (Map.Entry> entry : maps.entrySet()) {
         Map map0v = entry.getValue();
         Assert.assertEquals(map0v.get("fillFactor"), getm.get(entry.getKey() + ".fillFactor"));

From 70b8c7512e2448bee33d358c362b934cd63feb37 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Thu, 27 Feb 2025 08:06:07 +0100
Subject: [PATCH 35/40] HIVE-28059 : fix packaging;

---
 packaging/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packaging/pom.xml b/packaging/pom.xml
index f7c6ac7aee96..839356a0d977 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -426,7 +426,7 @@
     
     
       org.apache.hive
-      hive-standalone-metastore-rest-catalog
+      hive-metastore-rest-catalog
       ${project.version}
     
     

From 7f75025ed01fbbf3b0bf812f5f300d3b6a25f123 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Sun, 2 Mar 2025 11:51:01 +0100
Subject: [PATCH 36/40] HIVE-28059 : fix jar naming;

---
 packaging/pom.xml                             |  2 +-
 .../metastore-rest-catalog/pom.xml            | 24 +++----------------
 2 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/packaging/pom.xml b/packaging/pom.xml
index 839356a0d977..f7c6ac7aee96 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -426,7 +426,7 @@
     
     
       org.apache.hive
-      hive-metastore-rest-catalog
+      hive-standalone-metastore-rest-catalog
       ${project.version}
     
     
diff --git a/standalone-metastore/metastore-rest-catalog/pom.xml b/standalone-metastore/metastore-rest-catalog/pom.xml
index 7b09557ef878..3cb1cb9ccee5 100644
--- a/standalone-metastore/metastore-rest-catalog/pom.xml
+++ b/standalone-metastore/metastore-rest-catalog/pom.xml
@@ -17,7 +17,7 @@
     4.1.0-SNAPSHOT
   
   4.0.0
-  hive-metastore-rest-catalog
+  hive-standalone-metastore-rest-catalog
   Hive Metastore REST Catalog
   
     ..
@@ -44,26 +44,7 @@
       hive-iceberg-catalog
       ${hive.version}
     
-    
-      org.apache.iceberg
-      iceberg-bundled-guava
-      ${iceberg.version}
-    
-    
-    
+    
     
       org.apache.hive
       hive-standalone-metastore-common
@@ -82,6 +63,7 @@
       org.apache.httpcomponents.core5
       httpcore5
       5.2
+      test
     
     
       junit

From 4e09a68af41b134b4cbe2e03c3c2bf4d3306b296 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Sun, 2 Mar 2025 22:08:55 +0100
Subject: [PATCH 37/40] HIVE-28059 : fixing nits & javadoc;

---
 .../iceberg/rest/HMSCatalogFactory.java       | 33 ++++++++++++++++---
 .../iceberg/rest/HMSCatalogServlet.java       |  7 +++-
 .../apache/iceberg/rest/TestHMSCatalog.java   | 14 ++++----
 .../hadoop/hive/metastore/HiveMetaStore.java  |  6 ++--
 .../hive/metastore/PropertyServlet.java       |  8 ++---
 .../hive/metastore/ServletSecurity.java       |  8 +++--
 .../hive/metastore/ServletServerBuilder.java  | 32 ++++++++----------
 .../metastore/properties/HMSServletTest.java  | 20 ++++++-----
 .../metastore/properties/HMSServletTest1.java |  3 ++
 .../properties/HMSServletTest1A.java          |  4 +--
 .../metastore/properties/HMSServletTestA.java |  4 +--
 11 files changed, 86 insertions(+), 53 deletions(-)

diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
index 1d6d575ac6f2..76327d1fa80f 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
@@ -35,10 +35,13 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Iceberg Catalog server creator.
+ * Catalog & servlet factory.
  */
 public class HMSCatalogFactory {
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogFactory.class);
+  /**
+   * Convenience soft reference to last catalog.
+   */
   protected static final AtomicReference> catalogRef = new AtomicReference<>();
 
   public static Catalog getLastCatalog() {
@@ -55,6 +58,13 @@ protected static void setLastCatalog(Catalog catalog) {
   protected final String path;
   protected Catalog catalog;
 
+  /**
+   * Factory constructor.
+   * 
Called by the static method {@link HMSCatalogFactory#createServlet(Configuration)} that is
+   * declared in configuration and found through introspection.

+   * @param conf the configuration
+   * @param catalog the catalog
+   */
   protected HMSCatalogFactory(Configuration conf, Catalog catalog) {
     port = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PORT);
     path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.ICEBERG_CATALOG_SERVLET_PATH);
@@ -74,6 +84,10 @@ public Catalog getCatalog() {
     return catalog;
   }
 
+  /**
+   * Creates the catalog instance.
+   * @return the catalog
+   */
   protected Catalog createCatalog() {
     final Map properties = new TreeMap<>();
     MetastoreConf.setVar(configuration, MetastoreConf.ConfVars.THRIFT_URIS, "");
@@ -97,11 +111,21 @@ protected Catalog createCatalog() {
     return expiry > 0 ? new HMSCachingCatalog<>(catalog, expiry) : catalog;
   }
 
-  protected HttpServlet createServlet(Catalog catalog) throws IOException {
+  /**
+   * Creates the REST catalog servlet instance.
+   * @param catalog the Iceberg catalog
+   * @return the servlet
+   */
+  protected HttpServlet createServlet(Catalog catalog) {
     ServletSecurity security = new ServletSecurity(configuration);
     return security.proxy(new HMSCatalogServlet(new HMSCatalogAdapter(catalog)));
   }
 
+  /**
+   * Creates the REST catalog servlet instance.
+   * @return the servlet
+   * @throws IOException if creation fails
+   */
   protected HttpServlet createServlet() throws IOException {
     if (port >= 0 && path != null && !path.isEmpty()) {
       Catalog actualCatalog = catalog;
@@ -113,15 +137,16 @@ protected HttpServlet createServlet() throws IOException {
     }
     return null;
   }
-
   
   /**
    * Factory method to describe Iceberg servlet.
-   * 
This one is looked up through reflection to start from HMS.

+   * 
This method name is found through configuration as {@link MetastoreConf.ConfVars#ICEBERG_CATALOG_SERVLET_FACTORY}
+   * and looked up through reflection to start from HMS.

    *
    * @param configuration the configuration
    * @return the servlet descriptor instance
    */
+  @SuppressWarnings("unused")
   public static ServletServerBuilder.Descriptor createServlet(Configuration configuration) {
     try {
       HMSCatalogFactory hms = new HMSCatalogFactory(configuration, null);
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
index 4b0e6a47080c..e6ec84c99118 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
@@ -58,7 +58,12 @@ public class HMSCatalogServlet extends HttpServlet {
   public HMSCatalogServlet(HMSCatalogAdapter restCatalogAdapter) {
     this.restCatalogAdapter = restCatalogAdapter;
   }
-  
+
+  @Override
+  public String getServletName() {
+    return "HMS Catalog";
+  }
+
   @Override
   protected void service(HttpServletRequest request, HttpServletResponse response) {
     try {
diff --git a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
index 8b5a795e2c2d..7b05602fc123 100644
--- a/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
+++ b/standalone-metastore/metastore-rest-catalog/src/test/java/org/apache/iceberg/rest/TestHMSCatalog.java
@@ -22,7 +22,7 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
@@ -71,8 +71,8 @@ public void testCreateNamespaceHttp() throws Exception {
     Map nsrep = (Map) response;
     List> nslist = (List>) nsrep.get("namespaces");
     Assert.assertEquals(2, nslist.size());
-    Assert.assertTrue((nslist.contains(Arrays.asList("default"))));
-    Assert.assertTrue((nslist.contains(Arrays.asList("hivedb"))));
+    Assert.assertTrue((nslist.contains(Collections.singletonList("default"))));
+    Assert.assertTrue((nslist.contains(Collections.singletonList("hivedb"))));
     // succeed
     response = clientCall(jwt, url, "POST", false, "{ \"namespace\" : [ \""+ns+"\" ], "+
             "\"properties\":{ \"owner\": \"apache\", \"group\" : \"iceberg\" }"
@@ -94,9 +94,7 @@ public void testCreateNamespaceHttp() throws Exception {
 
     // quick check on metrics
     Map counters = reportMetricCounters("list_namespaces", "list_tables");
-    counters.entrySet().forEach(m->{
-      Assert.assertTrue(m.getKey(), m.getValue() > 0);
-    });
+    counters.forEach((key, value) -> Assert.assertTrue(key, value > 0));
   }
   
   private Schema getTestSchema() {
@@ -136,8 +134,8 @@ public void testCreateTableTxnBuilder() throws Exception {
       Assert.assertEquals(200, (int) eval(response, "json -> json.status"));
       List> nslist = (List>) eval(response, "json -> json.namespaces");
       Assert.assertEquals(2, nslist.size());
-      Assert.assertTrue((nslist.contains(Arrays.asList("default"))));
-      Assert.assertTrue((nslist.contains(Arrays.asList("hivedb"))));
+      Assert.assertTrue((nslist.contains(Collections.singletonList("default"))));
+      Assert.assertTrue((nslist.contains(Collections.singletonList("hivedb"))));
 
       // list tables in hivedb
       url = iceUri.resolve("namespaces/" + DB_NAME + "/tables").toURL();
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 503f15702c42..77812cc2cffe 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -768,6 +768,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
         throw e;
       }
     }
+
     // optionally create and start the property and Iceberg REST server
     ServletServerBuilder builder = new ServletServerBuilder(conf);
     ServletServerBuilder.Descriptor properties = PropertyServlet.createServlet(conf);
@@ -783,6 +784,7 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
         catalogServletPort = catalog.getPort();
       }
     }
+
     // main server
     thriftServer.start();
   }
@@ -799,10 +801,10 @@ static ServletServerBuilder.Descriptor createIcebergServlet(Configuration config
       Method iceStart = iceClazz.getMethod("createServlet", Configuration.class);
       return (ServletServerBuilder.Descriptor) iceStart.invoke(null, configuration);
     } catch (ClassNotFoundException xnf) {
-      LOG.warn("unable to start Iceberg REST Catalog server, missing jar?", xnf);
+      LOG.warn("Unable to start Iceberg REST Catalog server, missing jar?", xnf);
       return null;
     } catch (Exception e) {
-      LOG.error("unable to start Iceberg REST Catalog server", e);
+      LOG.error("Unable to start Iceberg REST Catalog server", e);
       return null;
     }
   }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index fab395df577a..ffb1be3058ad 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -66,7 +66,7 @@ public class PropertyServlet extends HttpServlet {
 
   @Override
   public String getServletName() {
-    return "HMS property";
+    return "HMS Property";
   }
   
   private String strError(String msg, Object...args) {
@@ -180,7 +180,7 @@ protected void doPost(HttpServletRequest request,
                 break;
               }
               default: {
-                throw new IllegalArgumentException("bad argument type " + action.getClass());
+                throw new IllegalArgumentException("Bad argument type " + action.getClass());
               }
             }
           }
@@ -317,12 +317,12 @@ public static ServletServerBuilder.Descriptor createServlet(Configuration config
         HttpServlet servlet = security.proxy(new PropertyServlet(configuration));
         return new ServletServerBuilder.Descriptor(port, path, servlet) {
           @Override public String toString() {
-            return "HMS property";
+            return "HMS Property";
           }
         };
       }
     } catch (Exception io) {
-      LOGGER.error("failed to create servlet ", io);
+      LOGGER.error("Failed to create servlet ", io);
     }
     return null;
   }
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
index 197c56e057c2..aee8d37b77b2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
@@ -125,12 +125,14 @@ public class ProxyServlet extends HttpServlet {
       this.delegate = delegate;
     }
 
-    @Override public void init() throws ServletException {
+    @Override
+    public void init() throws ServletException {
       ServletSecurity.this.init();
       delegate.init();
     }
 
-    @Override public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
+    @Override
+    public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
       execute(request, response, delegate::service);
     }
   }
@@ -281,7 +283,7 @@ static void loginServerPrincipal(Configuration conf) throws IOException {
    * @return null if no ssl in config, an instance otherwise
    * @throws IOException if getting password fails
    */
-  public static SslContextFactory createSslContextFactory(Configuration conf) throws IOException {
+  static SslContextFactory createSslContextFactory(Configuration conf) throws IOException {
     final boolean useSsl  = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL);
     if (!useSsl) {
       return null;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index 9f6ac2b76661..066c0c6a31f8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -35,7 +35,6 @@
 
 import javax.servlet.Servlet;
 import javax.servlet.http.HttpServlet;
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -43,6 +42,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Function;
 
 /**
@@ -53,7 +53,7 @@ public class ServletServerBuilder {
   /**
    * The configuration instance.
    */
-  protected final Configuration configuration;
+  private final Configuration configuration;
   /**
    * Keeping track of descriptors.
    */
@@ -64,7 +64,7 @@ public class ServletServerBuilder {
    *
    * @param conf the configuration
    */
-  protected ServletServerBuilder(Configuration conf) {
+  public ServletServerBuilder(Configuration conf) {
     this.configuration = conf;
   }
 
@@ -78,7 +78,7 @@ protected ServletServerBuilder(Configuration conf) {
   @SafeVarargs
   public static ServletServerBuilder builder(Configuration conf,
                                              Function... describe) {
-    List descriptors = new ArrayList();
+    List descriptors = new ArrayList<>();
     Arrays.asList(describe).forEach(functor -> {
       ServletServerBuilder.Descriptor descriptor = functor.apply(conf);
       if (descriptor != null) {
@@ -87,7 +87,7 @@ public static ServletServerBuilder builder(Configuration conf,
     });
     if (!descriptors.isEmpty()) {
       ServletServerBuilder builder = new ServletServerBuilder(conf);
-      descriptors.forEach(d -> builder.addServlet(d));
+      descriptors.forEach(builder::addServlet);
       return builder;
     }
     return null;
@@ -106,7 +106,7 @@ public static Server startServer(
           Logger logger,
           Configuration conf,
           Function... describe) {
-    return builder(conf, describe).start(logger);
+    return Objects.requireNonNull(builder(conf, describe)).start(logger);
   }
 
   public Configuration getConfiguration() {
@@ -147,9 +147,8 @@ public Descriptor addServlet(Descriptor descriptor) {
    * 
Default use configuration to determine thread-pool constants?

    *
    * @return the server instance
-   * @throws IOException if server creation fails
    */
-  protected Server createServer() throws IOException {
+  private Server createServer() {
     final int maxThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MAX);
     final int minThreads = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_MIN);
     final int idleTimeout = MetastoreConf.getIntVar(configuration, MetastoreConf.ConfVars.HTTPSERVER_THREADPOOL_IDLE);
@@ -166,9 +165,8 @@ protected Server createServer() throws IOException {
    * @param sslContextFactory the ssl factory
    * @param port              the port
    * @return the server connector listening to the port
-   * @throws IOException if server creation fails
    */
-  protected ServerConnector createConnector(Server server, SslContextFactory sslContextFactory, int port) throws IOException {
+  private ServerConnector createConnector(Server server, SslContextFactory sslContextFactory, int port) {
     final ServerConnector connector = new ServerConnector(server, sslContextFactory);
     connector.setPort(port);
     connector.setReuseAddress(true);
@@ -187,9 +185,8 @@ protected ServerConnector createConnector(Server server, SslContextFactory sslCo
    *
    * @param handlersMap the map of port to handlers
    * @param descriptor  the servlet descriptor
-   * @throws IOException
    */
-  protected void addServlet(Map handlersMap, Descriptor descriptor) throws IOException {
+  private void addServlet(Map handlersMap, Descriptor descriptor) {
     final int port = descriptor.getPort();
     final String path = descriptor.getPath();
     final HttpServlet servlet = descriptor.getServlet();
@@ -280,12 +277,10 @@ public Server start(Logger logger) {
         if (!server.isStarted()) {
           logger.error("Unable to start servlet server on {}", server.getURI());
         } else {
-          descriptorsMap.values().forEach(descriptor -> {
-            logger.info("Started {} servlet on {}:{}",
-                    descriptor.toString(),
-                    descriptor.getPort(),
-                    descriptor.getPath());
-          });
+          descriptorsMap.values().forEach(descriptor -> logger.info("Started {} servlet on {}:{}",
+                  descriptor.toString(),
+                  descriptor.getPort(),
+                  descriptor.getPath()));
         }
       }
       return server;
@@ -320,6 +315,7 @@ public Descriptor(int port, String path, HttpServlet servlet) {
 
     @Override
     public String toString() {
+      // can not use the servlet name since it is only valid after calling init()
       return servlet.getClass().getSimpleName() + ":" + port + "/" + path;
     }
 
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
index 51c4e26727d2..f196d66c33b3 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest.java
@@ -59,7 +59,7 @@
 public class HMSServletTest extends HMSTestBase {
   String path = null;
   Server servletServer = null;
-  int sport = -1;
+  int servletPort = -1;
   
   @Before
   public void setUp() throws Exception {
@@ -67,26 +67,28 @@ public void setUp() throws Exception {
     path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
   }
 
-  @Override protected int createServer(Configuration conf) throws Exception {
+  @Override
+  protected int createServer(Configuration conf) throws Exception {
     if (servletServer == null) {
       servletServer = PropertyServlet.startServer(conf);
       if (servletServer == null || !servletServer.isStarted()) {
         Assert.fail("http server did not start");
       }
-      sport = servletServer.getURI().getPort();
+      servletPort = servletServer.getURI().getPort();
     }
-    return sport;
+    return servletPort;
   }
 
   /**
    * Stops the server.
    * @param port the server port
    */
-  @Override protected void stopServer(int port) throws Exception {
+  @Override
+  protected void stopServer(int port) throws Exception {
     if (servletServer != null) {
       servletServer.stop();
       servletServer = null;
-      sport = -1;
+      servletPort = -1;
     }
   }
 
@@ -154,7 +156,7 @@ public Map getProperties(List selection) {
 
   @Test
   public void testServletEchoA() throws Exception {
-    URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
+    URL url = new URL("http://hive@localhost:" + servletPort + "/" + path + "/" + NS);
     Map json = Collections.singletonMap("method", "echo");
     String jwt = generateJWT();
     // succeed
@@ -186,7 +188,7 @@ public void testProperties0() throws Exception {
         .setScheme("http")
         .setUserInfo("hive")
         .setHost("localhost")
-        .setPort(sport)
+        .setPort(servletPort)
         .setPath("/" + path + "/" + NS)
         .setParameters(nvp)
         .build();
@@ -303,7 +305,7 @@ public static Object clientCall(String jwt, URL url, String method, Object arg)
    * @throws Exception
    */
   private HttpPost createPost(String jwt, String msgBody) {
-    HttpPost method = new HttpPost("http://hive@localhost:" + sport + "/" + path + "/" + NS);
+    HttpPost method = new HttpPost("http://hive@localhost:" + servletPort + "/" + path + "/" + NS);
     method.addHeader("Authorization", "Bearer " + jwt);
     method.addHeader("Content-Type", "application/json");
     method.addHeader("Accept", "application/json");
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
index 1a096e38f30c..b1c8b803dff9 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1.java
@@ -30,6 +30,7 @@
 import javax.servlet.http.HttpServletResponse;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -48,12 +49,14 @@ public class HMSServletTest1 extends HMSServletTest {
   public void tearDown() throws Exception {
     if (client instanceof AutoCloseable) {
       ((AutoCloseable) client).close();
+      client = null;
     }
     super.tearDown();
   }
 
   @Override
   protected PropertyClient createClient(Configuration conf, int sport) throws Exception {
+    String path = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.PROPERTIES_SERVLET_PATH);
     URL url = new URL("http://hive@localhost:" + sport + "/" + path + "/" + NS);
     String jwt = generateJWT();
     return new JSonHttpClient(jwt, url.toString());
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
index 1cf4b3e4e26e..fd58d53e1f19 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTest1A.java
@@ -50,8 +50,8 @@ protected int createServer(Configuration conf) throws Exception {
     if (servletServer == null || !servletServer.isStarted()) {
       Assert.fail("http server did not start");
     }
-    sport = HiveMetaStore.getPropertyServletPort();
-    return sport;
+    servletPort = HiveMetaStore.getPropertyServletPort();
+    return servletPort;
   }
 
   @Override
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
index 41a2ba06233d..3a8fb16028f0 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSServletTestA.java
@@ -49,8 +49,8 @@ protected int createServer(Configuration conf) throws Exception {
     if (servletServer == null || !servletServer.isStarted()) {
       Assert.fail("http server did not start");
     }
-    sport = HiveMetaStore.getPropertyServletPort();
-    return sport;
+    servletPort = HiveMetaStore.getPropertyServletPort();
+    return servletPort;
   }
 
   @Override

From 370a0e00bf2e9773abecbeef3032c3954133000f Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Mon, 3 Mar 2025 09:26:18 +0100
Subject: [PATCH 38/40] HIVE-28059 : fixing javadoc;

---
 .../org/apache/iceberg/rest/HMSCatalogFactory.java     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
index 76327d1fa80f..5add2e51564e 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
@@ -35,7 +35,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Catalog & servlet factory.
+ * Catalog & servlet factory.
  */
 public class HMSCatalogFactory {
   private static final Logger LOG = LoggerFactory.getLogger(HMSCatalogFactory.class);
@@ -103,12 +103,12 @@ protected Catalog createCatalog() {
     if (configExtWarehouse != null) {
       properties.put("external-warehouse", configExtWarehouse);
     }
-    final HiveCatalog catalog = new org.apache.iceberg.hive.HiveCatalog();
-    catalog.setConf(configuration);
+    final HiveCatalog hiveCatalog = new org.apache.iceberg.hive.HiveCatalog();
+    hiveCatalog.setConf(configuration);
     final String catalogName = MetastoreConf.getVar(configuration, MetastoreConf.ConfVars.CATALOG_DEFAULT);
-    catalog.initialize(catalogName, properties);
+    hiveCatalog.initialize(catalogName, properties);
     long expiry = MetastoreConf.getLongVar(configuration, MetastoreConf.ConfVars.ICEBERG_CATALOG_CACHE_EXPIRY);
-    return expiry > 0 ? new HMSCachingCatalog<>(catalog, expiry) : catalog;
+    return expiry > 0 ? new HMSCachingCatalog<>(hiveCatalog, expiry) : hiveCatalog;
   }
 
   /**

From d44bfe08c3d305a7cec7fc029ed759765ae09180 Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Thu, 6 Mar 2025 17:53:49 +0100
Subject: [PATCH 39/40] HIVE-28059 : latest nits (servlet names, etc); - fixing
 test failures;

---
 .../iceberg/rest/HMSCatalogAdapter.java       |  2 ++
 .../iceberg/rest/HMSCatalogFactory.java       |  6 +----
 .../iceberg/rest/HMSCatalogServlet.java       |  2 +-
 .../hadoop/hive/metastore/HiveMetaStore.java  |  8 +++----
 .../hive/metastore/PropertyServlet.java       |  6 +----
 .../hive/metastore/ServletSecurity.java       | 22 ++++++++++++++++++-
 .../hive/metastore/ServletServerBuilder.java  | 12 ++++++++--
 7 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
index cc2738008f75..dbf280396f1e 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogAdapter.java
@@ -297,12 +297,14 @@ private ConfigResponse config() {
   }
 
   private OAuthTokenResponse tokens(Object body) {
+    @SuppressWarnings("unchecked")
     Map request = (Map) castRequest(Map.class, body);
     String grantType = request.get(GRANT_TYPE);
     switch (grantType) {
       case CLIENT_CREDENTIALS:
         return OAuthTokenResponse.builder()
             .withToken("client-credentials-token:sub=" + request.get(CLIENT_ID))
+            .withIssuedTokenType(URN_OAUTH_ACCESS_TOKEN)
             .withTokenType(BEARER)
             .build();
 
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
index 5add2e51564e..1bddb3e6842d 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogFactory.java
@@ -152,11 +152,7 @@ public static ServletServerBuilder.Descriptor createServlet(Configuration config
       HMSCatalogFactory hms = new HMSCatalogFactory(configuration, null);
       HttpServlet servlet = hms.createServlet();
       if (servlet != null) {
-        return new ServletServerBuilder.Descriptor(hms.getPort(), hms.getPath(), servlet) {
-          @Override public String toString() {
-            return "Iceberg REST Catalog";
-          }
-        };
+        return new ServletServerBuilder.Descriptor(hms.getPort(), hms.getPath(), servlet);
       }
     } catch (IOException exception) {
       LOG.error("failed to create servlet ", exception);
diff --git a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
index e6ec84c99118..b164709149b1 100644
--- a/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
+++ b/standalone-metastore/metastore-rest-catalog/src/main/java/org/apache/iceberg/rest/HMSCatalogServlet.java
@@ -61,7 +61,7 @@ public HMSCatalogServlet(HMSCatalogAdapter restCatalogAdapter) {
 
   @Override
   public String getServletName() {
-    return "HMS Catalog";
+    return "Iceberg REST Catalog";
   }
 
   @Override
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 77812cc2cffe..edc91e7c51de 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -771,10 +771,8 @@ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
 
     // optionally create and start the property and Iceberg REST server
     ServletServerBuilder builder = new ServletServerBuilder(conf);
-    ServletServerBuilder.Descriptor properties = PropertyServlet.createServlet(conf);
-    builder.addServlet(properties);
-    ServletServerBuilder.Descriptor catalog = createIcebergServlet(conf);
-    builder.addServlet(catalog);
+    ServletServerBuilder.Descriptor properties = builder.addServlet(PropertyServlet.createServlet(conf));
+    ServletServerBuilder.Descriptor catalog = builder.addServlet(createIcebergServlet(conf));
     servletServer = builder.start(LOG);
     if (servletServer != null) {
       if (properties != null) {
@@ -812,7 +810,7 @@ static ServletServerBuilder.Descriptor createIcebergServlet(Configuration config
   /**
    * @param port where metastore server is running
    * @return metastore server instance URL. If the metastore server was bound to a configured
-   * host, return that appended by port. Otherwise return the externally visible URL of the local
+   * host, return that appended by port. Otherwise, return the externally visible URL of the local
    * host with the given port
    * @throws Exception
    */
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
index ffb1be3058ad..9437d2558f8a 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PropertyServlet.java
@@ -315,11 +315,7 @@ public static ServletServerBuilder.Descriptor createServlet(Configuration config
       if (port >= 0 && path != null && !path.isEmpty()) {
         ServletSecurity security = new ServletSecurity(configuration);
         HttpServlet servlet = security.proxy(new PropertyServlet(configuration));
-        return new ServletServerBuilder.Descriptor(port, path, servlet) {
-          @Override public String toString() {
-            return "HMS Property";
-          }
-        };
+        return new ServletServerBuilder.Descriptor(port, path, servlet);
       }
     } catch (Exception io) {
       LOGGER.error("Failed to create servlet ", io);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
index aee8d37b77b2..d0d48b04df75 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletSecurity.java
@@ -135,14 +135,34 @@ public void init() throws ServletException {
     public void service(HttpServletRequest request, HttpServletResponse response) throws IOException {
       execute(request, response, delegate::service);
     }
+
+    @Override
+    public String getServletName() {
+      try {
+        return delegate.getServletName();
+      } catch (IllegalStateException ill) {
+        return delegate.toString();
+      }
+    }
+
+    @Override
+    public String getServletInfo() {
+      return delegate.getServletInfo();
+    }
   }
 
   /**
    * Creates a proxy servlet.
    * @param servlet the servlet to serve within this security context
-   * @return a servlet instance
+   * @return a servlet instance or null if security initialization fails
    */
   public HttpServlet proxy(HttpServlet servlet) {
+    try {
+      init();
+    } catch (ServletException e) {
+      LOG.error("Unable to proxy security for servlet {}", servlet.toString(), e);
+      return null;
+    }
     return new ProxyServlet(servlet);
   }
 
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
index 066c0c6a31f8..7323845ae35f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ServletServerBuilder.java
@@ -315,8 +315,16 @@ public Descriptor(int port, String path, HttpServlet servlet) {
 
     @Override
     public String toString() {
-      // can not use the servlet name since it is only valid after calling init()
-      return servlet.getClass().getSimpleName() + ":" + port + "/" + path;
+      String name = null;
+      try {
+        name = servlet.getServletName() + ":" + port + "/" + path;
+      } catch (IllegalStateException ill) {
+        // ignore, it may happen if servlet config is not set (yet)
+      }
+      if (name == null) {
+        name = servlet.getClass().getSimpleName();
+      }
+      return name + ":" + port + "/" + path;
     }
 
     public int getPort() {

From 1e7a8da66b57681bdba225e152f78f4d287953bb Mon Sep 17 00:00:00 2001
From: Henrib 
Date: Fri, 7 Mar 2025 06:25:01 +0100
Subject: [PATCH 40/40] HIVE-28059 : fixing thrift over http servlet init;

---
 .../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index edc91e7c51de..6cd45e34aff1 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -482,7 +482,8 @@ public void setThreadFactory(ThreadFactory threadFactory) {
     IHMSHandler handler = HMSHandlerProxyFactory.getProxy(conf, baseHandler, false);
     processor = new ThriftHiveMetastore.Processor<>(handler);
     LOG.info("Starting DB backed MetaStore Server with generic processor");
-    ServletSecurity security = new ServletSecurity(conf);
+    boolean jwt = MetastoreConf.getVar(conf, ConfVars.THRIFT_METASTORE_AUTHENTICATION).equalsIgnoreCase("jwt");
+    ServletSecurity security = new ServletSecurity(conf, jwt);
     Servlet thriftHttpServlet = security.proxy(new TServlet(processor, protocolFactory));
 
     boolean directSqlEnabled = MetastoreConf.getBoolVar(conf, ConfVars.TRY_DIRECT_SQL);