HDDS-11465. Introducing Schema Versioning for Recon to Handle Fresh Installs and Upgrades.

hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/ReconSchemaGenerationModule.java

@@ -17,11 +17,7 @@
  */
 package org.hadoop.ozone.recon.codegen;
 
-import org.hadoop.ozone.recon.schema.ContainerSchemaDefinition;
-import org.hadoop.ozone.recon.schema.ReconTaskSchemaDefinition;
-import org.hadoop.ozone.recon.schema.ReconSchemaDefinition;
-import org.hadoop.ozone.recon.schema.StatsSchemaDefinition;
-import org.hadoop.ozone.recon.schema.UtilizationSchemaDefinition;
+import org.hadoop.ozone.recon.schema.*;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.multibindings.Multibinder;
@@ -40,5 +36,6 @@ protected void configure() {
     schemaBinder.addBinding().to(ContainerSchemaDefinition.class);
     schemaBinder.addBinding().to(ReconTaskSchemaDefinition.class);
     schemaBinder.addBinding().to(StatsSchemaDefinition.class);
+    schemaBinder.addBinding().to(SchemaVersionTableDefinition.class);
   }
 }

hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/schema/SchemaVersionTableDefinition.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.hadoop.ozone.recon.schema;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.TABLE_EXISTS_CHECK;
+
+/**
+ * Class for managing the schema of the SchemaVersion table.
+ */
+@Singleton
+public class SchemaVersionTableDefinition implements ReconSchemaDefinition {
+
+  public static final String SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION";
+  private final DataSource dataSource;
+  private DSLContext dslContext;
+
+  @Inject
+  public SchemaVersionTableDefinition(DataSource dataSource) {
+    this.dataSource = dataSource;
+  }
+
+  @Override
+  public void initializeSchema() throws SQLException {
+    Connection conn = dataSource.getConnection();
+    dslContext = DSL.using(conn);
+
+    if (!TABLE_EXISTS_CHECK.test(conn, SCHEMA_VERSION_TABLE_NAME)) {
+      createSchemaVersionTable();
+    }
+  }
+
+  /**
+   * Create the Schema Version table.
+   */
+  private void createSchemaVersionTable() throws SQLException {
+    dslContext.createTableIfNotExists(SCHEMA_VERSION_TABLE_NAME)
+        .column("version_number", SQLDataType.INTEGER.nullable(false))
+        .column("applied_on", SQLDataType.TIMESTAMP.defaultValue(DSL.currentTimestamp()))
+        .execute();
+  }
+
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSchemaVersionTableManager.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import com.google.inject.Inject;
+import org.jooq.DSLContext;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+
+import static org.jooq.impl.DSL.name;
+
+/**
+ * Manager for handling the Recon Schema Version table.
+ * This class provides methods to get and update the current schema version.
+ */
+public class ReconSchemaVersionTableManager {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ReconSchemaVersionTableManager.class);
+  public static final String RECON_SCHEMA_VERSION_TABLE_NAME = "RECON_SCHEMA_VERSION";
+  private final DSLContext dslContext;
+  private final DataSource dataSource;
+
+  @Inject
+  public ReconSchemaVersionTableManager(DataSource dataSource) throws SQLException {
+    this.dataSource = dataSource;
+    this.dslContext = DSL.using(dataSource.getConnection());
+  }
+
+  /**
+   * Get the current schema version from the RECON_SCHEMA_VERSION table.
+   * If the table is empty, or if it does not exist, it will return 0.
+   * @return The current schema version.
+   */
+  public int getCurrentSchemaVersion() {
+    try {
+      return dslContext.select(DSL.field(name("version_number")))
+          .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))
+          .fetchOptional()
+          .map(record -> record.get(
+              DSL.field(name("version_number"), Integer.class)))
+          .orElse(0); // Return 0 if no version is found
+    } catch (Exception e) {
+      LOG.error("Failed to fetch the current schema version.", e);
+      return 0; // Return 0 if there is an exception
+    }
+  }
+
+  /**
+   * Update the schema version in the RECON_SCHEMA_VERSION table after all tables are upgraded.
+   *
+   * @param newVersion The new version to set.
+   */
+  public void updateSchemaVersion(int newVersion) {
+    try {
+      boolean recordExists = dslContext.fetchExists(dslContext.selectOne()
+          .from(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME)));
+
+      if (recordExists) {
+        // Update the existing schema version record
+        dslContext.update(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))
+            .set(DSL.field(name("version_number")), newVersion)
+            .set(DSL.field(name("applied_on")), DSL.currentTimestamp())
+            .execute();
+        LOG.info("Updated schema version to '{}'.", newVersion);
+      } else {
+        // Insert a new schema version record
+        dslContext.insertInto(DSL.table(RECON_SCHEMA_VERSION_TABLE_NAME))
+            .columns(DSL.field(name("version_number")),
+                DSL.field(name("applied_on")))
+            .values(newVersion, DSL.currentTimestamp())
+            .execute();
+        LOG.info("Inserted new schema version '{}'.", newVersion);
+      }
+    } catch (Exception e) {
+      LOG.error("Failed to update schema version to '{}'.", newVersion, e);
+    }
+  }
+
+  /**
+   * Provides the data source used by this manager.
+   * @return The DataSource instance.
+   */
+  public DataSource getDataSource() {
+    return dataSource;
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java

@@ -42,6 +42,7 @@
 import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
 import org.apache.hadoop.ozone.recon.spi.StorageContainerServiceProvider;
 import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider;
+import org.apache.hadoop.ozone.recon.upgrade.ReconLayoutVersionManager;
 import org.apache.hadoop.ozone.util.OzoneNetUtils;
 import org.apache.hadoop.ozone.util.OzoneVersionInfo;
 import org.apache.hadoop.ozone.util.ShutdownHookManager;
@@ -138,10 +139,20 @@ public Void call() throws Exception {
 
       ReconSchemaManager reconSchemaManager =
           injector.getInstance(ReconSchemaManager.class);
+
       LOG.info("Creating Recon Schema.");
       reconSchemaManager.createReconSchema();
       LOG.debug("Recon schema creation done.");
 
+      // Handle Recon Schema Versioning
+      ReconSchemaVersionTableManager versionTableManager =
+          injector.getInstance(ReconSchemaVersionTableManager.class);
+
+      ReconLayoutVersionManager layoutVersionManager =
+          new ReconLayoutVersionManager(versionTableManager);
+      // Run the upgrade framework to finalize layout features if needed
+      layoutVersionManager.finalizeLayoutFeatures();
+
       this.reconSafeModeMgr = injector.getInstance(ReconSafeModeManager.class);
       this.reconSafeModeMgr.setInSafeMode(true);
       httpServer = injector.getInstance(ReconHttpServer.class);

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature1UpgradeAction.java

@@ -0,0 +1,14 @@
+package org.apache.hadoop.ozone.recon.upgrade;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Feature1UpgradeAction implements ReconUpgradeAction {
+  private static final Logger LOG = LoggerFactory.getLogger(Feature1UpgradeAction.class);
+  @Override
+  public void execute() throws Exception {
+    // Logic for upgrading to version 1
+    LOG.info("Executing Feature 1 upgrade:");
+    // Implement the database schema update or other upgrade logic here
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature2UpgradeAction.java

@@ -0,0 +1,15 @@
+package org.apache.hadoop.ozone.recon.upgrade;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Feature2UpgradeAction implements ReconUpgradeAction {
+  private static final Logger LOG = LoggerFactory.getLogger(Feature2UpgradeAction.class);
+
+  @Override
+  public void execute() throws Exception {
+    // Logic for upgrading to version 2
+    LOG.info("Executing Feature 2 upgrade");
+    // Implement the database schema update or other upgrade logic here
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/Feature3UpgradeAction.java

@@ -0,0 +1,14 @@
+package org.apache.hadoop.ozone.recon.upgrade;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Feature3UpgradeAction implements ReconUpgradeAction {
+  private static final Logger LOG = LoggerFactory.getLogger(Feature3UpgradeAction.class);
+  @Override
+  public void execute() throws Exception {
+    // Logic for upgrading to version 3
+    LOG.info("Executing Feature 3 upgrade");
+    // Implement the database schema update or other upgrade logic here
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/InitialLayoutUpgradeAction.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.upgrade;
+
+public class InitialLayoutUpgradeAction implements ReconUpgradeAction {
+
+  @Override
+  public void execute() throws Exception {
+    // No actions are performed for version 0.
+    // This version represents the introduction of schema versioning for Recon.
+    // Therefore, we do not perform any upgrade actions for now.
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/ReconLayoutFeature.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.upgrade;
+
+/**
+ * Enum representing Recon layout features with their version, description,
+ * and associated upgrade action to be executed during an upgrade.
+ */
+public enum ReconLayoutFeature {
+  INITIAL_VERSION(0, "Initial Layout Version", new InitialLayoutUpgradeAction()),
+  FEATURE_1(1, "Description for Feature 1", new Feature1UpgradeAction()),
+  FEATURE_2(2, "Description for Feature 2", new Feature2UpgradeAction()),
+  FEATURE_3(3, "Description for Feature 3", new Feature3UpgradeAction());
+
+  private final int version;
+  private final String description;
+  private final ReconUpgradeAction upgradeAction;
+
+  ReconLayoutFeature(int version, String description, ReconUpgradeAction upgradeAction) {
+    this.version = version;
+    this.description = description;
+    this.upgradeAction = upgradeAction;
+  }
+
+  public int getVersion() {
+    return version;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public ReconUpgradeAction getUpgradeAction() {
+    return upgradeAction;
+  }
+}