SOLR-16852: Let backups have custom key/values (#1739)

solr/CHANGES.txt

@@ -20,6 +20,8 @@ New Features
 
 * SOLR-15367: Convert "rid" functionality into a default Tracer (Alex Deparvu, David Smiley)
 
+* SOLR-16852: Backups now allow metadata to be added as key-values (Tomás Fernández Löbbe)
+
 Improvements
 ---------------------
 * SOLR-16490: `/admin/cores?action=backupcore` now has a v2 equivalent, available at

solr/api/src/java/org/apache/solr/client/api/model/CollectionBackupDetails.java

@@ -35,4 +35,5 @@ public class CollectionBackupDetails {
   public String configsetName;
 
   @JsonProperty public String collectionAlias;
+  @JsonProperty public Map<String, String> extraProperties;
 }

solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java

@@ -29,6 +29,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import org.apache.solr.cloud.api.collections.CollectionHandlingUtils.ShardRequestTracker;
 import org.apache.solr.common.SolrException;
@@ -68,6 +69,7 @@ public BackupCmd(CollectionCommandContext ccc) {
     this.ccc = ccc;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public void call(ClusterState state, ZkNodeProps message, NamedList<Object> results)
       throws Exception {
@@ -91,8 +93,11 @@ public void call(ClusterState state, ZkNodeProps message, NamedList<Object> resu
             .getCollection(collectionName)
             .getConfigName();
 
+    Map<String, String> customProps = (Map<String, String>) message.get("extraProperties");
+
     BackupProperties backupProperties =
-        BackupProperties.create(backupName, collectionName, extCollectionName, configName);
+        BackupProperties.create(
+            backupName, collectionName, extCollectionName, configName, customProps);
 
     CoreContainer cc = ccc.getCoreContainer();
     try (BackupRepository repository = cc.newBackupRepository(repo)) {
@@ -367,6 +372,9 @@ private NamedList<Object> aggregateResults(
     }
     aggRsp.add("indexVersion", backupProps.getIndexVersion());
     aggRsp.add("startTime", backupProps.getStartTime());
+    if (backupProps.getExtraProperties() != null) {
+      aggRsp.add("extraProperties", backupProps.getExtraProperties());
+    }
 
     // Optional options for backups
     Optional<Integer> indexFileCount = Optional.empty();

solr/core/src/java/org/apache/solr/core/backup/BackupProperties.java

@@ -48,26 +48,41 @@
  */
 public class BackupProperties {
 
+  private static final String EXTRA_PROPERTY_PREFIX = "property.";
+
   private double indexSizeMB;
   private int indexFileCount;
 
-  private Properties properties;
+  private final Properties properties;
+
+  private final Map<String, String> extraProperties;
 
-  private BackupProperties(Properties properties) {
+  private BackupProperties(Properties properties, Map<String, String> extraProperties) {
     this.properties = properties;
+    if (extraProperties == null) {
+      extraProperties = Map.of();
+    } else if (extraProperties.keySet().stream().anyMatch(String::isEmpty)) {
+      throw new IllegalArgumentException("Can't have an extra property with an empty key");
+    }
+    this.extraProperties = extraProperties;
   }
 
   public static BackupProperties create(
-      String backupName, String collectionName, String extCollectionName, String configName) {
-    Properties properties = new Properties();
+      String backupName,
+      String collectionName,
+      String extCollectionName,
+      String configName,
+      Map<String, String> extraProperties) {
+    final Properties properties = new Properties();
+
     properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
     properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
     properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);
     properties.put(CollectionAdminParams.COLL_CONF, configName);
     properties.put(BackupManager.START_TIME_PROP, Instant.now().toString());
     properties.put(BackupManager.INDEX_VERSION_PROP, Version.LATEST.toString());
 
-    return new BackupProperties(properties);
+    return new BackupProperties(properties, extraProperties);
   }
 
   public static Optional<BackupProperties> readFromLatest(
@@ -93,10 +108,29 @@ public static BackupProperties readFrom(
                 repository.openInput(backupPath, fileName, IOContext.DEFAULT)),
             StandardCharsets.UTF_8)) {
       props.load(is);
-      return new BackupProperties(props);
+      Map<String, String> extraProperties = extractExtraProperties(props);
+      return new BackupProperties(props, extraProperties);
     }
   }
 
+  private static Map<String, String> extractExtraProperties(Properties props) {
+    Map<String, String> extraProperties = new HashMap<>();
+    props
+        .entrySet()
+        .removeIf(
+            e -> {
+              String entryKey = e.getKey().toString();
+              if (entryKey.startsWith(EXTRA_PROPERTY_PREFIX)) {
+                extraProperties.put(
+                    entryKey.substring(EXTRA_PROPERTY_PREFIX.length()),
+                    String.valueOf(e.getValue()));
+                return true;
+              }
+              return false;
+            });
+    return extraProperties;
+  }
+
   public List<String> getAllShardBackupMetadataFiles() {
     return properties.entrySet().stream()
         .filter(entry -> entry.getKey().toString().endsWith(".md"))
@@ -128,10 +162,14 @@ private String getKeyForShardBackupId(String shardName) {
   }
 
   public void store(Writer propsWriter) throws IOException {
-    properties.put("indexSizeMB", String.valueOf(indexSizeMB));
-    properties.put("indexFileCount", String.valueOf(indexFileCount));
-    properties.put(BackupManager.END_TIME_PROP, Instant.now().toString());
-    properties.store(propsWriter, "Backup properties file");
+    Properties propertiesCopy = (Properties) properties.clone();
+    propertiesCopy.put("indexSizeMB", String.valueOf(indexSizeMB));
+    propertiesCopy.put("indexFileCount", String.valueOf(indexFileCount));
+    propertiesCopy.put(BackupManager.END_TIME_PROP, Instant.now().toString());
+    if (extraProperties != null && !extraProperties.isEmpty()) {
+      extraProperties.forEach((k, v) -> propertiesCopy.put(EXTRA_PROPERTY_PREFIX + k, v));
+    }
+    propertiesCopy.store(propsWriter, "Backup properties file");
   }
 
   public String getCollection() {
@@ -158,14 +196,20 @@ public String getIndexVersion() {
     return properties.getProperty(BackupManager.INDEX_VERSION_PROP);
   }
 
+  public Map<String, String> getExtraProperties() {
+    return extraProperties;
+  }
+
   public Map<String, Object> getDetails() {
     final Map<String, Object> result = new HashMap<>();
-    properties.entrySet().stream()
-        .forEach(entry -> result.put(entry.getKey().toString(), entry.getValue()));
+    properties.entrySet().forEach(entry -> result.put(entry.getKey().toString(), entry.getValue()));
     result.remove(BackupManager.BACKUP_NAME_PROP);
     result.remove(BackupManager.COLLECTION_NAME_PROP);
     result.put("indexSizeMB", Double.valueOf(properties.getProperty("indexSizeMB")));
     result.put("indexFileCount", Integer.valueOf(properties.getProperty("indexFileCount")));
+    if (extraProperties != null && !extraProperties.isEmpty()) {
+      result.put("extraProperties", extraProperties);
+    }
 
     Map<String, String> shardBackupIds = new HashMap<>();
     Iterator<String> keyIt = result.keySet().iterator();

solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackupAPI.java

@@ -22,6 +22,7 @@
 import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
 import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
 import static org.apache.solr.common.params.CollectionAdminParams.INDEX_BACKUP_STRATEGY;
+import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_PREFIX;
 import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
 import static org.apache.solr.common.params.CommonParams.NAME;
 import static org.apache.solr.common.params.CoreAdminParams.BACKUP_CONFIGSET;
@@ -31,6 +32,7 @@
 import static org.apache.solr.common.params.CoreAdminParams.COMMIT_NAME;
 import static org.apache.solr.common.params.CoreAdminParams.MAX_NUM_BACKUP_POINTS;
 import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;
+import static org.apache.solr.handler.admin.api.CreateCollection.copyPrefixedPropertiesWithoutPrefix;
 import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -166,6 +168,9 @@ public static CreateCollectionBackupRequestBody createRequestBodyFromV1Params(So
     requestBody.incremental = params.getBool(BACKUP_INCREMENTAL);
     requestBody.backupConfigset = params.getBool(BACKUP_CONFIGSET);
     requestBody.maxNumBackupPoints = params.getInt(MAX_NUM_BACKUP_POINTS);
+    requestBody.extraProperties =
+        copyPrefixedPropertiesWithoutPrefix(params, new HashMap<>(), PROPERTY_PREFIX);
+
     requestBody.async = params.get(ASYNC);
 
     return requestBody;
@@ -192,6 +197,7 @@ public static class CreateCollectionBackupRequestBody implements JacksonReflectM
     @JsonProperty public Boolean backupConfigset;
     @JsonProperty public Integer maxNumBackupPoints;
     @JsonProperty public String async;
+    @JsonProperty public Map<String, String> extraProperties;
   }
 
   public static class CreateCollectionBackupResponseBody
@@ -215,6 +221,7 @@ public static class CollectionBackupDetails implements JacksonReflectMapWriter {
     @JsonProperty public Integer indexFileCount;
     @JsonProperty public Integer uploadedIndexFileCount;
     @JsonProperty public Double indexSizeMB;
+    @JsonProperty public Map<String, String> extraProperties;
 
     @JsonProperty("uploadedIndexFileMB")
     public Double uploadedIndexSizeMB;

solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java

@@ -17,8 +17,19 @@
 
 package org.apache.solr.cloud.api.collections;
 
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
 import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.backup.repository.BackupRepository;
 import org.junit.BeforeClass;
+import org.junit.Test;
 
 // Backups do checksum validation against a footer value not present in 'SimpleText'
 @LuceneTestCase.SuppressCodecs({"SimpleText"})
@@ -85,4 +96,61 @@ public String getCollectionNamePrefix() {
   public String getBackupLocation() {
     return backupLocation;
   }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void testCustomProperties() throws Exception {
+    setTestSuffix("testCustomProperties");
+    final String backupCollectionName = getCollectionName();
+    final String restoreCollectionName = backupCollectionName + "_restore";
+
+    CloudSolrClient solrClient = cluster.getSolrClient();
+
+    CollectionAdminRequest.createCollection(backupCollectionName, "conf1", NUM_SHARDS, 1)
+        .process(solrClient);
+    int numDocs = indexDocs(backupCollectionName, true);
+    String backupName = BACKUPNAME_PREFIX + testSuffix;
+    try (BackupRepository repository =
+        cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(BACKUP_REPO_NAME)) {
+      String backupLocation = repository.getBackupLocation(getBackupLocation());
+      Properties extraProps = new Properties();
+      extraProps.putAll(Map.of("foo", "bar", "number", "12345"));
+      CollectionAdminRequest.backupCollection(backupCollectionName, backupName)
+          .setLocation(backupLocation)
+          .setExtraProperties(extraProps)
+          .setRepositoryName(BACKUP_REPO_NAME)
+          .processAndWait(cluster.getSolrClient(), 100);
+      CollectionAdminResponse response =
+          CollectionAdminRequest.listBackup(backupName)
+              .setBackupLocation(backupLocation)
+              .setBackupRepository(BACKUP_REPO_NAME)
+              .process(cluster.getSolrClient());
+      assertNotNull(response.getResponse().get("backups"));
+      assertTrue(response.getResponse().get("backups") instanceof List);
+      List<Map<String, Object>> backups =
+          (List<Map<String, Object>>) response.getResponse().get("backups");
+      assertEquals(1, backups.size());
+      Map<String, Object> backup0 = backups.get(0);
+      assertNotNull(backup0.get("extraProperties"));
+      assertTrue(backup0.get("extraProperties") instanceof Map);
+      Map<String, Object> extraProperties = (Map<String, Object>) backup0.get("extraProperties");
+      assertEquals("bar", extraProperties.get("foo"));
+      assertEquals("12345", extraProperties.get("number"));
+
+      CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
+          .setLocation(backupLocation)
+          .setRepositoryName(BACKUP_REPO_NAME)
+          .processAndWait(solrClient, 500);
+
+      AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+          restoreCollectionName, ZkStateReader.from(solrClient), false, false, 3);
+      assertEquals(
+          numDocs,
+          cluster
+              .getSolrClient()
+              .query(restoreCollectionName, new SolrQuery("*:*"))
+              .getResults()
+              .getNumFound());
+    }
+  }
 }

solr/core/src/test/org/apache/solr/cloud/api/collections/PurgeGraphTest.java

@@ -199,7 +199,11 @@ public void testUnreferencedShardMetadataFilesAreDeleted() throws Exception {
   private void createBackupIdFile(int backupId, String... shardNames) throws Exception {
     final BackupProperties createdProps =
         BackupProperties.create(
-            "someBackupName", "someCollectionName", "someExtCollectionName", "someConfigName");
+            "someBackupName",
+            "someCollectionName",
+            "someExtCollectionName",
+            "someConfigName",
+            null);
     for (String shardName : shardNames) {
       createdProps.putAndGetShardBackupIdFor(shardName, backupId);
     }

solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc

@@ -1668,11 +1668,19 @@ This parameter has no effect if `incremental=false` is specified.
 +
 [%autowidth,frame=none]
 |===
-|Optional |Default: true
+|Optional |Default: `true`
 |===
 +
 Indicates if configset files should be included with the index backup or not. Note that in order to restore a collection, the configset must either exist in ZooKeeper or be part of the backup. Only set this to `false` if you can restore configsets by other means external to Solr (i.e. you have it stored with your application source code, is part of your ZooKeeper backups, etc).
 
+`property.<propertyName>` (V1), `extraProperties` (V2)::
+[%autowidth,frame=none]
+|===
+|Optional |Default: none
+|===
++
+Allows storing additional key/value pairs for custom information related to the backup. In v2, the value is a map of key-value pairs.
+
 `incremental`::
 +
 [%autowidth,frame=none]

solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java

@@ -1111,6 +1111,7 @@ public static class Backup extends AsyncCollectionSpecificAdminRequest {
     protected boolean incremental = true;
     protected Optional<Integer> maxNumBackupPoints = Optional.empty();
     protected boolean backupConfigset = true;
+    protected Properties extraProperties;
 
     public Backup(String collection, String name) {
       super(CollectionAction.BACKUP, collection);
@@ -1205,9 +1206,17 @@ public Backup setBackupConfigset(boolean backupConfigset) {
       return this;
     }
 
+    public Backup setExtraProperties(Properties extraProperties) {
+      this.extraProperties = extraProperties;
+      return this;
+    }
+
     @Override
     public SolrParams getParams() {
       ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+      if (extraProperties != null) {
+        addProperties(params, extraProperties);
+      }
       params.set(CoreAdminParams.COLLECTION, collection);
       params.set(CoreAdminParams.NAME, name);
       params.set(CoreAdminParams.BACKUP_LOCATION, location); // note: optional