[Backport] Add support for project metadata (treasure-data#1802)

szyn · web-flow · commit e91111ef1ed7 · 2023-03-28T11:09:58.000+09:00
* Merge pull request st-tech#45 from treasure-data/WM-1639 WM-1639: Add DDL for project_metadata table * Merge pull request st-tech#47 from treasure-data/WM-1646-td-digdag WM-1646: Add support for adding project metadata * Merge pull request st-tech#69 from treasure-data/WM-1732 WM-1732: Allow ProjectService to pass AuthencatedResource userInfo
diff --git a/digdag-client/src/main/java/io/digdag/client/DigdagClient.java b/digdag-client/src/main/java/io/digdag/client/DigdagClient.java
@@ -452,7 +452,13 @@ public RestProject putProjectRevision(String projName, String revision, File bod
         return putProjectRevision(projName, revision, body, scheduleFrom, Collections.emptyList(), Optional.absent());
     }
 
-    public RestProject putProjectRevision(String projName, String revision, File body, Optional<Instant> scheduleFrom, List<String> clearSchedule, Optional<Boolean> clearAllSchedules )
+    public RestProject putProjectRevision(
+            String projName,
+            String revision,
+            File body,
+            Optional<Instant> scheduleFrom,
+            List<String> clearSchedule,
+            Optional<Boolean> clearAllSchedules)
         throws IOException
     {
         Preconditions.checkArgument(!Strings.isNullOrEmpty(projName), "projName");
diff --git a/digdag-core/src/main/java/io/digdag/core/database/DatabaseMigrator.java b/digdag-core/src/main/java/io/digdag/core/database/DatabaseMigrator.java
@@ -42,6 +42,7 @@ public class DatabaseMigrator
         new Migration_20191105105927_AddIndexToSessions(),
         new Migration_20200716114008_AddLastAttemptIdIndexToSessions(),
         new Migration_20200803184355_ReplacePartialIndexOnSessionAttempts(),
+        new Migration_20221021035534_CreateProjectMetadata(),
     })
     .sorted(Comparator.comparing(m -> m.getVersion()))
     .collect(Collectors.toList());
diff --git a/digdag-core/src/main/java/io/digdag/core/database/DatabaseProjectStoreManager.java b/digdag-core/src/main/java/io/digdag/core/database/DatabaseProjectStoreManager.java
@@ -13,6 +13,7 @@
 import io.digdag.core.repository.Project;
 import io.digdag.core.repository.ProjectControlStore;
 import io.digdag.core.repository.ProjectMap;
+import io.digdag.core.repository.ProjectMetadataMap;
 import io.digdag.core.repository.ProjectStore;
 import io.digdag.core.repository.ProjectStoreManager;
 import io.digdag.core.repository.ResourceConflictException;
@@ -48,6 +49,7 @@
 import java.sql.SQLException;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -477,6 +479,30 @@ public void insertRevisionArchiveData(int revId, byte[] data)
                 "revision archive=%d", revId);
         }
 
+        @DigdagTimed(value = "dpcst_", category = "db", appendMethodName = true)
+        @Override
+        public void insertProjectMetadata(int projId, int siteId, ProjectMetadataMap metadataMap)
+                throws ResourceConflictException
+        {
+            for (Map.Entry<String, Collection<String>> entry : metadataMap.toMap().entrySet()) {
+                String k = entry.getKey();
+                for (String v : entry.getValue()) {
+                    catchConflict(() -> {
+                                dao.insertProjectMetadata(siteId, projId, k, v);
+                                return true;
+                            },
+                            "metadata(key=%s, value=%s) in project id=%d", k, v, projId);
+                }
+            }
+        }
+
+        @DigdagTimed(value = "dpcst_", category = "db", appendMethodName = true)
+        @Override
+        public void deleteProjectMetadata(int projId)
+        {
+            dao.deleteProjectMetadata(projId);
+        }
+
         /**
          * Create a revision.
          *
@@ -942,6 +968,18 @@ List<StoredWorkflowDefinition> getWorkflowDefinitions(
                 " values (:revId, :data)")
         void insertRevisionArchiveData(@Bind("revId") int revId, @Bind("data") byte[] data);
 
+        @SqlUpdate("delete from project_metadata where project_id = :projId")
+        void deleteProjectMetadata(@Bind("projId") int projId);
+
+        @SqlUpdate("insert into project_metadata" +
+                " (site_id, project_id, key, value, created_at)" +
+                " values (:siteId, :projId, :key, :value, now())")
+        void insertProjectMetadata(
+                @Bind("siteId") int siteId,
+                @Bind("projId") int projId,
+                @Bind("key") String key,
+                @Bind("value") String value);
+
         @SqlUpdate("insert into workflow_definitions" +
                 " (revision_id, name, config_id)" +
                 " values (:revId, :name, :configId)")
diff --git a/digdag-core/src/main/java/io/digdag/core/database/migrate/Migration_20221021035534_CreateProjectMetadata.java b/digdag-core/src/main/java/io/digdag/core/database/migrate/Migration_20221021035534_CreateProjectMetadata.java
@@ -0,0 +1,25 @@
+package io.digdag.core.database.migrate;
+
+import org.skife.jdbi.v2.Handle;
+
+public class Migration_20221021035534_CreateProjectMetadata
+        implements Migration
+{
+    @Override
+    public void migrate(Handle handle, MigrationContext context)
+    {
+        handle.update(
+                context.newCreateTableBuilder("project_metadata")
+                .addIntId("id")
+                .addInt("site_id", "not null")
+                .addInt("project_id", "not null references projects (id)")
+                .addString("key", "not null")
+                .addString("value", "not null")
+                .addTimestamp("created_at", "not null")
+                .build());
+
+        handle.update("create unique index project_metadata_on_key_and_value_and_project_id_and_site_id on project_metadata(key, value, project_id, site_id)");
+        handle.update("create index project_metadata_on_site_id_and_key_and_value on project_metadata(site_id, key, value)");
+        handle.update("create index project_metadata_on_project_id_and_site_id_and_key on project_metadata(project_id, site_id, key)");
+    }
+}
diff --git a/digdag-core/src/main/java/io/digdag/core/repository/ProjectControl.java b/digdag-core/src/main/java/io/digdag/core/repository/ProjectControl.java
@@ -36,6 +36,7 @@ public static <T> T deleteProject(ProjectStore rs, int projId, DeleteProjectActi
             ProjectControl control = new ProjectControl(store, proj);
             T res = callback.call(control, proj);
             control.deleteSchedules();
+            control.deleteProjectMetadata();
             return res;
         });
     }
@@ -66,6 +67,17 @@ public void insertRevisionArchiveData(int revId, byte[] data)
         store.insertRevisionArchiveData(revId, data);
     }
 
+    public void insertProjectMetadata(ProjectMetadataMap projectMetadataMap)
+            throws ResourceConflictException
+    {
+        store.insertProjectMetadata(project.getId(), project.getSiteId(), projectMetadataMap);
+    }
+
+    public void deleteProjectMetadata()
+    {
+        store.deleteProjectMetadata(project.getId());
+    }
+
     public List<StoredWorkflowDefinition> insertWorkflowDefinitions(
             StoredRevision revision, List<WorkflowDefinition> defs,
             SchedulerManager srm, Instant currentTime)
diff --git a/digdag-core/src/main/java/io/digdag/core/repository/ProjectControlStore.java b/digdag-core/src/main/java/io/digdag/core/repository/ProjectControlStore.java
@@ -2,8 +2,7 @@
 
 import java.util.List;
 import java.time.ZoneId;
-import java.time.Instant;
-import com.google.common.base.Optional;
+
 import io.digdag.core.schedule.Schedule;
 import io.digdag.core.schedule.ScheduleStatus;
 import io.digdag.spi.ScheduleTime;
@@ -16,6 +15,11 @@ StoredRevision insertRevision(int projId, Revision revision)
     void insertRevisionArchiveData(int revId, byte[] data)
             throws ResourceConflictException;
 
+    void insertProjectMetadata(int projId, int siteId, ProjectMetadataMap metadataMap)
+            throws ResourceConflictException;
+
+    void deleteProjectMetadata(int projId);
+
     StoredWorkflowDefinition insertWorkflowDefinition(int projId, int revId, WorkflowDefinition workflow, ZoneId workflowTimeZone)
         throws ResourceConflictException;
 
diff --git a/digdag-core/src/main/java/io/digdag/core/repository/ProjectMetadataMap.java b/digdag-core/src/main/java/io/digdag/core/repository/ProjectMetadataMap.java
@@ -0,0 +1,37 @@
+package io.digdag.core.repository;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.LinkedHashMultimap;
+
+import java.util.Collection;
+import java.util.List;
+
+public class ProjectMetadataMap
+{
+    private final LinkedHashMultimap<String, String> map;
+
+    public ProjectMetadataMap(List<String> keyValueList)
+    {
+        LinkedHashMultimap<String, String> metadata = LinkedHashMultimap.create();
+        for (String keyValue : keyValueList) {
+            // format should be `key:value`
+            String[] kv = keyValue.split(":");
+            if (kv.length != 2) {
+                throw new IllegalArgumentException("Invalid project metadata format: " + keyValue);
+            }
+            metadata.put(kv[0], kv[1]);
+        }
+        this.map = metadata;
+    }
+
+    public ImmutableMap<String, Collection<String>> toMap()
+    {
+        return ImmutableMap.copyOf(map.asMap());
+    }
+
+    public List<String> getKeys()
+    {
+        return ImmutableList.copyOf(map.keySet());
+    }
+}
diff --git a/digdag-server/src/main/java/io/digdag/server/ServerModule.java b/digdag-server/src/main/java/io/digdag/server/ServerModule.java
@@ -31,6 +31,7 @@
 import io.digdag.server.rs.VersionResource;
 import io.digdag.server.rs.WorkflowResource;
 import io.digdag.server.rs.project.PutProjectsValidator;
+import io.digdag.server.service.ProjectService;
 import io.digdag.server.service.WorkflowService;
 import io.digdag.spi.AuthenticatedUser;
 import io.digdag.spi.Authenticator;
@@ -103,6 +104,7 @@ protected void bindSecrets()
     protected void bindResources(ApplicationBindingBuilder builder)
     {
         binder().bind(WorkflowService.class).in(Scopes.SINGLETON);
+        binder().bind(ProjectService.class).in(Scopes.SINGLETON);
         builder.addResources(
                 ProjectResource.class,
                 WorkflowResource.class,
diff --git a/digdag-server/src/main/java/io/digdag/server/rs/ProjectResource.java b/digdag-server/src/main/java/io/digdag/server/rs/ProjectResource.java
diff --git a/digdag-server/src/main/java/io/digdag/server/rs/RestModels.java b/digdag-server/src/main/java/io/digdag/server/rs/RestModels.java
diff --git a/digdag-server/src/main/java/io/digdag/server/service/ProjectService.java b/digdag-server/src/main/java/io/digdag/server/service/ProjectService.java
diff --git a/digdag-tests/src/test/java/utils/TestUtils.java b/digdag-tests/src/test/java/utils/TestUtils.java

Original file line number	Diff line number	Diff line change
`@@ -452,7 +452,13 @@ public RestProject putProjectRevision(String projName, String revision, File bod`
`452`	`452`	`return putProjectRevision(projName, revision, body, scheduleFrom, Collections.emptyList(), Optional.absent());`
`453`	`453`	`}`
`454`	`454`
`455`		`- public RestProject putProjectRevision(String projName, String revision, File body, Optional<Instant> scheduleFrom, List<String> clearSchedule, Optional<Boolean> clearAllSchedules )`
	`455`	`+ public RestProject putProjectRevision(`
	`456`	`+ String projName,`
	`457`	`+ String revision,`
	`458`	`+ File body,`
	`459`	`+ Optional<Instant> scheduleFrom,`
	`460`	`+ List<String> clearSchedule,`
	`461`	`+ Optional<Boolean> clearAllSchedules)`
`456`	`462`	`throws IOException`
`457`	`463`	`{`
`458`	`464`	`Preconditions.checkArgument(!Strings.isNullOrEmpty(projName), "projName");`