Merge branch 'main' of github.com:MarquezProject/marquez into prepare…

…-changelog-0.28.0

Signed-off-by: Michael Robinson <merobi@gmail.com>

.circleci/get-jdk17.sh

@@ -14,11 +14,9 @@
 #
 # Usage: $ ./get-jdk17.sh
 
-set -e
-
 wget -qO - https://adoptium.jfrog.io/adoptium/api/gpg/key/public | sudo apt-key add -
 sudo add-apt-repository --yes https://adoptium.jfrog.io/adoptium/deb
-sudo apt-get update && sudo apt-get install temurin-17-jdk
+sudo apt-get update --allow-releaseinfo-change && sudo apt-get install --yes temurin-17-jdk
 sudo update-alternatives --set java /usr/lib/jvm/temurin-17-jdk-amd64/bin/java
 sudo update-alternatives --set javac /usr/lib/jvm/temurin-17-jdk-amd64/bin/javac
 java -version

.github/workflows/headerchecker.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Check for headers
         run: |
           ok=1
-          readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.all }}')"
+          readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.added_modified }}')"
           for file in ${files[@]}; do 
             if [[ ($file == *".java") ]]; then
               if ! grep -q Copyright "$file"; then

.github/workflows/test-chart.yaml

@@ -16,15 +16,15 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Helm
-        uses: azure/setup-helm@v2.2
+        uses: azure/setup-helm@v3.4
 
       - name: Setup Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
 
       - name: Setup chart-testing
-        uses: helm/chart-testing-action@v2.3.0
+        uses: helm/chart-testing-action@v2.3.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed

CHANGELOG.md

@@ -2,21 +2,29 @@
 
 ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.28.0...HEAD)
 
-## [0.28.0](https://github.com/MarquezProject/marquez/compare/0.27.0...0.28.0) - 2022-11-15
+## [0.28.0](https://github.com/MarquezProject/marquez/compare/0.27.0...0.28.0) - 2022-11-21
 
 ### Added
 
 * Optimize current runs query for lineage API [`#2211`](https://github.com/MarquezProject/marquez/pull/2211) [@prachim-collab](https://github.com/prachim-collab)   
     *Add a simpler, alternate `getCurrentRuns` query that gets only simple runs from the database without the additional data from tables such as `run_args`, `job_context`, `facets`, etc., which required extra table joins.*
 * Add Code Quality, DCO and Governance docs to project [`#2237`](https://github.com/MarquezProject/marquez/pull/2237) [`#2241`](https://github.com/MarquezProject/marquez/pull/2241) [@merobi-hub](https://github.com/MarquezProject/marquez/commits?author=merobi-hub)  
     *Adds a number of standard governance and procedure docs to the project.*
+* Add possibility to soft-delete namespaces [`#2244`](https://github.com/MarquezProject/marquez/pull/2244) [@mobuchowski](https://github.com/mobuchowski)  
+    *Adds the ability to "hide" inactive namespaces. The namespaces are undeleted when a relevant OL event is received.*
+* Add search service proposal [`#2203`](https://github.com/MarquezProject/marquez/pull/2203) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  
+    *Proposes using ElasticSearch as a pluggable search service to enhance the search feature in Marquez and adding the ability to turn it off, as well. Includes ideas about what should be indexed and the requirements for the interface.*
 
 ### Fixed
 
 * Show facets even when dataset has no fields [`#2214`](https://github.com/MarquezProject/marquez/pull/2214) [@JDarDagran](https://github.com/JDarDagran)  
     *Changes the logic in the `DatasetInfo` component to always show facets so that dataset facets are visible in the UI even if no dataset fields have been set.* 
 * Appreciate column prefix when given for `ended_at` [`#2231`](https://github.com/MarquezProject/marquez/pull/2231) [@fm100](https://github.com/fm100)  
     *The `ended_at` column was always null when querying if `columnPrefix` was given for the mapper. Now, `columnPrefix` is included when checking for column existence.*
+* Fix bug keeping jobs from being properly deleted [`#2244`](https://github.com/MarquezProject/marquez/pull/2244) [@mobuchowski](https://github.com/mobuchowski)  
+    *It wasn't possible to delete jobs created from events that had a `ParentRunFacet`. Now it's possible.*
+* Fix symlink table column length ['#2217'](https://github.com/MarquezProject/marquez/pull/2217) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  
+    *The dataset's name column in the `dataset_symlinks` table was shorter than the column in the datasets table. Changes the existing V48 migration script to allow proper migration for users who did not upgrade yet, and adds an extra migration script to extend the column length for users who did upgrade but did not experience the issues.*
 
 
 ## [0.27.0](https://github.com/MarquezProject/marquez/compare/0.26.0...0.27.0) - 2022-10-24

api/src/main/java/marquez/api/JobResource.java

@@ -176,7 +176,8 @@ public Response delete(
             .findJobByName(namespaceName.getValue(), jobName.getValue())
             .orElseThrow(() -> new JobNotFoundException(jobName));
 
-    jobService.delete(namespaceName.getValue(), jobName.getValue());
+    // Should be simple name from `jobs_fqn`.
+    jobService.delete(namespaceName.getValue(), job.getSimpleName());
     return Response.ok(job).build();
   }
 

api/src/main/java/marquez/api/NamespaceResource.java

@@ -15,6 +15,7 @@
 import javax.validation.Valid;
 import javax.validation.constraints.Min;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.PUT;
@@ -77,6 +78,23 @@ public Response list(
     return Response.ok(new Namespaces(namespaces)).build();
   }
 
+  @Timed
+  @ResponseMetered
+  @ExceptionMetered
+  @DELETE
+  @Path("/namespaces/{namespace}")
+  @Produces(APPLICATION_JSON)
+  public Response delete(@PathParam("namespace") NamespaceName name) {
+    final Namespace namespace =
+        namespaceService
+            .findBy(name.getValue())
+            .orElseThrow(() -> new NamespaceNotFoundException(name));
+    datasetService.deleteByNamespaceName(namespace.getName().getValue());
+    jobService.deleteByNamespaceName(namespace.getName().getValue());
+    namespaceService.delete(namespace.getName().getValue());
+    return Response.ok(namespace).build();
+  }
+
   @Value
   static class Namespaces {
     @NonNull

api/src/main/java/marquez/db/Columns.java

@@ -55,6 +55,7 @@ private Columns() {}
   public static final String DATASET_NAME = "dataset_name";
   public static final String FACETS = "facets";
   public static final String TAGS = "tags";
+  public static final String IS_HIDDEN = "is_hidden";
 
   /* NAMESPACE ROW COLUMNS */
   public static final String CURRENT_OWNER_NAME = "current_owner_name";
@@ -197,6 +198,14 @@ public static boolean booleanOrDefault(
     return results.getBoolean(column);
   }
 
+  public static boolean booleanOrThrow(final ResultSet results, final String column)
+      throws SQLException {
+    if (results.getObject(column) == null) {
+      throw new IllegalArgumentException();
+    }
+    return results.getBoolean(column);
+  }
+
   public static int intOrThrow(final ResultSet results, final String column) throws SQLException {
     if (results.getObject(column) == null) {
       throw new IllegalArgumentException();

api/src/main/java/marquez/db/DatasetDao.java

@@ -292,15 +292,24 @@ DatasetRow upsert(
       String name,
       String physicalName);
 
+  @SqlUpdate(
+      """
+    UPDATE datasets d
+    SET is_hidden = true
+    FROM namespaces n
+    WHERE n.uuid=d.namespace_uuid
+    AND n.name=:namespaceName
+  """)
+  void deleteByNamespaceName(String namespaceName);
+
   @SqlQuery(
       """
-  UPDATE datasets
-  SET is_hidden = true
-  FROM dataset_symlinks, namespaces
-  WHERE dataset_symlinks.dataset_uuid = datasets.uuid
-  AND namespaces.uuid = dataset_symlinks.namespace_uuid
-  AND namespaces.name=:namespaceName AND dataset_symlinks.name=:name
-  RETURNING *
+    UPDATE datasets d
+    SET is_hidden = true
+    FROM namespaces n
+    WHERE n.uuid = d.namespace_uuid
+    AND n.name=:namespaceName AND d.name=:name
+    RETURNING *
   """)
   Optional<DatasetRow> delete(String namespaceName, String name);
 

api/src/main/java/marquez/db/JobDao.java

@@ -88,6 +88,16 @@ SELECT run_uuid, JSON_AGG(e.facets) AS facets
   """)
   void delete(String namespaceName, String name);
 
+  @SqlUpdate(
+      """
+  UPDATE jobs
+  SET is_hidden = true
+  FROM namespaces n
+  WHERE jobs.namespace_uuid = n.uuid
+  AND n.name = :namespaceName
+  """)
+  void deleteByNamespaceName(String namespaceName);
+
   default Optional<Job> findWithRun(String namespaceName, String jobName) {
     Optional<Job> job = findJobByName(namespaceName, jobName);
     job.ifPresent(

api/src/main/java/marquez/db/NamespaceDao.java

@@ -78,10 +78,20 @@ default Namespace upsertNamespaceMeta(
   @SqlQuery("SELECT * FROM namespaces ORDER BY name LIMIT :limit OFFSET :offset")
   List<Namespace> findAll(int limit, int offset);
 
+  @SqlQuery("UPDATE namespaces SET is_hidden=false WHERE name = :name RETURNING *")
+  NamespaceRow undelete(String name);
+
+  @SqlUpdate("UPDATE namespaces SET is_hidden=true WHERE name = :name")
+  void delete(String name);
+
   default NamespaceRow upsertNamespaceRow(
       UUID uuid, Instant now, String name, String currentOwnerName) {
     doUpsertNamespaceRow(uuid, now, name, currentOwnerName);
-    return findNamespaceByName(name).orElseThrow();
+    NamespaceRow namespaceRow = findNamespaceByName(name).orElseThrow();
+    if (namespaceRow.getIsHidden()) {
+      namespaceRow = undelete(namespaceRow.getName());
+    }
+    return namespaceRow;
   }
 
   /**
@@ -99,40 +109,47 @@ default NamespaceRow upsertNamespaceRow(
    * @param currentOwnerName
    */
   @SqlUpdate(
-      "INSERT INTO namespaces ( "
-          + "uuid, "
-          + "created_at, "
-          + "updated_at, "
-          + "name, "
-          + "current_owner_name "
-          + ") VALUES ("
-          + ":uuid, "
-          + ":now, "
-          + ":now, "
-          + ":name, "
-          + ":currentOwnerName) "
-          + "ON CONFLICT(name) DO NOTHING")
+      """
+    INSERT INTO namespaces (
+      uuid,
+      created_at,
+      updated_at,
+      name,
+      current_owner_name
+      ) VALUES (
+      :uuid,
+      :now,
+      :now,
+      :name,
+      :currentOwnerName)
+      ON CONFLICT(name) DO NOTHING
+  """)
   void doUpsertNamespaceRow(UUID uuid, Instant now, String name, String currentOwnerName);
 
   @SqlQuery(
-      "INSERT INTO namespaces ( "
-          + "uuid, "
-          + "created_at, "
-          + "updated_at, "
-          + "name, "
-          + "current_owner_name, "
-          + "description "
-          + ") VALUES ("
-          + ":uuid, "
-          + ":now, "
-          + ":now, "
-          + ":name, "
-          + ":currentOwnerName, "
-          + ":description "
-          + ") ON CONFLICT(name) DO "
-          + "UPDATE SET "
-          + "updated_at = EXCLUDED.updated_at "
-          + "RETURNING *")
+      """
+    INSERT INTO namespaces (
+      uuid,
+      created_at,
+      updated_at,
+      name,
+      current_owner_name,
+      description,
+      is_hidden
+      ) VALUES (
+      :uuid,
+      :now,
+      :now,
+      :name,
+      :currentOwnerName,
+      :description,
+      false
+      ) ON CONFLICT(name) DO
+      UPDATE SET
+      updated_at = EXCLUDED.updated_at,
+      is_hidden = false
+      RETURNING *
+    """)
   NamespaceRow upsertNamespaceRow(
       UUID uuid, Instant now, String name, String currentOwnerName, String description);
 

api/src/main/java/marquez/db/mappers/NamespaceMapper.java

@@ -5,6 +5,7 @@
 
 package marquez.db.mappers;
 
+import static marquez.db.Columns.booleanOrThrow;
 import static marquez.db.Columns.stringOrNull;
 import static marquez.db.Columns.stringOrThrow;
 import static marquez.db.Columns.timestampOrThrow;
@@ -28,6 +29,7 @@ public Namespace map(@NonNull ResultSet results, @NonNull StatementContext conte
         timestampOrThrow(results, Columns.CREATED_AT),
         timestampOrThrow(results, Columns.UPDATED_AT),
         OwnerName.of(stringOrThrow(results, Columns.CURRENT_OWNER_NAME)),
-        stringOrNull(results, Columns.DESCRIPTION));
+        stringOrNull(results, Columns.DESCRIPTION),
+        booleanOrThrow(results, Columns.IS_HIDDEN));
   }
 }

api/src/main/java/marquez/db/mappers/NamespaceRowMapper.java

@@ -5,6 +5,7 @@
 
 package marquez.db.mappers;
 
+import static marquez.db.Columns.booleanOrThrow;
 import static marquez.db.Columns.stringOrNull;
 import static marquez.db.Columns.stringOrThrow;
 import static marquez.db.Columns.timestampOrThrow;
@@ -28,6 +29,7 @@ public NamespaceRow map(@NonNull ResultSet results, @NonNull StatementContext co
         timestampOrThrow(results, Columns.UPDATED_AT),
         stringOrThrow(results, Columns.NAME),
         stringOrNull(results, Columns.DESCRIPTION),
-        stringOrThrow(results, Columns.CURRENT_OWNER_NAME));
+        stringOrThrow(results, Columns.CURRENT_OWNER_NAME),
+        booleanOrThrow(results, Columns.IS_HIDDEN));
   }
 }

api/src/main/java/marquez/db/models/NamespaceRow.java

@@ -20,6 +20,7 @@ public class NamespaceRow {
   @NonNull String name;
   @Nullable String description;
   @NonNull String currentOwnerName;
+  @NonNull Boolean isHidden;
 
   public Optional<String> getDescription() {
     return Optional.ofNullable(description);

api/src/main/java/marquez/service/models/Namespace.java

@@ -20,6 +20,7 @@ public class Namespace {
   @NonNull Instant updatedAt;
   @NonNull OwnerName ownerName;
   @Nullable String description;
+  @NonNull Boolean isHidden;
 
   public Optional<String> getDescription() {
     return Optional.ofNullable(description);

api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql

@@ -2,7 +2,7 @@
 
 CREATE TABLE dataset_symlinks (
   dataset_uuid      UUID,
-  name              VARCHAR(255) NOT NULL,
+  name              VARCHAR NOT NULL,
   namespace_uuid    UUID REFERENCES namespaces(uuid),
   type              VARCHAR(64),
   is_primary        BOOLEAN DEFAULT FALSE,

api/src/main/resources/marquez/db/migration/V52__alter_dataset_symlinks.sql

@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+ALTER TABLE dataset_symlinks ALTER COLUMN name TYPE VARCHAR;

api/src/main/resources/marquez/db/migration/V53__add_hidden_namespace.sql

@@ -0,0 +1 @@
+ALTER TABLE namespaces ADD COLUMN is_hidden BOOLEAN DEFAULT FALSE;