diff --git a/.github/workflows/headerchecker.yml b/.github/workflows/headerchecker.yml index db0c3a53b4..02986dd8b7 100644 --- a/.github/workflows/headerchecker.yml +++ b/.github/workflows/headerchecker.yml @@ -31,7 +31,7 @@ jobs: - name: Check for headers run: | ok=1 - readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.all }}')" + readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.added_modified }}')" for file in ${files[@]}; do if [[ ($file == *".java") ]]; then if ! grep -q Copyright "$file"; then diff --git a/CHANGELOG.md b/CHANGELOG.md index f75aed75fd..0b5de1de05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.27.0...HEAD) +### Added +* Add possibility to soft-delete namespaces [`#2244`](https://github.com/MarquezProject/marquez/pull/2244) [@mobuchowski](https://github.com/mobuchowski) + *Adds the ability to "hide" inactive namespaces. The namespaces are being undeleted when relevant OL event is received.* + +### Fixed +* Fix bug where job isn't properly deleted [`#2244`](https://github.com/MarquezProject/marquez/pull/2244) [@mobuchowski](https://github.com/mobuchowski) + *It wasn't possible to delete jobs created from events that had `ParentRunFacet`. Now it's possible.* + + ## [0.27.0](https://github.com/MarquezProject/marquez/compare/0.26.0...0.27.0) - 2022-10-24 ### Added diff --git a/api/src/main/java/marquez/api/JobResource.java b/api/src/main/java/marquez/api/JobResource.java index 199c4ac2e3..4230a26342 100644 --- a/api/src/main/java/marquez/api/JobResource.java +++ b/api/src/main/java/marquez/api/JobResource.java @@ -176,7 +176,8 @@ public Response delete( .findJobByName(namespaceName.getValue(), jobName.getValue()) .orElseThrow(() -> new JobNotFoundException(jobName)); - jobService.delete(namespaceName.getValue(), jobName.getValue()); + // Should be simple name from `jobs_fqn`. + jobService.delete(namespaceName.getValue(), job.getSimpleName()); return Response.ok(job).build(); } diff --git a/api/src/main/java/marquez/api/NamespaceResource.java b/api/src/main/java/marquez/api/NamespaceResource.java index bb81c4e5ec..4f14ef4f57 100644 --- a/api/src/main/java/marquez/api/NamespaceResource.java +++ b/api/src/main/java/marquez/api/NamespaceResource.java @@ -15,6 +15,7 @@ import javax.validation.Valid; import javax.validation.constraints.Min; import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.PUT; @@ -77,6 +78,23 @@ public Response list( return Response.ok(new Namespaces(namespaces)).build(); } + @Timed + @ResponseMetered + @ExceptionMetered + @DELETE + @Path("/namespaces/{namespace}") + @Produces(APPLICATION_JSON) + public Response delete(@PathParam("namespace") NamespaceName name) { + final Namespace namespace = + namespaceService + .findBy(name.getValue()) + .orElseThrow(() -> new NamespaceNotFoundException(name)); + datasetService.deleteByNamespaceName(namespace.getName().getValue()); + jobService.deleteByNamespaceName(namespace.getName().getValue()); + namespaceService.delete(namespace.getName().getValue()); + return Response.ok(namespace).build(); + } + @Value static class Namespaces { @NonNull diff --git a/api/src/main/java/marquez/db/Columns.java b/api/src/main/java/marquez/db/Columns.java index 73d95f10a9..21678a2a55 100644 --- a/api/src/main/java/marquez/db/Columns.java +++ b/api/src/main/java/marquez/db/Columns.java @@ -55,6 +55,7 @@ private Columns() {} public static final String DATASET_NAME = "dataset_name"; public static final String FACETS = "facets"; public static final String TAGS = "tags"; + public static final String IS_HIDDEN = "is_hidden"; /* NAMESPACE ROW COLUMNS */ public static final String CURRENT_OWNER_NAME = "current_owner_name"; @@ -197,6 +198,14 @@ public static boolean booleanOrDefault( return results.getBoolean(column); } + public static boolean booleanOrThrow(final ResultSet results, final String column) + throws SQLException { + if (results.getObject(column) == null) { + throw new IllegalArgumentException(); + } + return results.getBoolean(column); + } + public static int intOrThrow(final ResultSet results, final String column) throws SQLException { if (results.getObject(column) == null) { throw new IllegalArgumentException(); diff --git a/api/src/main/java/marquez/db/DatasetDao.java b/api/src/main/java/marquez/db/DatasetDao.java index 64f7faee25..5cc80392f9 100644 --- a/api/src/main/java/marquez/db/DatasetDao.java +++ b/api/src/main/java/marquez/db/DatasetDao.java @@ -292,15 +292,24 @@ DatasetRow upsert( String name, String physicalName); + @SqlUpdate( + """ + UPDATE datasets d + SET is_hidden = true + FROM namespaces n + WHERE n.uuid=d.namespace_uuid + AND n.name=:namespaceName + """) + void deleteByNamespaceName(String namespaceName); + @SqlQuery( """ - UPDATE datasets - SET is_hidden = true - FROM dataset_symlinks, namespaces - WHERE dataset_symlinks.dataset_uuid = datasets.uuid - AND namespaces.uuid = dataset_symlinks.namespace_uuid - AND namespaces.name=:namespaceName AND dataset_symlinks.name=:name - RETURNING * + UPDATE datasets d + SET is_hidden = true + FROM namespaces n + WHERE n.uuid = d.namespace_uuid + AND n.name=:namespaceName AND d.name=:name + RETURNING * """) Optional delete(String namespaceName, String name); diff --git a/api/src/main/java/marquez/db/JobDao.java b/api/src/main/java/marquez/db/JobDao.java index 1a5ea2ad64..6805673d66 100644 --- a/api/src/main/java/marquez/db/JobDao.java +++ b/api/src/main/java/marquez/db/JobDao.java @@ -88,6 +88,16 @@ SELECT run_uuid, JSON_AGG(e.facets) AS facets """) void delete(String namespaceName, String name); + @SqlUpdate( + """ + UPDATE jobs + SET is_hidden = true + FROM namespaces n + WHERE jobs.namespace_uuid = n.uuid + AND n.name = :namespaceName + """) + void deleteByNamespaceName(String namespaceName); + default Optional findWithRun(String namespaceName, String jobName) { Optional job = findJobByName(namespaceName, jobName); job.ifPresent( diff --git a/api/src/main/java/marquez/db/NamespaceDao.java b/api/src/main/java/marquez/db/NamespaceDao.java index a7f36e2336..03521cce51 100644 --- a/api/src/main/java/marquez/db/NamespaceDao.java +++ b/api/src/main/java/marquez/db/NamespaceDao.java @@ -78,10 +78,20 @@ default Namespace upsertNamespaceMeta( @SqlQuery("SELECT * FROM namespaces ORDER BY name LIMIT :limit OFFSET :offset") List findAll(int limit, int offset); + @SqlQuery("UPDATE namespaces SET is_hidden=false WHERE name = :name RETURNING *") + NamespaceRow undelete(String name); + + @SqlUpdate("UPDATE namespaces SET is_hidden=true WHERE name = :name") + void delete(String name); + default NamespaceRow upsertNamespaceRow( UUID uuid, Instant now, String name, String currentOwnerName) { doUpsertNamespaceRow(uuid, now, name, currentOwnerName); - return findNamespaceByName(name).orElseThrow(); + NamespaceRow namespaceRow = findNamespaceByName(name).orElseThrow(); + if (namespaceRow.getIsHidden()) { + namespaceRow = undelete(namespaceRow.getName()); + } + return namespaceRow; } /** @@ -99,40 +109,47 @@ default NamespaceRow upsertNamespaceRow( * @param currentOwnerName */ @SqlUpdate( - "INSERT INTO namespaces ( " - + "uuid, " - + "created_at, " - + "updated_at, " - + "name, " - + "current_owner_name " - + ") VALUES (" - + ":uuid, " - + ":now, " - + ":now, " - + ":name, " - + ":currentOwnerName) " - + "ON CONFLICT(name) DO NOTHING") + """ + INSERT INTO namespaces ( + uuid, + created_at, + updated_at, + name, + current_owner_name + ) VALUES ( + :uuid, + :now, + :now, + :name, + :currentOwnerName) + ON CONFLICT(name) DO NOTHING + """) void doUpsertNamespaceRow(UUID uuid, Instant now, String name, String currentOwnerName); @SqlQuery( - "INSERT INTO namespaces ( " - + "uuid, " - + "created_at, " - + "updated_at, " - + "name, " - + "current_owner_name, " - + "description " - + ") VALUES (" - + ":uuid, " - + ":now, " - + ":now, " - + ":name, " - + ":currentOwnerName, " - + ":description " - + ") ON CONFLICT(name) DO " - + "UPDATE SET " - + "updated_at = EXCLUDED.updated_at " - + "RETURNING *") + """ + INSERT INTO namespaces ( + uuid, + created_at, + updated_at, + name, + current_owner_name, + description, + is_hidden + ) VALUES ( + :uuid, + :now, + :now, + :name, + :currentOwnerName, + :description, + false + ) ON CONFLICT(name) DO + UPDATE SET + updated_at = EXCLUDED.updated_at, + is_hidden = false + RETURNING * + """) NamespaceRow upsertNamespaceRow( UUID uuid, Instant now, String name, String currentOwnerName, String description); diff --git a/api/src/main/java/marquez/db/mappers/NamespaceMapper.java b/api/src/main/java/marquez/db/mappers/NamespaceMapper.java index eb50079a73..4d51d9fd7f 100644 --- a/api/src/main/java/marquez/db/mappers/NamespaceMapper.java +++ b/api/src/main/java/marquez/db/mappers/NamespaceMapper.java @@ -5,6 +5,7 @@ package marquez.db.mappers; +import static marquez.db.Columns.booleanOrThrow; import static marquez.db.Columns.stringOrNull; import static marquez.db.Columns.stringOrThrow; import static marquez.db.Columns.timestampOrThrow; @@ -28,6 +29,7 @@ public Namespace map(@NonNull ResultSet results, @NonNull StatementContext conte timestampOrThrow(results, Columns.CREATED_AT), timestampOrThrow(results, Columns.UPDATED_AT), OwnerName.of(stringOrThrow(results, Columns.CURRENT_OWNER_NAME)), - stringOrNull(results, Columns.DESCRIPTION)); + stringOrNull(results, Columns.DESCRIPTION), + booleanOrThrow(results, Columns.IS_HIDDEN)); } } diff --git a/api/src/main/java/marquez/db/mappers/NamespaceRowMapper.java b/api/src/main/java/marquez/db/mappers/NamespaceRowMapper.java index 27541dd8ff..d3799a19e1 100644 --- a/api/src/main/java/marquez/db/mappers/NamespaceRowMapper.java +++ b/api/src/main/java/marquez/db/mappers/NamespaceRowMapper.java @@ -5,6 +5,7 @@ package marquez.db.mappers; +import static marquez.db.Columns.booleanOrThrow; import static marquez.db.Columns.stringOrNull; import static marquez.db.Columns.stringOrThrow; import static marquez.db.Columns.timestampOrThrow; @@ -28,6 +29,7 @@ public NamespaceRow map(@NonNull ResultSet results, @NonNull StatementContext co timestampOrThrow(results, Columns.UPDATED_AT), stringOrThrow(results, Columns.NAME), stringOrNull(results, Columns.DESCRIPTION), - stringOrThrow(results, Columns.CURRENT_OWNER_NAME)); + stringOrThrow(results, Columns.CURRENT_OWNER_NAME), + booleanOrThrow(results, Columns.IS_HIDDEN)); } } diff --git a/api/src/main/java/marquez/db/models/NamespaceRow.java b/api/src/main/java/marquez/db/models/NamespaceRow.java index de032cc382..58aa7bc17d 100644 --- a/api/src/main/java/marquez/db/models/NamespaceRow.java +++ b/api/src/main/java/marquez/db/models/NamespaceRow.java @@ -20,6 +20,7 @@ public class NamespaceRow { @NonNull String name; @Nullable String description; @NonNull String currentOwnerName; + @NonNull Boolean isHidden; public Optional getDescription() { return Optional.ofNullable(description); diff --git a/api/src/main/java/marquez/service/models/Namespace.java b/api/src/main/java/marquez/service/models/Namespace.java index d5a876f496..b51d359e4c 100644 --- a/api/src/main/java/marquez/service/models/Namespace.java +++ b/api/src/main/java/marquez/service/models/Namespace.java @@ -20,6 +20,7 @@ public class Namespace { @NonNull Instant updatedAt; @NonNull OwnerName ownerName; @Nullable String description; + @NonNull Boolean isHidden; public Optional getDescription() { return Optional.ofNullable(description); diff --git a/api/src/main/resources/marquez/db/migration/V53__add_hidden_namespace.sql b/api/src/main/resources/marquez/db/migration/V53__add_hidden_namespace.sql new file mode 100644 index 0000000000..1249955cf9 --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V53__add_hidden_namespace.sql @@ -0,0 +1 @@ +ALTER TABLE namespaces ADD COLUMN is_hidden BOOLEAN DEFAULT FALSE; diff --git a/api/src/test/java/marquez/DatasetIntegrationTest.java b/api/src/test/java/marquez/DatasetIntegrationTest.java index 718428285a..d0e31e5f8a 100644 --- a/api/src/test/java/marquez/DatasetIntegrationTest.java +++ b/api/src/test/java/marquez/DatasetIntegrationTest.java @@ -30,7 +30,9 @@ import marquez.client.models.DatasetId; import marquez.client.models.DatasetVersion; import marquez.client.models.DbTableMeta; +import marquez.client.models.Job; import marquez.client.models.JobMeta; +import marquez.client.models.Namespace; import marquez.client.models.Run; import marquez.client.models.RunMeta; import marquez.client.models.StreamVersion; @@ -139,15 +141,7 @@ public void testApp_getTableVersions() { .build())) .build(); - final CompletableFuture resp = - this.sendLineage(Utils.toJson(lineageEvent)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); + final CompletableFuture resp = sendEvent(lineageEvent); assertThat(resp.join()).isEqualTo(201); datasetFacets.setAdditional(inputFacets); @@ -170,15 +164,7 @@ public void testApp_getTableVersions() { .outputs(Collections.emptyList()) .build(); - final CompletableFuture readResp = - this.sendLineage(Utils.toJson(readEvent)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); + final CompletableFuture readResp = sendEvent(readEvent); assertThat(readResp.join()).isEqualTo(201); // update dataset facet to include input and output facets @@ -389,17 +375,7 @@ public void testApp_doesNotShowDeletedDataset() throws IOException { Collections.emptyList(), "the_producer"); - final CompletableFuture resp = - this.sendLineage(Utils.toJson(event)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); - - // Ensure the event was correctly rejected and a proper response code returned. + final CompletableFuture resp = sendEvent(event); assertThat(resp.join()).isEqualTo(201); client.deleteDataset(namespace, name); @@ -422,33 +398,14 @@ public void testApp_showsDeletedDatasetAfterReceivingNewVersion() throws IOExcep Collections.emptyList(), "the_producer"); - CompletableFuture resp = - this.sendLineage(Utils.toJson(event)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); - - // Ensure the event was correctly rejected and a proper response code returned. + CompletableFuture resp = sendEvent(event); assertThat(resp.join()).isEqualTo(201); client.deleteDataset(namespace, name); List datasets = client.listDatasets(namespace); assertThat(datasets).hasSize(0); - resp = - this.sendLineage(Utils.toJson(event)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); - + resp = sendEvent(event); assertThat(resp.join()).isEqualTo(201); datasets = client.listDatasets(namespace); @@ -495,4 +452,109 @@ public void testApp_getDatasetContainsColumnLineage() { assertThat(columnLineage).hasSize(1); assertThat(columnLineage.get(0).getInputFields()).hasSize(2); } + + @Test + public void testApp_doesNotShowDeletedDatasetAfterDeleteNamespace() throws IOException { + String namespace = "namespace"; + String name = "table"; + LineageEvent event = + new LineageEvent( + "COMPLETE", + Instant.now().atZone(ZoneId.systemDefault()), + new LineageEvent.Run(UUID.randomUUID().toString(), null), + new LineageEvent.Job("namespace", "job_name", null), + List.of(new LineageEvent.Dataset(namespace, name, LineageTestUtils.newDatasetFacet())), + Collections.emptyList(), + "the_producer"); + + final CompletableFuture resp = sendEvent(event); + assertThat(resp.join()).isEqualTo(201); + + client.deleteNamespace(namespace); + + List datasets = client.listDatasets(namespace); + assertThat(datasets).hasSize(0); + } + + @Test + public void testApp_doesNotShowDeletedDatasetAfterUndeleteNamespace() throws IOException { + String namespaceName = "namespace"; + String name = "table"; + + LineageEvent firstEvent = + new LineageEvent( + "COMPLETE", + Instant.now().atZone(ZoneId.systemDefault()), + new LineageEvent.Run(UUID.randomUUID().toString(), null), + new LineageEvent.Job(namespaceName, "job_name", null), + List.of( + new LineageEvent.Dataset(namespaceName, name, LineageTestUtils.newDatasetFacet())), + Collections.emptyList(), + "the_producer"); + + LineageEvent secondEvent = + new LineageEvent( + "COMPLETE", + Instant.now().atZone(ZoneId.systemDefault()), + new LineageEvent.Run(UUID.randomUUID().toString(), null), + new LineageEvent.Job(namespaceName, "second_job_name", null), + List.of( + new LineageEvent.Dataset( + namespaceName, name + "2", LineageTestUtils.newDatasetFacet())), + Collections.emptyList(), + "the_producer"); + + CompletableFuture resp = sendEvent(firstEvent); + assertThat(resp.join()).isEqualTo(201); + + resp = sendEvent(secondEvent); + assertThat(resp.join()).isEqualTo(201); + + List datasets = client.listDatasets(namespaceName); + assertThat(datasets).hasSize(2); + + client.deleteNamespace(namespaceName); + + List namespaces = client.listNamespaces(); + assertThat(namespaces) + .anySatisfy( + namespace -> { + assertThat(namespace.getIsHidden()).isTrue(); + assertThat(namespace.getName()).isEqualTo(namespaceName); + }); + + datasets = client.listDatasets(namespaceName); + assertThat(datasets).hasSize(0); + + List jobs = client.listJobs(namespaceName); + assertThat(jobs).hasSize(0); + + LineageEvent eventThatWillUndeleteNamespace = + new LineageEvent( + "COMPLETE", + Instant.now().atZone(ZoneId.systemDefault()), + new LineageEvent.Run(UUID.randomUUID().toString(), null), + new LineageEvent.Job(namespaceName, "job_name", null), + List.of( + new LineageEvent.Dataset(namespaceName, name, LineageTestUtils.newDatasetFacet())), + Collections.emptyList(), + "the_producer"); + + resp = sendEvent(eventThatWillUndeleteNamespace); + assertThat(resp.join()).isEqualTo(201); + + namespaces = client.listNamespaces(); + assertThat(namespaces) + .anySatisfy( + namespace -> { + assertThat(namespace.getIsHidden()).isFalse(); + assertThat(namespace.getName()).isEqualTo(namespaceName); + }); + + datasets = client.listDatasets(namespaceName); + assertThat(datasets).hasSize(1); + + jobs = client.listJobs(namespaceName); + assertThat(jobs).hasSize(1); + } } diff --git a/api/src/test/java/marquez/NamespaceIntegrationTest.java b/api/src/test/java/marquez/NamespaceIntegrationTest.java index 4f46dc1d8a..08d74d83e9 100644 --- a/api/src/test/java/marquez/NamespaceIntegrationTest.java +++ b/api/src/test/java/marquez/NamespaceIntegrationTest.java @@ -66,5 +66,21 @@ public void testApp_getNamespace() { assertThat(namespace.getUpdatedAt()).isNotNull(); assertThat(namespace.getCreatedAt()).isNotNull(); assertThat(namespace.getDescription().get()).isEqualTo(NAMESPACE_DESCRIPTION); + assertThat(namespace.getIsHidden()).isFalse(); + } + + @Test + public void testApp_deleteNamespace() { + NamespaceMeta namespaceMeta = + NamespaceMeta.builder().ownerName(OWNER_NAME).description(NAMESPACE_DESCRIPTION).build(); + client.createNamespace(NAMESPACE_NAME, namespaceMeta); + Namespace namespace = client.getNamespace(NAMESPACE_NAME); + assertThat(namespace.getName()).isEqualTo(NAMESPACE_NAME); + assertThat(namespace.getIsHidden()).isFalse(); + + client.deleteNamespace(NAMESPACE_NAME); + namespace = client.getNamespace(NAMESPACE_NAME); + assertThat(namespace.getName()).isEqualTo(NAMESPACE_NAME); + assertThat(namespace.getIsHidden()).isTrue(); } } diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java index bd5cc6f0c2..540399d184 100644 --- a/api/src/test/java/marquez/OpenLineageIntegrationTest.java +++ b/api/src/test/java/marquez/OpenLineageIntegrationTest.java @@ -5,6 +5,8 @@ package marquez; +import static marquez.db.LineageTestUtils.PRODUCER_URL; +import static marquez.db.LineageTestUtils.SCHEMA_URL; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -105,17 +107,7 @@ public void testSendOpenLineageBadArgument() throws IOException { Collections.emptyList(), "the_producer"); - final CompletableFuture resp = - this.sendLineage(Utils.toJson(event)) - .thenApply(HttpResponse::statusCode) - .whenComplete( - (val, error) -> { - if (error != null) { - Assertions.fail("Could not complete request"); - } - }); - - // Ensure the event was correctly rejected and a proper response code returned. + final CompletableFuture resp = sendEvent(event); assertThat(resp.join()).isEqualTo(400); } @@ -890,6 +882,65 @@ public void testFindEventBeforeAfterTime() { .isEqualTo(mapper.valueToTree(rawEvents.get(0))); } + @Test + public void testSendAndDeleteParentRunRelationshipFacet() { + marquez.service.models.LineageEvent.Run run = + new marquez.service.models.LineageEvent.Run( + UUID.randomUUID().toString(), + marquez.service.models.LineageEvent.RunFacet.builder() + .parent( + marquez.service.models.LineageEvent.ParentRunFacet.builder() + .run( + marquez.service.models.LineageEvent.RunLink.builder() + .runId(UUID.randomUUID().toString()) + .build()) + .job( + marquez.service.models.LineageEvent.JobLink.builder() + .name("parent") + .namespace(NAMESPACE_NAME) + .build()) + ._producer(PRODUCER_URL) + ._schemaURL(SCHEMA_URL) + .build()) + .build()); + marquez.service.models.LineageEvent.Job job = + marquez.service.models.LineageEvent.Job.builder() + .namespace(NAMESPACE_NAME) + .name(JOB_NAME) + .build(); + + marquez.service.models.LineageEvent event = + marquez.service.models.LineageEvent.builder() + .eventType("COMPLETE") + .eventTime(ZonedDateTime.of(2021, 1, 1, 0, 0, 0, 0, ZoneId.of("UTC"))) + .producer(PRODUCER_URL.toString()) + .run(run) + .job(job) + .inputs(Collections.emptyList()) + .outputs(Collections.emptyList()) + .build(); + + CompletableFuture resp = sendEvent(event); + assertThat(resp.join()).isEqualTo(201); + + List jobs = client.listJobs(NAMESPACE_NAME); + + String marquezJobName = String.format("parent.%s", JOB_NAME); + + assertThat(jobs.size()).isEqualTo(2); + assertThat(jobs) + .anySatisfy(returnedJob -> assertThat(returnedJob.getName()).isEqualTo("parent")) + .anySatisfy(returnedJob -> assertThat(returnedJob.getName()).isEqualTo(marquezJobName)); + + client.deleteJob(NAMESPACE_NAME, marquezJobName); + + jobs = client.listJobs(NAMESPACE_NAME); + assertThat(jobs.size()).isEqualTo(1); + assertThat(jobs) + .anySatisfy(returnedJob -> assertThat(returnedJob.getName()).isEqualTo("parent")) + .noneSatisfy(returnedJob -> assertThat(returnedJob.getName()).isEqualTo(marquezJobName)); + } + private CompletableFuture sendEvent(marquez.service.models.LineageEvent event) { return this.sendLineage(Utils.toJson(event)) .thenApply(HttpResponse::statusCode) diff --git a/api/src/test/java/marquez/PostgresContainer.java b/api/src/test/java/marquez/PostgresContainer.java index a74b4893a8..1d536694ce 100644 --- a/api/src/test/java/marquez/PostgresContainer.java +++ b/api/src/test/java/marquez/PostgresContainer.java @@ -11,7 +11,7 @@ import org.testcontainers.utility.DockerImageName; public final class PostgresContainer extends PostgreSQLContainer { - private static final DockerImageName POSTGRES = DockerImageName.parse("postgres:11.8"); + private static final DockerImageName POSTGRES = DockerImageName.parse("postgres:12.12"); private static final int JDBC = 5; private static final Map containers = new HashMap<>(); diff --git a/api/src/test/java/marquez/db/ColumnsTest.java b/api/src/test/java/marquez/db/ColumnsTest.java index 3798e03710..ae9d6ad4b1 100644 --- a/api/src/test/java/marquez/db/ColumnsTest.java +++ b/api/src/test/java/marquez/db/ColumnsTest.java @@ -307,4 +307,22 @@ public void testBooleanOrDefaultWhenNoValue() throws SQLException { final boolean actual = Columns.booleanOrDefault(results, column, true); assertThat(actual).isTrue(); } + + @Test + public void testBooleanOrThrow() throws SQLException { + final String column = "is_deleted"; + when(results.getObject(column)).thenReturn(true); + when(results.getBoolean(column)).thenReturn(true); + + final boolean actual = Columns.booleanOrThrow(results, column); + assertThat(actual).isTrue(); + } + + @Test + public void testBooleanOrThrowNoValue() throws SQLException { + final String column = "is_deleted"; + when(results.getObject(column)).thenReturn(null); + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> Columns.booleanOrThrow(results, column)); + } } diff --git a/api/src/test/java/marquez/db/DatasetDaoTest.java b/api/src/test/java/marquez/db/DatasetDaoTest.java index 15fa888c4e..2d0e06f4a4 100644 --- a/api/src/test/java/marquez/db/DatasetDaoTest.java +++ b/api/src/test/java/marquez/db/DatasetDaoTest.java @@ -394,6 +394,36 @@ public void testGetDatasets() { "http://test.schema/")); } + @Test + public void testDeleteDatasetByNamespaceDoesNotReturnFromDeletedNamespace() { + createLineageRow( + openLineageDao, + "writeJob", + "COMPLETE", + jobFacet, + Collections.emptyList(), + Collections.singletonList(newCommonDataset(Collections.emptyMap()))); + + createLineageRow( + openLineageDao, + "writeJob2", + "COMPLETE", + jobFacet, + Collections.emptyList(), + Collections.singletonList( + new Dataset( + NAMESPACE, + DATASET, + LineageEvent.DatasetFacets.builder() + .lifecycleStateChange( + new LineageEvent.LifecycleStateChangeFacet( + PRODUCER_URL, SCHEMA_URL, "DROP")) + .build()))); + + datasetDao.deleteByNamespaceName(NAMESPACE); + assertThat(datasetDao.findDatasetByName(NAMESPACE, DATASET)).isEmpty(); + } + @Test public void testGetSpecificDatasetReturnsDatasetIfDeleted() { createLineageRow( diff --git a/api/src/test/java/marquez/db/migrations/V44_2__BackfillAirflowParentRunsTest.java b/api/src/test/java/marquez/db/migrations/V44_2__BackfillAirflowParentRunsTest.java deleted file mode 100644 index fbffc59094..0000000000 --- a/api/src/test/java/marquez/db/migrations/V44_2__BackfillAirflowParentRunsTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2018-2022 contributors to the Marquez project - * SPDX-License-Identifier: Apache-2.0 - */ - -package marquez.db.migrations; - -import static marquez.db.LineageTestUtils.NAMESPACE; -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.core.JsonProcessingException; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.Instant; -import java.util.Optional; -import java.util.UUID; -import marquez.db.BackfillTestUtils; -import marquez.db.JobDao; -import marquez.db.NamespaceDao; -import marquez.db.OpenLineageDao; -import marquez.db.RunArgsDao; -import marquez.db.RunDao; -import marquez.db.models.NamespaceRow; -import marquez.jdbi.JdbiExternalPostgresExtension.FlywaySkipRepeatable; -import marquez.jdbi.JdbiExternalPostgresExtension.FlywayTarget; -import marquez.jdbi.MarquezJdbiExternalPostgresExtension; -import org.flywaydb.core.api.configuration.Configuration; -import org.flywaydb.core.api.migration.Context; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(MarquezJdbiExternalPostgresExtension.class) -// fix the flyway migration up to v44 since we depend on the database structure as it exists at this -// point in time. The migration will only ever be applied on a database at this version. -@FlywayTarget("44") -// As of the time of this migration, there were no repeatable migrations, so ignore any that are -// added -@FlywaySkipRepeatable() -class V44_2__BackfillAirflowParentRunsTest { - - static Jdbi jdbi; - private static OpenLineageDao openLineageDao; - private static JobDao jobDao; - private static RunArgsDao runArgsDao; - private static RunDao runDao; - - @BeforeAll - public static void setUpOnce(Jdbi jdbi) { - V44_2__BackfillAirflowParentRunsTest.jdbi = jdbi; - openLineageDao = jdbi.onDemand(OpenLineageDao.class); - jobDao = jdbi.onDemand(JobDao.class); - runArgsDao = jdbi.onDemand(RunArgsDao.class); - runDao = jdbi.onDemand(RunDao.class); - } - - @Test - public void testMigrateAirflowTasks() throws SQLException, JsonProcessingException { - String dagName = "airflowDag"; - String task1Name = dagName + ".task1"; - NamespaceDao namespaceDao = jdbi.onDemand(NamespaceDao.class); - Instant now = Instant.now(); - NamespaceRow namespace = - namespaceDao.upsertNamespaceRow(UUID.randomUUID(), now, NAMESPACE, "me"); - - BackfillTestUtils.writeNewEvent( - jdbi, task1Name, now, namespace, "schedule:00:00:00", task1Name); - BackfillTestUtils.writeNewEvent( - jdbi, "airflowDag.task2", now, namespace, "schedule:00:00:00", task1Name); - - BackfillTestUtils.writeNewEvent(jdbi, "a_non_airflow_task", now, namespace, null, null); - - jdbi.useHandle( - handle -> { - try { - new V44_2__BackfillAirflowParentRuns() - .migrate( - new Context() { - @Override - public Configuration getConfiguration() { - return null; - } - - @Override - public Connection getConnection() { - return handle.getConnection(); - } - }); - } catch (Exception e) { - throw new AssertionError("Unable to execute migration", e); - } - }); - Optional jobNameResult = - jdbi.withHandle( - h -> - h.createQuery( - """ - SELECT name FROM jobs_view - WHERE namespace_name=:namespace AND simple_name=:jobName - """) - .bind("namespace", NAMESPACE) - .bind("jobName", dagName) - .mapTo(String.class) - .findFirst()); - assertThat(jobNameResult).isPresent(); - } -} diff --git a/api/src/test/java/marquez/db/migrations/V44_3_BackfillJobsWithParentsTest.java b/api/src/test/java/marquez/db/migrations/V44_3_BackfillJobsWithParentsTest.java deleted file mode 100644 index bf33c08cb7..0000000000 --- a/api/src/test/java/marquez/db/migrations/V44_3_BackfillJobsWithParentsTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2018-2022 contributors to the Marquez project - * SPDX-License-Identifier: Apache-2.0 - */ - -package marquez.db.migrations; - -import static marquez.db.BackfillTestUtils.writeNewEvent; -import static marquez.db.LineageTestUtils.NAMESPACE; -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.core.JsonProcessingException; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.Instant; -import java.util.Optional; -import java.util.UUID; -import marquez.db.NamespaceDao; -import marquez.db.OpenLineageDao; -import marquez.db.models.NamespaceRow; -import marquez.db.models.RunRow; -import marquez.jdbi.JdbiExternalPostgresExtension.FlywaySkipRepeatable; -import marquez.jdbi.JdbiExternalPostgresExtension.FlywayTarget; -import marquez.jdbi.MarquezJdbiExternalPostgresExtension; -import org.flywaydb.core.api.configuration.Configuration; -import org.flywaydb.core.api.migration.Context; -import org.jdbi.v3.core.Jdbi; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(MarquezJdbiExternalPostgresExtension.class) -// fix the flyway migration up to v44 since we depend on the database structure as it exists at this -// point in time. The migration will only ever be applied on a database at this version. -@FlywayTarget("44") -// As of the time of this migration, there were no repeatable migrations, so ignore any that are -// added -@FlywaySkipRepeatable() -class V44_3_BackfillJobsWithParentsTest { - - static Jdbi jdbi; - private static OpenLineageDao openLineageDao; - - @BeforeAll - public static void setUpOnce(Jdbi jdbi) { - V44_3_BackfillJobsWithParentsTest.jdbi = jdbi; - openLineageDao = jdbi.onDemand(OpenLineageDao.class); - } - - @Test - public void testBackfill() throws SQLException, JsonProcessingException { - NamespaceDao namespaceDao = jdbi.onDemand(NamespaceDao.class); - Instant now = Instant.now(); - NamespaceRow namespace = - namespaceDao.upsertNamespaceRow(UUID.randomUUID(), now, NAMESPACE, "me"); - String parentName = "parentJob"; - RunRow parentRun = writeNewEvent(jdbi, parentName, now, namespace, null, null); - - String task1Name = "task1"; - writeNewEvent(jdbi, task1Name, now, namespace, parentRun.getUuid().toString(), parentName); - writeNewEvent(jdbi, "task2", now, namespace, parentRun.getUuid().toString(), parentName); - - jdbi.useHandle( - handle -> { - try { - Context context = - new Context() { - @Override - public Configuration getConfiguration() { - return null; - } - - @Override - public Connection getConnection() { - return handle.getConnection(); - } - }; - // apply migrations in order - new V44_1__UpdateRunsWithJobUUID().migrate(context); - new V44_3_BackfillJobsWithParents().migrate(context); - } catch (Exception e) { - throw new AssertionError("Unable to execute migration", e); - } - }); - - Optional jobName = - jdbi.withHandle( - h -> - h.createQuery("SELECT name FROM jobs_view WHERE simple_name=:jobName") - .bind("jobName", task1Name) - .mapTo(String.class) - .findFirst()); - assertThat(jobName).isPresent().get().isEqualTo(parentName + "." + task1Name); - } -} diff --git a/api/src/test/java/marquez/db/models/DbModelGenerator.java b/api/src/test/java/marquez/db/models/DbModelGenerator.java index 2b5de8320b..de2ea22774 100644 --- a/api/src/test/java/marquez/db/models/DbModelGenerator.java +++ b/api/src/test/java/marquez/db/models/DbModelGenerator.java @@ -26,7 +26,9 @@ private DbModelGenerator() {} /** Returns new {@link NamespaceRow} objects with a specified {@code limit}. */ public static List newNamespaceRows(int limit) { - return Stream.generate(() -> newNamespaceRow()).limit(limit).collect(toImmutableList()); + return Stream.generate(DbModelGenerator::newNamespaceRow) + .limit(limit) + .collect(toImmutableList()); } /** Returns a new {@link NamespaceRow} object. */ @@ -38,7 +40,8 @@ public static NamespaceRow newNamespaceRow() { now, newNamespaceName().getValue(), newDescription(), - newOwnerName().getValue()); + newOwnerName().getValue(), + false); } /** Returns a new {@code row} uuid. */ diff --git a/clients/java/src/main/java/marquez/client/MarquezClient.java b/clients/java/src/main/java/marquez/client/MarquezClient.java index c676c562eb..3972f868ac 100644 --- a/clients/java/src/main/java/marquez/client/MarquezClient.java +++ b/clients/java/src/main/java/marquez/client/MarquezClient.java @@ -230,6 +230,10 @@ public Dataset deleteDataset(@NonNull String namespaceName, @NonNull String data return Dataset.fromJson(bodyAsJson); } + public void deleteNamespace(@NonNull String namespaceName) { + http.delete(url.toNamespaceUrl(namespaceName)); + } + public DatasetVersion getDatasetVersion( @NonNull String namespaceName, @NonNull String datasetName, @NonNull String version) { final String bodyAsJson = diff --git a/clients/java/src/main/java/marquez/client/models/Namespace.java b/clients/java/src/main/java/marquez/client/models/Namespace.java index 9565ea5a59..37c7fa62fd 100644 --- a/clients/java/src/main/java/marquez/client/models/Namespace.java +++ b/clients/java/src/main/java/marquez/client/models/Namespace.java @@ -21,16 +21,20 @@ public final class Namespace extends NamespaceMeta { @Getter private final Instant createdAt; @Getter private final Instant updatedAt; + @Getter private final Boolean isHidden; + public Namespace( @NonNull final String name, @NonNull final Instant createdAt, @NonNull final Instant updatedAt, final String ownerName, - @Nullable final String description) { + @Nullable final String description, + @NonNull final Boolean isHidden) { super(ownerName, description); this.name = name; this.createdAt = createdAt; this.updatedAt = updatedAt; + this.isHidden = isHidden; } public static Namespace fromJson(@NonNull final String json) { diff --git a/clients/java/src/test/java/marquez/client/MarquezClientTest.java b/clients/java/src/test/java/marquez/client/MarquezClientTest.java index 4c2a725d4c..dce0192e87 100644 --- a/clients/java/src/test/java/marquez/client/MarquezClientTest.java +++ b/clients/java/src/test/java/marquez/client/MarquezClientTest.java @@ -118,7 +118,8 @@ public class MarquezClientTest { private static final String OWNER_NAME = newOwnerName(); private static final String NAMESPACE_DESCRIPTION = newDescription(); private static final Namespace NAMESPACE = - new Namespace(NAMESPACE_NAME, CREATED_AT, UPDATED_AT, OWNER_NAME, NAMESPACE_DESCRIPTION); + new Namespace( + NAMESPACE_NAME, CREATED_AT, UPDATED_AT, OWNER_NAME, NAMESPACE_DESCRIPTION, false); // SOURCE private static final String SOURCE_TYPE = newSourceType(); diff --git a/clients/java/src/test/java/marquez/client/MarquezHttpTest.java b/clients/java/src/test/java/marquez/client/MarquezHttpTest.java index 4116b988b7..c562c02380 100644 --- a/clients/java/src/test/java/marquez/client/MarquezHttpTest.java +++ b/clients/java/src/test/java/marquez/client/MarquezHttpTest.java @@ -181,7 +181,8 @@ public void testPut() throws Exception { final String ownerName = newOwnerName(); final String description = newDescription(); - final Namespace namespace = new Namespace(namespaceName, now, now, ownerName, description); + final Namespace namespace = + new Namespace(namespaceName, now, now, ownerName, description, false); final String json = JsonGenerator.newJsonFor(namespace); final ByteArrayInputStream stream = new ByteArrayInputStream(json.getBytes(UTF_8)); when(httpEntity.getContent()).thenReturn(stream); diff --git a/clients/java/src/test/java/marquez/client/models/JsonGenerator.java b/clients/java/src/test/java/marquez/client/models/JsonGenerator.java index 26cb75def9..81e06b016c 100644 --- a/clients/java/src/test/java/marquez/client/models/JsonGenerator.java +++ b/clients/java/src/test/java/marquez/client/models/JsonGenerator.java @@ -38,6 +38,7 @@ public static String newJsonFor(final Namespace namespace) { .put("updatedAt", ISO_INSTANT.format(namespace.getUpdatedAt())) .put("ownerName", namespace.getOwnerName()) .put("description", namespace.getDescription().orElse(null)) + .put("isHidden", namespace.getIsHidden()) .toString(); } diff --git a/clients/java/src/test/java/marquez/client/models/ModelGenerator.java b/clients/java/src/test/java/marquez/client/models/ModelGenerator.java index bc28f4be08..af1f0341d9 100644 --- a/clients/java/src/test/java/marquez/client/models/ModelGenerator.java +++ b/clients/java/src/test/java/marquez/client/models/ModelGenerator.java @@ -38,7 +38,7 @@ public static List newNamespaces(final int limit) { public static Namespace newNamespace() { final Instant now = newTimestamp(); - return new Namespace(newNamespaceName(), now, now, newOwnerName(), newDescription()); + return new Namespace(newNamespaceName(), now, now, newOwnerName(), newDescription(), false); } public static SourceMeta newSourceMeta() { diff --git a/codecov.yml b/codecov.yml index a9aec2ad72..363a04d59a 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,4 +1,11 @@ codecov: branch: main + +coverage: status: patch: off + +ignore: + - "api/src/main/java/marquez/db/migrations/V44_1__UpdateRunsWithJobUUID.java" + - "api/src/main/java/marquez/db/migrations/V44_2__BackfillAirflowParentRuns.java" + - "api/src/main/java/marquez/db/migrations/V44_3_BackfillJobsWithParents.java" \ No newline at end of file diff --git a/web/src/store/reducers/namespaces.ts b/web/src/store/reducers/namespaces.ts index 607b554723..a0e0518eeb 100644 --- a/web/src/store/reducers/namespaces.ts +++ b/web/src/store/reducers/namespaces.ts @@ -21,7 +21,7 @@ export default ( switch (type) { case FETCH_NAMESPACES_SUCCESS: return { - result: payload.namespaces, + result: payload.namespaces.filter(namespace => !namespace.isHidden), selectedNamespace: window.localStorage.getItem('selectedNamespace') && action.payload.namespaces.find( diff --git a/web/src/types/api.ts b/web/src/types/api.ts index d54cd1c92c..320bf02fd2 100644 --- a/web/src/types/api.ts +++ b/web/src/types/api.ts @@ -20,6 +20,7 @@ export interface Namespace { updatedAt: string ownerName: string description: string + isHidden: boolean } export interface Datasets {