-
Notifications
You must be signed in to change notification settings - Fork 324
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into update/datasets-sql
- Loading branch information
Showing
37 changed files
with
708 additions
and
166 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| API_PORT=5000 | ||
| API_ADMIN_PORT=5001 | ||
| WEB_PORT=3000 | ||
| TAG=0.36.0 | ||
| TAG=0.37.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| /* | ||
| * Copyright 2018-2023 contributors to the Marquez project | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package marquez.service.models; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonTypeInfo; | ||
| import com.fasterxml.jackson.annotation.JsonTypeInfo.As; | ||
| import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; | ||
| import com.fasterxml.jackson.databind.annotation.JsonTypeIdResolver; | ||
|
|
||
| @JsonTypeIdResolver(EventTypeResolver.class) | ||
| @JsonTypeInfo( | ||
| use = Id.CUSTOM, | ||
| include = As.EXISTING_PROPERTY, | ||
| property = "schemaURL", | ||
| defaultImpl = LineageEvent.class, | ||
| visible = true) | ||
| public class BaseEvent extends BaseJsonModel {} |
31 changes: 31 additions & 0 deletions
31
api/src/main/java/marquez/service/models/DatasetEvent.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| /* | ||
| * Copyright 2018-2023 contributors to the Marquez project | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package marquez.service.models; | ||
|
|
||
| import java.net.URI; | ||
| import java.time.ZonedDateTime; | ||
| import javax.validation.Valid; | ||
| import javax.validation.constraints.NotNull; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
| import lombok.ToString; | ||
|
|
||
| @Builder | ||
| @AllArgsConstructor | ||
| @NoArgsConstructor | ||
| @Setter | ||
| @Getter | ||
| @Valid | ||
| @ToString | ||
| public class DatasetEvent extends BaseEvent { | ||
| @NotNull private ZonedDateTime eventTime; | ||
| @Valid private LineageEvent.Dataset dataset; | ||
| @Valid @NotNull private String producer; | ||
| @Valid @NotNull private URI schemaURL; | ||
| } |
89 changes: 89 additions & 0 deletions
89
api/src/main/java/marquez/service/models/EventTypeResolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| /* | ||
| * Copyright 2018-2023 contributors to the Marquez project | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package marquez.service.models; | ||
|
|
||
| import static marquez.service.models.EventTypeResolver.EventSchemaURL.LINEAGE_EVENT; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; | ||
| import com.fasterxml.jackson.databind.DatabindContext; | ||
| import com.fasterxml.jackson.databind.JavaType; | ||
| import com.fasterxml.jackson.databind.jsontype.impl.TypeIdResolverBase; | ||
| import java.io.IOException; | ||
| import java.util.Arrays; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.Getter; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Slf4j | ||
| public class EventTypeResolver extends TypeIdResolverBase { | ||
|
|
||
| @AllArgsConstructor | ||
| public enum EventSchemaURL { | ||
| LINEAGE_EVENT( | ||
| "https://openlineage.io/spec/2-0-0/OpenLineage.json#/definitions/RunEvent", | ||
| LineageEvent.class), | ||
| DATASET_EVENT( | ||
| "https://openlineage.io/spec/2-0-0/OpenLineage.json#/definitions/DatasetEvent", | ||
| DatasetEvent.class), | ||
| JOB_EVENT( | ||
| "https://openlineage.io/spec/2-0-0/OpenLineage.json#/definitions/JobEvent", JobEvent.class); | ||
|
|
||
| @Getter private String schemaURL; | ||
|
|
||
| public String getName() { | ||
| int lastSlash = schemaURL.lastIndexOf('/'); | ||
| return schemaURL.substring(lastSlash, schemaURL.length()); | ||
| } | ||
|
|
||
| @Getter private Class<?> subType; | ||
| } | ||
|
|
||
| private JavaType superType; | ||
|
|
||
| @Override | ||
| public void init(JavaType baseType) { | ||
| superType = baseType; | ||
| } | ||
|
|
||
| @Override | ||
| public String idFromValue(Object value) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public String idFromValueAndType(Object value, Class<?> suggestedType) { | ||
| return null; | ||
| } | ||
|
|
||
| @Override | ||
| public JavaType typeFromId(DatabindContext context, String id) throws IOException { | ||
| if (id == null) { | ||
| return context.constructSpecializedType(superType, LINEAGE_EVENT.subType); | ||
| } | ||
|
|
||
| int lastSlash = id.lastIndexOf('/'); | ||
|
|
||
| if (lastSlash < 0) { | ||
| return context.constructSpecializedType(superType, LINEAGE_EVENT.subType); | ||
| } | ||
|
|
||
| String type = id.substring(lastSlash, id.length()); | ||
|
|
||
| Class<?> subType = | ||
| Arrays.stream(EventSchemaURL.values()) | ||
| .filter(s -> s.getName().equals(type)) | ||
| .findAny() | ||
| .map(EventSchemaURL::getSubType) | ||
| .orElse(LINEAGE_EVENT.subType); | ||
|
|
||
| return context.constructSpecializedType(superType, subType); | ||
| } | ||
|
|
||
| @Override | ||
| public Id getMechanism() { | ||
| return null; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /* | ||
| * Copyright 2018-2023 contributors to the Marquez project | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package marquez.service.models; | ||
|
|
||
| import java.net.URI; | ||
| import java.time.ZonedDateTime; | ||
| import java.util.List; | ||
| import javax.validation.Valid; | ||
| import javax.validation.constraints.NotNull; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
| import lombok.ToString; | ||
|
|
||
| @Builder | ||
| @AllArgsConstructor | ||
| @NoArgsConstructor | ||
| @Setter | ||
| @Getter | ||
| @Valid | ||
| @ToString | ||
| public class JobEvent extends BaseEvent { | ||
| @NotNull private ZonedDateTime eventTime; | ||
| @Valid @NotNull private LineageEvent.Job job; | ||
| @Valid private List<LineageEvent.Dataset> inputs; | ||
| @Valid private List<LineageEvent.Dataset> outputs; | ||
| @Valid @NotNull private String producer; | ||
| @Valid @NotNull private URI schemaURL; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.