Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow message_name field for protobuf ingestion #9480

Merged
merged 6 commits into from
Dec 19, 2023
Merged

Allow message_name field for protobuf ingestion #9480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
614800a
add messageName
eboneil Dec 18, 2023
c1b91fe
update test package
eboneil Dec 18, 2023
cd93e7e
add test file
eboneil Dec 18, 2023
9789418
test with updated jackson version
eboneil Dec 19, 2023
009e4f1
remove old file
eboneil Dec 19, 2023
4b240e8
update build.gradle
eboneil Dec 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions metadata-integration/java/datahub-protobuf/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ dependencies {
implementation externalDependency.commonsCli
implementation externalDependency.httpAsyncClient
implementation externalDependency.slf4jApi
implementation externalDependency.jacksonCore
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok
testImplementation externalDependency.junitJupiterApi
testRuntimeOnly externalDependency.junitJupiterEngine
testImplementation externalDependency.testng
}

import java.nio.file.Paths
Expand All @@ -61,10 +61,7 @@ jacocoTestReport {
dependsOn test // tests are required to run before generating the report
}

test {
useJUnit()
finalizedBy jacocoTestReport
}
test.finalizedBy jacocoTestReport


task checkShadowJar(type: Exec) {
Expand Down
11 changes: 11 additions & 0 deletions metadata-integration/java/datahub-protobuf/src/main/java/datahub/protobuf/Proto2DataHub.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ public class Proto2DataHub {
"[Optional if using --directory] The protobuf source file. Typically a .proto file.")
.build();

private static final Option OPTION_MESSAGE_NAME =
Option.builder()
.longOpt("message_name")
.hasArg()
.desc("[Optional] The protobuf message name to read from.")
.build();

private static final Option OPTION_DIR =
Option.builder()
.longOpt("directory")
Expand Down Expand Up @@ -166,6 +173,7 @@ static class AppConfig {
private final String dataPlatform;
private final String protoc;
private final String inputFile;
private final String messageName;
private final String inputDir;
private final TransportOptions transport;
private final String filename;
Expand All @@ -191,6 +199,7 @@ static class AppConfig {
dataPlatform = cli.getOptionValue(OPTION_DATAHUB_PLATFORM, "kafka").toLowerCase(Locale.ROOT);
protoc = cli.getOptionValue(OPTION_DESCRIPTOR);
inputFile = cli.getOptionValue(OPTION_FILE, null);
messageName = cli.getOptionValue(OPTION_MESSAGE_NAME, null);
transport =
TransportOptions.valueOf(
cli.getOptionValue(OPTION_TRANSPORT, "rest").toUpperCase(Locale.ROOT));
Expand Down Expand Up @@ -250,6 +259,7 @@ public static void main(String[] args) throws Exception {
.addOption(OPTION_DATAHUB_TOKEN)
.addOption(OPTION_DESCRIPTOR)
.addOption(OPTION_FILE)
.addOption(OPTION_MESSAGE_NAME)
.addOption(OPTION_DIR)
.addOption(OPTION_EXCLUDE_PATTERN)
.addOption(OPTION_DATAHUB_USER)
Expand Down Expand Up @@ -354,6 +364,7 @@ public static void main(String[] args) throws Exception {
.setGithubOrganization(config.githubOrg)
.setSlackTeamId(config.slackId)
.setSubType(config.subType)
.setMessageName(config.messageName)
.build();

dataset
Expand Down
6 changes: 3 additions & 3 deletions ...integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufDatasetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package datahub.protobuf;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;

import com.linkedin.common.FabricType;
import com.linkedin.common.GlobalTags;
Expand Down Expand Up @@ -34,7 +34,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufDatasetTest {

Expand Down
4 changes: 2 additions & 2 deletions ...a-integration/java/datahub-protobuf/src/test/java/datahub/protobuf/ProtobufUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@

import static datahub.protobuf.TestFixtures.getTestProtobufFileSet;
import static datahub.protobuf.TestFixtures.getTestProtoc;
import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos;
import com.google.protobuf.ExtensionRegistry;
import datahub.protobuf.model.ProtobufGraph;
import java.io.IOException;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufUtilsTest {

Expand Down
4 changes: 2 additions & 2 deletions ...egration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufEnumTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package datahub.protobuf.model;

import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos.DescriptorProto;
import com.google.protobuf.DescriptorProtos.EnumDescriptorProto;
Expand All @@ -11,7 +11,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufEnumTest {

Expand Down
4 changes: 2 additions & 2 deletions ...gration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufFieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package datahub.protobuf.model;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos.DescriptorProto;
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
Expand All @@ -22,7 +22,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Set;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufFieldTest {
private static final DescriptorProto EXPECTED_MESSAGE_PROTO =
Expand Down
4 changes: 2 additions & 2 deletions ...gration/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufGraphTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@

import static datahub.protobuf.TestFixtures.getTestProtobufFileSet;
import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufGraphTest {

Expand Down
4 changes: 2 additions & 2 deletions ...ation/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufMessageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package datahub.protobuf.model;

import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos.DescriptorProto;
import com.google.protobuf.DescriptorProtos.FileDescriptorProto;
Expand All @@ -11,7 +11,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufMessageTest {

Expand Down
4 changes: 2 additions & 2 deletions ...on/java/datahub-protobuf/src/test/java/datahub/protobuf/model/ProtobufOneOfFieldTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package datahub.protobuf.model;

import static org.junit.jupiter.api.Assertions.*;
import static org.testng.Assert.*;

import com.google.protobuf.DescriptorProtos.DescriptorProto;
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
Expand All @@ -12,7 +12,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufOneOfFieldTest {

Expand Down
4 changes: 2 additions & 2 deletions ...ation/java/datahub-protobuf/src/test/java/datahub/protobuf/visitors/VisitContextTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import static datahub.protobuf.TestFixtures.getTestProtobufFileSet;
import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.testng.Assert.assertNotEquals;

import com.google.protobuf.DescriptorProtos.FileDescriptorSet;
import datahub.protobuf.model.FieldTypeEdge;
Expand All @@ -13,7 +13,7 @@
import java.util.Set;
import java.util.stream.Collectors;
import org.jgrapht.GraphPath;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class VisitContextTest {

Expand Down
4 changes: 2 additions & 2 deletions .../datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DatasetVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package datahub.protobuf.visitors.dataset;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.urn.DatasetUrn;
import com.linkedin.data.template.RecordTemplate;
Expand All @@ -14,7 +14,7 @@
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class DatasetVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...ahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DescriptionVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
package datahub.protobuf.visitors.dataset;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import datahub.protobuf.model.ProtobufGraph;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class DescriptionVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...a/datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/DomainVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.urn.Urn;
import datahub.protobuf.model.ProtobufGraph;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class DomainVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...tobuf/src/test/java/datahub/protobuf/visitors/dataset/InstitutionalMemoryVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
package datahub.protobuf.visitors.dataset;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.InstitutionalMemoryMetadata;
import com.linkedin.common.url.Url;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class InstitutionalMemoryVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...otobuf/src/test/java/datahub/protobuf/visitors/dataset/KafkaTopicPropertyVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.data.template.StringMap;
import com.linkedin.dataset.DatasetProperties;
Expand All @@ -11,7 +11,7 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class KafkaTopicPropertyVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...atahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/OwnershipVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.Owner;
import com.linkedin.common.OwnershipSource;
Expand All @@ -14,7 +14,7 @@
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class OwnershipVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...datahub-protobuf/src/test/java/datahub/protobuf/visitors/dataset/PropertyVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@
import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static java.util.Map.entry;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.data.template.StringMap;
import com.linkedin.dataset.DatasetProperties;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class PropertyVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...-protobuf/src/test/java/datahub/protobuf/visitors/dataset/TermAssociationVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.GlossaryTermAssociation;
import com.linkedin.common.urn.GlossaryTermUrn;
import java.io.IOException;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class TermAssociationVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...obuf/src/test/java/datahub/protobuf/visitors/field/ProtobufExtensionFieldVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package datahub.protobuf.visitors.field;

import static datahub.protobuf.TestFixtures.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.common.GlobalTags;
import com.linkedin.common.GlossaryTermAssociation;
Expand All @@ -23,7 +23,7 @@
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class ProtobufExtensionFieldVisitorTest {

Expand Down
4 changes: 2 additions & 2 deletions ...atahub-protobuf/src/test/java/datahub/protobuf/visitors/field/SchemaFieldVisitorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import static datahub.protobuf.TestFixtures.getTestProtobufGraph;
import static datahub.protobuf.TestFixtures.getVisitContextBuilder;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.testng.Assert.assertEquals;

import com.linkedin.schema.NumberType;
import com.linkedin.schema.SchemaField;
Expand All @@ -15,7 +15,7 @@
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
import org.testng.annotations.Test;

public class SchemaFieldVisitorTest {

Expand Down
Loading
Loading