feat(propagation): Add models for schema field docs, tags, terms (#2959)

metadata-models/src/main/pegasus/com/linkedin/common/Documentation.pdl

@@ -0,0 +1,32 @@
+namespace com.linkedin.common
+
+/**
+ * Aspect used for storing all applicable documentations on assets.
+ * This aspect supports multiple documentations from different sources.
+ * There is an implicit assumption that there is only one documentation per
+   source.
+ * For example, if there are two documentations from the same source, the
+   latest one will overwrite the previous one.
+ * If there are two documentations from different sources, both will be
+   stored.
+  * Future evolution considerations:
+  * The first entity that uses this aspect is Schema Field. We will expand this
+    aspect to other entities eventually.
+  * The values of the documentation are not currently searchable. This will be
+    changed once this aspect develops opinion on which documentation entry is
+    the authoritative one.
+  * Ensuring that there is only one documentation per source is a business
+    rule that is not enforced by the aspect yet. This will currently be enforced by the
+    application that uses this aspect. We will eventually enforce this rule in
+    the aspect using AspectMutators.
+ */
+@Aspect = {
+  "name": "documentation"
+}
+record Documentation {
+
+  /**
+   * Documentations associated with this asset. We could be receiving docs from different sources
+   */
+  documentations: array[DocumentationAssociation]
+}

metadata-models/src/main/pegasus/com/linkedin/common/DocumentationAssociation.pdl

@@ -0,0 +1,30 @@
+namespace com.linkedin.common
+
+/**
+ * Properties of applied documentation including the attribution of the doc
+ */
+record DocumentationAssociation {
+ /**
+  * Description of this asset
+  */
+  documentation: string
+
+  /**
+   * Information about who, why, and how this metadata was applied
+   */
+  @Searchable = {
+    "/time": {
+        "fieldName": "documentationAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/actor": {
+        "fieldName": "documentationAttributionActors",
+        "fieldType": "URN"
+      },
+    "/source": {
+        "fieldName": "documentationAttributionSources",
+        "fieldType": "URN"
+      },
+  }
+  attribution: optional MetadataAttribution
+}

metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl

@@ -30,4 +30,23 @@ record GlossaryTermAssociation {
    */
   context: optional string
 
+  /**
+   * Information about who, why, and how this metadata was applied
+   */
+  @Searchable = {
+    "/time": {
+        "fieldName": "termAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/actor": {
+        "fieldName": "termAttributionActors",
+        "fieldType": "URN"
+      },
+    "/source": {
+        "fieldName": "termAttributionSources",
+        "fieldType": "URN"
+      },
+  }
+  attribution: optional MetadataAttribution
+
 }

metadata-models/src/main/pegasus/com/linkedin/common/MetadataAttribution.pdl

@@ -0,0 +1,29 @@
+namespace com.linkedin.common
+
+/**
+ * Information about who, why, and how this metadata was applied
+ */
+record MetadataAttribution {
+  /**
+   * When this metadata was updated.
+   */
+  time: Time
+
+  /**
+   * The entity (e.g. a member URN) responsible for applying the assocated metadata. This can
+   * either be a user (in case of UI edits) or the datahub system for automation.
+   */
+  actor: Urn
+
+  /**
+   * The DataHub source responsible for applying the associated metadata. This will only be filled out
+   * when a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.
+   */
+  source: optional Urn
+
+  /**
+   * The details associated with why this metadata was applied. For example, this could include
+   * the actual regex rule, sql statement, ingestion pipeline ID, etc.
+   */
+  sourceDetail: map[string, string] = { }
+}

metadata-models/src/main/pegasus/com/linkedin/common/TagAssociation.pdl

@@ -14,4 +14,23 @@ record TagAssociation {
    * Additional context about the association
    */
   context: optional string
+
+  /**
+   * Information about who, why, and how this metadata was applied
+   */
+  @Searchable = {
+    "/time": {
+        "fieldName": "tagAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/actor": {
+        "fieldName": "tagAttributionActors",
+        "fieldType": "URN"
+      },
+    "/source": {
+        "fieldName": "tagAttributionSources",
+        "fieldType": "URN"
+      },
+  }
+  attribution: optional MetadataAttribution
 }

metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl

@@ -36,7 +36,19 @@ record EditableSchemaFieldInfo {
       "fieldName": "editedFieldTags",
       "fieldType": "URN",
       "boostScore": 0.5
-    }
+    },
+    "/tags/*/attribution/time": {
+        "fieldName": "editedFieldTagAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/tags/*/attribution/actor": {
+        "fieldName": "editedFieldTagAttributionActors",
+        "fieldType": "URN"
+      },
+    "/tags/*/attribution/source": {
+        "fieldName": "editedFieldTagAttributionSources",
+        "fieldType": "URN"
+      },
   }
   globalTags: optional GlobalTags
 
@@ -54,7 +66,19 @@ record EditableSchemaFieldInfo {
       "fieldName": "editedFieldGlossaryTerms",
       "fieldType": "URN",
       "boostScore": 0.5
-    }
+    },
+    "/terms/*/attribution/time": {
+        "fieldName": "editedFieldTermAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/terms/*/attribution/actor": {
+        "fieldName": "editedFieldTermAttributionActors",
+        "fieldType": "URN"
+      },
+    "/terms/*/attribution/source": {
+        "fieldName": "editedFieldTermAttributionSources",
+        "fieldType": "URN"
+      },
   }
   glossaryTerms: optional GlossaryTerms
 }

metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl

@@ -96,7 +96,19 @@ record SchemaField {
       "fieldName": "fieldTags",
       "fieldType": "URN",
       "boostScore": 0.5
-    }
+    },
+    "/tags/*/attribution/time": {
+        "fieldName": "fieldTagAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/tags/*/attribution/actor": {
+        "fieldName": "fieldTagAttributionActors",
+        "fieldType": "URN"
+      },
+    "/tags/*/attribution/source": {
+        "fieldName": "fieldTagAttributionSources",
+        "fieldType": "URN"
+      },
   }
   globalTags: optional GlobalTags
 
@@ -114,7 +126,19 @@ record SchemaField {
       "fieldName": "fieldGlossaryTerms",
       "fieldType": "URN",
       "boostScore": 0.5
-    }
+    },
+    "/terms/*/attribution/time": {
+        "fieldName": "fieldTermAttributionDates",
+        "fieldType": "DATETIME"
+      },
+    "/terms/*/attribution/actor": {
+        "fieldName": "fieldTermAttributionActors",
+        "fieldType": "URN"
+      },
+    "/terms/*/attribution/source": {
+        "fieldName": "fieldTermAttributionSources",
+        "fieldType": "URN"
+      },
   }
   glossaryTerms: optional GlossaryTerms
 

metadata-models/src/main/resources/entity-registry.yml

@@ -454,6 +454,7 @@ entities:
       - structuredProperties
       - forms
       - businessAttributes
+      - documentation
   - name: globalSettings
     doc: Global settings for an the platform
     category: internal
@@ -670,4 +671,4 @@ plugins:
       spring:
         enabled: true
       packageScan:
-        - com.linkedin.gms.factory.plugins
+        - com.linkedin.gms.factory.plugins