feat(build): adding support for python codegen for all aspects, not j…

…ust the snapshot ones (datahub-project#3299)

metadata-ingestion/scripts/codegen.sh

@@ -6,7 +6,24 @@ OUTDIR=./src/datahub/metadata
 # Note: this assumes that datahub has already been built with `./gradlew build`.
 DATAHUB_ROOT=..
 SCHEMAS_ROOT="$DATAHUB_ROOT/metadata-events/mxe-schemas/src/renamed/avro/com/linkedin"
-FILES="$SCHEMAS_ROOT/mxe/MetadataChangeEvent.avsc $SCHEMAS_ROOT/mxe/MetadataChangeProposal.avsc $SCHEMAS_ROOT/usage/UsageAggregation.avsc $SCHEMAS_ROOT/dataset/DatasetProfile.avsc $SCHEMAS_ROOT/dataset/DatasetUsageStatistics.avsc"
+FILES="$SCHEMAS_ROOT/mxe/MetadataChangeEvent.avsc $SCHEMAS_ROOT/mxe/MetadataChangeProposal.avsc $SCHEMAS_ROOT/usage/UsageAggregation.avsc"
+# Since we depend on jq, check if jq is installed
+if ! which jq; then
+   echo "jq is not installed. Please install jq and rerun (https://stedolan.github.io/jq/)"
+   exit 1
+fi
+
+find $SCHEMAS_ROOT -name "*.avsc" | sort | while read file
+do
+# Add all other files that are aspects but not included in the above
+        if (jq '.Aspect' -e $file > /dev/null)
+        then
+            FILES="${FILES} ${file}"
+        fi
+        echo $FILES > /tmp/codegen_files.txt
+done
+
+FILES=$(cat /tmp/codegen_files.txt)
 
 rm -r $OUTDIR || true
 python scripts/avro_codegen.py $FILES $OUTDIR

metadata-ingestion/src/datahub/metadata/schema.avsc

@@ -6044,6 +6044,27 @@
     ],
     "doc": "Usage data for a given resource, rolled up into a bucket."
   },
+  "com.linkedin.pegasus2avro.chart.ChartInfo",
+  "com.linkedin.pegasus2avro.chart.ChartQuery",
+  "com.linkedin.pegasus2avro.chart.EditableChartProperties",
+  "com.linkedin.pegasus2avro.common.BrowsePaths",
+  "com.linkedin.pegasus2avro.common.Cost",
+  "com.linkedin.pegasus2avro.common.Deprecation",
+  "com.linkedin.pegasus2avro.common.GlobalTags",
+  "com.linkedin.pegasus2avro.common.GlossaryTerms",
+  "com.linkedin.pegasus2avro.common.InstitutionalMemory",
+  "com.linkedin.pegasus2avro.common.Ownership",
+  "com.linkedin.pegasus2avro.common.Status",
+  "com.linkedin.pegasus2avro.dashboard.DashboardInfo",
+  "com.linkedin.pegasus2avro.dashboard.EditableDashboardProperties",
+  "com.linkedin.pegasus2avro.datajob.DataFlowInfo",
+  "com.linkedin.pegasus2avro.datajob.DataJobInfo",
+  "com.linkedin.pegasus2avro.datajob.DataJobInputOutput",
+  "com.linkedin.pegasus2avro.datajob.EditableDataFlowProperties",
+  "com.linkedin.pegasus2avro.datajob.EditableDataJobProperties",
+  "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo",
+  "com.linkedin.pegasus2avro.dataprocess.DataProcessInfo",
+  "com.linkedin.pegasus2avro.dataset.DatasetDeprecation",
   {
     "type": "record",
     "Aspect": {
@@ -6347,6 +6368,8 @@
     ],
     "doc": "Stats corresponding to datasets"
   },
+  "com.linkedin.pegasus2avro.dataset.DatasetProperties",
+  "com.linkedin.pegasus2avro.dataset.DatasetUpstreamLineage",
   {
     "type": "record",
     "Aspect": {
@@ -6493,5 +6516,55 @@
       }
     ],
     "doc": "Stats corresponding to dataset's usage."
-  }
+  },
+  "com.linkedin.pegasus2avro.dataset.EditableDatasetProperties",
+  "com.linkedin.pegasus2avro.dataset.UpstreamLineage",
+  "com.linkedin.pegasus2avro.glossary.GlossaryNodeInfo",
+  "com.linkedin.pegasus2avro.glossary.GlossaryRelatedTerms",
+  "com.linkedin.pegasus2avro.glossary.GlossaryTermInfo",
+  "com.linkedin.pegasus2avro.identity.CorpGroupInfo",
+  "com.linkedin.pegasus2avro.identity.CorpUserEditableInfo",
+  "com.linkedin.pegasus2avro.identity.CorpUserInfo",
+  "com.linkedin.pegasus2avro.identity.GroupMembership",
+  "com.linkedin.pegasus2avro.metadata.key.ChartKey",
+  "com.linkedin.pegasus2avro.metadata.key.CorpGroupKey",
+  "com.linkedin.pegasus2avro.metadata.key.CorpUserKey",
+  "com.linkedin.pegasus2avro.metadata.key.DashboardKey",
+  "com.linkedin.pegasus2avro.metadata.key.DataFlowKey",
+  "com.linkedin.pegasus2avro.metadata.key.DataHubPolicyKey",
+  "com.linkedin.pegasus2avro.metadata.key.DataJobKey",
+  "com.linkedin.pegasus2avro.metadata.key.DataPlatformKey",
+  "com.linkedin.pegasus2avro.metadata.key.DataProcessKey",
+  "com.linkedin.pegasus2avro.metadata.key.DatasetKey",
+  "com.linkedin.pegasus2avro.metadata.key.GlossaryNodeKey",
+  "com.linkedin.pegasus2avro.metadata.key.GlossaryTermKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLFeatureKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLFeatureTableKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLModelDeploymentKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLModelGroupKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLModelKey",
+  "com.linkedin.pegasus2avro.metadata.key.MLPrimaryKeyKey",
+  "com.linkedin.pegasus2avro.metadata.key.SchemaFieldKey",
+  "com.linkedin.pegasus2avro.metadata.key.TagKey",
+  "com.linkedin.pegasus2avro.ml.metadata.CaveatsAndRecommendations",
+  "com.linkedin.pegasus2avro.ml.metadata.EthicalConsiderations",
+  "com.linkedin.pegasus2avro.ml.metadata.EvaluationData",
+  "com.linkedin.pegasus2avro.ml.metadata.IntendedUse",
+  "com.linkedin.pegasus2avro.ml.metadata.MLFeatureProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.MLFeatureTableProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.MLHyperParam",
+  "com.linkedin.pegasus2avro.ml.metadata.MLMetric",
+  "com.linkedin.pegasus2avro.ml.metadata.MLModelDeploymentProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.MLModelFactorPrompts",
+  "com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.MLModelProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties",
+  "com.linkedin.pegasus2avro.ml.metadata.Metrics",
+  "com.linkedin.pegasus2avro.ml.metadata.QuantitativeAnalyses",
+  "com.linkedin.pegasus2avro.ml.metadata.SourceCode",
+  "com.linkedin.pegasus2avro.ml.metadata.TrainingData",
+  "com.linkedin.pegasus2avro.policy.DataHubPolicyInfo",
+  "com.linkedin.pegasus2avro.schema.EditableSchemaMetadata",
+  "com.linkedin.pegasus2avro.schema.SchemaMetadata",
+  "com.linkedin.pegasus2avro.tag.TagProperties"
 ]