[skip ci] Metadata: Remove leading underscore (#29024)

* DNC

* Add test models

* Add model test

* Remove underscore from metadata files

* Regenerate models

* Add test to check for key transformation

* Allow additional fields on metadata

* Delete transform

airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py

@@ -16,7 +16,7 @@
 
 from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER, ICON_FILE_NAME
 from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, validate_and_load
-from metadata_service.utils import to_json_sanitized_dict
+from metadata_service.models.transform import to_json_sanitized_dict
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
 
 from dataclasses import dataclass

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py

@@ -5,13 +5,13 @@
 
 from typing import Optional
 
-from pydantic import BaseModel, Extra, Field
+from pydantic import BaseModel, Extra
 from typing_extensions import Literal
 
 
 class AirbyteInternal(BaseModel):
     class Config:
         extra = Extra.allow
 
-    field_sl: Optional[Literal[100, 200, 300]] = Field(None, alias="_sl")
-    field_ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = Field(None, alias="_ql")
+    sl: Optional[Literal[100, 200, 300]] = None
+    ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py

@@ -112,8 +112,8 @@ class AirbyteInternal(BaseModel):
     class Config:
         extra = Extra.allow
 
-    field_sl: Optional[Literal[100, 200, 300]] = Field(None, alias="_sl")
-    field_ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = Field(None, alias="_ql")
+    sl: Optional[Literal[100, 200, 300]] = None
+    ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
 class JobTypeResourceLimit(BaseModel):
@@ -185,7 +185,7 @@ class Config:
 
 class Data(BaseModel):
     class Config:
-        extra = Extra.forbid
+        extra = Extra.allow
 
     name: str
     icon: Optional[str] = None
@@ -224,7 +224,7 @@ class Config:
     normalizationConfig: Optional[NormalizationDestinationDefinitionConfig] = None
     suggestedStreams: Optional[SuggestedStreams] = None
     resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None
-    field_ab_internal: Optional[AirbyteInternal] = Field(None, alias="_ab_internal")
+    ab_internal: Optional[AirbyteInternal] = None
 
 
 class ConnectorMetadataDefinitionV0(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py

@@ -102,8 +102,8 @@ class AirbyteInternal(BaseModel):
     class Config:
         extra = Extra.allow
 
-    field_sl: Optional[Literal[100, 200, 300]] = Field(None, alias="_sl")
-    field_ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = Field(None, alias="_ql")
+    sl: Optional[Literal[100, 200, 300]] = None
+    ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
 class JobTypeResourceLimit(BaseModel):
@@ -190,4 +190,4 @@ class Config:
     )
     allowedHosts: Optional[AllowedHosts] = None
     releases: Optional[ConnectorReleases] = None
-    field_ab_internal: Optional[AirbyteInternal] = Field(None, alias="_ab_internal")
+    ab_internal: Optional[AirbyteInternal] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py

@@ -94,8 +94,8 @@ class AirbyteInternal(BaseModel):
     class Config:
         extra = Extra.allow
 
-    field_sl: Optional[Literal[100, 200, 300]] = Field(None, alias="_sl")
-    field_ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = Field(None, alias="_ql")
+    sl: Optional[Literal[100, 200, 300]] = None
+    ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
 class JobTypeResourceLimit(BaseModel):
@@ -179,4 +179,4 @@ class Config:
         description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach",
     )
     releases: Optional[ConnectorReleases] = None
-    field_ab_internal: Optional[AirbyteInternal] = Field(None, alias="_ab_internal")
+    ab_internal: Optional[AirbyteInternal] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py

@@ -102,8 +102,8 @@ class AirbyteInternal(BaseModel):
     class Config:
         extra = Extra.allow
 
-    field_sl: Optional[Literal[100, 200, 300]] = Field(None, alias="_sl")
-    field_ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = Field(None, alias="_ql")
+    sl: Optional[Literal[100, 200, 300]] = None
+    ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None
 
 
 class SuggestedStreams(BaseModel):
@@ -197,7 +197,7 @@ class Config:
         description="Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach",
     )
     releases: Optional[ConnectorReleases] = None
-    field_ab_internal: Optional[AirbyteInternal] = Field(None, alias="_ab_internal")
+    ab_internal: Optional[AirbyteInternal] = None
 
 
 class ConnectorRegistryDestinationDefinition(BaseModel):
@@ -244,7 +244,7 @@ class Config:
     )
     allowedHosts: Optional[AllowedHosts] = None
     releases: Optional[ConnectorReleases] = None
-    field_ab_internal: Optional[AirbyteInternal] = Field(None, alias="_ab_internal")
+    ab_internal: Optional[AirbyteInternal] = None
 
 
 class ConnectorRegistryV0(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AirbyteInternal.yaml

@@ -6,13 +6,13 @@ description: Fields for internal use only
 type: object
 additionalProperties: true
 properties:
-  _sl:
+  sl:
     type: integer
     enum:
       - 100
       - 200
       - 300
-  _ql:
+  ql:
     type: integer
     enum:
       - 100

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml

@@ -25,7 +25,7 @@ properties:
       - githubIssueLabel
       - connectorSubtype
       - releaseStage
-    additionalProperties: false
+    additionalProperties: true
     properties:
       name:
         type: string
@@ -105,5 +105,5 @@ properties:
         "$ref": SuggestedStreams.yaml
       resourceRequirements:
         "$ref": ActorDefinitionResourceRequirements.yaml
-      _ab_internal:
+      ab_internal:
         "$ref": AirbyteInternal.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml

@@ -70,5 +70,5 @@ properties:
     "$ref": AllowedHosts.yaml
   releases:
     "$ref": ConnectorReleases.yaml
-  _ab_internal:
+  ab_internal:
     "$ref": AirbyteInternal.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml

@@ -72,5 +72,5 @@ properties:
     type: integer
   releases:
     "$ref": ConnectorReleases.yaml
-  _ab_internal:
+  ab_internal:
     "$ref": AirbyteInternal.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/transform.py

@@ -0,0 +1,62 @@
+import json
+from pydantic import BaseModel
+
+def _apply_default_pydantic_kwargs(kwargs: dict) -> dict:
+    """A helper function to apply default kwargs to pydantic models.
+
+    Args:
+        kwargs (dict): the kwargs to apply
+
+    Returns:
+        dict: the kwargs with defaults applied
+    """
+    default_kwargs = {
+        "by_alias": True, # Ensure that the original field name from the jsonschema is used in the event it begins with an underscore (e.g. ab_internal)
+        "exclude_none": True, # Exclude fields that are None
+    }
+
+    return {**default_kwargs, **kwargs}
+
+def to_json_sanitized_dict(pydantic_model_obj: BaseModel, **kwargs) -> dict:
+    """A helper function to convert a pydantic model to a sanitized dict.
+
+    Without this pydantic dictionary may contain values that are not JSON serializable.
+
+    Args:
+        pydantic_model_obj (BaseModel): a pydantic model
+
+    Returns:
+        dict: a sanitized dictionary
+    """
+
+    return json.loads(to_json(pydantic_model_obj, **kwargs))
+
+def to_json(pydantic_model_obj: BaseModel, **kwargs) -> str:
+    """A helper function to convert a pydantic model to a json string.
+
+    Without this pydantic dictionary may contain values that are not JSON serializable.
+
+    Args:
+        pydantic_model_obj (BaseModel): a pydantic model
+
+    Returns:
+        str: a json string
+    """
+    kwargs = _apply_default_pydantic_kwargs(kwargs)
+
+    return pydantic_model_obj.json(**kwargs)
+
+def to_dict(pydantic_model_obj: BaseModel, **kwargs) -> dict:
+    """A helper function to convert a pydantic model to a dict.
+
+    Without this pydantic dictionary may contain values that are not JSON serializable.
+
+    Args:
+        pydantic_model_obj (BaseModel): a pydantic model
+
+    Returns:
+        dict: a dict
+    """
+    kwargs = _apply_default_pydantic_kwargs(kwargs)
+
+    return pydantic_model_obj.dict(**kwargs)