Expected Impact Calibration

ludwig/schema/decoders/base.py

@@ -15,44 +15,91 @@
 class BaseDecoderConfig(schema_utils.BaseMarshmallowConfig, ABC):
     """Base class for decoders."""
 
-    type: str
+    type: str = schema_utils.StringOptions(
+        ["regressor", "classifier", "projector", "generator", "tagger"],
+        description="The type of decoder to use.",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["type"],
+    )
 
     fc_layers: List[Dict[str, Any]] = schema_utils.DictList(
-        default=None, description="List of dictionaries containing the parameters for each fully connected layer."
+        default=None,
+        description="List of dictionaries containing the parameters for each fully connected layer.",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_layers"],
     )
 
     num_fc_layers: int = schema_utils.NonNegativeInteger(
-        default=0, description="Number of fully-connected layers if fc_layers not specified."
+        default=0,
+        description="Number of fully-connected layers if fc_layers not specified.",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["num_fc_layers"],
     )
 
-    fc_output_size: int = schema_utils.PositiveInteger(default=256, description="Output size of fully connected stack.")
+    fc_output_size: int = schema_utils.PositiveInteger(
+        default=256,
+        description="Output size of fully connected stack.",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_output_size"],
+    )
 
     fc_use_bias: bool = schema_utils.Boolean(
-        default=True, description="Whether the layer uses a bias vector in the fc_stack."
+        default=True,
+        description="Whether the layer uses a bias vector in the fc_stack.",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_use_bias"],
     )
 
-    fc_weights_initializer: Union[str, Dict] = schema_utils.InitializerOrDict(
-        default="xavier_uniform", description="The weights initializer to use for the layers in the fc_stack"
+    fc_weights_initializer: Union[str, Dict] = schema_utils.OneOfOptionsField(
+        default="xavier_uniform",
+        description="The weights initializer to use for the layers in the fc_stack",
+        field_options=[
+            schema_utils.InitializerOptions(
+                description="Preconfigured initializer to use for the layers in the fc_stack.",
+                parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_weights_initializer"],
+            ),
+            schema_utils.Dict(
+                description="Custom initializer to use for the layers in the fc_stack.",
+                parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_weights_initializer"],
+            ),
+        ],
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_weights_initializer"],
     )
 
-    fc_bias_initializer: Union[str, Dict] = schema_utils.InitializerOrDict(
-        default="zeros", description="The bias initializer to use for the layers in the fc_stack"
+    fc_bias_initializer: Union[str, Dict] = schema_utils.OneOfOptionsField(
+        default="zeros",
+        description="The bias initializer to use for the layers in the fc_stack",
+        field_options=[
+            schema_utils.InitializerOptions(
+                description="Preconfigured bias initializer to use for the layers in the fc_stack.",
+                parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_bias_initializer"],
+            ),
+            schema_utils.Dict(
+                description="Custom bias initializer to use for the layers in the fc_stack.",
+                parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_bias_initializer"],
+            ),
+        ],
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_bias_initializer"],
     )
 
     fc_norm: str = schema_utils.StringOptions(
-        ["batch", "layer"], description="The normalization to use for the layers in the fc_stack"
+        ["batch", "layer"],
+        description="The normalization to use for the layers in the fc_stack",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_norm"],
     )
 
     fc_norm_params: dict = schema_utils.Dict(
-        description="The additional parameters for the normalization in the fc_stack"
+        description="The additional parameters for the normalization in the fc_stack",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_norm_params"],
     )
 
     fc_activation: str = schema_utils.ActivationOptions(
-        default="relu", description="The activation to use for the layers in the fc_stack"
+        default="relu",
+        description="The activation to use for the layers in the fc_stack",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_activation"],
     )
 
     fc_dropout: float = schema_utils.FloatRange(
-        default=0.0, min=0, max=1, description="The dropout rate to use for the layers in the fc_stack"
+        default=0.0,
+        min=0,
+        max=1,
+        description="The dropout rate to use for the layers in the fc_stack",
+        parameter_metadata=DECODER_METADATA["BaseDecoder"]["fc_dropout"],
     )
 
 
@@ -70,11 +117,13 @@ def module_name(cls):
         "passthrough",
         description="The passthrough decoder simply returns the raw numerical values coming from the combiner as "
         "outputs",
+        parameter_metadata=DECODER_METADATA["PassthroughDecoder"]["type"],
     )
 
     input_size: int = schema_utils.PositiveInteger(
         default=1,
         description="Size of the input to the decoder.",
+        parameter_metadata=DECODER_METADATA["PassthroughDecoder"]["input_size"],
     )
 
 

ludwig/schema/encoders/base.py

@@ -54,56 +54,67 @@ def module_name():
         min=0,
         max=1,
         description="Dropout rate.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["dropout"],
     )
 
     activation: str = schema_utils.StringOptions(
         ["elu", "leakyRelu", "logSigmoid", "relu", "sigmoid", "tanh", "softmax"],
         default="relu",
         description="Activation function to apply to the output.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["activation"],
     )
 
     input_size: int = schema_utils.PositiveInteger(
         default=None,
         description="Size of the input to the dense encoder.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["input_size"],
     )
 
     output_size: int = schema_utils.PositiveInteger(
         default=256,
         description="Size of the output of the feature.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["output_size"],
     )
 
     use_bias: bool = schema_utils.Boolean(
         default=True,
         description="Whether the layer uses a bias vector.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["use_bias"],
     )
 
     bias_initializer: Union[str, dict] = schema_utils.InitializerOptions(
         default="zeros",
         description="Initializer for the bias vector.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["bias_initializer"],
     )
 
     weights_initializer: Union[str, dict] = schema_utils.InitializerOptions(
         description="Initializer for the weight matrix.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["weights_initializer"],
     )
 
     norm: str = schema_utils.StringOptions(
         ["batch", "layer"],
         allow_none=True,
         default=None,
         description="Normalization to use in the dense layer.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["norm"],
     )
 
     norm_params: dict = schema_utils.Dict(
         default=None,
         description="Parameters for normalization if norm is either batch or layer.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["norm_params"],
     )
 
     num_layers: int = schema_utils.PositiveInteger(
         default=1,
         description="Number of stacked fully connected layers that the input to the feature passes through.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["num_layers"],
     )
 
     fc_layers: List[dict] = schema_utils.DictList(
         default=None,
         description="List of fully connected layers to use in the encoder.",
+        parameter_metadata=ENCODER_METADATA["DenseEncoder"]["fc_layers"],
     )

ludwig/schema/features/binary_feature.py

@@ -18,6 +18,7 @@
     output_config_registry,
     output_mixin_registry,
 )
+from ludwig.schema.metadata import FEATURE_METADATA
 from ludwig.schema.metadata.parameter_metadata import INTERNAL_ONLY
 from ludwig.schema.utils import BaseMarshmallowConfig
 
@@ -73,6 +74,7 @@ class BinaryOutputFeatureConfig(BaseOutputFeatureConfig, BinaryOutputFeatureConf
     calibration: bool = schema_utils.Boolean(
         default=False,
         description="Calibrate the model's output probabilities using temperature scaling.",
+        parameter_metadata=FEATURE_METADATA[BINARY]["calibration"],
     )
 
     default_validation_metric: str = schema_utils.StringOptions(
@@ -85,19 +87,22 @@ class BinaryOutputFeatureConfig(BaseOutputFeatureConfig, BinaryOutputFeatureConf
     dependencies: list = schema_utils.List(
         default=[],
         description="List of input features that this feature depends on.",
+        parameter_metadata=FEATURE_METADATA[BINARY]["dependencies"],
     )
 
     preprocessing: BasePreprocessingConfig = PreprocessingDataclassField(feature_type="binary_output")
 
     reduce_dependencies: str = schema_utils.ReductionOptions(
         default="sum",
         description="How to reduce the dependencies of the output feature.",
+        parameter_metadata=FEATURE_METADATA[BINARY]["reduce_dependencies"],
     )
 
     reduce_input: str = schema_utils.ReductionOptions(
         default="sum",
         description="How to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first "
         "dimension (second if you count the batch dimension)",
+        parameter_metadata=FEATURE_METADATA[BINARY]["reduce_input"],
     )
 
     threshold: float = schema_utils.FloatRange(
@@ -106,4 +111,5 @@ class BinaryOutputFeatureConfig(BaseOutputFeatureConfig, BinaryOutputFeatureConf
         max=1,
         description="The threshold used to convert output probabilities to predictions. Predicted probabilities greater"
         "than or equal to threshold are mapped to True.",
+        parameter_metadata=FEATURE_METADATA[BINARY]["threshold"],
     )

ludwig/schema/features/category_feature.py

@@ -18,6 +18,7 @@
     output_config_registry,
     output_mixin_registry,
 )
+from ludwig.schema.metadata import FEATURE_METADATA
 from ludwig.schema.metadata.parameter_metadata import INTERNAL_ONLY
 from ludwig.schema.utils import BaseMarshmallowConfig
 
@@ -75,6 +76,7 @@ class CategoryOutputFeatureConfig(BaseOutputFeatureConfig, CategoryOutputFeature
     calibration: bool = schema_utils.Boolean(
         default=False,
         description="Calibrate the model's output probabilities using temperature scaling.",
+        parameter_metadata=FEATURE_METADATA[CATEGORY]["calibration"],
     )
 
     default_validation_metric: str = schema_utils.StringOptions(
@@ -87,24 +89,28 @@ class CategoryOutputFeatureConfig(BaseOutputFeatureConfig, CategoryOutputFeature
     dependencies: list = schema_utils.List(
         default=[],
         description="List of input features that this feature depends on.",
+        parameter_metadata=FEATURE_METADATA[CATEGORY]["dependencies"],
     )
 
     preprocessing: BasePreprocessingConfig = PreprocessingDataclassField(feature_type="category_output")
 
     reduce_dependencies: str = schema_utils.ReductionOptions(
         default="sum",
         description="How to reduce the dependencies of the output feature.",
+        parameter_metadata=FEATURE_METADATA[CATEGORY]["reduce_dependencies"],
     )
 
     reduce_input: str = schema_utils.ReductionOptions(
         default="sum",
         description="How to reduce an input that is not a vector, but a matrix or a higher order tensor, on the first "
         "dimension (second if you count the batch dimension)",
+        parameter_metadata=FEATURE_METADATA[CATEGORY]["reduce_input"],
     )
 
     top_k: int = schema_utils.NonNegativeInteger(
         default=3,
         description="Determines the parameter k, the number of categories to consider when computing the top_k "
         "measure. It computes accuracy but considering as a match if the true category appears in the "
         "first k predicted categories ranked by decoder's confidence.",
+        parameter_metadata=FEATURE_METADATA[CATEGORY]["top_k"],
     )