From 0b4851e062a298025e6bfeb4ed22b28102fdc08d Mon Sep 17 00:00:00 2001 From: Mathieu St-Louis Date: Wed, 13 Sep 2023 19:25:17 -0700 Subject: [PATCH 1/5] Add model monitor 2023-08-01-preview changes --- .../preview/2023-08-01-preview/mfe.json | 300 +++++++++--------- 1 file changed, 146 insertions(+), 154 deletions(-) diff --git a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json index eae30eafed85..67f0c5cda8b0 100644 --- a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json +++ b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json @@ -12399,16 +12399,6 @@ }, "additionalProperties": false }, - "AzMonMonitoringAlertNotificationSettings": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/MonitoringAlertNotificationSettingsBase" - } - ], - "x-ms-discriminator-value": "AzureMonitor", - "additionalProperties": false - }, "AzureBlobDatastore": { "description": "Azure Blob datastore configuration.", "type": "object", @@ -14939,6 +14929,15 @@ ], "x-nullable": true }, + "featureImportanceSettings": { + "description": "The settings for computing feature importance.", + "$ref": "#/definitions/FeatureImportanceSettings", + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, "features": { "description": "The feature filter which identifies which feature to calculate drift over.", "$ref": "#/definitions/MonitoringFeatureFilterBase", @@ -15683,27 +15682,6 @@ }, "additionalProperties": false }, - "EmailMonitoringAlertNotificationSettings": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/MonitoringAlertNotificationSettingsBase" - } - ], - "properties": { - "emailNotificationSetting": { - "description": "Configuration for notification.", - "$ref": "#/definitions/NotificationSetting", - "x-ms-mutability": [ - "create", - "read" - ], - "x-nullable": true - } - }, - "x-ms-discriminator-value": "Email", - "additionalProperties": false - }, "EmailNotificationEnableType": { "description": "Enum to determine the email notification type.", "enum": [ @@ -16447,6 +16425,44 @@ }, "additionalProperties": false }, + "FeatureImportanceMode": { + "description": "The mode of operation for computing feature importance.", + "enum": [ + "Enabled", + "Disabled" + ], + "type": "string", + "x-ms-enum": { + "name": "FeatureImportanceMode", + "modelAsString": true, + "values": [ + { + "value": "Enabled", + "description": "Enables computing feature importance within a signal." + }, + { + "value": "Disabled", + "description": "Disables computing feature importance within a signal." + } + ] + }, + "additionalProperties": false + }, + "FeatureImportanceSettings": { + "type": "object", + "properties": { + "mode": { + "description": "The mode of operation for computing feature importance.", + "$ref": "#/definitions/FeatureImportanceMode" + }, + "targetColumnn": { + "description": "The name of the target column within the input data asset.", + "type": "string", + "x-nullable": true + } + }, + "additionalProperties": false + }, "FeatureLags": { "description": "Flag for generating lags for the numeric features.", "enum": [ @@ -17427,7 +17443,7 @@ "x-ms-identifiers": [] }, "productionData": { - "description": "Gets or sets the target data for computing metrics.", + "description": "Gets or sets the production data for computing metrics.", "type": "array", "items": { "description": "Monitoring input data base definition.", @@ -17442,7 +17458,7 @@ }, "samplingRate": { "format": "double", - "description": "[Required] The sample rate of the target data, should be greater than 0 and at most 1.", + "description": "[Required] The sample rate of the production data, should be greater than 0 and at most 1.", "type": "number", "x-ms-mutability": [ "create", @@ -17462,7 +17478,7 @@ "x-ms-discriminator-value": "GenerationSafetyQuality", "additionalProperties": false }, - "GenerationTokenStatisticsMetric": { + "GenerationTokenUsageMetric": { "description": "Generation token statistics metric enum.", "enum": [ "TotalTokenCount", @@ -17470,7 +17486,7 @@ ], "type": "string", "x-ms-enum": { - "name": "GenerationTokenStatisticsMetric", + "name": "GenerationTokenUsageMetric", "modelAsString": true, "values": [ { @@ -17483,7 +17499,7 @@ }, "additionalProperties": false }, - "GenerationTokenStatisticsMetricThreshold": { + "GenerationTokenUsageMetricThreshold": { "description": "Generation token statistics metric threshold definition.", "required": [ "metric" @@ -17492,7 +17508,7 @@ "properties": { "metric": { "description": "[Required] Gets or sets the feature attribution metric to calculate.", - "$ref": "#/definitions/GenerationTokenStatisticsMetric", + "$ref": "#/definitions/GenerationTokenUsageMetric", "x-ms-mutability": [ "create", "read" @@ -17510,8 +17526,8 @@ }, "additionalProperties": false }, - "GenerationTokenStatisticsSignal": { - "description": "Generation token statistics signal definition.", + "GenerationTokenUsageSignal": { + "description": "Generation token usage signal definition.", "required": [ "metricThresholds", "samplingRate" @@ -17527,7 +17543,7 @@ "description": "[Required] Gets or sets the metrics to calculate and the corresponding thresholds.", "type": "array", "items": { - "$ref": "#/definitions/GenerationTokenStatisticsMetricThreshold" + "$ref": "#/definitions/GenerationTokenUsageMetricThreshold" }, "x-ms-mutability": [ "create", @@ -17536,17 +17552,22 @@ "x-ms-identifiers": [] }, "productionData": { - "description": "Gets or sets the target data for computing metrics.", - "$ref": "#/definitions/MonitoringInputDataBase", + "description": "Gets or sets the production data for computing metrics.", + "type": "array", + "items": { + "description": "Monitoring input data base definition.", + "$ref": "#/definitions/MonitoringInputDataBase" + }, "x-ms-mutability": [ "create", "read" ], - "x-nullable": true + "x-nullable": true, + "x-ms-identifiers": [] }, "samplingRate": { "format": "double", - "description": "[Required] The sample rate of the target data, should be greater than 0 and at most 1.", + "description": "[Required] The sample rate of the production data, should be greater than 0 and at most 1.", "type": "number", "x-ms-mutability": [ "create", @@ -20890,9 +20911,9 @@ ], "type": "object", "properties": { - "alertNotificationSetting": { + "alertNotificationSettings": { "description": "The monitor's notification settings.", - "$ref": "#/definitions/MonitoringAlertNotificationSettingsBase", + "$ref": "#/definitions/MonitorNotificationSettings", "x-ms-mutability": [ "create", "read" @@ -20931,6 +20952,21 @@ }, "additionalProperties": false }, + "MonitorNotificationSettings": { + "type": "object", + "properties": { + "emailNotificationSettings": { + "description": "Configuration for notification.", + "$ref": "#/definitions/NotificationSetting", + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + } + }, + "additionalProperties": false + }, "MonitorServerlessSparkCompute": { "description": "Monitor serverless spark compute definition.", "required": [ @@ -20977,45 +21013,6 @@ "x-ms-discriminator-value": "ServerlessSpark", "additionalProperties": false }, - "MonitoringAlertNotificationSettingsBase": { - "required": [ - "alertNotificationType" - ], - "type": "object", - "properties": { - "alertNotificationType": { - "description": "[Required] Specifies the type of signal to monitor.", - "$ref": "#/definitions/MonitoringAlertNotificationType", - "x-ms-mutability": [ - "create", - "read" - ] - } - }, - "discriminator": "alertNotificationType" - }, - "MonitoringAlertNotificationType": { - "enum": [ - "AzureMonitor", - "Email" - ], - "type": "string", - "x-ms-enum": { - "name": "MonitoringAlertNotificationType", - "modelAsString": true, - "values": [ - { - "value": "AzureMonitor", - "description": "Settings for Azure Monitor based alerting." - }, - { - "value": "Email", - "description": "Settings for AML email notifications." - } - ] - }, - "additionalProperties": false - }, "MonitoringDataSegment": { "type": "object", "properties": { @@ -21173,7 +21170,7 @@ "description": "Monitoring input data type enum.", "enum": [ "Static", - "Trailing", + "Rolling", "Fixed" ], "type": "string", @@ -21186,8 +21183,8 @@ "description": "An input data with a fixed window size." }, { - "value": "Trailing", - "description": "An input data which trailing relatively to the monitor's current run." + "value": "Rolling", + "description": "An input data which rolls relatively to the monitor's current run time." }, { "value": "Fixed", @@ -21219,23 +21216,23 @@ }, "additionalProperties": false }, - "MonitoringNotificationMode": { + "MonitoringNotificationType": { "enum": [ - "Disabled", - "Enabled" + "AzureMonitor", + "AmlNotification" ], "type": "string", "x-ms-enum": { - "name": "MonitoringNotificationMode", + "name": "MonitoringNotificationType", "modelAsString": true, "values": [ { - "value": "Disabled", - "description": "Disabled notifications will not produce emails/metrics leveraged for alerting." + "value": "AzureMonitor", + "description": "Enables notifications through Azure Monitor by posting metrics to the workspace's Azure Monitor instance." }, { - "value": "Enabled", - "description": "Enabled notification will produce emails/metrics leveraged for alerting." + "value": "AmlNotification", + "description": "Enables email notifications through AML notifications." } ] }, @@ -21247,14 +21244,18 @@ ], "type": "object", "properties": { - "mode": { + "notificationTypes": { "description": "The current notification mode for this signal.", - "default": "Disabled", - "$ref": "#/definitions/MonitoringNotificationMode", + "type": "array", + "items": { + "$ref": "#/definitions/MonitoringNotificationType" + }, "x-ms-mutability": [ "create", "read" - ] + ], + "x-nullable": true, + "x-ms-identifiers": [] }, "properties": { "description": "Property dictionary. Properties can be added, but not removed or altered.", @@ -23300,7 +23301,6 @@ "PredictionDriftMonitoringSignal": { "required": [ "metricThresholds", - "modelType", "productionData", "referenceData" ], @@ -23323,14 +23323,6 @@ ], "x-ms-identifiers": [] }, - "modelType": { - "description": "[Required] The type of the model monitored.", - "$ref": "#/definitions/MonitoringModelType", - "x-ms-mutability": [ - "create", - "read" - ] - }, "productionData": { "description": "[Required] The data which drift will be calculated for.", "$ref": "#/definitions/MonitoringInputDataBase", @@ -24076,6 +24068,50 @@ }, "additionalProperties": false }, + "RollingInputData": { + "description": "Rolling input data definition.", + "required": [ + "windowOffset", + "windowSize" + ], + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/MonitoringInputDataBase" + } + ], + "properties": { + "preprocessingComponentId": { + "description": "The ARM resource ID of the component resource used to preprocess the data.", + "type": "string", + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, + "windowOffset": { + "format": "duration", + "description": "[Required] The time offset between the end of the data window and the monitor's current run time.", + "type": "string", + "x-ms-mutability": [ + "create", + "read" + ] + }, + "windowSize": { + "format": "duration", + "description": "[Required] The size of the trailing data window.", + "type": "string", + "x-ms-mutability": [ + "create", + "read" + ] + } + }, + "x-ms-discriminator-value": "Rolling", + "additionalProperties": false + }, "RollingRateType": { "enum": [ "Year", @@ -26308,50 +26344,6 @@ "x-ms-discriminator-value": "TopNByAttribution", "additionalProperties": false }, - "TrailingInputData": { - "description": "Trailing input data definition.", - "required": [ - "windowOffset", - "windowSize" - ], - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/MonitoringInputDataBase" - } - ], - "properties": { - "preprocessingComponentId": { - "description": "The ARM resource ID of the component resource used to preprocess the data.", - "type": "string", - "x-ms-mutability": [ - "create", - "read" - ], - "x-nullable": true - }, - "windowOffset": { - "format": "duration", - "description": "[Required] The time offset between the end of the data window and the monitor's current run time.", - "type": "string", - "x-ms-mutability": [ - "create", - "read" - ] - }, - "windowSize": { - "format": "duration", - "description": "[Required] The size of the trailing data window.", - "type": "string", - "x-ms-mutability": [ - "create", - "read" - ] - } - }, - "x-ms-discriminator-value": "Trailing", - "additionalProperties": false - }, "TrainingMode": { "description": "Training mode dictates whether to use distributed training or not", "enum": [ @@ -26859,4 +26851,4 @@ "description": "Azure Active Directory OAuth2 Flow." } } -} +} \ No newline at end of file From f6e6465d443f411496410dcd99c138d7a25c0eeb Mon Sep 17 00:00:00 2001 From: Mathieu St-Louis Date: Wed, 13 Sep 2023 19:36:47 -0700 Subject: [PATCH 2/5] update --- .../preview/2023-08-01-preview/mfe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json index 67f0c5cda8b0..260e8e6ece9d 100644 --- a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json +++ b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json @@ -16455,7 +16455,7 @@ "description": "The mode of operation for computing feature importance.", "$ref": "#/definitions/FeatureImportanceMode" }, - "targetColumnn": { + "targetColumn": { "description": "The name of the target column within the input data asset.", "type": "string", "x-nullable": true From 3d7d3b3371812632e7cd5afcb1ca383a4ba348f5 Mon Sep 17 00:00:00 2001 From: Mathieu St-Louis Date: Wed, 13 Sep 2023 20:21:57 -0700 Subject: [PATCH 3/5] run prettier --- .../preview/2023-08-01-preview/mfe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json index 260e8e6ece9d..ac4effa00029 100644 --- a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json +++ b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json @@ -26851,4 +26851,4 @@ "description": "Azure Active Directory OAuth2 Flow." } } -} \ No newline at end of file +} From 4d0f81e4e0862d7d71af89d400d4b53c46f9a940 Mon Sep 17 00:00:00 2001 From: Mathieu St-Louis Date: Thu, 14 Sep 2023 14:46:04 -0700 Subject: [PATCH 4/5] update comments --- .../preview/2023-08-01-preview/mfe.json | 94 +++++++++++++++---- 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json index ac4effa00029..f3df6708cd10 100644 --- a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json +++ b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json @@ -15118,6 +15118,15 @@ ], "x-nullable": true }, + "featureImportanceSettings": { + "description": "The settings for computing feature importance.", + "$ref": "#/definitions/FeatureImportanceSettings", + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, "features": { "description": "The features to calculate drift over.", "$ref": "#/definitions/MonitoringFeatureFilterBase", @@ -16305,6 +16314,27 @@ } ], "properties": { + "featureDataTypeOverride": { + "description": "A dictionary that maps feature names to their respective data types.", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/MonitoringFeatureDataType" + }, + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, + "featureImportanceSettings": { + "description": "The settings for computing feature importance.", + "$ref": "#/definitions/FeatureImportanceSettings", + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, "metricThreshold": { "description": "[Required] A list of metrics to calculate and their associated thresholds.", "$ref": "#/definitions/FeatureAttributionMetricThreshold", @@ -16428,21 +16458,21 @@ "FeatureImportanceMode": { "description": "The mode of operation for computing feature importance.", "enum": [ - "Enabled", - "Disabled" + "Disabled", + "Enabled" ], "type": "string", "x-ms-enum": { "name": "FeatureImportanceMode", "modelAsString": true, "values": [ - { - "value": "Enabled", - "description": "Enables computing feature importance within a signal." - }, { "value": "Disabled", "description": "Disables computing feature importance within a signal." + }, + { + "value": "Enabled", + "description": "Enables computing feature importance within a signal." } ] }, @@ -16453,6 +16483,7 @@ "properties": { "mode": { "description": "The mode of operation for computing feature importance.", + "default": "Disabled", "$ref": "#/definitions/FeatureImportanceMode" }, "targetColumn": { @@ -20628,7 +20659,7 @@ ] }, "productionData": { - "description": "[Required] The data produced by the production service which drift will be calculated for.", + "description": "[Required] The data produced by the production service which performance will be calculated for.", "type": "array", "items": { "description": "Monitoring input data base definition.", @@ -20641,7 +20672,7 @@ "x-ms-identifiers": [] }, "referenceData": { - "description": "[Required] The data to calculate drift against.", + "description": "[Required] The reference data used as the basis to calculate model performance.", "$ref": "#/definitions/MonitoringInputDataBase", "x-ms-mutability": [ "create", @@ -20952,12 +20983,31 @@ }, "additionalProperties": false }, + "MonitorEmailNotificationSettings": { + "type": "object", + "properties": { + "emails": { + "description": "This is the email recipient list which has a limitation of 499 characters in total concat with comma seperator.", + "type": "array", + "items": { + "type": "string" + }, + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true, + "x-ms-identifiers": [] + } + }, + "additionalProperties": false + }, "MonitorNotificationSettings": { "type": "object", "properties": { "emailNotificationSettings": { - "description": "Configuration for notification.", - "$ref": "#/definitions/NotificationSetting", + "description": "The AML notification email settings.", + "$ref": "#/definitions/MonitorEmailNotificationSettings", "x-ms-mutability": [ "create", "read" @@ -21218,21 +21268,21 @@ }, "MonitoringNotificationType": { "enum": [ - "AzureMonitor", - "AmlNotification" + "AmlNotification", + "AzureMonitor" ], "type": "string", "x-ms-enum": { "name": "MonitoringNotificationType", "modelAsString": true, "values": [ - { - "value": "AzureMonitor", - "description": "Enables notifications through Azure Monitor by posting metrics to the workspace's Azure Monitor instance." - }, { "value": "AmlNotification", "description": "Enables email notifications through AML notifications." + }, + { + "value": "AzureMonitor", + "description": "Enables notifications through Azure Monitor by posting metrics to the workspace's Azure Monitor instance." } ] }, @@ -23311,6 +23361,18 @@ } ], "properties": { + "featureDataTypeOverride": { + "description": "A dictionary that maps feature names to their respective data types.", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/MonitoringFeatureDataType" + }, + "x-ms-mutability": [ + "create", + "read" + ], + "x-nullable": true + }, "metricThresholds": { "description": "[Required] A list of metrics to calculate and their associated thresholds.", "type": "array", From 57a46dcc0214353afd0fa09c8fa4b43bca77060c Mon Sep 17 00:00:00 2001 From: Mathieu St-Louis Date: Thu, 14 Sep 2023 15:08:04 -0700 Subject: [PATCH 5/5] add --- .../preview/2023-08-01-preview/mfe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json index f3df6708cd10..a5e6afc11558 100644 --- a/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json +++ b/specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/preview/2023-08-01-preview/mfe.json @@ -20987,7 +20987,7 @@ "type": "object", "properties": { "emails": { - "description": "This is the email recipient list which has a limitation of 499 characters in total concat with comma seperator.", + "description": "This is the email recipient list which has a limitation of 499 characters in total.", "type": "array", "items": { "type": "string"