Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Get the cross validation macro to work with non-default column names #291

Merged
merged 7 commits into from
Jun 5, 2018
Merged

Get the cross validation macro to work with non-default column names #291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
52aba23
Add label/grou/weight column name arguments to CV and train-test macros
yaeldMS Jun 1, 2018
7826c71
Fix unit test.
yaeldMS Jun 4, 2018
235ff2d
Merge branch 'master' of https://github.com/dotnet/machinelearning in…
yaeldMS Jun 4, 2018
43a8bbe
Merge.
yaeldMS Jun 4, 2018
40ae3c2
Update CSharp API.
yaeldMS Jun 4, 2018
4e62e14
Fix EntryPointCatalog test.
yaeldMS Jun 4, 2018
c862d36
Address PR comments.
yaeldMS Jun 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/Microsoft.ML.Data/DataView/AppendRowsDataView.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,8 @@ protected Delegate CreateGetter(int col)
public ValueGetter<TValue> GetGetter<TValue>(int col)
{
Ch.Check(IsColumnActive(col), "The column must be active against the defined predicate.");
if (!(Getters[col] is ValueGetter<TValue>))
throw Ch.Except($"Invalid TValue in GetGetter: '{typeof(TValue)}'");
return Getters[col] as ValueGetter<TValue>;
}

Expand Down
70 changes: 62 additions & 8 deletions src/Microsoft.ML.Data/EntryPoints/EntryPointNode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,9 +473,9 @@ public float Cost
}
}

public EntryPointNode(IHostEnvironment env, ModuleCatalog moduleCatalog, RunContext context,
private EntryPointNode(IHostEnvironment env, IChannel ch, ModuleCatalog moduleCatalog, RunContext context,
string id, string entryPointName, JObject inputs, JObject outputs, bool checkpoint = false,
string stageId = "", float cost = float.NaN)
string stageId = "", float cost = float.NaN, string label = null, string group = null, string weight = null)
{
Contracts.AssertValue(env);
env.AssertNonEmpty(id);
Expand All @@ -497,6 +497,7 @@ public EntryPointNode(IHostEnvironment env, ModuleCatalog moduleCatalog, RunCont
_inputMap = new Dictionary<ParameterBinding, VariableBinding>();
_inputBindingMap = new Dictionary<string, List<ParameterBinding>>();
_inputBuilder = new InputBuilder(_host, _entryPoint.InputType, moduleCatalog);

// REVIEW: This logic should move out of Node eventually and be delegated to
// a class that can nest to handle Components with variables.
if (inputs != null)
Expand All @@ -508,6 +509,51 @@ public EntryPointNode(IHostEnvironment env, ModuleCatalog moduleCatalog, RunCont
if (missing.Length > 0)
throw _host.Except($"The following required inputs were not provided: {String.Join(", ", missing)}");

var inputInstance = _inputBuilder.GetInstance();
var warning = "Different {0} column specified in trainer and in macro: '{1}', '{2}'." +
" Using column '{2}'. To column use '{1}' instead, please specify this name in" +
"the trainer node arguments.";
if (!string.IsNullOrEmpty(label) && Utils.Size(_entryPoint.InputKinds) > 0 &&
_entryPoint.InputKinds.Contains(typeof(CommonInputs.ITrainerInputWithLabel)))
{
var labelColField = _inputBuilder.GetFieldNameOrNull("LabelColumn");
ch.AssertNonEmpty(labelColField);
var labelColFieldType = _inputBuilder.GetFieldTypeOrNull(labelColField);
ch.Assert(labelColFieldType == typeof(string));
var inputLabel = inputInstance.GetType().GetField(labelColField).GetValue(inputInstance);
if (label != (string)inputLabel)
ch.Warning(warning, "label", label, inputLabel);
else
_inputBuilder.TrySetValue(labelColField, label);
}
if (!string.IsNullOrEmpty(group) && Utils.Size(_entryPoint.InputKinds) > 0 &&
_entryPoint.InputKinds.Contains(typeof(CommonInputs.ITrainerInputWithGroupId)))
{
var groupColField = _inputBuilder.GetFieldNameOrNull("GroupIdColumn");
ch.AssertNonEmpty(groupColField);
var groupColFieldType = _inputBuilder.GetFieldTypeOrNull(groupColField);
ch.Assert(groupColFieldType == typeof(string));
var inputGroup = inputInstance.GetType().GetField(groupColField).GetValue(inputInstance);
if (group != (Optional<string>)inputGroup)
ch.Warning(warning, "group Id", label, inputGroup);
else
_inputBuilder.TrySetValue(groupColField, label);
}
if (!string.IsNullOrEmpty(weight) && Utils.Size(_entryPoint.InputKinds) > 0 &&
(_entryPoint.InputKinds.Contains(typeof(CommonInputs.ITrainerInputWithWeight)) ||
_entryPoint.InputKinds.Contains(typeof(CommonInputs.IUnsupervisedTrainerWithWeight))))
{
var weightColField = _inputBuilder.GetFieldNameOrNull("WeightColumn");
ch.AssertNonEmpty(weightColField);
var weightColFieldType = _inputBuilder.GetFieldTypeOrNull(weightColField);
ch.Assert(weightColFieldType == typeof(string));
var inputWeight = inputInstance.GetType().GetField(weightColField).GetValue(inputInstance);
if (weight != (Optional<string>)inputWeight)
ch.Warning(warning, "weight", label, inputWeight);
else
_inputBuilder.TrySetValue(weightColField, label);
}

// Validate outputs.
_outputHelper = new OutputHelper(_host, _entryPoint.OutputType);
_outputMap = new Dictionary<string, string>();
Expand Down Expand Up @@ -550,10 +596,15 @@ public static EntryPointNode Create(
var inputBuilder = new InputBuilder(env, info.InputType, catalog);
var outputHelper = new OutputHelper(env, info.OutputType);

var entryPointNode = new EntryPointNode(env, catalog, context, context.GenerateId(entryPointName), entryPointName,
inputBuilder.GetJsonObject(arguments, inputBindingMap, inputMap),
outputHelper.GetJsonObject(outputMap), checkpoint, stageId, cost);
return entryPointNode;
using (var ch = env.Start("Create EntryPointNode"))
{
var entryPointNode = new EntryPointNode(env, ch, catalog, context, context.GenerateId(entryPointName), entryPointName,
inputBuilder.GetJsonObject(arguments, inputBindingMap, inputMap),
outputHelper.GetJsonObject(outputMap), checkpoint, stageId, cost);

ch.Done();
return entryPointNode;
}
}

public static EntryPointNode Create(
Expand Down Expand Up @@ -850,7 +901,8 @@ private object BuildParameterValue(List<ParameterBinding> bindings)
throw _host.ExceptNotImpl("Unsupported ParameterBinding");
}

public static List<EntryPointNode> ValidateNodes(IHostEnvironment env, RunContext context, JArray nodes, ModuleCatalog moduleCatalog)
public static List<EntryPointNode> ValidateNodes(IHostEnvironment env, RunContext context, JArray nodes,
ModuleCatalog moduleCatalog, string label = null, string group = null, string weight = null)
{
Contracts.AssertValue(env);
env.AssertValue(context);
Expand Down Expand Up @@ -890,8 +942,10 @@ public static List<EntryPointNode> ValidateNodes(IHostEnvironment env, RunContex
ch.Warning("Node '{0}' has unexpected fields that are ignored: {1}", id, string.Join(", ", unexpectedFields.Select(x => x.Name)));
}

result.Add(new EntryPointNode(env, moduleCatalog, context, id, name, inputs, outputs, checkpoint, stageId, cost));
result.Add(new EntryPointNode(env, ch, moduleCatalog, context, id, name, inputs, outputs, checkpoint, stageId, cost, label, group, weight));
}

ch.Done();
}
return result;
}
Expand Down
7 changes: 5 additions & 2 deletions src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@

using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reflection;
using Microsoft.ML.Runtime.CommandLine;
using Microsoft.ML.Runtime.Internal.Utilities;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.Internal.Utilities;
using Newtonsoft.Json.Linq;

namespace Microsoft.ML.Runtime.EntryPoints.JsonUtils
Expand Down Expand Up @@ -405,7 +404,11 @@ private static object ParseJsonValue(IExceptionContext ectx, Type type, Attribut
return null;

if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Optional<>) || type.GetGenericTypeDefinition() == typeof(Nullable<>)))
{
if (type.GetGenericTypeDefinition() == typeof(Optional<>) && value.HasValues)
value = value.Values().FirstOrDefault();
type = type.GetGenericArguments()[0];
}

if (type.IsGenericType && (type.GetGenericTypeDefinition() == typeof(Var<>)))
{
Expand Down
49 changes: 47 additions & 2 deletions src/Microsoft.ML/CSharpApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2165,6 +2165,16 @@ public sealed partial class CrossValidationResultsCombiner
/// </summary>
public string LabelColumn { get; set; } = "Label";

/// <summary>
/// Column to use for example weight
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> WeightColumn { get; set; }

/// <summary>
/// Column to use for grouping
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> GroupColumn { get; set; }

/// <summary>
/// Specifies the trainer kind, which determines the evaluator to be used.
/// </summary>
Expand Down Expand Up @@ -2270,6 +2280,21 @@ public sealed partial class CrossValidator
/// </summary>
public Models.MacroUtilsTrainerKinds Kind { get; set; } = Models.MacroUtilsTrainerKinds.SignatureBinaryClassifierTrainer;

/// <summary>
/// Column to use for labels
/// </summary>
public string LabelColumn { get; set; } = "Label";

/// <summary>
/// Column to use for example weight
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> WeightColumn { get; set; }

/// <summary>
/// Column to use for grouping
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> GroupColumn { get; set; }


public sealed class Output
{
Expand Down Expand Up @@ -3456,6 +3481,21 @@ public sealed partial class TrainTestEvaluator
/// </summary>
public bool IncludeTrainingMetrics { get; set; } = false;

/// <summary>
/// Column to use for labels
/// </summary>
public string LabelColumn { get; set; } = "Label";

/// <summary>
/// Column to use for example weight
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> WeightColumn { get; set; }

/// <summary>
/// Column to use for grouping
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> GroupColumn { get; set; }
Copy link
Contributor

@george-microsoft george-microsoft Jun 4, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there any default which makes sense here for GroupColumn, such as Label has?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The GroupColumn argument has a default value (see in TrainTestMacro.cs). I think this might be an issue with the CSharpCodeGenerator. I'll open an issue.

In reply to: 192864697 [](ancestors = 192864697)



public sealed class Output
{
Expand Down Expand Up @@ -6173,7 +6213,7 @@ public enum KMeansPlusPlusTrainerInitAlgorithm
/// <summary>
/// K-means is a popular clustering algorithm. With K-means, the data is clustered into a specified number of clusters in order to minimize the within-cluster sum of squares. K-means++ improves upon K-means by using a better method for choosing the initial cluster centers.
/// </summary>
public sealed partial class KMeansPlusPlusClusterer : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem
public sealed partial class KMeansPlusPlusClusterer : Microsoft.ML.Runtime.EntryPoints.CommonInputs.IUnsupervisedTrainerWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem
{


Expand Down Expand Up @@ -6208,6 +6248,11 @@ public sealed partial class KMeansPlusPlusClusterer : Microsoft.ML.Runtime.Entry
/// </summary>
public int? NumThreads { get; set; }

/// <summary>
/// Column to use for example weight
/// </summary>
public Microsoft.ML.Runtime.EntryPoints.Optional<string> WeightColumn { get; set; }

/// <summary>
/// The data to be used for training
/// </summary>
Expand Down Expand Up @@ -7024,7 +7069,7 @@ namespace Trainers
/// <summary>
/// Train an PCA Anomaly model.
/// </summary>
public sealed partial class PcaAnomalyDetector : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem
public sealed partial class PcaAnomalyDetector : Microsoft.ML.Runtime.EntryPoints.CommonInputs.IUnsupervisedTrainerWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem
{


Expand Down
40 changes: 34 additions & 6 deletions src/Microsoft.ML/Runtime/EntryPoints/CrossValidationMacro.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,15 @@ public sealed class Arguments
// (and the same for the TrainTest macro). I currently do not know how to do this, so this should be revisited in the future.
[Argument(ArgumentType.Required, HelpText = "Specifies the trainer kind, which determines the evaluator to be used.", SortOrder = 8)]
public MacroUtils.TrainerKinds Kind = MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for labels", ShortName = "lab", SortOrder = 10)]
public string LabelColumn = DefaultColumnNames.Label;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for example weight", ShortName = "weight", SortOrder = 11)]
public Optional<string> WeightColumn = Optional<string>.Implicit(DefaultColumnNames.Weight);

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for grouping", ShortName = "group", SortOrder = 12)]
public Optional<string> GroupColumn = Optional<string>.Implicit(DefaultColumnNames.GroupId);
}

// REVIEW: This output would be much better as an array of CommonOutputs.ClassificationEvaluateOutput,
Expand Down Expand Up @@ -121,6 +130,12 @@ public sealed class CombineMetricsInput
[Argument(ArgumentType.AtMostOnce, HelpText = "The label column name", ShortName = "Label", SortOrder = 5)]
public string LabelColumn = DefaultColumnNames.Label;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for example weight", ShortName = "weight", SortOrder = 6)]
public Optional<string> WeightColumn = Optional<string>.Implicit(DefaultColumnNames.Weight);

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Column to use for grouping", ShortName = "group", SortOrder = 12)]
public Optional<string> GroupColumn = Optional<string>.Implicit(DefaultColumnNames.GroupId);

[Argument(ArgumentType.Required, HelpText = "Specifies the trainer kind, which determines the evaluator to be used.", SortOrder = 6)]
public MacroUtils.TrainerKinds Kind = MacroUtils.TrainerKinds.SignatureBinaryClassifierTrainer;
}
Expand Down Expand Up @@ -188,7 +203,10 @@ public static CommonOutputs.MacroOutput<Output> CrossValidate(
var args = new TrainTestMacro.Arguments
{
Nodes = new JArray(graph.Select(n => n.ToJson()).ToArray()),
TransformModel = null
TransformModel = null,
LabelColumn = input.LabelColumn,
GroupColumn = input.GroupColumn,
WeightColumn = input.WeightColumn
};

if (transformModelVarName != null)
Expand Down Expand Up @@ -356,6 +374,9 @@ public static CommonOutputs.MacroOutput<Output> CrossValidate(

var combineArgs = new CombineMetricsInput();
combineArgs.Kind = input.Kind;
combineArgs.LabelColumn = input.LabelColumn;
combineArgs.WeightColumn = input.WeightColumn;
combineArgs.GroupColumn = input.GroupColumn;

// Set the input bindings for the CombineMetrics entry point.
var combineInputBindingMap = new Dictionary<string, List<ParameterBinding>>();
Expand Down Expand Up @@ -383,10 +404,12 @@ public static CommonOutputs.MacroOutput<Output> CrossValidate(
var combineInstanceMetric = new Var<IDataView>();
combineInstanceMetric.VarName = node.GetOutputVariableName(nameof(Output.PerInstanceMetrics));
combineOutputMap.Add(nameof(Output.PerInstanceMetrics), combineInstanceMetric.VarName);
var combineConfusionMatrix = new Var<IDataView>();
combineConfusionMatrix.VarName = node.GetOutputVariableName(nameof(Output.ConfusionMatrix));
combineOutputMap.Add(nameof(TrainTestMacro.Output.ConfusionMatrix), combineConfusionMatrix.VarName);

if (confusionMatricesOutput != null)
{
var combineConfusionMatrix = new Var<IDataView>();
combineConfusionMatrix.VarName = node.GetOutputVariableName(nameof(Output.ConfusionMatrix));
combineOutputMap.Add(nameof(TrainTestMacro.Output.ConfusionMatrix), combineConfusionMatrix.VarName);
}
subGraphNodes.AddRange(EntryPointNode.ValidateNodes(env, node.Context, exp.GetNodes(), node.Catalog));
subGraphNodes.Add(EntryPointNode.Create(env, "Models.CrossValidationResultsCombiner", combineArgs, node.Catalog, node.Context, combineInputBindingMap, combineInputMap, combineOutputMap));
return new CommonOutputs.MacroOutput<Output>() { Nodes = subGraphNodes };
Expand All @@ -398,7 +421,12 @@ public static CombinedOutput CombineMetrics(IHostEnvironment env, CombineMetrics
var eval = GetEvaluator(env, input.Kind);

var perInst = EvaluateUtils.ConcatenatePerInstanceDataViews(env, eval, true, true, input.PerInstanceMetrics.Select(
idv => RoleMappedData.Create(idv, RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, input.LabelColumn))).ToArray(),
idv => RoleMappedData.CreateOpt(idv, new[]
{
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Label, input.LabelColumn),
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Weight, input.WeightColumn.Value),
RoleMappedSchema.CreatePair(RoleMappedSchema.ColumnRole.Group, input.GroupColumn.Value)
})).ToArray(),
out var variableSizeVectorColumnNames);

var warnings = input.Warnings != null ? new List<IDataView>(input.Warnings) : new List<IDataView>();
Expand Down
Loading