Skip to content

Commit

Permalink
feat: add metadata filtering to SearchRuns (#9611)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AmanuelAaron
AmanuelAaron authored Jul 15, 2024
1 parent 0709cab commit 4eeb4db
Show file tree
Hide file tree
Showing 7 changed files with 203 additions and 3 deletions.
2 changes: 2 additions & 0 deletions harness/determined/common/api/bindings.py
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

94 changes: 94 additions & 0 deletions master/internal/api_runs_intg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,40 @@ func TestSearchRunsFilter(t *testing.T) {
HParams: hyperparameters2,
}, task2.TaskID))

resp, err = api.SearchRuns(ctx, req)
require.NoError(t, err)
require.Len(t, resp.Runs, 2)

rawMetadata := map[string]any{
"string_key": "a",
"number_key": 1,
"nested": map[string]any{
"string_key": "a",
"number_key": 1,
},
}
metadata := newProtoStruct(t, rawMetadata)
_, err = api.PostRunMetadata(ctx, &apiv1.PostRunMetadataRequest{
RunId: resp.Runs[0].Id,
Metadata: metadata,
})
require.NoError(t, err)

rawMetadata = map[string]any{
"string_key": "b",
"number_key": 2,
"nested": map[string]any{
"string_key": "b",
"number_key": 2,
},
}
metadata = newProtoStruct(t, rawMetadata)
_, err = api.PostRunMetadata(ctx, &apiv1.PostRunMetadataRequest{
RunId: resp.Runs[1].Id,
Metadata: metadata,
})
require.NoError(t, err)

tests := map[string]struct {
expectedNumRuns int
filter string
Expand Down Expand Up @@ -363,6 +397,66 @@ func TestSearchRunsFilter(t *testing.T) {
`"location":"LOCATION_TYPE_RUN_HYPERPARAMETERS","operator":"<=","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataEmpty": {
expectedNumRuns: 0,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"isEmpty","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNotEmpty": {
expectedNumRuns: 2,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"notEmpty","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataContains": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.string_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"contains","type":"COLUMN_TYPE_TEXT","value":"a"}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNotContains": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.string_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"notContains","type":"COLUMN_TYPE_TEXT","value":"a"}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataOperator": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"<=","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNestedEmpty": {
expectedNumRuns: 0,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.nested.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"isEmpty","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNestedNotEmpty": {
expectedNumRuns: 2,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.nested.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"notEmpty","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNestedContains": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.nested.string_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"contains","type":"COLUMN_TYPE_TEXT","value":"a"}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNestedNotContains": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.nested.string_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"notContains","type":"COLUMN_TYPE_TEXT","value":"a"}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
"MetadataNestedOperator": {
expectedNumRuns: 1,
filter: `{"filterGroup":{"children":[{"columnName":"metadata.nested.number_key","kind":"field",` +
`"location":"LOCATION_TYPE_RUN_METADATA","operator":"<=","type":"COLUMN_TYPE_NUMBER","value":1}],` +
`"conjunction":"and","kind":"group"},"showArchived":false}`,
},
}

for testCase, testVars := range tests {
Expand Down
94 changes: 94 additions & 0 deletions master/internal/experiment_filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,98 @@ func runColumnNameToSQL(columnName string) (string, error) {
return col, nil
}

func runMetadataToSQL(c string, filterColumnType *string, filterValue *interface{},
op *operator, q *bun.SelectQuery,
fc *filterConjunction,
) (*bun.SelectQuery, error) {
queryColumnType := projectv1.ColumnType_COLUMN_TYPE_UNSPECIFIED.String()
var o operator
var queryValue interface{}
if filterValue == nil && op != nil && *op != empty && *op != notEmpty {
return nil, fmt.Errorf("metadata field defined without value and without a valid operator")
}
o = *op
if o != empty && o != notEmpty {
queryValue = *filterValue
}
if filterColumnType != nil {
queryColumnType = *filterColumnType
}
var queryArgs []interface{}
runHparam := strings.TrimPrefix(c, "metadata.")
oSQL, err := o.toSQL()
if err != nil {
return nil, err
}
var queryString string
switch o {
case empty:
queryString = fmt.Sprintf(`r.id NOT IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s')`, runHparam)
case notEmpty:
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s')`, runHparam)
case contains:
queryArgs = append(queryArgs, queryValue)
switch queryColumnType {
case projectv1.ColumnType_COLUMN_TYPE_NUMBER.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND (number_value=%s OR float_value=%s))`,
runHparam, "?", "?")
queryArgs = append(queryArgs, queryValue)
case projectv1.ColumnType_COLUMN_TYPE_TEXT.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND string_value LIKE %s)`,
runHparam, "?")
default:
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND boolean_value=%s)`,
runHparam, "?")
}
case doesNotContain:
queryArgs = append(queryArgs, queryValue)
switch queryColumnType {
case projectv1.ColumnType_COLUMN_TYPE_NUMBER.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND (number_value!=%s AND float_value!=%s))`,
runHparam, "?", "?")
queryArgs = append(queryArgs, queryValue)
case projectv1.ColumnType_COLUMN_TYPE_TEXT.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND string_value NOT LIKE %s)`,
runHparam, "?")
default:
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND boolean_value!=%s)`,
runHparam, "?")
}
default:
queryArgs = append(queryArgs, bun.Safe(oSQL), queryValue)
switch queryColumnType {
case projectv1.ColumnType_COLUMN_TYPE_NUMBER.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND (integer_value %s %s OR float_value %s %s))`,
runHparam, "?", "?", "?", "?")
queryArgs = append(queryArgs, bun.Safe(oSQL), queryValue)
case projectv1.ColumnType_COLUMN_TYPE_TEXT.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND string_value %s %s)`,
runHparam, "?", "?")
case projectv1.ColumnType_COLUMN_TYPE_DATE.String():
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND timestamp_value %s %s)`,
runHparam, "?", "?")
default:
queryString = fmt.Sprintf(`r.id IN (SELECT run_id FROM runs_metadata_index WHERE flat_key='%s'
AND boolean_value %s %s)`,
runHparam, "?", "?")
}
}

if fc != nil && *fc == or {
return q.WhereOr(queryString, queryArgs...), nil
}
return q.Where(queryString, queryArgs...), nil
}

func runHpToSQL(c string, filterColumnType *string, filterValue *interface{},
op *operator, q *bun.SelectQuery,
fc *filterConjunction,
Expand Down Expand Up @@ -557,6 +649,8 @@ func (e experimentFilter) toSQL(q *bun.SelectQuery,
return hpToSQL(e.ColumnName, e.Type, e.Value, e.Operator, q, c)
case projectv1.LocationType_LOCATION_TYPE_RUN_HYPERPARAMETERS.String():
return runHpToSQL(e.ColumnName, e.Type, e.Value, e.Operator, q, c)
case projectv1.LocationType_LOCATION_TYPE_RUN_METADATA.String():
return runMetadataToSQL(e.ColumnName, e.Type, e.Value, e.Operator, q, c)
}
case group:
var co string
Expand Down
10 changes: 8 additions & 2 deletions proto/pkg/projectv1/project.pb.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions proto/src/determined/project/v1/project.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ enum LocationType {
LOCATION_TYPE_RUN = 6;
// Column is located in the hyperparameter of the run
LOCATION_TYPE_RUN_HYPERPARAMETERS = 7;
// Column is located on the run's arbitrary metadata
LOCATION_TYPE_RUN_METADATA = 8;
}

// ColumnType indicates the type of data under the column
Expand Down
1 change: 1 addition & 0 deletions webui/react/src/ioTypes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,7 @@ export const ioLocationType: io.Type<V1LocationType> = io.keyof({
[V1LocationType.UNSPECIFIED]: null,
[V1LocationType.RUN]: null,
[V1LocationType.RUNHYPERPARAMETERS]: null,
[V1LocationType.RUNMETADATA]: null,
});
export const ioColumnType: io.Type<V1ColumnType> = io.keyof({
[V1ColumnType.DATE]: null,
Expand Down
3 changes: 2 additions & 1 deletion webui/react/src/services/api-ts-sdk/api.ts
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 4eeb4db

Please sign in to comment.