From b7fcb0546032c09e1e3965f79e100c8fa3d6534f Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 11:19:14 -0500 Subject: [PATCH 1/7] add tabulardatabyfilteroptions --- app/data_client.go | 59 ++++++++++++++++++++++++++++------------- app/data_client_test.go | 31 +++++++++++----------- 2 files changed, 56 insertions(+), 34 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 6fc201aaf76..2b69bf1c873 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -151,6 +151,20 @@ type Annotations struct { Bboxes []BoundingBox } +// TabularDataByFilterOptions contains optional parameters for TabularDataByFilter. +type TabularDataByFilterOptions struct { + // No Filter implies all tabular data. + Filter *Filter + // Limit defaults to 50 if unspecified. + Limit int + // Last indicates the object identifier of the Last-returned data. This is returned by calls to TabularDataByFilter as the `Last` value. + // If provided, the server will return the next data entries after the Last object identifier. + Last string + SortOrder Order + CountOnly bool + IncludeInternalData bool +} + // TabularDataReturn represents the result of a TabularDataByFilter query. // It contains the retrieved tabular data and associated metadata, // the total number of entries retrieved (Count), and the ID of the last returned page (Last). @@ -320,21 +334,28 @@ func BsonToGo(rawData [][]byte) ([]map[string]interface{}, error) { } // TabularDataByFilter queries tabular data and metadata based on given filters. -func (d *DataClient) TabularDataByFilter( - ctx context.Context, - filter Filter, - limit int, - last string, - sortOrder Order, - countOnly bool, - includeInternalData bool, -) (TabularDataReturn, error) { +func (d *DataClient) TabularDataByFilter(ctx context.Context, opts *TabularDataByFilterOptions) (TabularDataReturn, error) { + var filter *pb.Filter + var limit uint64 + var last string + var order pb.Order + var countOnly, includeInternalData bool + if opts != nil { + limit = uint64(opts.Limit) + last = opts.Last + order = orderToProto(opts.SortOrder) + countOnly = opts.CountOnly + includeInternalData = opts.IncludeInternalData + if opts.Filter != nil { + filter = filterToProto(opts.Filter) + } + } resp, err := d.dataClient.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ DataRequest: &pb.DataRequest{ - Filter: filterToProto(filter), - Limit: uint64(limit), + Filter: filter, + Limit: limit, Last: last, - SortOrder: orderToProto(sortOrder), + SortOrder: order, }, CountOnly: countOnly, IncludeInternalData: includeInternalData, @@ -397,7 +418,7 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationID string // BinaryDataByFilter queries binary data and metadata based on given filters. func (d *DataClient) BinaryDataByFilter( ctx context.Context, - filter Filter, + filter *Filter, limit int, sortOrder Order, last string, @@ -461,7 +482,7 @@ func (d *DataClient) DeleteTabularData(ctx context.Context, organizationID strin // DeleteBinaryDataByFilter deletes binary data based on given filters. // It returns the number of binary datapoints deleted. -func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter Filter) (int, error) { +func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *Filter) (int, error) { resp, err := d.dataClient.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ Filter: filterToProto(filter), IncludeInternalData: true, @@ -494,7 +515,7 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string } // AddTagsToBinaryDataByFilter adds string tags, unless the tags are already present, to binary data based on the given filter. -func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter Filter) error { +func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter *Filter) error { _, err := d.dataClient.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{ Filter: filterToProto(filter), Tags: tags, @@ -520,7 +541,7 @@ func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, // RemoveTagsFromBinaryDataByFilter removes the specified string tags from binary data that match the given filter. // It returns the number of binary files from which tags were removed. func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, - tags []string, filter Filter, + tags []string, filter *Filter, ) (int, error) { resp, err := d.dataClient.RemoveTagsFromBinaryDataByFilter(ctx, &pb.RemoveTagsFromBinaryDataByFilterRequest{ Filter: filterToProto(filter), @@ -534,7 +555,7 @@ func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, // TagsByFilter retrieves all unique tags associated with the data that match the specified filter. // It returns the list of these unique tags. -func (d *DataClient) TagsByFilter(ctx context.Context, filter Filter) ([]string, error) { +func (d *DataClient) TagsByFilter(ctx context.Context, filter *Filter) ([]string, error) { resp, err := d.dataClient.TagsByFilter(ctx, &pb.TagsByFilterRequest{ Filter: filterToProto(filter), }) @@ -585,7 +606,7 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID( // BoundingBoxLabelsByFilter retrieves all unique string labels for bounding boxes that match the specified filter. // It returns a list of these labels. -func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter Filter) ([]string, error) { +func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *Filter) ([]string, error) { resp, err := d.dataClient.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{ Filter: filterToProto(filter), }) @@ -1174,7 +1195,7 @@ func binaryIDsToProto(binaryIDs []BinaryID) []*pb.BinaryID { return protoBinaryIDs } -func filterToProto(filter Filter) *pb.Filter { +func filterToProto(filter *Filter) *pb.Filter { return &pb.Filter{ ComponentName: filter.ComponentName, ComponentType: filter.ComponentType, diff --git a/app/data_client_test.go b/app/data_client_test.go index 260e5b88ed7..3834edb2f5c 100644 --- a/app/data_client_test.go +++ b/app/data_client_test.go @@ -182,7 +182,7 @@ func binaryMetadataToProto(binaryMetadata BinaryMetadata) *pb.BinaryMetadata { func dataRequestToProto(dataRequest DataRequest) *pb.DataRequest { return &pb.DataRequest{ - Filter: filterToProto(dataRequest.Filter), + Filter: filterToProto(&dataRequest.Filter), Limit: uint64(dataRequest.Limit), Last: dataRequest.Last, SortOrder: orderToProto(dataRequest.SortOrder), @@ -230,6 +230,7 @@ func TestDataClient(t *testing.T) { BboxLabels: bboxLabels, DatasetID: datasetID, } + pbFilter := filterToProto(&filter) binaryMetadata := BinaryMetadata{ ID: binaryMetaID, @@ -278,9 +279,9 @@ func TestDataClient(t *testing.T) { Metadata: []*pb.CaptureMetadata{captureMetadataToProto(tabularMetadata)}, }, nil } - resp, _ := client.TabularDataByFilter( - context.Background(), filter, limit, last, - dataRequest.SortOrder, countOnly, includeInternalData) + resp, _ := client.TabularDataByFilter(context.Background(), &TabularDataByFilterOptions{ + &filter, limit, last, dataRequest.SortOrder, countOnly, includeInternalData, + }) test.That(t, resp.TabularData[0], test.ShouldResemble, tabularData) test.That(t, resp.Count, test.ShouldEqual, count) test.That(t, resp.Last, test.ShouldEqual, last) @@ -354,7 +355,7 @@ func TestDataClient(t *testing.T) { }, nil } resp, _ := client.BinaryDataByFilter( - context.Background(), filter, limit, dataRequest.SortOrder, + context.Background(), &filter, limit, dataRequest.SortOrder, last, includeBinary, countOnly, includeInternalData) test.That(t, resp.BinaryData[0], test.ShouldResemble, binaryData) test.That(t, resp.Count, test.ShouldEqual, count) @@ -395,13 +396,13 @@ func TestDataClient(t *testing.T) { grpcClient.DeleteBinaryDataByFilterFunc = func(ctx context.Context, in *pb.DeleteBinaryDataByFilterRequest, opts ...grpc.CallOption, ) (*pb.DeleteBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Filter, test.ShouldResemble, pbFilter) test.That(t, in.IncludeInternalData, test.ShouldBeTrue) return &pb.DeleteBinaryDataByFilterResponse{ DeletedCount: pbCount, }, nil } - resp, _ := client.DeleteBinaryDataByFilter(context.Background(), filter) + resp, _ := client.DeleteBinaryDataByFilter(context.Background(), &filter) test.That(t, resp, test.ShouldEqual, count) }) @@ -433,11 +434,11 @@ func TestDataClient(t *testing.T) { grpcClient.AddTagsToBinaryDataByFilterFunc = func(ctx context.Context, in *pb.AddTagsToBinaryDataByFilterRequest, opts ...grpc.CallOption, ) (*pb.AddTagsToBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Filter, test.ShouldResemble, pbFilter) test.That(t, in.Tags, test.ShouldResemble, tags) return &pb.AddTagsToBinaryDataByFilterResponse{}, nil } - client.AddTagsToBinaryDataByFilter(context.Background(), tags, filter) + client.AddTagsToBinaryDataByFilter(context.Background(), tags, &filter) }) t.Run("RemoveTagsFromBinaryDataByIDs", func(t *testing.T) { @@ -458,13 +459,13 @@ func TestDataClient(t *testing.T) { grpcClient.RemoveTagsFromBinaryDataByFilterFunc = func(ctx context.Context, in *pb.RemoveTagsFromBinaryDataByFilterRequest, opts ...grpc.CallOption, ) (*pb.RemoveTagsFromBinaryDataByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Filter, test.ShouldResemble, pbFilter) test.That(t, in.Tags, test.ShouldResemble, tags) return &pb.RemoveTagsFromBinaryDataByFilterResponse{ DeletedCount: pbCount, }, nil } - resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, filter) + resp, _ := client.RemoveTagsFromBinaryDataByFilter(context.Background(), tags, &filter) test.That(t, resp, test.ShouldEqual, count) }) @@ -472,12 +473,12 @@ func TestDataClient(t *testing.T) { grpcClient.TagsByFilterFunc = func(ctx context.Context, in *pb.TagsByFilterRequest, opts ...grpc.CallOption, ) (*pb.TagsByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Filter, test.ShouldResemble, pbFilter) return &pb.TagsByFilterResponse{ Tags: tags, }, nil } - resp, _ := client.TagsByFilter(context.Background(), filter) + resp, _ := client.TagsByFilter(context.Background(), &filter) test.That(t, resp, test.ShouldResemble, tags) }) @@ -527,12 +528,12 @@ func TestDataClient(t *testing.T) { grpcClient.BoundingBoxLabelsByFilterFunc = func(ctx context.Context, in *pb.BoundingBoxLabelsByFilterRequest, opts ...grpc.CallOption, ) (*pb.BoundingBoxLabelsByFilterResponse, error) { - test.That(t, in.Filter, test.ShouldResemble, filterToProto(filter)) + test.That(t, in.Filter, test.ShouldResemble, pbFilter) return &pb.BoundingBoxLabelsByFilterResponse{ Labels: expectedBBoxLabelsPb, }, nil } - resp, _ := client.BoundingBoxLabelsByFilter(context.Background(), filter) + resp, _ := client.BoundingBoxLabelsByFilter(context.Background(), &filter) test.That(t, resp, test.ShouldResemble, expectedBBoxLabels) }) t.Run("UpdateBoundingBox", func(t *testing.T) { From db611ac823cf489cd2cfc8ceff78db0a80d3d920 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 11:36:38 -0500 Subject: [PATCH 2/7] add binarydatabyfilter options --- app/data_client.go | 43 +++++++++++++++++++++++++---------------- app/data_client_test.go | 7 ++++--- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 2b69bf1c873..7cc7d1eaa8f 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -151,14 +151,15 @@ type Annotations struct { Bboxes []BoundingBox } -// TabularDataByFilterOptions contains optional parameters for TabularDataByFilter. -type TabularDataByFilterOptions struct { - // No Filter implies all tabular data. +// DataByFilterOptions contains optional parameters for TabularDataByFilter and BinaryDataByFilter. +type DataByFilterOptions struct { + // No Filter implies all data. Filter *Filter // Limit defaults to 50 if unspecified. Limit int - // Last indicates the object identifier of the Last-returned data. This is returned by calls to TabularDataByFilter as the `Last` value. - // If provided, the server will return the next data entries after the Last object identifier. + // Last indicates the object identifier of the Last-returned data. + // This is returned by calls to TabularDataByFilter and BinaryDataByFilter as the `Last` value. + // If provided, the server will return the next data entries after the last object identifier. Last string SortOrder Order CountOnly bool @@ -334,7 +335,7 @@ func BsonToGo(rawData [][]byte) ([]map[string]interface{}, error) { } // TabularDataByFilter queries tabular data and metadata based on given filters. -func (d *DataClient) TabularDataByFilter(ctx context.Context, opts *TabularDataByFilterOptions) (TabularDataReturn, error) { +func (d *DataClient) TabularDataByFilter(ctx context.Context, opts *DataByFilterOptions) (TabularDataReturn, error) { var filter *pb.Filter var limit uint64 var last string @@ -417,21 +418,29 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationID string // BinaryDataByFilter queries binary data and metadata based on given filters. func (d *DataClient) BinaryDataByFilter( - ctx context.Context, - filter *Filter, - limit int, - sortOrder Order, - last string, - includeBinary bool, - countOnly bool, - includeInternalData bool, + ctx context.Context, includeBinary bool, opts *DataByFilterOptions, ) (BinaryDataReturn, error) { + var filter *pb.Filter + var limit uint64 + var last string + var order pb.Order + var countOnly, includeInternalData bool + if opts != nil { + limit = uint64(opts.Limit) + last = opts.Last + order = orderToProto(opts.SortOrder) + countOnly = opts.CountOnly + includeInternalData = opts.IncludeInternalData + if opts.Filter != nil { + filter = filterToProto(opts.Filter) + } + } resp, err := d.dataClient.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ DataRequest: &pb.DataRequest{ - Filter: filterToProto(filter), - Limit: uint64(limit), + Filter: filter, + Limit: limit, Last: last, - SortOrder: orderToProto(sortOrder), + SortOrder: order, }, IncludeBinary: includeBinary, CountOnly: countOnly, diff --git a/app/data_client_test.go b/app/data_client_test.go index 3834edb2f5c..b2e2fe71f6f 100644 --- a/app/data_client_test.go +++ b/app/data_client_test.go @@ -279,7 +279,7 @@ func TestDataClient(t *testing.T) { Metadata: []*pb.CaptureMetadata{captureMetadataToProto(tabularMetadata)}, }, nil } - resp, _ := client.TabularDataByFilter(context.Background(), &TabularDataByFilterOptions{ + resp, _ := client.TabularDataByFilter(context.Background(), &DataByFilterOptions{ &filter, limit, last, dataRequest.SortOrder, countOnly, includeInternalData, }) test.That(t, resp.TabularData[0], test.ShouldResemble, tabularData) @@ -355,8 +355,9 @@ func TestDataClient(t *testing.T) { }, nil } resp, _ := client.BinaryDataByFilter( - context.Background(), &filter, limit, dataRequest.SortOrder, - last, includeBinary, countOnly, includeInternalData) + context.Background(), includeBinary, &DataByFilterOptions{ + &filter, limit, last, dataRequest.SortOrder, countOnly, includeInternalData, + }) test.That(t, resp.BinaryData[0], test.ShouldResemble, binaryData) test.That(t, resp.Count, test.ShouldEqual, count) test.That(t, resp.Last, test.ShouldEqual, last) From 06f4b65780adb54d4eeb7e59051054b856c47771 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 11:59:32 -0500 Subject: [PATCH 3/7] add updateboundingbox options --- app/data_client.go | 105 ++++++++++++++++++++++------------------ app/data_client_test.go | 29 ++++++----- 2 files changed, 76 insertions(+), 58 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 7cc7d1eaa8f..e8469f0465f 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -151,21 +151,6 @@ type Annotations struct { Bboxes []BoundingBox } -// DataByFilterOptions contains optional parameters for TabularDataByFilter and BinaryDataByFilter. -type DataByFilterOptions struct { - // No Filter implies all data. - Filter *Filter - // Limit defaults to 50 if unspecified. - Limit int - // Last indicates the object identifier of the Last-returned data. - // This is returned by calls to TabularDataByFilter and BinaryDataByFilter as the `Last` value. - // If provided, the server will return the next data entries after the last object identifier. - Last string - SortOrder Order - CountOnly bool - IncludeInternalData bool -} - // TabularDataReturn represents the result of a TabularDataByFilter query. // It contains the retrieved tabular data and associated metadata, // the total number of entries retrieved (Count), and the ID of the last returned page (Last). @@ -250,8 +235,23 @@ type FileData struct { Data []byte } -// BinaryOptions represents optional parameters for the BinaryDataCaptureUpload method. -type BinaryOptions struct { +// DataByFilterOptions contains optional parameters for TabularDataByFilter and BinaryDataByFilter. +type DataByFilterOptions struct { + // No Filter implies all data. + Filter *Filter + // Limit defaults to 50 if unspecified. + Limit int + // Last indicates the object identifier of the Last-returned data. + // This is returned by calls to TabularDataByFilter and BinaryDataByFilter as the `Last` value. + // If provided, the server will return the next data entries after the last object identifier. + Last string + SortOrder Order + CountOnly bool + IncludeInternalData bool +} + +// BinaryDataCaptureUploadOptions represents optional parameters for the BinaryDataCaptureUpload method. +type BinaryDataCaptureUploadOptions struct { Type *DataType FileName *string MethodParameters map[string]interface{} @@ -259,8 +259,8 @@ type BinaryOptions struct { DataRequestTimes *[2]time.Time } -// TabularOptions represents optional parameters for the TabularDataCaptureUpload method. -type TabularOptions struct { +// TabularDataCaptureUploadOptions represents optional parameters for the TabularDataCaptureUpload method. +type TabularDataCaptureUploadOptions struct { Type *DataType FileName *string MethodParameters map[string]interface{} @@ -268,8 +268,8 @@ type TabularOptions struct { Tags []string } -// StreamingOptions represents optional parameters for the StreamingDataCaptureUpload method. -type StreamingOptions struct { +// StreamingDataCaptureUploadOptions represents optional parameters for the StreamingDataCaptureUpload method. +type StreamingDataCaptureUploadOptions struct { ComponentType *string ComponentName *string MethodName *string @@ -291,6 +291,15 @@ type FileUploadOptions struct { Tags []string } +// UpdateBoundingBoxOptions contains optional parameters for UpdateBoundingBox. +type UpdateBoundingBoxOptions struct { + Label *string + XMinNormalized *float64 + YMinNormalized *float64 + XMaxNormalized *float64 + YMaxNormalized *float64 +} + // Dataset contains the information of a dataset. type Dataset struct { ID string @@ -461,7 +470,7 @@ func (d *DataClient) BinaryDataByFilter( } // BinaryDataByIDs queries binary data and metadata based on given IDs. -func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIDs []BinaryID) ([]BinaryData, error) { +func (d *DataClient) BinaryDataByIDs(ctx context.Context, binaryIDs []*BinaryID) ([]BinaryData, error) { resp, err := d.dataClient.BinaryDataByIDs(ctx, &pb.BinaryDataByIDsRequest{ IncludeBinary: true, BinaryIds: binaryIDsToProto(binaryIDs), @@ -489,7 +498,7 @@ func (d *DataClient) DeleteTabularData(ctx context.Context, organizationID strin return int(resp.DeletedCount), nil } -// DeleteBinaryDataByFilter deletes binary data based on given filters. +// DeleteBinaryDataByFilter deletes binary data based on given filters. If filter is empty, delete all data. // It returns the number of binary datapoints deleted. func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *Filter) (int, error) { resp, err := d.dataClient.DeleteBinaryDataByFilter(ctx, &pb.DeleteBinaryDataByFilterRequest{ @@ -504,7 +513,7 @@ func (d *DataClient) DeleteBinaryDataByFilter(ctx context.Context, filter *Filte // DeleteBinaryDataByIDs deletes binary data based on given IDs. // It returns the number of binary datapoints deleted. -func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIDs []BinaryID) (int, error) { +func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIDs []*BinaryID) (int, error) { resp, err := d.dataClient.DeleteBinaryDataByIDs(ctx, &pb.DeleteBinaryDataByIDsRequest{ BinaryIds: binaryIDsToProto(binaryIDs), }) @@ -515,7 +524,7 @@ func (d *DataClient) DeleteBinaryDataByIDs(ctx context.Context, binaryIDs []Bina } // AddTagsToBinaryDataByIDs adds string tags, unless the tags are already present, to binary data based on given IDs. -func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIDs []BinaryID) error { +func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string, binaryIDs []*BinaryID) error { _, err := d.dataClient.AddTagsToBinaryDataByIDs(ctx, &pb.AddTagsToBinaryDataByIDsRequest{ BinaryIds: binaryIDsToProto(binaryIDs), Tags: tags, @@ -524,6 +533,7 @@ func (d *DataClient) AddTagsToBinaryDataByIDs(ctx context.Context, tags []string } // AddTagsToBinaryDataByFilter adds string tags, unless the tags are already present, to binary data based on the given filter. +// If no filter is given, all data will be tagged. func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []string, filter *Filter) error { _, err := d.dataClient.AddTagsToBinaryDataByFilter(ctx, &pb.AddTagsToBinaryDataByFilterRequest{ Filter: filterToProto(filter), @@ -535,7 +545,7 @@ func (d *DataClient) AddTagsToBinaryDataByFilter(ctx context.Context, tags []str // RemoveTagsFromBinaryDataByIDs removes string tags from binary data based on given IDs. // It returns the number of binary files which had tags removed. func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, - tags []string, binaryIDs []BinaryID, + tags []string, binaryIDs []*BinaryID, ) (int, error) { resp, err := d.dataClient.RemoveTagsFromBinaryDataByIDs(ctx, &pb.RemoveTagsFromBinaryDataByIDsRequest{ BinaryIds: binaryIDsToProto(binaryIDs), @@ -548,6 +558,7 @@ func (d *DataClient) RemoveTagsFromBinaryDataByIDs(ctx context.Context, } // RemoveTagsFromBinaryDataByFilter removes the specified string tags from binary data that match the given filter. +// If no filter is given, all data will be untagged. // It returns the number of binary files from which tags were removed. func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, tags []string, filter *Filter, @@ -563,7 +574,7 @@ func (d *DataClient) RemoveTagsFromBinaryDataByFilter(ctx context.Context, } // TagsByFilter retrieves all unique tags associated with the data that match the specified filter. -// It returns the list of these unique tags. +// It returns the list of these unique tags. If no filter is given, all data tags are returned. func (d *DataClient) TagsByFilter(ctx context.Context, filter *Filter) ([]string, error) { resp, err := d.dataClient.TagsByFilter(ctx, &pb.TagsByFilterRequest{ Filter: filterToProto(filter), @@ -579,7 +590,7 @@ func (d *DataClient) TagsByFilter(ctx context.Context, filter *Filter) ([]string // All normalized coordinates (xMin, yMin, xMax, yMax) must be float values in the range [0, 1]. func (d *DataClient) AddBoundingBoxToImageByID( ctx context.Context, - binaryID BinaryID, + binaryID *BinaryID, label string, xMinNormalized float64, yMinNormalized float64, @@ -604,7 +615,7 @@ func (d *DataClient) AddBoundingBoxToImageByID( func (d *DataClient) RemoveBoundingBoxFromImageByID( ctx context.Context, bboxID string, - binaryID BinaryID, + binaryID *BinaryID, ) error { _, err := d.dataClient.RemoveBoundingBoxFromImageByID(ctx, &pb.RemoveBoundingBoxFromImageByIDRequest{ BinaryId: binaryIDToProto(binaryID), @@ -614,7 +625,7 @@ func (d *DataClient) RemoveBoundingBoxFromImageByID( } // BoundingBoxLabelsByFilter retrieves all unique string labels for bounding boxes that match the specified filter. -// It returns a list of these labels. +// It returns a list of these labels. If no filter is given, all labels are returned. func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *Filter) ([]string, error) { resp, err := d.dataClient.BoundingBoxLabelsByFilter(ctx, &pb.BoundingBoxLabelsByFilterRequest{ Filter: filterToProto(filter), @@ -628,15 +639,17 @@ func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *Filt // UpdateBoundingBox updates the bounding box for a given bbox ID for the file represented by the binary ID, // modifying its label and position using optional normalized coordinates (xMin, yMin, xMax, yMax), // where all coordinates must be in the range [0, 1]. -func (d *DataClient) UpdateBoundingBox(ctx context.Context, - binaryID BinaryID, - bboxID string, - label *string, // optional - xMinNormalized *float64, // optional - yMinNormalized *float64, // optional - xMaxNormalized *float64, // optional - yMaxNormalized *float64, // optional -) error { +func (d *DataClient) UpdateBoundingBox(ctx context.Context, binaryID *BinaryID, bboxID string, opts *UpdateBoundingBoxOptions) error { + var label *string + var xMinNormalized, yMinNormalized, xMaxNormalized, yMaxNormalized *float64 + if opts != nil { + label = opts.Label + xMinNormalized = opts.XMinNormalized + yMinNormalized = opts.YMinNormalized + xMaxNormalized = opts.XMaxNormalized + yMaxNormalized = opts.YMaxNormalized + } + _, err := d.dataClient.UpdateBoundingBox(ctx, &pb.UpdateBoundingBoxRequest{ BinaryId: binaryIDToProto(binaryID), BboxId: bboxID, @@ -682,7 +695,7 @@ func (d *DataClient) ConfigureDatabaseUser( // AddBinaryDataToDatasetByIDs adds the binary data with the given binary IDs to the dataset. func (d *DataClient) AddBinaryDataToDatasetByIDs( ctx context.Context, - binaryIDs []BinaryID, + binaryIDs []*BinaryID, datasetID string, ) error { _, err := d.dataClient.AddBinaryDataToDatasetByIDs(ctx, &pb.AddBinaryDataToDatasetByIDsRequest{ @@ -695,7 +708,7 @@ func (d *DataClient) AddBinaryDataToDatasetByIDs( // RemoveBinaryDataFromDatasetByIDs removes the binary data with the given binary IDs from the dataset. func (d *DataClient) RemoveBinaryDataFromDatasetByIDs( ctx context.Context, - binaryIDs []BinaryID, + binaryIDs []*BinaryID, datasetID string, ) error { _, err := d.dataClient.RemoveBinaryDataFromDatasetByIDs(ctx, &pb.RemoveBinaryDataFromDatasetByIDsRequest{ @@ -714,7 +727,7 @@ func (d *DataClient) BinaryDataCaptureUpload( componentName string, methodName string, fileExtension string, - options *BinaryOptions, + options *BinaryDataCaptureUploadOptions, ) (string, error) { var sensorMetadata SensorMetadata if options.DataRequestTimes != nil && len(options.DataRequestTimes) == 2 { @@ -762,7 +775,7 @@ func (d *DataClient) TabularDataCaptureUpload( componentName string, methodName string, dataRequestTimes [][2]time.Time, - options *TabularOptions, + options *TabularDataCaptureUploadOptions, ) (string, error) { if len(dataRequestTimes) != len(tabularData) { return "", errors.New("dataRequestTimes and tabularData lengths must be equal") @@ -832,7 +845,7 @@ func (d *DataClient) StreamingDataCaptureUpload( data []byte, partID string, fileExt string, - options *StreamingOptions, + options *StreamingDataCaptureUploadOptions, ) (string, error) { uploadMetadata := UploadMetadata{ PartID: partID, @@ -1188,7 +1201,7 @@ func tabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) T } } -func binaryIDToProto(binaryID BinaryID) *pb.BinaryID { +func binaryIDToProto(binaryID *BinaryID) *pb.BinaryID { return &pb.BinaryID{ FileId: binaryID.FileID, OrganizationId: binaryID.OrganizationID, @@ -1196,7 +1209,7 @@ func binaryIDToProto(binaryID BinaryID) *pb.BinaryID { } } -func binaryIDsToProto(binaryIDs []BinaryID) []*pb.BinaryID { +func binaryIDsToProto(binaryIDs []*BinaryID) []*pb.BinaryID { var protoBinaryIDs []*pb.BinaryID for _, binaryID := range binaryIDs { protoBinaryIDs = append(protoBinaryIDs, binaryIDToProto(binaryID)) diff --git a/app/data_client_test.go b/app/data_client_test.go index b2e2fe71f6f..6d985b6dcd0 100644 --- a/app/data_client_test.go +++ b/app/data_client_test.go @@ -77,7 +77,8 @@ var ( OrganizationID: organizationID, LocationID: locationID, } - binaryIDs = []BinaryID{binaryID} + pbBinaryID = binaryIDToProto(&binaryID) + binaryIDs = []*BinaryID{&binaryID} binaryDataByte = []byte("BYTE") sqlQuery = "SELECT * FROM readings WHERE organization_id='e76d1b3b-0468-4efd-bb7f-fb1d2b352fcb' LIMIT 1" rawData = []map[string]interface{}{ @@ -488,7 +489,7 @@ func TestDataClient(t *testing.T) { in *pb.AddBoundingBoxToImageByIDRequest, opts ...grpc.CallOption, ) (*pb.AddBoundingBoxToImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.BinaryId, test.ShouldResemble, pbBinaryID) test.That(t, in.Label, test.ShouldEqual, bboxLabel) test.That(t, in.XMinNormalized, test.ShouldEqual, annotations.Bboxes[0].XMinNormalized) test.That(t, in.YMinNormalized, test.ShouldEqual, annotations.Bboxes[0].YMinNormalized) @@ -500,7 +501,7 @@ func TestDataClient(t *testing.T) { }, nil } resp, _ := client.AddBoundingBoxToImageByID( - context.Background(), binaryID, bboxLabel, annotations.Bboxes[0].XMinNormalized, + context.Background(), &binaryID, bboxLabel, annotations.Bboxes[0].XMinNormalized, annotations.Bboxes[0].YMinNormalized, annotations.Bboxes[0].XMaxNormalized, annotations.Bboxes[0].YMaxNormalized) test.That(t, resp, test.ShouldResemble, annotations.Bboxes[0].ID) }) @@ -509,12 +510,12 @@ func TestDataClient(t *testing.T) { grpcClient.RemoveBoundingBoxFromImageByIDFunc = func(ctx context.Context, in *pb.RemoveBoundingBoxFromImageByIDRequest, opts ...grpc.CallOption, ) (*pb.RemoveBoundingBoxFromImageByIDResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.BinaryId, test.ShouldResemble, pbBinaryID) test.That(t, in.BboxId, test.ShouldEqual, annotations.Bboxes[0].ID) return &pb.RemoveBoundingBoxFromImageByIDResponse{}, nil } - client.RemoveBoundingBoxFromImageByID(context.Background(), annotations.Bboxes[0].ID, binaryID) + client.RemoveBoundingBoxFromImageByID(context.Background(), annotations.Bboxes[0].ID, &binaryID) }) t.Run("BoundingBoxLabelsByFilter", func(t *testing.T) { @@ -542,7 +543,7 @@ func TestDataClient(t *testing.T) { grpcClient.UpdateBoundingBoxFunc = func(ctx context.Context, in *pb.UpdateBoundingBoxRequest, opts ...grpc.CallOption, ) (*pb.UpdateBoundingBoxResponse, error) { - test.That(t, in.BinaryId, test.ShouldResemble, binaryIDToProto(binaryID)) + test.That(t, in.BinaryId, test.ShouldResemble, pbBinaryID) test.That(t, in.BboxId, test.ShouldResemble, annotationsPb.Bboxes[0].Id) test.That(t, *in.Label, test.ShouldEqual, annotationsPb.Bboxes[0].Label) test.That(t, *in.XMinNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].XMinNormalized) @@ -551,9 +552,13 @@ func TestDataClient(t *testing.T) { test.That(t, *in.YMaxNormalized, test.ShouldEqual, annotationsPb.Bboxes[0].YMaxNormalized) return &pb.UpdateBoundingBoxResponse{}, nil } - client.UpdateBoundingBox(context.Background(), binaryID, annotations.Bboxes[0].ID, &annotationsPb.Bboxes[0].Label, - &annotationsPb.Bboxes[0].XMinNormalized, &annotationsPb.Bboxes[0].YMinNormalized, - &annotationsPb.Bboxes[0].XMaxNormalized, &annotationsPb.Bboxes[0].YMaxNormalized) + client.UpdateBoundingBox(context.Background(), &binaryID, annotations.Bboxes[0].ID, &UpdateBoundingBoxOptions{ + &annotationsPb.Bboxes[0].Label, + &annotationsPb.Bboxes[0].XMinNormalized, + &annotationsPb.Bboxes[0].YMinNormalized, + &annotationsPb.Bboxes[0].XMaxNormalized, + &annotationsPb.Bboxes[0].YMaxNormalized, + }) }) t.Run("GetDatabaseConnection", func(t *testing.T) { @@ -625,7 +630,7 @@ func TestDataSyncClient(t *testing.T) { t.Run("BinaryDataCaptureUpload", func(t *testing.T) { uploadMetadata.Type = DataTypeBinarySensor - options := BinaryOptions{ + options := BinaryDataCaptureUploadOptions{ Type: &binaryDataType, FileName: &fileName, MethodParameters: methodParameters, @@ -671,7 +676,7 @@ func TestDataSyncClient(t *testing.T) { TimeRequested: timestamppb.New(start), TimeReceived: timestamppb.New(end), } - options := TabularOptions{ + options := TabularDataCaptureUploadOptions{ Type: &binaryDataType, FileName: &fileName, MethodParameters: methodParameters, @@ -713,7 +718,7 @@ func TestDataSyncClient(t *testing.T) { }) t.Run("StreamingDataCaptureUpload", func(t *testing.T) { - options := StreamingOptions{ + options := StreamingDataCaptureUploadOptions{ ComponentType: &componentType, ComponentName: &componentName, MethodName: &method, From 145f44e8500aa9577490e33263df47ee64b3cdd7 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 12:18:15 -0500 Subject: [PATCH 4/7] use optional parameters correctly --- app/data_client.go | 55 +++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index e8469f0465f..04a65bc6257 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -345,28 +345,22 @@ func BsonToGo(rawData [][]byte) ([]map[string]interface{}, error) { // TabularDataByFilter queries tabular data and metadata based on given filters. func (d *DataClient) TabularDataByFilter(ctx context.Context, opts *DataByFilterOptions) (TabularDataReturn, error) { - var filter *pb.Filter - var limit uint64 - var last string - var order pb.Order + dataReq := pb.DataRequest{} var countOnly, includeInternalData bool if opts != nil { - limit = uint64(opts.Limit) - last = opts.Last - order = orderToProto(opts.SortOrder) + dataReq.Filter = filterToProto(opts.Filter) + if opts.Limit != 0 { + dataReq.Limit = uint64(opts.Limit) + } + if opts.Last != "" { + dataReq.Last = opts.Last + } + dataReq.SortOrder = orderToProto(opts.SortOrder) countOnly = opts.CountOnly includeInternalData = opts.IncludeInternalData - if opts.Filter != nil { - filter = filterToProto(opts.Filter) - } } resp, err := d.dataClient.TabularDataByFilter(ctx, &pb.TabularDataByFilterRequest{ - DataRequest: &pb.DataRequest{ - Filter: filter, - Limit: limit, - Last: last, - SortOrder: order, - }, + DataRequest: &dataReq, CountOnly: countOnly, IncludeInternalData: includeInternalData, }) @@ -429,28 +423,22 @@ func (d *DataClient) TabularDataByMQL(ctx context.Context, organizationID string func (d *DataClient) BinaryDataByFilter( ctx context.Context, includeBinary bool, opts *DataByFilterOptions, ) (BinaryDataReturn, error) { - var filter *pb.Filter - var limit uint64 - var last string - var order pb.Order + dataReq := pb.DataRequest{} var countOnly, includeInternalData bool if opts != nil { - limit = uint64(opts.Limit) - last = opts.Last - order = orderToProto(opts.SortOrder) + dataReq.Filter = filterToProto(opts.Filter) + if opts.Limit != 0 { + dataReq.Limit = uint64(opts.Limit) + } + if opts.Last != "" { + dataReq.Last = opts.Last + } + dataReq.SortOrder = orderToProto(opts.SortOrder) countOnly = opts.CountOnly includeInternalData = opts.IncludeInternalData - if opts.Filter != nil { - filter = filterToProto(opts.Filter) - } } resp, err := d.dataClient.BinaryDataByFilter(ctx, &pb.BinaryDataByFilterRequest{ - DataRequest: &pb.DataRequest{ - Filter: filter, - Limit: limit, - Last: last, - SortOrder: order, - }, + DataRequest: &dataReq, IncludeBinary: includeBinary, CountOnly: countOnly, IncludeInternalData: includeInternalData, @@ -1218,6 +1206,9 @@ func binaryIDsToProto(binaryIDs []*BinaryID) []*pb.BinaryID { } func filterToProto(filter *Filter) *pb.Filter { + if filter == nil { + return nil + } return &pb.Filter{ ComponentName: filter.ComponentName, ComponentType: filter.ComponentType, From 091f35160823d94195b27941768804a0c7f6706f Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 12:36:39 -0500 Subject: [PATCH 5/7] fix comments --- app/data_client.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/app/data_client.go b/app/data_client.go index 04a65bc6257..2dcfeddc699 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -294,6 +294,8 @@ type FileUploadOptions struct { // UpdateBoundingBoxOptions contains optional parameters for UpdateBoundingBox. type UpdateBoundingBoxOptions struct { Label *string + + // Normalized coordinates where all coordinates must be in the range [0, 1]. XMinNormalized *float64 YMinNormalized *float64 XMaxNormalized *float64 @@ -624,9 +626,7 @@ func (d *DataClient) BoundingBoxLabelsByFilter(ctx context.Context, filter *Filt return resp.Labels, nil } -// UpdateBoundingBox updates the bounding box for a given bbox ID for the file represented by the binary ID, -// modifying its label and position using optional normalized coordinates (xMin, yMin, xMax, yMax), -// where all coordinates must be in the range [0, 1]. +// UpdateBoundingBox updates the bounding box for a given bbox ID for the file represented by the binary ID. func (d *DataClient) UpdateBoundingBox(ctx context.Context, binaryID *BinaryID, bboxID string, opts *UpdateBoundingBoxOptions) error { var label *string var xMinNormalized, yMinNormalized, xMaxNormalized, yMaxNormalized *float64 @@ -1190,6 +1190,9 @@ func tabularDataFromProto(proto *pb.TabularData, metadata *pb.CaptureMetadata) T } func binaryIDToProto(binaryID *BinaryID) *pb.BinaryID { + if binaryID == nil { + return nil + } return &pb.BinaryID{ FileId: binaryID.FileID, OrganizationId: binaryID.OrganizationID, From 92537c75334dfee6235840b85d7b8cc271633bf5 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 12:47:57 -0500 Subject: [PATCH 6/7] make lint --- app/data_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data_client.go b/app/data_client.go index 2dcfeddc699..e7e21fe0769 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -293,7 +293,7 @@ type FileUploadOptions struct { // UpdateBoundingBoxOptions contains optional parameters for UpdateBoundingBox. type UpdateBoundingBoxOptions struct { - Label *string + Label *string // Normalized coordinates where all coordinates must be in the range [0, 1]. XMinNormalized *float64 From cd9ea444cb5510bebdcd7d028464fd6374d1d539 Mon Sep 17 00:00:00 2001 From: purplenicole730 Date: Tue, 3 Dec 2024 13:28:51 -0500 Subject: [PATCH 7/7] clarify comment --- app/data_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data_client.go b/app/data_client.go index e7e21fe0769..80162ca8857 100644 --- a/app/data_client.go +++ b/app/data_client.go @@ -239,7 +239,7 @@ type FileData struct { type DataByFilterOptions struct { // No Filter implies all data. Filter *Filter - // Limit defaults to 50 if unspecified. + // Limit is the maximum number of entries to include in a page. Limit defaults to 50 if unspecified. Limit int // Last indicates the object identifier of the Last-returned data. // This is returned by calls to TabularDataByFilter and BinaryDataByFilter as the `Last` value.