Add the ability to filter certain properties in the deal list

README.md

@@ -279,6 +279,9 @@ By default the [docker-compose.yaml](./docker-compose.yaml) does not expose any
 ```
 docker exec -it lotus lotus auth api-info --perm=admin
 FULLNODE_API_INFO=eyJ...ms4:/dns/lotus/tcp/1234/http
+
+docker exec -it lotus-miner lotus-miner auth api-info --perm=admin
+MINER_API_INFO=eyJ...UlI:/dns/lotus-miner/tcp/2345/http
 ```
 3. Change the `dns/lotus/tcp/1234/http` to `ip4/<127.0.0.1 or container's IP>/tcp/1234/http` for the use in `FULLNODE_API_INFO`.
 

db/deals.go

@@ -39,6 +39,13 @@ type dealAccessor struct {
 	def  map[string]fielddef.FieldDefinition
 }
 
+type FilterOptions struct {
+	Checkpoint   *string
+	IsOffline    *bool
+	TransferType *string
+	IsVerified   *bool
+}
+
 func (d *DealsDB) newDealDef(deal *types.ProviderDealState) *dealAccessor {
 	return newDealAccessor(d.db, deal)
 }
@@ -229,14 +236,26 @@ func (d *DealsDB) BySignedProposalCID(ctx context.Context, proposalCid cid.Cid)
 	return d.scanRow(row)
 }
 
-func (d *DealsDB) Count(ctx context.Context, query string) (int, error) {
+func (d *DealsDB) Count(ctx context.Context, query string, filter *FilterOptions) (int, error) {
 	whereArgs := []interface{}{}
 	where := "SELECT count(*) FROM Deals"
 	if query != "" {
 		searchWhere, searchArgs := withSearchQuery(query)
 		where += " WHERE " + searchWhere
 		whereArgs = append(whereArgs, searchArgs...)
 	}
+
+	if filter != nil {
+		filterWhere, filterArgs := withSearchFilter(*filter)
+
+		if query != "" {
+			where += " AND "
+		} else {
+			where += " WHERE "
+		}
+		where += filterWhere
+		whereArgs = append(whereArgs, filterArgs...)
+	}
 	row := d.db.QueryRowContext(ctx, where, whereArgs...)
 
 	var count int
@@ -252,7 +271,7 @@ func (d *DealsDB) ListCompleted(ctx context.Context) ([]*types.ProviderDealState
 	return d.list(ctx, 0, 0, "Checkpoint = ?", dealcheckpoints.Complete.String())
 }
 
-func (d *DealsDB) List(ctx context.Context, query string, cursor *graphql.ID, offset int, limit int) ([]*types.ProviderDealState, error) {
+func (d *DealsDB) List(ctx context.Context, query string, filter *FilterOptions, cursor *graphql.ID, offset int, limit int) ([]*types.ProviderDealState, error) {
 	where := ""
 	whereArgs := []interface{}{}
 
@@ -272,9 +291,51 @@ func (d *DealsDB) List(ctx context.Context, query string, cursor *graphql.ID, of
 		whereArgs = append(whereArgs, searchArgs...)
 	}
 
+	if filter != nil {
+		if where != "" {
+			where += " AND "
+		}
+
+		filterWhere, filterArgs := withSearchFilter(*filter)
+		where += filterWhere
+		whereArgs = append(whereArgs, filterArgs...)
+	}
+
 	return d.list(ctx, offset, limit, where, whereArgs...)
 }
 
+func withSearchFilter(filter FilterOptions) (string, []interface{}) {
+	whereArgs := []interface{}{}
+	statements := []string{}
+
+	if filter.Checkpoint != nil {
+		statements = append(statements, "Checkpoint = ?")
+		whereArgs = append(whereArgs, *filter.Checkpoint)
+	}
+
+	if filter.IsOffline != nil {
+		statements = append(statements, "IsOffline = ?")
+		whereArgs = append(whereArgs, *filter.IsOffline)
+	}
+
+	if filter.TransferType != nil {
+		statements = append(statements, "TransferType = ?")
+		whereArgs = append(whereArgs, *filter.TransferType)
+	}
+
+	if filter.IsVerified != nil {
+		statements = append(statements, "VerifiedDeal = ?")
+		whereArgs = append(whereArgs, *filter.IsVerified)
+	}
+
+	if len(statements) == 0 {
+		return "", whereArgs
+	}
+
+	where := "(" + strings.Join(statements, " AND ") + ")"
+	return where, whereArgs
+}
+
 var searchFields = []string{"ID", "PieceCID", "ClientAddress", "ProviderAddress", "ClientPeerID", "DealDataRoot", "PublishCID", "SignedProposalCID"}
 
 func withSearchQuery(query string) (string, []interface{}) {

db/deals_test.go

@@ -2,16 +2,40 @@ package db
 
 import (
 	"context"
-	"github.com/filecoin-project/boost/db/migrations"
 	"testing"
 	"time"
 
+	"github.com/filecoin-project/boost/db/migrations"
+
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints"
 	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/stretchr/testify/require"
 )
 
+func ToFilterOptions(filters map[string]interface{}) *FilterOptions {
+	filter := &FilterOptions{}
+
+	cp, ok := filters["Checkpoint"].(string)
+	if ok {
+		filter.Checkpoint = &cp
+	}
+	io, ok := filters["IsOffline"].(bool)
+	if ok {
+		filter.IsOffline = &io
+	}
+	tt, ok := filters["TransferType"].(string)
+	if ok {
+		filter.TransferType = &tt
+	}
+	vd, ok := filters["IsVerified"].(bool)
+	if ok {
+		filter.IsVerified = &vd
+	}
+
+	return filter
+}
+
 func TestDealsDB(t *testing.T) {
 	req := require.New(t)
 	ctx := context.Background()
@@ -45,16 +69,16 @@ func TestDealsDB(t *testing.T) {
 	req.NoError(err)
 	req.Equal(deal.DealUuid, storedDealBySignedPropCid.DealUuid)
 
-	dealList, err := db.List(ctx, "", nil, 0, 0)
+	dealList, err := db.List(ctx, "", nil, nil, 0, 0)
 	req.NoError(err)
 	req.Len(dealList, len(deals))
 
-	limitedDealList, err := db.List(ctx, "", nil, 1, 1)
+	limitedDealList, err := db.List(ctx, "", nil, nil, 1, 1)
 	req.NoError(err)
 	req.Len(limitedDealList, 1)
 	req.Equal(dealList[1].DealUuid, limitedDealList[0].DealUuid)
 
-	count, err := db.Count(ctx, "")
+	count, err := db.Count(ctx, "", nil)
 	req.NoError(err)
 	req.Equal(len(deals), count)
 
@@ -120,62 +144,96 @@ func TestDealsDBSearch(t *testing.T) {
 	label, err := deals[0].ClientDealProposal.Proposal.Label.ToString()
 	req.NoError(err)
 	tcs := []struct {
-		name  string
-		value string
-		count int
+		name   string
+		value  string
+		filter *FilterOptions
+		count  int
 	}{{
-		name:  "search error",
-		value: "data-transfer failed",
-		count: 1,
+		name:   "search error",
+		value:  "data-transfer failed",
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "search error with padding",
-		value: "  data-transfer failed\n\t ",
-		count: 1,
+		name:   "search error with padding",
+		value:  "  data-transfer failed\n\t ",
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "Deal UUID",
-		value: deals[0].DealUuid.String(),
-		count: 1,
+		name:   "Deal UUID",
+		value:  deals[0].DealUuid.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "piece CID",
-		value: deals[0].ClientDealProposal.Proposal.PieceCID.String(),
-		count: 1,
+		name:   "piece CID",
+		value:  deals[0].ClientDealProposal.Proposal.PieceCID.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "client address",
-		value: deals[0].ClientDealProposal.Proposal.Client.String(),
-		count: 1,
+		name:   "client address",
+		value:  deals[0].ClientDealProposal.Proposal.Client.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "provider address",
-		value: deals[0].ClientDealProposal.Proposal.Provider.String(),
-		count: len(deals),
+		name:   "provider address",
+		value:  deals[0].ClientDealProposal.Proposal.Provider.String(),
+		filter: nil,
+		count:  len(deals),
 	}, {
-		name:  "client peer ID",
-		value: deals[0].ClientPeerID.String(),
-		count: 1,
+		name:   "client peer ID",
+		value:  deals[0].ClientPeerID.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "deal data root",
-		value: deals[0].DealDataRoot.String(),
-		count: 1,
+		name:   "deal data root",
+		value:  deals[0].DealDataRoot.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "publish CID",
-		value: deals[0].PublishCID.String(),
-		count: 1,
+		name:   "publish CID",
+		value:  deals[0].PublishCID.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "signed proposal CID",
-		value: signedPropCid.String(),
-		count: 1,
+		name:   "signed proposal CID",
+		value:  signedPropCid.String(),
+		filter: nil,
+		count:  1,
 	}, {
-		name:  "label",
-		value: label,
-		count: 1,
+		name:   "label",
+		value:  label,
+		filter: nil,
+		count:  1,
+	}, {
+		name:  "filter out isOffline",
+		value: "",
+		filter: ToFilterOptions(map[string]interface{}{
+			"IsOffline": false,
+		}),
+		count: 0,
+	}, {
+		name:  "filter isOffline",
+		value: "",
+		filter: ToFilterOptions(map[string]interface{}{
+			"IsOffline": true,
+		}),
+		count: 5,
+	}, {
+		name:  "filter isOffline and IndexedAndAnnounced (in sealing)",
+		value: "",
+		filter: ToFilterOptions(map[string]interface{}{
+			"IsOffline":  true,
+			"Checkpoint": dealcheckpoints.IndexedAndAnnounced.String(),
+		}),
+		count: 0,
 	}}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
-			count, err := db.Count(ctx, tc.value)
+			count, err := db.Count(ctx, tc.value, tc.filter)
 			req.NoError(err)
 			req.Equal(tc.count, count)
 
 			searchStart := time.Now()
-			searchRes, err := db.List(ctx, tc.value, nil, 0, 0)
+			searchRes, err := db.List(ctx, tc.value, tc.filter, nil, 0, 0)
 			searchElapsed := time.Since(searchStart)
 			req.NoError(err)
 			req.Len(searchRes, tc.count)
@@ -186,3 +244,29 @@ func TestDealsDBSearch(t *testing.T) {
 		})
 	}
 }
+
+func TestWithSearchFilter(t *testing.T) {
+	req := require.New(t)
+
+	fo := ToFilterOptions(map[string]interface{}{
+		"Checkpoint":      "Accepted",
+		"IsOffline":       true,
+		"NotAValidFilter": 123,
+	})
+	where, whereArgs := withSearchFilter(*fo)
+	expectedArgs := []interface{}{
+		"Accepted",
+		true,
+	}
+	req.Equal("(Checkpoint = ? AND IsOffline = ?)", where)
+	req.Equal(expectedArgs, whereArgs)
+
+	fo = ToFilterOptions(map[string]interface{}{
+		"IsOffline":       nil,
+		"NotAValidFilter": nil,
+	})
+	where, whereArgs = withSearchFilter(*fo)
+
+	req.Equal("", where)
+	req.Equal(0, len(whereArgs))
+}

docker/devnet/docker-compose.yaml

@@ -15,10 +15,12 @@ services:
   lotus:
     container_name: lotus
     image: ${LOTUS_IMAGE}
-    # ports:
-    #   - "1234:1234"
+    ports:
+      - "1234:1234"
+      - "9090:9090"
     environment:
       - LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/1234/http
+      - LOTUS_LIBP2P_LISTENADDRESSES=/ip4/0.0.0.0/tcp/9090
     restart: unless-stopped
     logging: *default-logging
     volumes:

docker/devnet/lotus/Dockerfile

@@ -14,7 +14,8 @@ LABEL org.opencontainers.image.version=$BUILD_VERSION \
       summary="This image is used to host the lotus dev service" \
       description="This image is used to host the lotus dev service"
 
-EXPOSE 1234 
+EXPOSE 1234
+EXPOSE 9090
 ENV LOTUS_SKIP_GENESIS_CHECK=_yes_
 ENV GENESIS_PATH=/var/lib/genesis
 ENV SECTOR_SIZE=8388608