merge main into piece dir (#1076)

* feat: client flag to not announce deals (#1051) * client flag to announce deals * fix itest * apply suggestions * test: fix announce tests * refactor: remove dead code * fix: TestCancelTransferForTransferredDealFails * fix var names * fix typo * refactor test * fix double negative * change flag names * change defaults value to false Co-authored-by: Dirk McCormick <dirkmdev@gmail.com> * fix: vacuum db after log cleanup (#1059) * vacuum db * fix syntax * vacuum at startup * log error * fix: run deal filter for offline deals (#1067) * run deal filter for offline deals * move filters to common func * fix: deal status on transfer complete (#1066) * fix: sort sealing worker list by date (#1074) Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>
filecoin-project · Jan 13, 2023 · 0bfdcdf · 0bfdcdf
1 parent 08b1d92
commit 0bfdcdf
Show file tree

Hide file tree

Showing 28 changed files with 494 additions and 226 deletions.
diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz
diff --git a/cmd/boost/deal_cmd.go b/cmd/boost/deal_cmd.go
@@ -78,14 +78,19 @@ var dealFlags = []cli.Flag{
 		Value: true,
 	},
 	&cli.BoolFlag{
-		Name:  "fast-retrieval",
-		Usage: "indicates that data should be available for fast retrieval",
-		Value: true,
+		Name:  "remove-unsealed-copy",
+		Usage: "indicates that an unsealed copy of the sector in not required for fast retrieval",
+		Value: false,
 	},
 	&cli.StringFlag{
 		Name:  "wallet",
 		Usage: "wallet address to be used to initiate the deal",
 	},
+	&cli.BoolFlag{
+		Name:  "skip-ipni-announce",
+		Usage: "indicates that deal index should not be announced to the IPNI(Network Indexer)",
+		Value: false,
+	},
 }
 
 var dealCmd = &cli.Command{
@@ -256,7 +261,8 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error {
 		DealDataRoot:       rootCid,
 		IsOffline:          !isOnline,
 		Transfer:           transfer,
-		FastRetrieval:      cctx.Bool("fast-retrieval"),
+		RemoveUnsealedCopy: cctx.Bool("remove-unsealed-copy"),
+		SkipIPNIAnnounce:   cctx.Bool("skip-ipni-announce"),
 	}
 
 	log.Debugw("about to submit deal proposal", "uuid", dealUuid.String())

diff --git a/db/deals.go b/db/deals.go
@@ -79,6 +79,7 @@ func newDealAccessor(db *sql.DB, deal *types.ProviderDealState) *dealAccessor {
 			"Error":                 &fielddef.FieldDef{F: &deal.Err},
 			"Retry":                 &fielddef.FieldDef{F: &deal.Retry},
 			"FastRetrieval":         &fielddef.FieldDef{F: &deal.FastRetrieval},
+			"AnnounceToIPNI":        &fielddef.FieldDef{F: &deal.AnnounceToIPNI},
 
 			// Needed so the deal can be looked up by signed proposal cid
 			"SignedProposalCID": &fielddef.SignedPropFieldDef{Prop: deal.ClientDealProposal},

diff --git a/db/fixtures.go b/db/fixtures.go
@@ -84,15 +84,16 @@ func GenerateNDeals(count int) ([]types.ProviderDealState, error) {
 					Params: []byte(fmt.Sprintf(`{"url":"http://files.org/file%d.car"}`, rand.Intn(1000))),
 					Size:   uint64(rand.Intn(10000)),
 				},
-				ChainDealID:   abi.DealID(rand.Intn(10000)),
-				PublishCID:    &publishCid,
-				SectorID:      abi.SectorNumber(rand.Intn(10000)),
-				Offset:        abi.PaddedPieceSize(rand.Intn(1000000)),
-				Length:        abi.PaddedPieceSize(rand.Intn(1000000)),
-				Checkpoint:    dealcheckpoints.Accepted,
-				Retry:         types.DealRetryAuto,
-				Err:           dealErr,
-				FastRetrieval: false,
+				ChainDealID:    abi.DealID(rand.Intn(10000)),
+				PublishCID:     &publishCid,
+				SectorID:       abi.SectorNumber(rand.Intn(10000)),
+				Offset:         abi.PaddedPieceSize(rand.Intn(1000000)),
+				Length:         abi.PaddedPieceSize(rand.Intn(1000000)),
+				Checkpoint:     dealcheckpoints.Accepted,
+				Retry:          types.DealRetryAuto,
+				Err:            dealErr,
+				FastRetrieval:  false,
+				AnnounceToIPNI: false,
 			}
 
 			deals = append(deals, deal)

diff --git a/db/migrations/20230104230242_deal_announce_to_ipni.sql b/db/migrations/20230104230242_deal_announce_to_ipni.sql
@@ -0,0 +1,10 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE Deals
+    ADD AnnounceToIPNI BOOL;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query';
+-- +goose StatementEnd
diff --git a/db/migrations/20230104230411_deal_announce_to_ipni.go b/db/migrations/20230104230411_deal_announce_to_ipni.go
@@ -0,0 +1,24 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigration(upSetdealsAnnounceToIPNI, downSetdealsAnnounceToIPNI)
+}
+
+func upSetdealsAnnounceToIPNI(tx *sql.Tx) error {
+	_, err := tx.Exec("UPDATE Deals SET AnnounceToIPNI=?;", true)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func downSetdealsAnnounceToIPNI(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}
diff --git a/db/migrations_tests/deals_announce_to_ipni_test.go b/db/migrations_tests/deals_announce_to_ipni_test.go
@@ -0,0 +1,46 @@
+package migrations_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/boost/db"
+	"github.com/filecoin-project/boost/db/migrations"
+	"github.com/pressly/goose/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDealAnnounceToIPNI(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+
+	sqldb := db.CreateTestTmpDB(t)
+	req.NoError(db.CreateAllBoostTables(ctx, sqldb, sqldb))
+
+	// Run migrations up to the one that adds the AnnounceToIPNI field to Deals
+	goose.SetBaseFS(migrations.EmbedMigrations)
+	req.NoError(goose.SetDialect("sqlite3"))
+	req.NoError(goose.UpTo(sqldb, ".", 20230104230242))
+
+	// Generate 1 deal
+	dealsDB := db.NewDealsDB(sqldb)
+	deals, err := db.GenerateNDeals(1)
+	req.NoError(err)
+
+	// Insert the deals in DB
+	err = dealsDB.Insert(ctx, &deals[0])
+	require.NoError(t, err)
+
+	// Get deal state
+	dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.False(t, dealState.AnnounceToIPNI)
+
+	//Run migration
+	req.NoError(goose.UpByOne(sqldb, "."))
+
+	// Check the deal state again
+	dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.True(t, dealState.AnnounceToIPNI)
+}
diff --git a/db/migrations_tests/deals_fast_retrieval_test.go b/db/migrations_tests/deals_fast_retrieval_test.go
@@ -22,25 +22,41 @@ func TestDealFastRetrieval(t *testing.T) {
 	req.NoError(goose.SetDialect("sqlite3"))
 	req.NoError(goose.UpTo(sqldb, ".", 20221124191002))
 
-	// Generate 2 deals
-	dealsDB := db.NewDealsDB(sqldb)
+	// Generate 1 deal
 	deals, err := db.GenerateNDeals(1)
 	req.NoError(err)
 
-	// Insert the deals in DB
-	err = dealsDB.Insert(ctx, &deals[0])
-	require.NoError(t, err)
+	deal := deals[0]
+
+	_, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize",
+                   "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch",
+                   "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot",
+                   "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID",
+                   "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID") 
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+		deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(),
+		deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline,
+		deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test",
+		deal.ClientDealProposal.Proposal.StartEpoch, deal.ClientDealProposal.Proposal.EndEpoch, deal.ClientDealProposal.Proposal.StoragePricePerEpoch.Uint64(),
+		deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(),
+		deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID,
+		deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint, deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"))
 
-	// Get deal state
-	dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
 	require.NoError(t, err)
-	require.False(t, dealState.FastRetrieval)
 
 	//Run migration
 	req.NoError(goose.UpByOne(sqldb, "."))
 
-	// Check the deal state again
-	dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
+	rows, err := sqldb.Query("SELECT FastRetrieval FROM Deals WHERE ID =?", deal.DealUuid)
 	require.NoError(t, err)
-	require.True(t, dealState.FastRetrieval)
+	defer rows.Close()
+
+	for rows.Next() {
+
+		var ft bool
+
+		err = rows.Scan(&ft)
+		require.NoError(t, err)
+		require.True(t, ft)
+	}
 }
diff --git a/db/migrations_tests/storagetagged_set_host_test.go b/db/migrations_tests/storagetagged_set_host_test.go
@@ -24,13 +24,6 @@ func TestStorageTaggedSetHost(t *testing.T) {
 	req.NoError(goose.SetDialect("sqlite3"))
 	req.NoError(goose.UpTo(sqldb, ".", 20220908122510))
 
-	// Generate 2 deals
-	dealsDB := db.NewDealsDB(sqldb)
-
-	// Add FastRetrieval to allow tests to works
-	_, err := sqldb.Exec(`ALTER TABLE Deals ADD FastRetrieval BOOL`)
-	require.NoError(t, err)
-
 	deals, err := db.GenerateNDeals(2)
 	req.NoError(err)
 
@@ -44,8 +37,21 @@ func TestStorageTaggedSetHost(t *testing.T) {
 			Params: []byte(fmt.Sprintf(`{"url":"http://%s/file.car"}`, getHost(i))),
 			Size:   uint64(1024),
 		}
-		err = dealsDB.Insert(ctx, &deal)
-		req.NoError(err)
+		_, err = sqldb.Exec(`INSERT INTO Deals ("ID", "CreatedAt", "DealProposalSignature", "PieceCID", "PieceSize",
+                   "VerifiedDeal", "IsOffline", "ClientAddress", "ProviderAddress","Label", "StartEpoch", "EndEpoch",
+                   "StoragePricePerEpoch", "ProviderCollateral", "ClientCollateral", "ClientPeerID", "DealDataRoot",
+                   "InboundFilePath", "TransferType", "TransferParams", "TransferSize", "ChainDealID", "PublishCID",
+                   "SectorID", "Offset", "Length", "Checkpoint", "CheckpointAt", "Error", "Retry", "SignedProposalCID") 
+                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+			deal.DealUuid, deal.CreatedAt, []byte("test"), deal.ClientDealProposal.Proposal.PieceCID.String(),
+			deal.ClientDealProposal.Proposal.PieceSize, deal.ClientDealProposal.Proposal.VerifiedDeal, deal.IsOffline,
+			deal.ClientDealProposal.Proposal.Client.String(), deal.ClientDealProposal.Proposal.Provider.String(), "test",
+			deal.ClientDealProposal.Proposal.StartEpoch, deal.ClientDealProposal.Proposal.EndEpoch, deal.ClientDealProposal.Proposal.StoragePricePerEpoch.Uint64(),
+			deal.ClientDealProposal.Proposal.ProviderCollateral.Int64(), deal.ClientDealProposal.Proposal.ClientCollateral.Uint64(), deal.ClientPeerID.String(),
+			deal.DealDataRoot.String(), deal.InboundFilePath, deal.Transfer.Type, deal.Transfer.Params, deal.Transfer.Size, deal.ChainDealID,
+			deal.PublishCID.String(), deal.SectorID, deal.Offset, deal.Length, deal.Checkpoint, deal.CheckpointAt, deal.Err, deal.Retry, []byte("test"))
+
+		require.NoError(t, err)
 	}
 
 	// Simulate tagging a deal

diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md
@@ -396,7 +396,8 @@ Response:
   "Err": "string value",
   "Retry": "auto",
   "NBytesReceived": 9,
-  "FastRetrieval": true
+  "FastRetrieval": true,
+  "AnnounceToIPNI": true
 }
 ```
 
@@ -462,7 +463,8 @@ Response:
   "Err": "string value",
   "Retry": "auto",
   "NBytesReceived": 9,
-  "FastRetrieval": true
+  "FastRetrieval": true,
+  "AnnounceToIPNI": true
 }
 ```
 
@@ -507,7 +509,8 @@ Inputs:
       "Params": "Ynl0ZSBhcnJheQ==",
       "Size": 42
     },
-    "FastRetrieval": true
+    "RemoveUnsealedCopy": true,
+    "SkipIPNIAnnounce": true
   }
 ]
 ```

diff --git a/gql/resolver.go b/gql/resolver.go
@@ -5,7 +5,9 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"math"
 
+	"github.com/dustin/go-humanize"
 	"github.com/filecoin-project/boost/db"
 	"github.com/filecoin-project/boost/fundmanager"
 	gqltypes "github.com/filecoin-project/boost/gql/types"
@@ -143,6 +145,7 @@ func (r *resolver) Deals(ctx context.Context, args dealsArgs) (*dealListResolver
 
 	resolvers := make([]*dealResolver, 0, len(deals))
 	for _, deal := range deals {
+		deal.NBytesReceived = int64(r.provider.NBytesReceived(deal.DealUuid))
 		resolvers = append(resolvers, newDealResolver(&deal, r.provider, r.dealsDB, r.logsDB, r.spApi))
 	}
 
@@ -376,6 +379,14 @@ func (dr *dealResolver) IsVerified() bool {
 	return dr.ProviderDealState.ClientDealProposal.Proposal.VerifiedDeal
 }
 
+func (dr *dealResolver) KeepUnsealedCopy() bool {
+	return dr.ProviderDealState.FastRetrieval
+}
+
+func (dr *dealResolver) AnnounceToIPNI() bool {
+	return dr.ProviderDealState.AnnounceToIPNI
+}
+
 func (dr *dealResolver) ProposalLabel() (string, error) {
 	l := dr.ProviderDealState.ClientDealProposal.Proposal.Label
 	if l.IsString() {
@@ -512,16 +523,25 @@ func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints.
 		if dr.IsOffline {
 			return "Awaiting Offline Data Import"
 		}
+		var pct uint64 = math.MaxUint64
+		if dr.ProviderDealState.Transfer.Size > 0 {
+			pct = (100 * dr.transferred) / dr.ProviderDealState.Transfer.Size
+		}
 		switch {
 		case dr.transferred == 0 && !dr.provider.IsTransferStalled(dr.DealUuid):
 			return "Transfer Queued"
-		case dr.transferred == 100:
-			return "Transfer Complete"
+		case pct == 100:
+			return "Verifying Commp"
 		default:
-			pct := (100 * dr.transferred) / dr.ProviderDealState.Transfer.Size
 			isStalled := dr.provider.IsTransferStalled(dr.DealUuid)
 			if isStalled {
-				return fmt.Sprintf("Transfer stalled at %d%% ", pct)
+				if pct == math.MaxUint64 {
+					return fmt.Sprintf("Transfer stalled at %s", humanize.Bytes(dr.transferred))
+				}
+				return fmt.Sprintf("Transfer stalled at %d%%", pct)
+			}
+			if pct == math.MaxUint64 {
+				return fmt.Sprintf("Transferring %s", humanize.Bytes(dr.transferred))
 			}
 			return fmt.Sprintf("Transferring %d%%", pct)
 		}

diff --git a/gql/schema.graphql b/gql/schema.graphql
@@ -45,6 +45,8 @@ type Deal {
   PieceCid: String!
   PieceSize: Uint64!
   IsVerified: Boolean!
+  AnnounceToIPNI: Boolean!
+  KeepUnsealedCopy: Boolean!
   ProposalLabel: String!
   ProviderCollateral: Uint64!
   ClientCollateral: Uint64!

diff --git a/itests/framework/framework.go b/itests/framework/framework.go
@@ -541,7 +541,8 @@ func (f *TestFramework) MakeDummyDeal(dealUuid uuid.UUID, carFilepath string, ro
 			Params: transferParamsJSON,
 			Size:   uint64(carFileinfo.Size()),
 		},
-		FastRetrieval: true,
+		RemoveUnsealedCopy: false,
+		SkipIPNIAnnounce:   false,
 	}
 
 	return f.Client.StorageDeal(f.ctx, dealParams, peerID)

diff --git a/react/src/DealDetail.js b/react/src/DealDetail.js
@@ -149,6 +149,14 @@ export function DealDetail(props) {
                     <th>Verified</th>
                     <td>{deal.IsVerified ? 'Yes' : 'No'}</td>
                 </tr>
+                <tr>
+                    <th>Keep Unsealed Copy</th>
+                    <td>{deal.KeepUnsealedCopy ? 'Yes' : 'No'}</td>
+                </tr>
+                <tr>
+                    <th>Announce To IPNI</th>
+                    <td>{deal.AnnounceToIPNI ? 'Yes' : 'No'}</td>
+                </tr>
                 <tr>
                     <th>Piece CID</th>
                     <td><Link to={'/inspect/'+deal.PieceCid}>{deal.PieceCid}</Link></td>

diff --git a/react/src/Mpool.js b/react/src/Mpool.js
@@ -55,7 +55,7 @@ function MpoolMessage(props) {
         </tr>
         <tr key={i+"nonce"}>
             <td>Nonce</td>
-            <td>{msg.Nonce}</td>
+            <td>{msg.Nonce+''}</td>
         </tr>
         <tr key={i+"value"}>
             <td>Value</td>