Remove file on storage termination (#219)

* feat(tasks): remove file on storage termination

recover hard drive space once a storage deal terminates by deleting the generated file

* fix(integration tests): attempt to fix integration tests

integration_tests/01_storage_retrieval_ok.sh

@@ -5,15 +5,15 @@ source "$my_dir/header.sh"
 
 export DEALBOT_MINER_ADDRESS=t01000
 
-dealbot storage-deal
+dealbot storage-deal 2>&1 | tee dealbot.log
 
 returnValue=$?
 if [[ $returnValue -ne 0 ]]; then
 	echo "expected storage-deal to succeed, but it returned exit code != 0"
 	exit 1
 fi
 
-CID=$(lotus client local | tail -1 | awk '{print $2}')
+CID=$(cat dealbot.log | grep datacid | sed 's/.*datacid": "//' | sed 's/"}//')
 
 dealbot retrieval-deal --cid=$CID
 

integration_tests/02_controller_daemon.sh

@@ -76,7 +76,7 @@ if ! grep -q 'INFO.*controller.*storage task.*"miner": "t01000",.*"size": 1024,.
 	exit 1
 fi
 
-CID=$(lotus client local | tail -1 | awk '{print $2}')
+CID=$(cat dealbot-daemon.log | grep datacid | sed 's/.*datacid": "//' | sed 's/"}//')
 
 # Also queue a retrieval task of the data we just stored.
 curl --header "Content-Type: application/json" \

tasks/storage_deal.go

@@ -54,6 +54,10 @@ func MakeStorageDeal(ctx context.Context, config NodeConfig, node api.FullNode,
 		task: task,
 	}
 
+	defer func() {
+		_ = de.cleanupDeal()
+	}()
+
 	defaultTimeout := stageTimeouts[defaultStorageStageTimeoutName]
 	getStageCtx := func(stage string) (context.Context, context.CancelFunc) {
 		timeout, ok := stageTimeouts[stage]
@@ -251,6 +255,7 @@ func (de *storageDealExecutor) importFile(l logFunc) (err error) {
 }
 
 func (de *storageDealExecutor) executeAndMonitorDeal(ctx context.Context, updateStage UpdateStage, stageTimeouts map[string]time.Duration) error {
+
 	stage := "ProposeDeal"
 	dealStage := CommonStages[stage]()
 	err := updateStage(ctx, stage, dealStage)
@@ -379,6 +384,26 @@ func (de *storageDealExecutor) executeAndMonitorDeal(ctx context.Context, update
 	return nil
 }
 
+func (de *storageDealExecutor) cleanupDeal() error {
+	if de.importRes != nil {
+		// clear out the lotus import store for CIDs in this deal
+		err := de.node.ClientRemoveImport(de.ctx, de.importRes.ImportID)
+		if err != nil {
+			return err
+		}
+		de.importRes = nil
+	}
+	if de.fileName != "" {
+		// as a last step no matter what ended up happening with the deal, delete the generated file off to reclaim disk space
+		err := os.Remove(filepath.Join(de.config.DataDir, de.fileName))
+		if err != nil {
+			return err
+		}
+		de.fileName = ""
+	}
+	return nil
+}
+
 func mktime(t time.Time) _Time {
 	return _Time{x: t.UnixNano()}
 }