Add changes from maser to release/v14

Cargo.toml

@@ -95,6 +95,7 @@ pretty_env_logger = "0.4.0"
 serde_repr = "0.1.8"
 unsigned-varint = "0.7.1"
 rand_chacha = "0.3.1"
+const-hex = "1.11.3"
 
 # Crypto
 libsecp256k1 = { version = "0.7.1", default-features = false }

actors/market/src/state.rs

@@ -626,16 +626,7 @@ impl State {
 
         // Flush if any of the requested sectors were found.
         if flush {
-            if sector_deals.is_empty() {
-                // Remove from top-level map
-                provider_sectors
-                    .delete(&provider)
-                    .with_context_code(ExitCode::USR_ILLEGAL_STATE, || {
-                        format!("failed to delete sector deals for {}", provider)
-                    })?;
-            } else {
-                save_provider_sector_deals(&mut provider_sectors, provider, &mut sector_deals)?;
-            }
+            save_provider_sector_deals(&mut provider_sectors, provider, &mut sector_deals)?;
             self.save_provider_sectors(&mut provider_sectors)?;
         }
 
@@ -651,6 +642,7 @@ impl State {
     ) -> Result<(), ActorError> {
         let mut provider_sectors = self.load_provider_sectors(store)?;
         for (provider, sector_deal_ids) in provider_sector_deal_ids {
+            let mut flush = false;
             let mut sector_deals = load_provider_sector_deals(store, &provider_sectors, *provider)?;
             for (sector_number, deals_to_remove) in sector_deal_ids {
                 let existing_deal_ids = sector_deals
@@ -662,20 +654,39 @@ impl State {
                     // pretty fast.
                     // Loading into a HashSet could be an improvement for large collections of deals
                     // in a single sector being removed at one time.
-                    let new_deals = existing_deal_ids
+                    let new_deals: Vec<_> = existing_deal_ids
                         .iter()
                         .filter(|deal_id| !deals_to_remove.contains(*deal_id))
                         .cloned()
                         .collect();
-
-                    sector_deals
-                        .set(sector_number, new_deals)
-                        .with_context_code(ExitCode::USR_ILLEGAL_STATE, || {
-                            format!("failed to set sector deals for {} {}", provider, sector_number)
-                        })?;
+                    flush = true;
+
+                    if new_deals.is_empty() {
+                        sector_deals.delete(sector_number).with_context_code(
+                            ExitCode::USR_ILLEGAL_STATE,
+                            || {
+                                format!(
+                                    "failed to delete sector deals for {} {}",
+                                    provider, sector_number
+                                )
+                            },
+                        )?;
+                    } else {
+                        sector_deals.set(sector_number, new_deals).with_context_code(
+                            ExitCode::USR_ILLEGAL_STATE,
+                            || {
+                                format!(
+                                    "failed to set sector deals for {} {}",
+                                    provider, sector_number
+                                )
+                            },
+                        )?;
+                    }
                 }
             }
-            save_provider_sector_deals(&mut provider_sectors, *provider, &mut sector_deals)?;
+            if flush {
+                save_provider_sector_deals(&mut provider_sectors, *provider, &mut sector_deals)?;
+            }
         }
         self.save_provider_sectors(&mut provider_sectors)?;
         Ok(())
@@ -1273,7 +1284,15 @@ fn save_provider_sector_deals<BS>(
 where
     BS: Blockstore,
 {
-    let sectors_root = sector_deals.flush()?;
-    provider_sectors.set(&provider, sectors_root)?;
+    if sector_deals.is_empty() {
+        provider_sectors
+            .delete(&provider)
+            .with_context_code(ExitCode::USR_ILLEGAL_STATE, || {
+                format!("failed to delete sector deals for {}", provider)
+            })?;
+    } else {
+        let sectors_root = sector_deals.flush()?;
+        provider_sectors.set(&provider, sectors_root)?;
+    }
     Ok(())
 }

actors/market/tests/activate_deal_failures.rs

@@ -107,7 +107,7 @@ fn reject_proposal_expired() {
         ExitCode::OK,
     );
     cron_tick(&rt);
-    assert_deal_deleted(&rt, deal_id, &deal, 0);
+    assert_deal_deleted(&rt, deal_id, &deal, 0, true);
 
     assert_activation_failure(&rt, deal_id, &deal, 1, sector_expiry, EX_DEAL_EXPIRED);
 }

actors/market/tests/batch_activate_deals.rs

@@ -46,7 +46,7 @@ fn activate_deals_one_sector() {
     assert!(res.activation_results.all_ok());
 
     // Deal IDs are stored under the sector, in correct order.
-    assert_eq!(deal_ids, get_sector_deal_ids(&rt, PROVIDER_ID, &[1]));
+    assert_eq!(deal_ids, get_sector_deal_ids(&rt, PROVIDER_ID, &[1]).unwrap());
 
     for id in deal_ids.iter() {
         let state = get_deal_state(&rt, *id);
@@ -392,7 +392,7 @@ fn activate_new_deals_in_existing_sector() {
     assert_eq!(0, get_deal_state(&rt, deal_ids[2]).sector_start_epoch);
 
     // All deals stored under the sector, in order.
-    assert_eq!(deal_ids, get_sector_deal_ids(&rt, PROVIDER_ID, &[sector_number]));
+    assert_eq!(deal_ids, get_sector_deal_ids(&rt, PROVIDER_ID, &[sector_number]).unwrap());
     check_state(&rt);
 }
 

actors/market/tests/cron_tick_deal_expiry.rs

@@ -49,7 +49,7 @@ fn deal_is_correctly_processed_if_first_cron_after_expiry() {
     assert!(slashed.is_zero());
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
     check_state(&rt);
 }
 
@@ -124,7 +124,7 @@ fn regular_payments_till_deal_expires_and_then_locked_funds_are_unlocked() {
     assert!(slashed.is_zero());
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
     check_state(&rt);
 }
 
@@ -154,7 +154,7 @@ fn payment_for_a_deal_if_deal_is_already_expired_before_a_cron_tick() {
     assert_eq!((end - start) * &deal_proposal.storage_price_per_epoch, pay);
     assert!(slashed.is_zero());
 
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
 
     // running cron tick again doesn't do anything
     cron_tick_no_change(&rt, CLIENT_ADDR, PROVIDER_ADDR);
@@ -203,7 +203,7 @@ fn expired_deal_should_unlock_the_remaining_client_and_provider_locked_balance_a
     assert!(provider_acct.locked.is_zero());
 
     // deal should be deleted
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
     check_state(&rt);
 }
 

actors/market/tests/cron_tick_deal_slashing.rs

@@ -102,7 +102,7 @@ fn deal_is_slashed() {
         rt.set_epoch(cron_tick_epoch);
         cron_tick(&rt);
 
-        assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+        assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
 
         check_state(&rt);
     }
@@ -142,7 +142,7 @@ fn deal_is_slashed_at_the_end_epoch_should_not_be_slashed_and_should_be_consider
     assert!(slashed.is_zero());
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER, true);
 
     check_state(&rt);
 }
@@ -192,7 +192,7 @@ fn deal_payment_and_slashing_correctly_processed_in_same_crontick() {
     cron_tick(&rt);
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
     check_state(&rt);
 }
 
@@ -249,9 +249,9 @@ fn slash_multiple_deals_in_the_same_epoch() {
     rt.set_epoch(epoch + 1);
     cron_tick(&rt);
 
-    assert_deal_deleted(&rt, deal_id1, &deal_proposal1, SECTOR_NUMBER);
-    assert_deal_deleted(&rt, deal_id2, &deal_proposal2, SECTOR_NUMBER);
-    assert_deal_deleted(&rt, deal_id3, &deal_proposal3, SECTOR_NUMBER);
+    assert_deal_deleted(&rt, deal_id1, &deal_proposal1, SECTOR_NUMBER, true);
+    assert_deal_deleted(&rt, deal_id2, &deal_proposal2, SECTOR_NUMBER, true);
+    assert_deal_deleted(&rt, deal_id3, &deal_proposal3, SECTOR_NUMBER, true);
     check_state(&rt);
 }
 
@@ -319,7 +319,7 @@ fn regular_payments_till_deal_is_slashed_and_then_slashing_is_processed() {
     cron_tick(&rt);
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER, true);
     check_state(&rt);
 }
 
@@ -372,6 +372,6 @@ fn regular_payments_till_deal_expires_and_then_we_attempt_to_slash_it_but_it_wil
     assert!(slashed.is_zero());
 
     // deal should be deleted as it should have expired
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, SECTOR_NUMBER, true);
     check_state(&rt);
 }

actors/market/tests/cron_tick_timedout_deals.rs

@@ -55,7 +55,7 @@ fn timed_out_deal_is_slashed_and_deleted() {
     assert_eq!(c_escrow, client_acct.balance);
     assert!(client_acct.locked.is_zero());
     assert_account_zero(&rt, PROVIDER_ADDR);
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, 0);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, 0, true);
     check_state(&rt);
 }
 
@@ -126,7 +126,7 @@ fn publishing_timed_out_deal_again_should_work_after_cron_tick_as_it_should_no_l
         ExitCode::OK,
     );
     cron_tick(&rt);
-    assert_deal_deleted(&rt, deal_id, &deal_proposal, 0);
+    assert_deal_deleted(&rt, deal_id, &deal_proposal, 0, true);
 
     // now publishing should work
     generate_and_publish_deal(&rt, CLIENT_ADDR, &MinerAddresses::default(), START_EPOCH, END_EPOCH);
@@ -192,8 +192,8 @@ fn timed_out_and_verified_deals_are_slashed_deleted() {
     cron_tick_no_change(&rt, CLIENT_ADDR, PROVIDER_ADDR);
 
     assert_account_zero(&rt, PROVIDER_ADDR);
-    assert_deal_deleted(&rt, deal_ids[0], &deal1, 0);
-    assert_deal_deleted(&rt, deal_ids[1], &deal2, 0);
-    assert_deal_deleted(&rt, deal_ids[2], &deal3, 0);
+    assert_deal_deleted(&rt, deal_ids[0], &deal1, 0, true);
+    assert_deal_deleted(&rt, deal_ids[1], &deal2, 0, true);
+    assert_deal_deleted(&rt, deal_ids[2], &deal3, 0, true);
     check_state(&rt);
 }

actors/market/tests/deal_termination.rs

@@ -106,7 +106,7 @@ fn deal_is_terminated() {
         assert_eq!(tc.termination_payment, pay);
         assert_eq!(deal_proposal.provider_collateral, slashed);
 
-        assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number);
+        assert_deal_deleted(&rt, deal_id, &deal_proposal, sector_number, true);
 
         // assert that trying to settle is always a no-op after termination
 
@@ -197,7 +197,7 @@ fn settle_payments_then_terminate_deal_in_the_same_epoch() {
         &[sector_number],
         &[deal_id],
     );
-    assert_deal_deleted(&rt, deal_id, &proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &proposal, sector_number, true);
 
     // end state should be equivalent to only calling termination
     let client_after = get_balance(&rt, &CLIENT_ADDR);
@@ -245,7 +245,7 @@ fn terminate_a_deal_then_settle_it_in_the_same_epoch() {
     );
     let ret = settle_deal_payments(&rt, PROVIDER_ADDR, &[deal_id], &[], &[]);
     assert_eq!(ret.results.codes(), vec![EX_DEAL_EXPIRED]);
-    assert_deal_deleted(&rt, deal_id, &proposal, sector_number);
+    assert_deal_deleted(&rt, deal_id, &proposal, sector_number, true);
 
     check_state(&rt);
 }