drop txo field from txos table in db

.gitignore

@@ -78,4 +78,4 @@ dbml-error.log
 
 __pycache__
 
-/tools/*.bin
+*sgx_linux_x64_sdk_2.9.101.2.bin

Cargo.toml

@@ -57,4 +57,5 @@ x25519-dalek = { git = "https://github.com/mobilecoinfoundation/x25519-dalek.git
 
 # Override diesel dependency with our fork, in order to use a version of libsqlite3-sys that has bundled-sqlcipher. This allows us to
 # statically link SQLite.
+# If updating here, also update in README
 diesel = { git = "https://github.com/mobilecoinofficial/diesel", rev = "026f6379715d27c8be48396e5ca9059f4a263198" }

README.md

@@ -407,12 +407,33 @@ See [CONTRIBUTING](./CONTRIBUTING.md).
 To add or edit tables:
 
 1. Ensure that you have `diesel_cli` installed and that it is using the current sqlite
-   version: `cargo install --git="https://github.com/mobilecoinofficial/diesel" --rev="22a4a4b973db2b7aadaf088b3279dbbe52176896" diesel_cli --no-default-features --features sqlite`
+   version:  
+
+   ```
+   cargo install --git="https://github.com/mobilecoinofficial/diesel" --rev="026f6379715d27c8be48396e5ca9059f4a263198" diesel_cli --no-default-features --features sqlite
+   ```
+
 1. `cd full-service`
+
+1. Create an empty version of the base db `diesel migration run --database-url $MIGRATION_TEST_DB`
+
 1. Create a migration with `diesel migration generate <migration_name>`
+
 1. Edit the migrations/<migration_name>/up.sql and down.sql.
-1. Run the migration with `diesel migration run --database-url /tmp/db.db`, and test delete
-   with `diesel migration redo --database-url /tmp/db.db`
+
+1. Run the migration with `diesel migration run --database-url $MIGRATION_TEST_DB`, and test the 
+   inverse operation with `diesel migration redo --database-url $MIGRATION_TEST_DB`
+
+Make sure that the following is still present in `schema.rs` before commiting changes.
+``` 
+table! {
+    __diesel_schema_migrations(version) {
+        version -> Text,
+        run_on -> Timestamp,
+    }
+}
+```
+
 
 Note that full-service/diesel.toml provides the path to the schema.rs which will be updated in a migration.
 

full-service/migrations/2022-10-11-050914_drop_txo_from_txos/down.sql

@@ -0,0 +1 @@
+ALTER TABLE txos ADD COLUMN txo BLOB NOT NULL;

full-service/migrations/2022-10-11-050914_drop_txo_from_txos/up.sql

@@ -0,0 +1 @@
+ALTER TABLE txos DROP COLUMN txo;

full-service/src/db/models.rs

@@ -74,8 +74,6 @@ pub struct Txo {
     pub public_key: Vec<u8>,
     /// The serialized e_fog_hint of the TxOut.
     pub e_fog_hint: Vec<u8>,
-    /// The serialized TxOut.
-    pub txo: Vec<u8>,
     /// The receiving subaddress, if known.
     pub subaddress_index: Option<i64>,
     /// Pre-computed key image for this Txo, or None if the Txo is orphaned.
@@ -104,7 +102,6 @@ pub struct NewTxo<'a> {
     pub target_key: &'a [u8],
     pub public_key: &'a [u8],
     pub e_fog_hint: &'a [u8],
-    pub txo: &'a [u8],
     pub subaddress_index: Option<i64>,
     pub key_image: Option<&'a [u8]>,
     pub received_block_index: Option<i64>,

full-service/src/db/schema.rs

@@ -71,7 +71,6 @@ table! {
         target_key -> Binary,
         public_key -> Binary,
         e_fog_hint -> Binary,
-        txo -> Binary,
         subaddress_index -> Nullable<BigInt>,
         key_image -> Nullable<Binary>,
         received_block_index -> Nullable<BigInt>,

full-service/src/db/transaction_log.rs

@@ -578,7 +578,7 @@ mod tests {
     use mc_account_keys::{PublicAddress, CHANGE_SUBADDRESS_INDEX};
     use mc_common::logger::{test_with_logger, Logger};
     use mc_ledger_db::Ledger;
-    use mc_transaction_core::{tokens::Mob, Token};
+    use mc_transaction_core::{ring_signature::KeyImage, tokens::Mob, Token};
     use rand::{rngs::StdRng, SeedableRng};
 
     use crate::{
@@ -588,9 +588,9 @@ mod tests {
             transaction_builder::WalletTransactionBuilder,
         },
         test_utils::{
-            add_block_from_transaction_log, add_block_with_tx_outs, builder_for_random_recipient,
-            get_resolver_factory, get_test_ledger, manually_sync_account,
-            random_account_with_seed_values, WalletDbTestContext, MOB,
+            add_block_with_tx_outs, builder_for_random_recipient, get_resolver_factory,
+            get_test_ledger, manually_sync_account, random_account_with_seed_values,
+            WalletDbTestContext, MOB,
         },
         util::b58::b58_encode_public_address,
     };
@@ -719,10 +719,20 @@ mod tests {
         // The subaddress will also be set once received.
         assert_eq!(change_details.subaddress_index, None,);
 
-        add_block_from_transaction_log(
+        let key_images: Vec<KeyImage> = tx_proposal
+            .input_txos
+            .iter()
+            .map(|txo| txo.key_image.clone())
+            .collect();
+
+        // Note: This block doesn't contain the fee output.
+        add_block_with_tx_outs(
             &mut ledger_db,
-            &wallet_db.get_conn().unwrap(),
-            &tx_log,
+            &[
+                tx_proposal.change_txos[0].tx_out.clone(),
+                tx_proposal.payload_txos[0].tx_out.clone(),
+            ],
+            &key_images,
             &mut rng,
         );
 
@@ -1128,8 +1138,8 @@ mod tests {
         add_block_with_tx_outs(
             &mut ledger_db,
             &[
-                mc_util_serial::decode(&change_details.txo).unwrap(),
-                mc_util_serial::decode(&output_details.txo).unwrap(),
+                tx_proposal.change_txos[0].tx_out.clone(),
+                tx_proposal.payload_txos[0].tx_out.clone(),
             ],
             &[
                 mc_util_serial::decode(&input_details0.key_image.unwrap()).unwrap(),

full-service/src/db/txo.rs

@@ -372,7 +372,6 @@ impl TxoModel for Txo {
                     target_key: &mc_util_serial::encode(&txo.target_key),
                     public_key: &mc_util_serial::encode(&txo.public_key),
                     e_fog_hint: &mc_util_serial::encode(&txo.e_fog_hint),
-                    txo: &mc_util_serial::encode(&txo),
                     subaddress_index: subaddress_index.map(|i| i as i64),
                     key_image: key_image_bytes.as_deref(),
                     received_block_index: Some(received_block_index as i64),
@@ -411,7 +410,6 @@ impl TxoModel for Txo {
             target_key: &mc_util_serial::encode(&output_txo.tx_out.target_key),
             public_key: &mc_util_serial::encode(&output_txo.tx_out.public_key),
             e_fog_hint: &mc_util_serial::encode(&output_txo.tx_out.e_fog_hint),
-            txo: &mc_util_serial::encode(&output_txo.tx_out),
             subaddress_index: None,
             key_image: None,
             received_block_index: None,
@@ -1430,10 +1428,10 @@ mod tests {
             transaction_builder::WalletTransactionBuilder,
         },
         test_utils::{
-            add_block_with_db_txos, add_block_with_tx, add_block_with_tx_outs,
-            create_test_minted_and_change_txos, create_test_received_txo,
-            create_test_txo_for_recipient, get_resolver_factory, get_test_ledger,
-            manually_sync_account, random_account_with_seed_values, WalletDbTestContext, MOB,
+            add_block_with_tx, add_block_with_tx_outs, create_test_minted_and_change_txos,
+            create_test_received_txo, create_test_txo_for_recipient, get_resolver_factory,
+            get_test_ledger, manually_sync_account, random_account_with_seed_values,
+            WalletDbTestContext, MOB,
         },
         WalletDb,
     };
@@ -1509,7 +1507,6 @@ mod tests {
             target_key: mc_util_serial::encode(&for_alice_txo.target_key),
             public_key: mc_util_serial::encode(&for_alice_txo.public_key),
             e_fog_hint: mc_util_serial::encode(&for_alice_txo.e_fog_hint),
-            txo: mc_util_serial::encode(&for_alice_txo),
             subaddress_index: Some(0),
             key_image: Some(mc_util_serial::encode(&for_alice_key_image)),
             received_block_index: Some(12),
@@ -1538,7 +1535,7 @@ mod tests {
         // have not yet assigned. At the DB layer, we accomplish this by
         // constructing the output txos, then logging sent and received for this
         // account.
-        let transaction_log = create_test_minted_and_change_txos(
+        let (transaction_log, tx_proposal) = create_test_minted_and_change_txos(
             alice_account_key.clone(),
             alice_account_key.subaddress(4),
             33 * MOB,
@@ -1556,10 +1553,12 @@ mod tests {
         assert_eq!(minted_txo.value as u64, 33 * MOB);
         assert_eq!(change_txo.value as u64, 967 * MOB - Mob::MINIMUM_FEE);
 
-        add_block_with_db_txos(
+        add_block_with_tx_outs(
             &mut ledger_db,
-            &wallet_db,
-            &[minted_txo.id.clone(), change_txo.id.clone()],
+            &[
+                tx_proposal.change_txos[0].tx_out.clone(),
+                tx_proposal.payload_txos[0].tx_out.clone(),
+            ],
             &[KeyImage::from(for_alice_key_image)],
             &mut rng,
         );
@@ -1754,7 +1753,7 @@ mod tests {
         )
         .unwrap();
 
-        let transaction_log = create_test_minted_and_change_txos(
+        let (transaction_log, tx_proposal) = create_test_minted_and_change_txos(
             alice_account_key.clone(),
             bob_account_key.subaddress(0),
             72 * MOB,
@@ -1773,10 +1772,12 @@ mod tests {
         assert_eq!(change_txo.value as u64, 928 * MOB - (2 * Mob::MINIMUM_FEE));
 
         // Add the minted Txos to the ledger
-        add_block_with_db_txos(
+        add_block_with_tx_outs(
             &mut ledger_db,
-            &wallet_db,
-            &[minted_txo.id.clone(), change_txo.id.clone()],
+            &[
+                tx_proposal.change_txos[0].tx_out.clone(),
+                tx_proposal.payload_txos[0].tx_out.clone(),
+            ],
             &[KeyImage::from(for_bob_key_image)],
             &mut rng,
         );
@@ -2093,7 +2094,7 @@ mod tests {
 
         assert_eq!(txos.len(), 12);
 
-        let transaction_log = create_test_minted_and_change_txos(
+        let (transaction_log, _) = create_test_minted_and_change_txos(
             src_account.clone(),
             recipient,
             1 * MOB,

full-service/src/json_rpc/v2/api/wallet.rs

@@ -442,9 +442,21 @@ where
                 )
                 .map_err(format_error)?;
 
-            let unverified_txos_encoded: Vec<String> = unverified_txos
+            let unverified_tx_results: Result<Vec<mc_transaction_core::tx::TxOut>, JsonRPCError> =
+                unverified_txos
+                    .iter()
+                    .map(
+                        |(txo, _)| -> Result<mc_transaction_core::tx::TxOut, JsonRPCError> {
+                            service.get_txo_object(&txo.id).map_err(format_error)
+                        },
+                    )
+                    .collect::<Result<Vec<_>, JsonRPCError>>();
+
+            let unverified_tx_results = unverified_tx_results?;
+
+            let unverified_txos_encoded: Vec<String> = unverified_tx_results
                 .iter()
-                .map(|(txo, _)| hex::encode(&txo.txo))
+                .map(|txo_obj| hex::encode(&mc_util_serial::encode(txo_obj)))
                 .collect();
 
             JsonCommandResponse::create_view_only_account_sync_request {

full-service/src/service/ledger.rs

@@ -17,7 +17,7 @@ use mc_connection::{
 };
 use mc_crypto_keys::CompressedRistrettoPublic;
 use mc_fog_report_validation::FogPubkeyResolver;
-use mc_ledger_db::Ledger;
+use mc_ledger_db::{Ledger, LedgerDB};
 use mc_ledger_sync::NetworkState;
 use mc_transaction_core::{
     ring_signature::KeyImage,
@@ -176,8 +176,10 @@ where
     fn get_txo_object(&self, txo_id_hex: &str) -> Result<TxOut, LedgerServiceError> {
         let conn = self.get_conn()?;
         let txo_details = Txo::get(txo_id_hex, &conn)?;
-
-        let txo: TxOut = mc_util_serial::decode(&txo_details.txo)?;
+        let txo = self.ledger_db.get_tx_out_by_index(
+            self.ledger_db
+                .get_tx_out_index_by_public_key(&txo_details.public_key()?)?,
+        )?;
         Ok(txo)
     }
 
@@ -295,3 +297,12 @@ where
         Ok(self.ledger_db.get_block_index_by_tx_out_index(index)?)
     }
 }
+
+pub fn get_tx_out_by_public_key(
+    ledger_db: &LedgerDB,
+    public_key: &CompressedRistrettoPublic,
+) -> Result<TxOut, LedgerServiceError> {
+    let txo_index = ledger_db.get_tx_out_index_by_public_key(public_key)?;
+    let txo = ledger_db.get_tx_out_by_index(txo_index)?;
+    Ok(txo)
+}

full-service/src/service/receipt.rs

@@ -291,6 +291,7 @@ mod tests {
             account::AccountService,
             address::AddressService,
             confirmation_number::ConfirmationService,
+            ledger::get_tx_out_by_public_key,
             transaction::{TransactionMemo, TransactionService},
             transaction_log::TransactionLogService,
             txo::TxoService,
@@ -477,8 +478,12 @@ mod tests {
             .expect("Could not decode pubkey");
         assert_eq!(receipt.public_key, txo_pubkey);
         assert_eq!(receipt.tombstone_block, 23); // Ledger seeded with 12 blocks at tx construction, then one appended + 10
-        let txo: TxOut =
-            mc_util_serial::decode(&txos_and_statuses[0].0.txo).expect("Could not decode txo");
+        let public_key = txos_and_statuses[0]
+            .0
+            .public_key()
+            .expect("Could not get CompressedRistrettoPublic from txo");
+        let txo: TxOut = get_tx_out_by_public_key(&ledger_db, &public_key)
+            .expect("Could not get the txo from the ledger.");
         assert_eq!(&receipt.amount, txo.get_masked_amount().unwrap());
         assert_eq!(receipt.confirmation, confirmations[0].confirmation);
     }

full-service/src/service/transaction.rs

@@ -538,8 +538,8 @@ mod tests {
             transaction_log::TransactionLogService,
         },
         test_utils::{
-            add_block_from_transaction_log, add_block_to_ledger_db, get_test_ledger,
-            manually_sync_account, setup_wallet_service, MOB,
+            add_block_to_ledger_db, add_block_with_tx_outs, get_test_ledger, manually_sync_account,
+            setup_wallet_service, MOB,
         },
         util::b58::b58_encode_public_address,
     };
@@ -770,7 +770,7 @@ mod tests {
             .unwrap();
 
         // Send a transaction from Alice to Bob
-        let (transaction_log, _associated_txos, _value_map, _tx_proposal) = service
+        let (transaction_log, _associated_txos, _value_map, tx_proposal) = service
             .build_sign_and_submit_transaction(
                 &alice.id,
                 &[(
@@ -793,8 +793,22 @@ mod tests {
         // workaround.
         {
             log::info!(logger, "Adding block from transaction log");
-            let conn = service.get_conn().unwrap();
-            add_block_from_transaction_log(&mut ledger_db, &conn, &transaction_log, &mut rng);
+            let key_images: Vec<KeyImage> = tx_proposal
+                .input_txos
+                .iter()
+                .map(|txo| txo.key_image.clone())
+                .collect();
+
+            // Note: This block doesn't contain the fee output.
+            add_block_with_tx_outs(
+                &mut ledger_db,
+                &[
+                    tx_proposal.change_txos[0].tx_out.clone(),
+                    tx_proposal.payload_txos[0].tx_out.clone(),
+                ],
+                &key_images,
+                &mut rng,
+            );
         }
 
         manually_sync_account(
@@ -853,7 +867,7 @@ mod tests {
         assert_eq!(bob_balance_pmob.unspent, 42000000000000);
 
         // Bob should now be able to send to Alice
-        let (transaction_log, _associated_txos, _value_map, _tx_proposal) = service
+        let (_, _, _, tx_proposal) = service
             .build_sign_and_submit_transaction(
                 &bob.id,
                 &[(
@@ -875,9 +889,23 @@ mod tests {
         // workaround.
 
         {
-            log::info!(logger, "Adding block from transaction log");
-            let conn = service.get_conn().unwrap();
-            add_block_from_transaction_log(&mut ledger_db, &conn, &transaction_log, &mut rng);
+            log::info!(logger, "Adding block from transaction proposal");
+            let key_images: Vec<KeyImage> = tx_proposal
+                .input_txos
+                .iter()
+                .map(|txo| txo.key_image.clone())
+                .collect();
+
+            // Note: This block doesn't contain the fee output.
+            add_block_with_tx_outs(
+                &mut ledger_db,
+                &[
+                    tx_proposal.change_txos[0].tx_out.clone(),
+                    tx_proposal.payload_txos[0].tx_out.clone(),
+                ],
+                &key_images,
+                &mut rng,
+            );
         }
 
         manually_sync_account(