Merge #2267 #2297 #2302

2267: Add CLI option for transaction TTL r=rvl a=rvl

### Issue Number

ADP-93 / #1840

### Overview

- [x] Hide shelley-specific CLI options in `cardano-wallet-jormungandr` (fixes #2169)
- [x] Add option `cardano-wallet transaction create [--ttl=SECONDS]`
- [ ] Update wiki page after merging.

### Comments

- Based on PR #2262 branch - merge that first.


2297: Enable parallel integration tests in hydra r=Anviking a=Anviking

# Issue Number

ADP-466

# Overview

<!-- Detail in a few bullet points the work accomplished in this PR -->

- [x] Pass in `-j 8` 
- [x] Only run CLI tests in parallel if `CI` env var isn't set
- [x] Clean up some debug printing in tests

# Comments


<!-- Additional comments or screenshots to attach if any -->

<!-- 
Don't forget to:

 ✓ Self-review your changes to make sure nothing unexpected slipped through
 ✓ Assign yourself to the PR
 ✓ Assign one or several reviewer(s)
 ✓ Once created, link this PR to its corresponding ticket
 ✓ Assign the PR to a corresponding milestone
 ✓ Acknowledge any changes required to the Wiki
-->


2302: Fix DB migration tests r=rvl a=piotr-iohk

# Issue Number

<!-- Put here a reference to the issue this PR relates to and which requirements it tackles -->


# Overview

- 098f6e5
  Fix DB migration tests



# Comments

post release


Co-authored-by: Rodney Lorrimar <rodney.lorrimar@iohk.io>
Co-authored-by: Johannes Lund <johannes.lund@iohk.io>
Co-authored-by: Piotr Stachyra <piotr.stachyra@iohk.io>

lib/cli/cardano-wallet-cli.cabal

@@ -47,6 +47,7 @@ library
     , servant-client-core
     , text
     , text-class
+    , time
     , optparse-applicative
   hs-source-dirs:
       src

lib/cli/src/Cardano/CLI.hs

@@ -35,6 +35,7 @@ module Cardano.CLI
     , cmdWalletCreate
     , cmdByronWalletCreate
     , cmdTransaction
+    , cmdTransactionJormungandr
     , cmdAddress
     , cmdStakePool
     , cmdNetwork
@@ -155,6 +156,8 @@ import Cardano.Wallet.Api.Types
     )
 import Cardano.Wallet.Network
     ( ErrNetworkUnavailable (..) )
+import Cardano.Wallet.Orphans
+    ()
 import Cardano.Wallet.Primitive.AddressDerivation
     ( Depth (..)
     , DerivationType (..)
@@ -206,6 +209,8 @@ import Data.Text.Class
     ( FromText (..), TextDecodingError (..), ToText (..), showT )
 import Data.Text.Read
     ( decimal )
+import Data.Time.Clock
+    ( NominalDiffTime )
 import Data.Void
     ( Void )
 import Fmt
@@ -682,19 +687,38 @@ cmdWalletGetUtxoStatistics mkClient =
                             Commands - 'transaction'
 -------------------------------------------------------------------------------}
 
+data TransactionFeatures = NoShelleyFeatures | ShelleyFeatures
+    deriving (Show, Eq)
+
 -- | cardano-wallet transaction
 cmdTransaction
     :: ToJSON wallet
     => TransactionClient
     -> WalletClient wallet
     -> Mod CommandFields (IO ())
-cmdTransaction mkTxClient mkWalletClient =
+cmdTransaction = cmdTransactionBase ShelleyFeatures
+
+-- | cardano-wallet-jormungandr transaction
+cmdTransactionJormungandr
+    :: ToJSON wallet
+    => TransactionClient
+    -> WalletClient wallet
+    -> Mod CommandFields (IO ())
+cmdTransactionJormungandr = cmdTransactionBase NoShelleyFeatures
+
+cmdTransactionBase
+    :: ToJSON wallet
+    => TransactionFeatures
+    -> TransactionClient
+    -> WalletClient wallet
+    -> Mod CommandFields (IO ())
+cmdTransactionBase isShelley mkTxClient mkWalletClient =
     command "transaction" $ info (helper <*> cmds) $ mempty
         <> progDesc "About transactions"
   where
     cmds = subparser $ mempty
-        <> cmdTransactionCreate mkTxClient mkWalletClient
-        <> cmdTransactionFees mkTxClient mkWalletClient
+        <> cmdTransactionCreate isShelley mkTxClient mkWalletClient
+        <> cmdTransactionFees isShelley mkTxClient mkWalletClient
         <> cmdTransactionList mkTxClient
         <> cmdTransactionSubmit mkTxClient
         <> cmdTransactionForget mkTxClient
@@ -706,23 +730,31 @@ data TransactionCreateArgs t = TransactionCreateArgs
     , _id :: WalletId
     , _payments :: NonEmpty Text
     , _metadata :: ApiTxMetadata
+    , _timeToLive :: Maybe NominalDiffTime
     }
 
+whenShelley :: a -> Parser a -> TransactionFeatures -> Parser a
+whenShelley j s = \case
+    NoShelleyFeatures -> pure j
+    ShelleyFeatures -> s
+
 cmdTransactionCreate
     :: ToJSON wallet
-    => TransactionClient
+    => TransactionFeatures
+    -> TransactionClient
     -> WalletClient wallet
     -> Mod CommandFields (IO ())
-cmdTransactionCreate mkTxClient mkWalletClient =
+cmdTransactionCreate isShelley mkTxClient mkWalletClient =
     command "create" $ info (helper <*> cmd) $ mempty
         <> progDesc "Create and submit a new transaction."
   where
     cmd = fmap exec $ TransactionCreateArgs
         <$> portOption
         <*> walletIdArgument
         <*> fmap NE.fromList (some paymentOption)
-        <*> metadataOption
-    exec (TransactionCreateArgs wPort wId wAddressAmounts md) = do
+        <*> whenShelley (ApiTxMetadata Nothing) metadataOption isShelley
+        <*> whenShelley Nothing timeToLiveOption isShelley
+    exec (TransactionCreateArgs wPort wId wAddressAmounts md ttl) = do
         wPayments <- either (fail . getTextDecodingError) pure $
             traverse (fromText @(AddressAmount Text)) wAddressAmounts
         res <- sendRequest wPort $ getWallet mkWalletClient $ ApiT wId
@@ -736,26 +768,29 @@ cmdTransactionCreate mkTxClient mkWalletClient =
                         [ "payments" .= wPayments
                         , "passphrase" .= ApiT wPwd
                         , "metadata" .= md
+                        , "time_to_live" .= ttl
                         ]
                     )
             Left _ ->
                 handleResponse Aeson.encodePretty res
 
 cmdTransactionFees
     :: ToJSON wallet
-    => TransactionClient
+    => TransactionFeatures
+    -> TransactionClient
     -> WalletClient wallet
     -> Mod CommandFields (IO ())
-cmdTransactionFees mkTxClient mkWalletClient =
+cmdTransactionFees isShelley mkTxClient mkWalletClient =
     command "fees" $ info (helper <*> cmd) $ mempty
         <> progDesc "Estimate fees for a transaction."
   where
     cmd = fmap exec $ TransactionCreateArgs
         <$> portOption
         <*> walletIdArgument
         <*> fmap NE.fromList (some paymentOption)
-        <*> metadataOption
-    exec (TransactionCreateArgs wPort wId wAddressAmounts md) = do
+        <*> whenShelley (ApiTxMetadata Nothing) metadataOption isShelley
+        <*> whenShelley Nothing timeToLiveOption isShelley
+    exec (TransactionCreateArgs wPort wId wAddressAmounts md ttl) = do
         wPayments <- either (fail . getTextDecodingError) pure $
             traverse (fromText @(AddressAmount Text)) wAddressAmounts
         res <- sendRequest wPort $ getWallet mkWalletClient $ ApiT wId
@@ -767,6 +802,7 @@ cmdTransactionFees mkTxClient mkWalletClient =
                     (Aeson.object
                         [ "payments" .= wPayments
                         , "metadata" .= md
+                        , "time_to_live" .= ttl
                         ])
             Left _ ->
                 handleResponse Aeson.encodePretty res
@@ -1338,7 +1374,7 @@ walletIdArgument :: Parser WalletId
 walletIdArgument = argumentT $ mempty
     <> metavar "WALLET_ID"
 
--- | <stake=STAKE>
+-- | [--stake=STAKE]
 stakeOption :: Parser (Maybe Coin)
 stakeOption = optional $ optionT $ mempty
     <> long "stake"
@@ -1368,7 +1404,7 @@ transactionSubmitPayloadArgument = argumentT $ mempty
     <> metavar "BINARY_BLOB"
     <> help "hex-encoded binary blob of externally-signed transaction."
 
--- | <metadata=JSON>
+-- | [--metadata=JSON]
 --
 -- Note: we decode the JSON just so that we can validate more client-side.
 metadataOption :: Parser ApiTxMetadata
@@ -1383,6 +1419,15 @@ metadataOption = option txMetadataReader $ mempty
 txMetadataReader :: ReadM ApiTxMetadata
 txMetadataReader = eitherReader (Aeson.eitherDecode' . BL8.pack)
 
+-- | [--ttl=DURATION]
+timeToLiveOption :: Parser (Maybe NominalDiffTime)
+timeToLiveOption = optional $ optionT $ mempty
+    <> long "ttl"
+    <> metavar "DURATION"
+    <> help ("Time-to-live value. "
+             <> "Expressed in seconds with a trailing 's'. "
+             <> "Default is 3600s (2 hours).")
+
 -- | <address=ADDRESS>
 addressIdArgument :: Parser Text
 addressIdArgument = argumentT $ mempty

lib/cli/test/unit/Cardano/CLISpec.hs

@@ -267,6 +267,7 @@ spec = do
         ["transaction", "create", "--help"] `shouldShowUsage`
             [ "Usage:  transaction create [--port INT] WALLET_ID"
             , "                           --payment PAYMENT [--metadata JSON]"
+            , "                           [--ttl DURATION]"
             , "  Create and submit a new transaction."
             , ""
             , "Available options:"
@@ -280,11 +281,14 @@ spec = do
             , "                           metadata as a JSON object. The value"
             , "                           must match the schema defined in the"
             , "                           cardano-wallet OpenAPI specification."
+            , "  --ttl DURATION           Time-to-live value. Expressed in"
+            , "                           seconds with a trailing 's'. Default"
+            , "                           is 3600s (2 hours)."
             ]
 
         ["transaction", "fees", "--help"] `shouldShowUsage`
             [ "Usage:  transaction fees [--port INT] WALLET_ID --payment PAYMENT"
-            , "                         [--metadata JSON]"
+            , "                         [--metadata JSON] [--ttl DURATION]"
             , "  Estimate fees for a transaction."
             , ""
             , "Available options:"
@@ -298,6 +302,9 @@ spec = do
             , "                           metadata as a JSON object. The value"
             , "                           must match the schema defined in the"
             , "                           cardano-wallet OpenAPI specification."
+            , "  --ttl DURATION           Time-to-live value. Expressed in"
+            , "                           seconds with a trailing 's'. Default"
+            , "                           is 3600s (2 hours)."
             ]
 
         ["transaction", "list", "--help"] `shouldShowUsage`

lib/core-integration/src/Test/Integration/Scenario/API/Shelley/StakePools.hs

@@ -974,7 +974,6 @@ spec = describe "SHELLEY_STAKE_POOLS" $ do
                 Right pools@[pool1,_pool2,pool3] <-
                     snd <$> listPools ctx arbitraryStake
                 let rewards = view (#metrics . #nonMyopicMemberRewards)
-                print (rewards <$> pools) -- FIXME temporary
                 (rewards <$> pools) `shouldBe`
                     (rewards <$> sortOn (Down . rewards) pools)
                 -- Make sure the rewards are not all equal:

lib/core-integration/src/Test/Integration/Scenario/API/Shelley/Transactions.hs

@@ -901,12 +901,13 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do
                 | i <- [0..127] ]
             bytes = [json|{ "bytes": #{T.replicate 64 "a"} }|]
         let payload = addTxMetadata txMeta basePayload
-        liftIO $ print payload
         r <- request @ApiFee ctx
             (Link.getTransactionFee @'Shelley wa) Default payload
 
-        expectResponseCode HTTP.status403 r
-        expectErrorMessage errMsg403TxTooLarge r
+        verify r
+            [ expectResponseCode HTTP.status403
+            , expectErrorMessage errMsg403TxTooLarge
+            ]
 
     it "TRANS_EXTERNAL_01 - Single Output Transaction - Shelley witnesses" $ \ctx -> runResourceT $ do
         wFaucet <- fixtureWallet ctx
@@ -2358,7 +2359,6 @@ spec = describe "SHELLEY_TRANSACTIONS" $ do
         eventually "rewards are transferred from self to self" $ do
             rW <- request @ApiWallet ctx
                 (Link.getWallet @'Shelley wSrc) Default payload
-            print rW
             verify rW
                 [ expectField (#balance . #getApiT . #available)
                     (.> (wSrc ^. #balance . #getApiT . #available))

lib/core/src/Cardano/Wallet/Primitive/SyncProgress.hs

@@ -26,6 +26,8 @@ import Cardano.Wallet.Primitive.Types
     ( BlockHeader (..), SlotNo (..) )
 import Control.DeepSeq
     ( NFData (..) )
+import Data.Bifunctor
+    ( bimap )
 import Data.Quantity
     ( Percentage (..), Quantity (..), mkPercentage )
 import Data.Ratio
@@ -40,10 +42,6 @@ import GHC.Generics
     ( Generic )
 import GHC.Stack
     ( HasCallStack )
-import Safe
-    ( readMay )
-
-import qualified Data.Text as T
 
 data SyncProgress
     = Ready
@@ -82,17 +80,10 @@ mkSyncTolerance =
     pico = 1_000_000_000_000
 
 instance ToText SyncTolerance where
-    toText (SyncTolerance t) = T.pack (show t)
+    toText (SyncTolerance t) = toText t
 
 instance FromText SyncTolerance where
-    fromText t = case T.splitOn "s" t of
-        [v,""] ->
-            maybe
-                (Left errSyncTolerance)
-                (Right . mkSyncTolerance)
-                (readMay $ T.unpack v)
-        _ ->
-            Left errSyncTolerance
+    fromText = bimap (const errSyncTolerance) SyncTolerance . fromText
       where
         errSyncTolerance = TextDecodingError $ unwords
             [ "Cannot parse given time duration. Here are a few examples of"

lib/jormungandr/exe/cardano-wallet-jormungandr.hs

@@ -37,7 +37,7 @@ import Cardano.CLI
     , cmdMnemonic
     , cmdNetwork
     , cmdStakePool
-    , cmdTransaction
+    , cmdTransactionJormungandr
     , cmdVersion
     , cmdWallet
     , cmdWalletCreate
@@ -180,7 +180,7 @@ main = withUtf8Encoding $ do
         <> cmdServe
         <> cmdMnemonic
         <> cmdWallet cmdWalletCreate walletClient
-        <> cmdTransaction transactionClient walletClient
+        <> cmdTransactionJormungandr transactionClient walletClient
         <> cmdAddress addressClient
         <> cmdStakePool @ApiStakePool stakePoolClient
         <> cmdNetwork networkClient

lib/jormungandr/test/integration/Test/Integration/Jormungandr/Scenario/CLI/Server.hs

@@ -50,6 +50,7 @@ import Test.Integration.Framework.DSL
     ( Context (..)
     , KnownCommand (..)
     , cardanoWalletCLI
+    , counterexample
     , expectPathEventuallyExist
     , proc'
     )
@@ -168,8 +169,8 @@ spec = do
                     threadDelay (10 * oneSecond)
                 hClose hLogs
                 logged <- T.unpack <$> TIO.readFile logs
-                putStrLn logged
-                logged `shouldContain` "Logging shutdown"
+                counterexample logged $
+                    logged `shouldContain` "Logging shutdown"
 
         describe "LOGGING - Exits nicely on wrong genesis hash" $  do
             let hashes =

lib/shelley/test/integration/Main.hs

@@ -106,6 +106,8 @@ import Test.Integration.Framework.Context
     ( Context (..), PoolGarbageCollectionEvent (..) )
 import Test.Integration.Framework.DSL
     ( KnownCommand (..) )
+import Test.Utils.Paths
+    ( inNixBuild )
 
 import qualified Cardano.Pool.DB as Pool
 import qualified Cardano.Pool.DB.Sqlite as Pool
@@ -141,9 +143,11 @@ instance KnownCommand Shelley where
 main :: forall t n . (t ~ Shelley, n ~ 'Mainnet) => IO ()
 main = withUtf8Encoding $ withTracers $ \tracers -> do
     hSetBuffering stdout LineBuffering
+    nix <- inNixBuild
     hspec $ do
-        describe "No backend required" $ do
-            describe "Miscellaneous CLI tests" $ parallel (MiscellaneousCLI.spec @t)
+        describe "No backend required" $
+            parallelIf (not nix) $ describe "Miscellaneous CLI tests" $
+                MiscellaneousCLI.spec @t
         specWithServer tracers $ do
             parallel $ describe "API Specifications" $ do
                 Addresses.spec @n
@@ -162,13 +166,22 @@ main = withUtf8Encoding $ withTracers $ \tracers -> do
                 ByronTransactions.spec @n
                 ByronHWWallets.spec @n
                 Settings.spec @n
-            parallel $ describe "CLI Specifications" $ do
+
+            -- Hydra runs tests with code coverage enabled. CLI tests run
+            -- multiple processes. These processes can try to write to the
+            -- same .tix file simultaneously, causing errors.
+            --
+            -- Because of this, don't run the CLI tests in parallel in hydra.
+            parallelIf (not nix) $ describe "CLI Specifications" $ do
                 AddressesCLI.spec @n
                 TransactionsCLI.spec @n
                 WalletsCLI.spec @n
                 HWWalletsCLI.spec @n
                 PortCLI.spec @t
                 NetworkCLI.spec @t
+  where
+    parallelIf :: forall a. Bool -> SpecWith a -> SpecWith a
+    parallelIf flag = if flag then parallel else id
 
 specWithServer
     :: (Tracer IO TestsLog, Tracers IO)