Skip to content

Commit 97550d2

Browse files
m-Peterm-Peter
committed
Add a config flag for enabling the signing keys release frequency improvement
1 parent 87ecd69 commit 97550d2

File tree

8 files changed

+100
-49
lines changed

8 files changed

+100
-49
lines changed

‎bootstrap/bootstrap.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
243243
}
244244
}
245245

246-
b.keystore = keystore.New(ctx, accountKeys, b.client, b.logger)
246+
b.keystore = keystore.New(ctx, accountKeys, b.client, b.config, b.logger)
247247

248248
// create transaction pool
249249
var txPool requester.TxPool

‎cmd/run/cmd.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,7 @@ func init() {
282282
Cmd.Flags().StringVar(&cloudKMSLocationID, "coa-cloud-kms-location-id", "", "The location ID where the key ring is grouped into, e.g. 'global'")
283283
Cmd.Flags().StringVar(&cloudKMSKeyRingID, "coa-cloud-kms-key-ring-id", "", "The key ring ID where the KMS keys exist, e.g. 'tx-signing'")
284284
Cmd.Flags().StringVar(&cloudKMSKey, "coa-cloud-kms-key", "", `Name of the KMS key and its version, e.g. "gw-key-6@1"`)
285+
Cmd.Flags().BoolVar(&cfg.COATxLookupEnabled, "coa-tx-lookup-enabled", false, "Release COA signing keys by looking up their tx result status. Increases capacity of the available COA signing keys.")
285286
Cmd.Flags().StringVar(&walletKey, "wallet-api-key", "", "ECDSA private key used for wallet APIs. WARNING: This should only be used locally or for testing, never in production.")
286287
Cmd.Flags().IntVar(&cfg.MetricsPort, "metrics-port", 9091, "Port for the metrics server")
287288
Cmd.Flags().BoolVar(&cfg.IndexOnly, "index-only", false, "Run the gateway in index-only mode which only allows querying the state and indexing, but disallows sending transactions.")

‎config/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ type Config struct {
5656
COAKey crypto.PrivateKey
5757
// COACloudKMSKey is a Cloud KMS key that will be used for signing transactions.
5858
COACloudKMSKey *flowGoKMS.Key
59+
// COATxLookupEnabled sets whether COA signing keys are released by looking up
60+
// their transaction result status from ANs. Increases capacity of the available
61+
// COA signing keys.
62+
COATxLookupEnabled bool
5963
// GasPrice is a fixed gas price that will be used when submitting transactions.
6064
GasPrice *big.Int
6165
// EnforceGasPrice defines whether the minimum gas price should be enforced.

‎services/ingestion/event_subscriber.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,12 @@ func (r *RPCEventSubscriber) subscribe(ctx context.Context, height uint64) <-cha
176176
for _, evt := range blockEvents.Events {
177177
r.keyLock.NotifyTransaction(evt.TransactionID)
178178
}
179-
r.keyLock.NotifyBlock(blockEvents.BlockID)
179+
r.keyLock.NotifyBlock(
180+
flow.BlockHeader{
181+
ID: blockEvents.BlockID,
182+
Height: blockEvents.Height,
183+
},
184+
)
180185

181186
eventsChan <- evmEvents
182187

‎services/ingestion/event_subscriber_test.go‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/onflow/flow-go-sdk/access"
1111
gethCommon "github.com/onflow/go-ethereum/common"
1212

13+
"github.com/onflow/flow-evm-gateway/config"
1314
"github.com/onflow/flow-evm-gateway/models"
1415
errs "github.com/onflow/flow-evm-gateway/models/errors"
1516
"github.com/onflow/flow-evm-gateway/services/requester"
@@ -46,7 +47,8 @@ func Test_Subscribing(t *testing.T) {
4647

4748
ctx := context.Background()
4849
logger := zerolog.Nop()
49-
ks := keystore.New(ctx, nil, client, logger)
50+
cfg := config.Config{COATxLookupEnabled: true}
51+
ks := keystore.New(ctx, nil, client, cfg, logger)
5052

5153
subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
5254

@@ -90,7 +92,8 @@ func Test_MissingBlockEvent(t *testing.T) {
9092

9193
ctx := context.Background()
9294
logger := zerolog.Nop()
93-
ks := keystore.New(ctx, nil, client, logger)
95+
cfg := config.Config{COATxLookupEnabled: true}
96+
ks := keystore.New(ctx, nil, client, cfg, logger)
9497

9598
subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
9699

@@ -196,7 +199,8 @@ func Test_SubscribingWithRetryOnError(t *testing.T) {
196199

197200
ctx := context.Background()
198201
logger := zerolog.Nop()
199-
ks := keystore.New(ctx, nil, client, logger)
202+
cfg := config.Config{COATxLookupEnabled: true}
203+
ks := keystore.New(ctx, nil, client, cfg, logger)
200204

201205
subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
202206

@@ -263,7 +267,8 @@ func Test_SubscribingWithRetryOnErrorMultipleBlocks(t *testing.T) {
263267

264268
ctx := context.Background()
265269
logger := zerolog.Nop()
266-
ks := keystore.New(ctx, nil, client, logger)
270+
cfg := config.Config{COATxLookupEnabled: true}
271+
ks := keystore.New(ctx, nil, client, cfg, logger)
267272

268273
subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
269274

@@ -329,7 +334,8 @@ func Test_SubscribingWithRetryOnErrorEmptyBlocks(t *testing.T) {
329334

330335
ctx := context.Background()
331336
logger := zerolog.Nop()
332-
ks := keystore.New(ctx, nil, client, logger)
337+
cfg := config.Config{COATxLookupEnabled: true}
338+
ks := keystore.New(ctx, nil, client, cfg, logger)
333339

334340
subscriber := NewRPCEventSubscriber(logger, client, flowGo.Previewnet, ks, 1)
335341

‎services/requester/keystore/key_store.go‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"sync"
77

8+
"github.com/onflow/flow-evm-gateway/config"
89
flowsdk "github.com/onflow/flow-go-sdk"
910
"github.com/onflow/flow-go-sdk/access"
1011
"github.com/onflow/flow-go/model/flow"
@@ -17,16 +18,17 @@ const accountKeyBlockExpiration = flow.DefaultTransactionExpiry
1718

1819
type KeyLock interface {
1920
NotifyTransaction(txID flowsdk.Identifier)
20-
NotifyBlock(blockID flowsdk.Identifier)
21+
NotifyBlock(blockHeader flowsdk.BlockHeader)
2122
}
2223

2324
type KeyStore struct {
2425
client access.Client
26+
config config.Config
2527
availableKeys chan *AccountKey
2628
usedKeys map[flowsdk.Identifier]*AccountKey
2729
size int
2830
keyMu sync.Mutex
29-
blockChan chan flowsdk.Identifier
31+
blockChan chan flowsdk.BlockHeader
3032
logger zerolog.Logger
3133

3234
// Signal channel used to prevent blocking writes
@@ -40,19 +42,21 @@ func New(
4042
ctx context.Context,
4143
keys []*AccountKey,
4244
client access.Client,
45+
config config.Config,
4346
logger zerolog.Logger,
4447
) *KeyStore {
4548
totalKeys := len(keys)
4649

4750
ks := &KeyStore{
4851
client: client,
52+
config: config,
4953
availableKeys: make(chan *AccountKey, totalKeys),
5054
usedKeys: map[flowsdk.Identifier]*AccountKey{},
5155
size: totalKeys,
5256
// `KeyStore.NotifyBlock` is called for each new Flow block,
53-
// so we use a buffered channel to write the new block IDs
57+
// so we use a buffered channel to write the new block headers
5458
// to the `blockChan`, and read them through `processLockedKeys`.
55-
blockChan: make(chan flowsdk.Identifier, 100),
59+
blockChan: make(chan flowsdk.BlockHeader, 100),
5660
logger: logger,
5761
done: make(chan struct{}),
5862
}
@@ -106,15 +110,15 @@ func (k *KeyStore) NotifyTransaction(txID flowsdk.Identifier) {
106110

107111
// NotifyBlock is called to notify the KeyStore of a newly ingested block.
108112
// Pending transactions older than a threshold number of blocks are removed.
109-
func (k *KeyStore) NotifyBlock(blockID flowsdk.Identifier) {
113+
func (k *KeyStore) NotifyBlock(blockHeader flowsdk.BlockHeader) {
110114
select {
111115
case <-k.done:
112116
k.logger.Warn().Msg(
113117
"received `NotifyBlock` while the server is shutting down",
114118
)
115119
return
116120
default:
117-
k.blockChan <- blockID
121+
k.blockChan <- blockHeader
118122
}
119123
}
120124

@@ -152,7 +156,7 @@ func (k *KeyStore) processLockedKeys(ctx context.Context) {
152156
case <-ctx.Done():
153157
close(k.done)
154158
return
155-
case blockID := <-k.blockChan:
159+
case blockHeader := <-k.blockChan:
156160
// TODO: If calling `k.client.GetTransactionResultsByBlockID` for each
157161
// new block, seems to be problematic for ANs, we can take an approach
158162
// like the one below:
@@ -173,37 +177,37 @@ func (k *KeyStore) processLockedKeys(ctx context.Context) {
173177
continue
174178
}
175179

176-
txResults, err := k.client.GetTransactionResultsByBlockID(ctx, blockID)
177-
if err != nil {
178-
k.logger.Error().Err(err).Msgf(
179-
"failed to get transaction results for block ID: %s",
180-
blockID.Hex(),
181-
)
182-
continue
180+
txResults := []*flowsdk.TransactionResult{}
181+
var err error
182+
if k.config.COATxLookupEnabled {
183+
txResults, err = k.client.GetTransactionResultsByBlockID(ctx, blockHeader.ID)
184+
if err != nil {
185+
k.logger.Error().Err(err).Msgf(
186+
"failed to get transaction results for block ID: %s",
187+
blockHeader.ID.Hex(),
188+
)
189+
continue
190+
}
183191
}
184192

185-
k.releasekeys(txResults)
193+
k.releasekeys(blockHeader.Height, txResults)
186194
}
187195
}
188196
}
189197

190-
// releasekeys accepts a slice of `TransactionResult` objects and
191-
// releases the account keys used for signing the given transactions.
198+
// releasekeys accepts a block height and a slice of `TransactionResult`
199+
// objects and releases the account keys used for signing the given
200+
// transactions.
192201
// It also releases the account keys which were last locked more than
193202
// or equal to `accountKeyBlockExpiration` blocks in the past.
194-
func (k *KeyStore) releasekeys(txResults []*flowsdk.TransactionResult) {
195-
if len(txResults) == 0 {
196-
return
197-
}
198-
203+
func (k *KeyStore) releasekeys(blockHeight uint64, txResults []*flowsdk.TransactionResult) {
199204
k.keyMu.Lock()
200205
defer k.keyMu.Unlock()
201206

202207
for _, txResult := range txResults {
203208
k.unsafeUnlockKey(txResult.TransactionID)
204209
}
205210

206-
blockHeight := txResults[0].BlockHeight
207211
for txID, key := range k.usedKeys {
208212
if blockHeight-key.lastLockedBlock.Load() >= accountKeyBlockExpiration {
209213
k.unsafeUnlockKey(txID)

‎services/requester/keystore/key_store_test.go‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"testing"
88
"time"
99

10+
"github.com/onflow/flow-evm-gateway/config"
1011
"github.com/onflow/flow-evm-gateway/services/testutils"
1112
sdk "github.com/onflow/flow-go-sdk"
1213
"github.com/onflow/flow-go-sdk/access/mocks"
@@ -29,7 +30,13 @@ func TestTake(t *testing.T) {
2930
keys = append(keys, NewAccountKey(*accountKey, addrGenerator.New(), signer))
3031
}
3132

32-
ks := New(context.Background(), keys, testutils.SetupClientForRange(1, 100), zerolog.Nop())
33+
ks := New(
34+
context.Background(),
35+
keys,
36+
testutils.SetupClientForRange(1, 100),
37+
config.Config{COATxLookupEnabled: true},
38+
zerolog.Nop(),
39+
)
3340

3441
t.Run("Take with no metadata updates", func(t *testing.T) {
3542
key, err := ks.Take()
@@ -110,7 +117,13 @@ func TestTake(t *testing.T) {
110117
}, nil
111118
},
112119
}
113-
ks := New(context.Background(), keys, client, zerolog.Nop())
120+
ks := New(
121+
context.Background(),
122+
keys,
123+
client,
124+
config.Config{COATxLookupEnabled: true},
125+
zerolog.Nop(),
126+
)
114127

115128
key, err := ks.Take()
116129
require.NoError(t, err)
@@ -126,7 +139,12 @@ func TestTake(t *testing.T) {
126139
assert.Equal(t, key, ks.usedKeys[txID])
127140

128141
// notify for one block before the expiration block, key should still be reserved
129-
ks.NotifyBlock(blockIDNonExpired)
142+
ks.NotifyBlock(
143+
sdk.BlockHeader{
144+
ID: blockIDNonExpired,
145+
Height: blockHeight + accountKeyBlockExpiration - 1,
146+
},
147+
)
130148

131149
// Give some time to allow the KeyStore to check for the
132150
// transaction result statuses in the background.
@@ -136,7 +154,12 @@ func TestTake(t *testing.T) {
136154
assert.Equal(t, key, ks.usedKeys[txID])
137155

138156
// notify for the expiration block
139-
ks.NotifyBlock(identifierFixture())
157+
ks.NotifyBlock(
158+
sdk.BlockHeader{
159+
ID: identifierFixture(),
160+
Height: blockHeight + accountKeyBlockExpiration,
161+
},
162+
)
140163

141164
// Give some time to allow the KeyStore to check for the
142165
// transaction result statuses in the background.
@@ -164,6 +187,7 @@ func TestKeySigning(t *testing.T) {
164187
NewAccountKey(*accountKey, address, signer),
165188
},
166189
testutils.SetupClientForRange(1, 100),
190+
config.Config{COATxLookupEnabled: true},
167191
zerolog.Nop(),
168192
)
169193

@@ -211,7 +235,13 @@ func TestConcurrentUse(t *testing.T) {
211235
keys = append(keys, key)
212236
}
213237

214-
ks := New(context.Background(), keys, testutils.SetupClientForRange(1, 100), zerolog.Nop())
238+
ks := New(
239+
context.Background(),
240+
keys,
241+
testutils.SetupClientForRange(1, 100),
242+
config.Config{COATxLookupEnabled: true},
243+
zerolog.Nop(),
244+
)
215245

216246
g := errgroup.Group{}
217247
g.SetLimit(concurrentTxCount)

‎tests/key_store_release_test.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,21 @@ func Test_KeyStoreSigningKeysRelease(t *testing.T) {
5555
require.NoError(t, err)
5656

5757
cfg := config.Config{
58-
DatabaseDir: t.TempDir(),
59-
AccessNodeHost: grpcHost,
60-
RPCPort: 8545,
61-
RPCHost: "127.0.0.1",
62-
FlowNetworkID: "flow-emulator",
63-
EVMNetworkID: types.FlowEVMPreviewNetChainID,
64-
Coinbase: eoaTestAccount,
65-
COAAddress: *createdAddr,
66-
COAKey: privateKey,
67-
GasPrice: new(big.Int).SetUint64(0),
68-
EnforceGasPrice: true,
69-
LogLevel: zerolog.DebugLevel,
70-
LogWriter: testLogWriter(),
71-
TxStateValidation: config.LocalIndexValidation,
58+
DatabaseDir: t.TempDir(),
59+
AccessNodeHost: grpcHost,
60+
RPCPort: 8545,
61+
RPCHost: "127.0.0.1",
62+
FlowNetworkID: "flow-emulator",
63+
EVMNetworkID: types.FlowEVMPreviewNetChainID,
64+
Coinbase: eoaTestAccount,
65+
COAAddress: *createdAddr,
66+
COAKey: privateKey,
67+
COATxLookupEnabled: true,
68+
GasPrice: new(big.Int).SetUint64(0),
69+
EnforceGasPrice: true,
70+
LogLevel: zerolog.DebugLevel,
71+
LogWriter: testLogWriter(),
72+
TxStateValidation: config.LocalIndexValidation,
7273
}
7374

7475
rpcTester := &rpcTest{

0 commit comments

Comments
 (0)