Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add token owners to RPC and archive db #10989

Merged
merged 4 commits into from
May 17, 2022
Merged

Add token owners to RPC and archive db #10989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8499ab4
Add token owners to RPC and archive db
psteckler May 17, 2022
513726e
fix compile errors
psteckler May 17, 2022
e289417
Merge branch 'develop' into feature/token-owners-in-db
psteckler May 17, 2022
315014d
dedup tokens before lookup
psteckler May 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion src/app/archive/archive_lib/diff.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ module Transition_frontier = struct
; accounts_accessed : (int * Mina_base.Account.Stable.Latest.t) list
; accounts_created :
(Account_id.Stable.Latest.t * Currency.Fee.Stable.Latest.t) list
; tokens_used :
(Token_id.Stable.Latest.t * Account_id.Stable.Latest.t option) list
; sender_receipt_chains_from_parent_ledger :
(Account_id.Stable.Latest.t * Receipt.Chain_hash.Stable.Latest.t)
list
Expand Down Expand Up @@ -71,8 +73,8 @@ module Builder = struct
in
let block_with_hash = Mina_block.Validated.forget validated_block in
let block = With_hash.data block_with_hash in
let account_ids_accessed = Mina_block.account_ids_accessed block in
let accounts_accessed =
let account_ids_accessed = Mina_block.account_ids_accessed block in
List.filter_map account_ids_accessed ~f:(fun acct_id ->
(* an accessed account may not be the ledger *)
let%bind.Option index =
Expand All @@ -95,10 +97,20 @@ module Builder = struct
~previous_block_state_hash ) ~f:(fun acct_id ->
(acct_id, account_creation_fee) )
in
let tokens_used =
let unique_tokens =
List.map account_ids_accessed ~f:Account_id.token_id
|> List.dedup_and_sort ~compare:Token_id.compare
in
List.map unique_tokens ~f:(fun token_id ->
let owner = Mina_ledger.Ledger.token_owner ledger token_id in
(token_id, owner) )
in
Transition_frontier.Breadcrumb_added
{ block = With_hash.map ~f:External_transition.compose block_with_hash
; accounts_accessed
; accounts_created
; tokens_used
; sender_receipt_chains_from_parent_ledger
}

Expand Down
7 changes: 5 additions & 2 deletions src/app/archive/archive_lib/extensional.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,9 @@ module Zkapp_command = struct
end

module Block = struct
(* in accounts_accessed, the int is the ledger index *)
(* in accounts_accessed, the int is the ledger index
in tokens_used, the account id is the token owner
*)
type t =
{ state_hash : State_hash.Stable.Latest.t
; parent_hash : State_hash.Stable.Latest.t
Expand All @@ -96,7 +98,8 @@ module Block = struct
; accounts_accessed : (int * Account.Stable.Latest.t) list
; accounts_created :
(Account_id.Stable.Latest.t * Currency.Fee.Stable.Latest.t) list
(* TODO: list of created token ids and owners *)
; tokens_used :
(Token_id.Stable.Latest.t * Account_id.Stable.Latest.t option) list
}
[@@deriving yojson, equal, bin_io_unversioned]
end
112 changes: 54 additions & 58 deletions src/app/archive/archive_lib/processor.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,18 @@ module Public_key = struct
public_key
end

module Token_owners = struct
(* hash table of token owners, updated for each block *)
let owner_tbl : Account_id.t Token_id.Table.t = Token_id.Table.create ()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How would this work when the archive process restarts? Should we initialize the table with the tokens from the database?

Copy link
Member Author

@psteckler psteckler May 17, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're only adding tokens to the db when there's a transaction in a block that mentions them. In that case, those tokens will be in the tokens_used list, and hence, in this table.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, tokens section in zkapps integration test should be sufficient, right? since it exercises the replayer flow now

We have to add that, right?


let add_if_doesn't_exist token_id owner =
match Token_id.Table.add owner_tbl ~key:token_id ~data:owner with
| `Ok | `Duplicate ->
()

let find_owner token_id = Token_id.Table.find owner_tbl token_id
end

module Token = struct
type t =
{ value : string
Expand Down Expand Up @@ -82,11 +94,10 @@ module Token = struct
let find_opt (module Conn : CONNECTION) =
make_finder Conn.find_opt Caqti_request.find_opt

let add_if_doesn't_exist (module Conn : CONNECTION)
~(owner : Account_id.t option) token_id =
let add_if_doesn't_exist (module Conn : CONNECTION) token_id =
let open Deferred.Result.Let_syntax in
let value = Token_id.(to_string token_id) in
match owner with
match Token_owners.find_owner token_id with
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not finding an owner doesn't mean this is default token, right? (See the comment in Token_owners module definition)

Copy link
Member Author

@psteckler psteckler May 17, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There should not be any tokens other than the default that have no owner, unless there's a bug, hence the asserts.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be safer to have the table contain options, so that a lookup should never fail.

| None ->
assert (Token_id.(equal default) token_id) ;
Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int)
Expand Down Expand Up @@ -165,16 +176,14 @@ module Account_identifiers = struct

let table_name = "account_identifiers"

let add_if_doesn't_exist (module Conn : CONNECTION) ~token_owner account_id =
let add_if_doesn't_exist (module Conn : CONNECTION) account_id =
let open Deferred.Result.Let_syntax in
let pk = Account_id.public_key account_id in
(* this token_id is Token_id.t *)
let token_id = Account_id.token_id account_id in
let%bind public_key_id = Public_key.add_if_doesn't_exist (module Conn) pk in
(* this token_id is a Postgresql table id *)
let%bind token_id =
Token.add_if_doesn't_exist (module Conn) ~owner:token_owner token_id
in
let%bind token_id = Token.add_if_doesn't_exist (module Conn) token_id in
let t = { public_key_id; token_id } in
Mina_caqti.select_insert_into_cols ~select:("id", Caqti_type.int)
~table_name ~cols:(Fields.names, typ)
Expand Down Expand Up @@ -1367,11 +1376,8 @@ module Zkapp_other_party_body = struct
=
let open Deferred.Result.Let_syntax in
let account_identifier = Account_id.create body.public_key body.token_id in
(* TODO: TEMP!!!! get real token owner *)
let%bind account_identifier_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_identifier
Account_identifiers.add_if_doesn't_exist (module Conn) account_identifier
in
let%bind update_id =
Zkapp_updates.add_if_doesn't_exist (module Conn) body.update
Expand Down Expand Up @@ -1519,9 +1525,7 @@ module Zkapp_fee_payer_body = struct
Account_id.create body.public_key Token_id.default
in
let%bind account_identifier_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_identifier
Account_identifiers.add_if_doesn't_exist (module Conn) account_identifier
in
let%bind update_id =
Zkapp_updates.add_if_doesn't_exist (module Conn) body.update
Expand Down Expand Up @@ -1677,27 +1681,21 @@ module User_command = struct
let token_id = Signed_command.fee_token t in
assert (Token_id.(equal default) token_id) ;
let acct_id = Account_id.create pk token_id in
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None acct_id
Account_identifiers.add_if_doesn't_exist (module Conn) acct_id
in
let%bind source_id =
let pk = Signed_command.source_pk t in
let token_id = Signed_command.token t in
assert (Token_id.(equal default) token_id) ;
let acct_id = Account_id.create pk token_id in
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None acct_id
Account_identifiers.add_if_doesn't_exist (module Conn) acct_id
in
let%map receiver_id =
let pk = Signed_command.receiver_pk t in
let token_id = Signed_command.token t in
assert (Token_id.(equal default) token_id) ;
let acct_id = Account_id.create pk token_id in
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None acct_id
Account_identifiers.add_if_doesn't_exist (module Conn) acct_id
in
{ fee_payer_id; source_id; receiver_id }

Expand Down Expand Up @@ -1764,21 +1762,20 @@ module User_command = struct
~f:(Fn.compose Unsigned.UInt64.to_int64 Currency.Amount.to_uint64)
in
let open Deferred.Result.Let_syntax in
(* TODO : use real token owners *)
let%bind fee_payer_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None user_cmd.fee_payer
user_cmd.fee_payer
in
let%bind source_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None user_cmd.source
user_cmd.source
in
let%bind receiver_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None user_cmd.receiver
user_cmd.receiver
in
Conn.find
(Caqti_request.find typ Caqti_type.int
Expand Down Expand Up @@ -1860,16 +1857,6 @@ module User_command = struct
{ zkapp_fee_payer_body_id; zkapp_other_parties_ids; memo; hash }
end

let as_signed_command (t : User_command.t) : Mina_base.Signed_command.t =
match t with
| Signed_command c ->
c
| Parties _ ->
let `Needs_some_work_for_zkapps_on_mainnet =
Mina_base.Util.todo_zkapps
in
failwith "TODO"

let via (t : User_command.t) : [ `Parties | `Ident ] =
match t with Signed_command _ -> `Ident | Parties _ -> `Parties

Expand Down Expand Up @@ -1935,7 +1922,7 @@ module Internal_command = struct
let%bind receiver_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None internal_cmd.receiver
internal_cmd.receiver
in
Conn.find
(Caqti_request.find typ Caqti_type.int
Expand Down Expand Up @@ -2004,9 +1991,7 @@ module Fee_transfer = struct
(Fee_transfer.Single.receiver_pk t)
Token_id.default
in
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_id
Account_identifiers.add_if_doesn't_exist (module Conn) account_id
in
Conn.find
(Caqti_request.find typ Caqti_type.int
Expand Down Expand Up @@ -2052,9 +2037,7 @@ module Coinbase = struct
let account_id =
Account_id.create (Coinbase.receiver_pk t) Token_id.default
in
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_id
Account_identifiers.add_if_doesn't_exist (module Conn) account_id
in
Conn.find
(Caqti_request.find typ Caqti_type.int
Expand Down Expand Up @@ -2303,6 +2286,14 @@ module Block_and_zkapp_command = struct
~cols:[ "block_id"; "zkapp_command_id"; "sequence_no" ]
() ) )
(block_id, zkapp_command_id, sequence_no)

let all_from_block (module Conn : CONNECTION) ~block_id =
let comma_cols = String.concat Fields.names ~sep:"," in
Conn.collect_list
(Caqti_request.collect Caqti_type.int typ
(Mina_caqti.select_cols ~table_name ~select:comma_cols
~cols:[ "block_id" ] () ) )
block_id
end

module Zkapp_account = struct
Expand Down Expand Up @@ -2432,11 +2423,8 @@ module Accounts_accessed = struct
(ledger_index, (account : Account.t)) =
let open Deferred.Result.Let_syntax in
let account_id = Account_id.create account.public_key account.token_id in
(* TODO!!! TEMP!!!! need to get real token owner *)
let%bind account_identifier_id =
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_id
Account_identifiers.add_if_doesn't_exist (module Conn) account_id
in
match%bind find_opt (module Conn) ~block_id ~account_identifier_id with
| Some result ->
Expand Down Expand Up @@ -2531,10 +2519,7 @@ module Accounts_created = struct
creation_fee =
let open Deferred.Result.Let_syntax in
let%bind account_identifier_id =
(* TODO: TEMP!!!! -- add real token owner *)
Account_identifiers.add_if_doesn't_exist
(module Conn)
~token_owner:None account_id
Account_identifiers.add_if_doesn't_exist (module Conn) account_id
in
let creation_fee =
Currency.Fee.to_uint64 creation_fee |> Unsigned.UInt64.to_int64
Expand Down Expand Up @@ -3289,7 +3274,7 @@ let retry ~f ~logger ~error_str retries =
go retries

let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash
~delete_older_than ~accounts_accessed ~accounts_created block =
~delete_older_than ~accounts_accessed ~accounts_created ~tokens_used block =
let state_hash = hash block in

(* the block itself is added in a single transaction with a transaction block
Expand All @@ -3300,6 +3285,15 @@ let add_block_aux ?(retries = 3) ~logger ~pool ~add_block ~hash
transaction block
*)
let add () =
[%log info]
"Populating token owners table for block with state hash $state_hash"
~metadata:[ ("state_hash", Mina_base.State_hash.to_yojson state_hash) ] ;
List.iter tokens_used ~f:(fun (token_id, owner) ->
match owner with
| None ->
()
| Some acct_id ->
Token_owners.add_if_doesn't_exist token_id acct_id ) ;
Caqti_async.Pool.use
(fun (module Conn : CONNECTION) ->
let%bind res =
Expand Down Expand Up @@ -3424,26 +3418,28 @@ let add_block_aux_precomputed ~constraint_constants ~logger ?retries ~pool
~hash:(fun block ->
(block.Precomputed.protocol_state |> Protocol_state.hashes).state_hash )
~accounts_accessed:block.Precomputed.accounts_accessed
~accounts_created:block.Precomputed.accounts_created block
~accounts_created:block.Precomputed.accounts_created
~tokens_used:block.Precomputed.tokens_used block

let add_block_aux_extensional ~logger ?retries ~pool ~delete_older_than block =
add_block_aux ~logger ?retries ~pool ~delete_older_than
~add_block:Block.add_from_extensional
~hash:(fun (block : Extensional.Block.t) -> block.state_hash)
~accounts_accessed:block.Extensional.Block.accounts_accessed
~accounts_created:block.Extensional.Block.accounts_created block
~accounts_created:block.Extensional.Block.accounts_created
~tokens_used:block.Extensional.Block.tokens_used block

let run pool reader ~constraint_constants ~logger ~delete_older_than :
unit Deferred.t =
Strict_pipe.Reader.iter reader ~f:(function
| Diff.Transition_frontier
(Breadcrumb_added { block; accounts_accessed; accounts_created; _ })
-> (
(Breadcrumb_added
{ block; accounts_accessed; accounts_created; tokens_used; _ } ) -> (
let add_block = Block.add_if_doesn't_exist ~constraint_constants in
let hash = State_hash.With_state_hashes.state_hash in
match%bind
add_block_aux ~logger ~pool ~delete_older_than ~hash ~add_block
~accounts_accessed ~accounts_created
~accounts_accessed ~accounts_created ~tokens_used
(With_hash.map ~f:External_transition.decompose block)
with
| Error e ->
Expand Down
Loading