From dc4d90d3cdcefc1eafdb7f222343e4971f36f4cf Mon Sep 17 00:00:00 2001 From: Rogerio Pontual <44991200+jyeshe@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:53:17 +0200 Subject: [PATCH] chore: upgrade to node 6.11 (#1483) * chore: upgrade to node 6.11 * fix: get node for accounts tree * test: add mock needed for contract calls --- .github/workflows/dockerhub.yml | 9 +- Dockerfile | 2 +- lib/ae_mdw/application.ex | 2 +- lib/ae_mdw/contract.ex | 3 +- lib/ae_mdw/db/int_transfer.ex | 9 +- lib/ae_mdw/db/sync/block.ex | 2 +- lib/ae_mdw/node/db.ex | 11 +- .../20230217145700_add_template_edition.ex | 26 ----- .../20230329160010_index_inner_tx.ex | 43 ------- .../20230404095010_update_total_supply.ex | 38 ------ .../20230412085004_index_event_supply.ex | 84 -------------- .../20230413115004_index_logs_count.ex | 57 --------- .../20230428120101_aex9_balance_account.ex | 44 ------- priv/migrations/20230502090101_aexn_count.ex | 52 --------- ...1_reindex_int_transfers_with_new_format.ex | 88 -------------- ...00_fix_inactive_name_owner_deactivation.ex | 30 ----- .../20230506120000_dedup_balance_account.ex | 95 --------------- .../20230515090000_contract_event_logs.ex | 29 ----- ...15110000_index_oracle_queries_responses.ex | 53 --------- ...fix_internal_oracle_responses_transfers.ex | 109 ------------------ .../20230516120000_add_inner_tx_type_index.ex | 43 ------- ...120000_add_inactive_channels_activation.ex | 33 ------ priv/migrations/20230612101501_aexn_count.ex | 61 ---------- scripts/test.sh | 1 + test/ae_mdw/contracts_test.exs | 68 ++++++----- test/test_helper.exs | 7 +- 26 files changed, 72 insertions(+), 927 deletions(-) delete mode 100644 priv/migrations/20230217145700_add_template_edition.ex delete mode 100644 priv/migrations/20230329160010_index_inner_tx.ex delete mode 100644 priv/migrations/20230404095010_update_total_supply.ex delete mode 100644 priv/migrations/20230412085004_index_event_supply.ex delete mode 100644 priv/migrations/20230413115004_index_logs_count.ex delete mode 100644 priv/migrations/20230428120101_aex9_balance_account.ex delete mode 100644 priv/migrations/20230502090101_aexn_count.ex delete mode 100644 priv/migrations/20230503090101_reindex_int_transfers_with_new_format.ex delete mode 100644 priv/migrations/20230503120000_fix_inactive_name_owner_deactivation.ex delete mode 100644 priv/migrations/20230506120000_dedup_balance_account.ex delete mode 100644 priv/migrations/20230515090000_contract_event_logs.ex delete mode 100644 priv/migrations/20230515110000_index_oracle_queries_responses.ex delete mode 100644 priv/migrations/20230515120000_fix_internal_oracle_responses_transfers.ex delete mode 100644 priv/migrations/20230516120000_add_inner_tx_type_index.ex delete mode 100644 priv/migrations/20230519120000_add_inactive_channels_activation.ex delete mode 100644 priv/migrations/20230612101501_aexn_count.ex diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 35049f723..920e07ba2 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -2,6 +2,8 @@ name: Dockerhub Pipeline on: push: + branches: + - develop tags: - v* @@ -17,13 +19,18 @@ jobs: with: fetch-depth: 100 + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Extract metadata for docker id: meta uses: docker/metadata-action@v4 with: images: aeternity/ae_mdw tags: | - type=raw,value=master,enable={{is_default_branch}} + type=raw,value=${{ steps.extract_branch.outputs.branch }},enable=true type=sha,prefix=,enable={{is_default_branch}} type=semver,pattern={{version}},enable=true diff --git a/Dockerfile b/Dockerfile index 21e61b17a..b81e66894 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ WORKDIR /home/aeternity/node ARG DEV_MODE="false" ENV DEV_MODE=${DEV_MODE} ENV NODEROOT=/home/aeternity/node/local -ARG NODE_VERSION=6.8.1 +ARG NODE_VERSION=6.11.0 ARG NODE_URL=https://github.com/aeternity/aeternity/releases/download/v${NODE_VERSION}/aeternity-v${NODE_VERSION}-ubuntu-x86_64.tar.gz ENV NODEDIR=/home/aeternity/node/local/rel/aeternity RUN mkdir -p ./local/rel/aeternity/data/mnesia diff --git a/lib/ae_mdw/application.ex b/lib/ae_mdw/application.ex index 59d280fd0..a6ec5d069 100644 --- a/lib/ae_mdw/application.ex +++ b/lib/ae_mdw/application.ex @@ -69,7 +69,7 @@ defmodule AeMdw.Application do defp init(:meta) do {:ok, chain_state_code} = Extract.AbsCode.module(:aec_chain_state) - [:header, :hash, :type] = NodeHelper.record_keys(chain_state_code, :node) + [:header, :hash, :type | _txs] = NodeHelper.record_keys(chain_state_code, :node) {:ok, aetx_code} = Extract.AbsCode.module(:aetx) {:ok, aeser_code} = Extract.AbsCode.module(:aeser_api_encoder) diff --git a/lib/ae_mdw/contract.ex b/lib/ae_mdw/contract.ex index b909b0142..c8c59b1d8 100644 --- a/lib/ae_mdw/contract.ex +++ b/lib/ae_mdw/contract.ex @@ -433,9 +433,8 @@ defmodule AeMdw.Contract do if txs_taken != [] do header = :aec_blocks.to_header(micro_block) - {:ok, hash} = :aec_headers.hash_header(header) consensus = :aec_headers.consensus_module(header) - node = {:node, header, hash, :micro} + node = :aec_chain_state.wrap_block(micro_block) time = :aec_block_insertion.node_time(node) prev_hash = :aec_block_insertion.node_prev_hash(node) prev_key_hash = :aec_block_insertion.node_prev_key_hash(node) diff --git a/lib/ae_mdw/db/int_transfer.ex b/lib/ae_mdw/db/int_transfer.ex index 214ebf1c9..c8bb12df6 100644 --- a/lib/ae_mdw/db/int_transfer.ex +++ b/lib/ae_mdw/db/int_transfer.ex @@ -27,21 +27,24 @@ defmodule AeMdw.Db.IntTransfer do @typep optional_txi_idx() :: Txs.optional_txi_idx() @typep height_optional_txi_idx() :: {Blocks.height(), optional_txi_idx()} @typep kind_suffix() :: :lock_name | :spend_name | :refund_name | :earn_oracle | :refund_oracle + @opaque key_block() :: tuple() @fee_kinds [:lock_name, :spend_name, :refund_name, :earn_oracle, :refund_oracle] @reward_block_kind "reward_block" @reward_dev_kind "reward_dev" - @spec block_rewards_mutations(Blocks.height(), Blocks.key_header(), Blocks.block_hash()) :: [ + @spec block_rewards_mutations(key_block()) :: [ Mutation.t() ] - def block_rewards_mutations(height, key_header, key_hash) do + def block_rewards_mutations(key_block) do + height = :aec_blocks.height(key_block) delay = :aec_governance.beneficiary_reward_delay() dev_benefs = Enum.map(:aec_dev_reward.beneficiaries(), &elem(&1, 0)) {devs_rewards, miners_rewards} = - {:node, key_header, key_hash, :key} + key_block + |> :aec_chain_state.wrap_block() |> :aec_chain_state.grant_fees(:aec_trees.new(), delay, false, nil) |> :aec_trees.accounts() |> :aeu_mtrees.to_list() diff --git a/lib/ae_mdw/db/sync/block.ex b/lib/ae_mdw/db/sync/block.ex index 546c23db9..b06d9f342 100644 --- a/lib/ae_mdw/db/sync/block.ex +++ b/lib/ae_mdw/db/sync/block.ex @@ -95,7 +95,7 @@ defmodule AeMdw.Db.Sync.Block do block_rewards_mutation = if height >= AE.min_block_reward_height() and :aec_consensus_hc != :aec_consensus.get_genesis_consensus_module() do - IntTransfer.block_rewards_mutations(height, kb_header, kb_hash) + IntTransfer.block_rewards_mutations(key_block) end gen_mutations = [ diff --git a/lib/ae_mdw/node/db.ex b/lib/ae_mdw/node/db.ex index 5bd95da39..b9d36ea8c 100644 --- a/lib/ae_mdw/node/db.ex +++ b/lib/ae_mdw/node/db.ex @@ -274,15 +274,18 @@ defmodule AeMdw.Node.Db do defp block_accounts_tree(mb_hash) do {:value, micro_block} = :aec_db.find_block(mb_hash) - header = :aec_blocks.to_header(micro_block) - {:ok, hash} = :aec_headers.hash_header(header) - consensus_mod = :aec_headers.consensus_module(header) - node = {:node, header, hash, :micro} + + node = :aec_chain_state.wrap_block(micro_block) prev_hash = :aec_block_insertion.node_prev_hash(node) {:value, trees_in, _tree, _difficulty, _fees, _fraud} = :aec_db.find_block_state_and_data(prev_hash, true) + consensus_mod = + micro_block + |> :aec_blocks.to_header() + |> :aec_headers.consensus_module() + node |> consensus_mod.state_pre_transform_micro_node(trees_in) |> :aec_trees.accounts() diff --git a/priv/migrations/20230217145700_add_template_edition.ex b/priv/migrations/20230217145700_add_template_edition.ex deleted file mode 100644 index 457a8a708..000000000 --- a/priv/migrations/20230217145700_add_template_edition.ex +++ /dev/null @@ -1,26 +0,0 @@ -defmodule AeMdw.Migrations.AddTemplateEdition do - @moduledoc """ - Adds template edition to NftTemplateToken. - """ - - alias AeMdw.Collection - alias AeMdw.Db.Model - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - write_mutations = - state - |> Collection.stream(Model.NftTemplateToken, nil) - |> Stream.map(&State.fetch!(state, Model.NftTemplateToken, &1)) - |> Stream.filter(&match?({:nft_template_token, _key, _txi, _log_idx}, &1)) - |> Enum.map(&WriteMutation.new(Model.NftTemplateToken, Tuple.append(&1, "1"))) - - _state = State.commit(state, write_mutations) - - {:ok, length(write_mutations)} - end -end diff --git a/priv/migrations/20230329160010_index_inner_tx.ex b/priv/migrations/20230329160010_index_inner_tx.ex deleted file mode 100644 index 7686cf51f..000000000 --- a/priv/migrations/20230329160010_index_inner_tx.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule AeMdw.Migrations.IndexGameta do - @moduledoc """ - Index GaMetaTx. - """ - - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.WriteFieldsMutation - alias AeMdw.Collection - alias AeMdw.Db.Model - alias AeMdw.Db.Sync.InnerTx - alias AeMdw.Node.Db - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - write_mutations = - [:ga_meta_tx, :paying_for_tx] - |> Stream.flat_map(fn tx_type -> - key_boundary = {{tx_type, 1, <<>>, 0}, {tx_type, 1, AeMdw.Util.max_256bit_bin(), nil}} - - state - |> Collection.stream(Model.Field, :forward, key_boundary, nil) - |> Stream.flat_map(fn {_tx_type, _pos, _pubkey, txi} -> - Model.tx(block_index: block_index, id: hash) = State.fetch!(state, Model.Tx, txi) - {_block_hash, ^tx_type, _signed_tx, tx_rec} = Db.get_tx_data(hash) - inner_signed_tx = InnerTx.signed_tx(tx_type, tx_rec) - {inner_type, inner_tx} = :aetx.specialize_type(:aetx_sign.tx(inner_signed_tx)) - - [ - WriteMutation.new(Model.Type, Model.type(index: {tx_type, txi})), - WriteFieldsMutation.new(inner_type, inner_tx, block_index, txi, tx_type) - ] - end) - end) - |> Enum.to_list() - - _state = State.commit(state, write_mutations) - - {:ok, length(write_mutations)} - end -end diff --git a/priv/migrations/20230404095010_update_total_supply.ex b/priv/migrations/20230404095010_update_total_supply.ex deleted file mode 100644 index 3d12fc9c4..000000000 --- a/priv/migrations/20230404095010_update_total_supply.ex +++ /dev/null @@ -1,38 +0,0 @@ -defmodule AeMdw.Migrations.UpdateTotalSupply do - @moduledoc """ - Updates totalstats total supply incrementing with the lima contracts amount. - """ - - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - with [%{amount: amount}] <- :aec_fork_block_settings.lima_contracts(), - {:ok, {last_height, _mbi}} <- State.prev(state, Model.Block, nil), - %{4 => lima_height} <- - :aec_hard_forks.protocols_from_network_id(:aec_governance.get_network_id()) do - mutations = - Enum.map(lima_height..last_height, fn height -> - m_stat = - Model.total_stat(total_supply: total_supply) = - State.fetch!(state, Model.TotalStat, height) - - WriteMutation.new( - Model.TotalStat, - Model.total_stat(m_stat, total_supply: total_supply + amount) - ) - end) - - _state = State.commit(state, mutations) - - {:ok, length(mutations)} - else - _no_lima -> - {:ok, 0} - end - end -end diff --git a/priv/migrations/20230412085004_index_event_supply.ex b/priv/migrations/20230412085004_index_event_supply.ex deleted file mode 100644 index 681283ebe..000000000 --- a/priv/migrations/20230412085004_index_event_supply.ex +++ /dev/null @@ -1,84 +0,0 @@ -defmodule AeMdw.Migrations.IndexEventSupply do - # credo:disable-for-this-file - @moduledoc """ - Index aex9 intial and event supply. - """ - - alias AeMdw.AexnContracts - alias AeMdw.Collection - alias AeMdw.Node.Db, as: NodeDb - alias AeMdw.Db.Model - alias AeMdw.Db.Origin - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.State - - require Model - - @mint_hash <<215, 0, 247, 67, 100, 22, 167, 140, 76, 197, 95, 144, 242, 214, 49, 111, 60, 169, - 26, 213, 244, 50, 59, 170, 72, 182, 90, 72, 178, 84, 251, 35>> - - @burn_hash <<131, 150, 191, 31, 191, 94, 29, 68, 10, 143, 62, 247, 169, 46, 221, 88, 138, 150, - 176, 154, 87, 110, 105, 73, 173, 237, 42, 252, 105, 193, 146, 6>> - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - key_boundary = {{@mint_hash, 0, 0, 0}, {@mint_hash, nil, nil, nil}} - mint_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - key_boundary = {{@burn_hash, 0, 0, 0}, {@burn_hash, nil, nil, nil}} - burn_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - - write_mutations = - [mint_stream, burn_stream] - |> Collection.merge(:forward) - |> Stream.flat_map(fn {event_hash, call_txi, create_txi, log_idx} -> - contract_pk = Origin.pubkey(state, {:contract, create_txi}) || <<>> - - if AexnContracts.is_aex9?(contract_pk) do - case State.fetch!(state, Model.ContractLog, {create_txi, call_txi, event_hash, log_idx}) do - Model.contract_log(args: [_account_pk, <>]) -> - if event_hash == @mint_hash, - do: [{contract_pk, amount}], - else: [{contract_pk, -amount}] - - _other -> - [] - end - else - [] - end - end) - |> Enum.group_by(fn {pk, _amount} -> pk end) - |> Enum.flat_map(fn {pk, list} -> - create_txi = Origin.tx_index!(state, {:contract, pk}) - Model.tx(block_index: {kbi, mbi}) = State.fetch!(state, Model.Tx, create_txi) - Model.block(hash: next_kb_hash) = State.fetch!(state, Model.Block, {kbi + 1, -1}) - mb_hash = NodeDb.get_next_hash(next_kb_hash, mbi) - - create_sum = - case NodeDb.aex9_balances(pk, {:micro, kbi, mb_hash}) do - {:ok, addr_map} -> - addr_map |> Enum.map(fn {_addr, balance} -> balance end) |> Enum.sum() - - {:error, _reason} -> - 0 - end - - event_sum = list |> Enum.map(fn {_pk, amount} -> amount end) |> Enum.sum() - - [ - WriteMutation.new( - Model.Aex9ContractBalance, - Model.aex9_contract_balance(index: pk, amount: event_sum) - ), - WriteMutation.new( - Model.Aex9InitialSupply, - Model.aex9_initial_supply(index: pk, amount: create_sum) - ) - ] - end) - - _state = State.commit(state, write_mutations) - - {:ok, length(write_mutations)} - end -end diff --git a/priv/migrations/20230413115004_index_logs_count.ex b/priv/migrations/20230413115004_index_logs_count.ex deleted file mode 100644 index 4b08e6244..000000000 --- a/priv/migrations/20230413115004_index_logs_count.ex +++ /dev/null @@ -1,57 +0,0 @@ -defmodule AeMdw.Migrations.IndexLogsCount do - # credo:disable-for-this-file - @moduledoc """ - Index aex9 logs count. - """ - - alias AeMdw.AexnContracts - alias AeMdw.Collection - alias AeMdw.Db.Model - alias AeMdw.Db.Origin - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.State - alias AeMdw.Stats - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - mint_hash = :aec_hash.blake2b_256_hash("Mint") - burn_hash = :aec_hash.blake2b_256_hash("Burn") - swap_hash = :aec_hash.blake2b_256_hash("Swap") - transfer_hash = :aec_hash.blake2b_256_hash("Transfer") - - key_boundary = {{mint_hash, 0, 0, 0}, {mint_hash, nil, nil, nil}} - mint_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - key_boundary = {{burn_hash, 0, 0, 0}, {burn_hash, nil, nil, nil}} - burn_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - key_boundary = {{swap_hash, 0, 0, 0}, {swap_hash, nil, nil, nil}} - swap_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - key_boundary = {{transfer_hash, 0, 0, 0}, {transfer_hash, nil, nil, nil}} - transfer_stream = Collection.stream(state, Model.EvtContractLog, :forward, key_boundary, nil) - - write_mutations = - [mint_stream, burn_stream, swap_stream, transfer_stream] - |> Enum.flat_map(fn stream -> - Enum.flat_map(stream, fn {_event_hash, _call_txi, create_txi, _log_idx} -> - contract_pk = Origin.pubkey(state, {:contract, create_txi}) || <<>> - - if AexnContracts.is_aex9?(contract_pk) do - [{contract_pk, 1}] - else - [] - end - end) - end) - |> Enum.group_by(fn {pk, _amount} -> pk end) - |> Enum.map(fn {contract_pk, list} -> - key = Stats.aex9_logs_count_key(contract_pk) - count = list |> Enum.map(fn {_pk, 1} -> 1 end) |> Enum.sum() - WriteMutation.new(Model.Stat, Model.stat(index: key, payload: count)) - end) - - _state = State.commit(state, write_mutations) - - {:ok, length(write_mutations)} - end -end diff --git a/priv/migrations/20230428120101_aex9_balance_account.ex b/priv/migrations/20230428120101_aex9_balance_account.ex deleted file mode 100644 index de399498a..000000000 --- a/priv/migrations/20230428120101_aex9_balance_account.ex +++ /dev/null @@ -1,44 +0,0 @@ -defmodule AeMdw.Migrations.Aex9BalanceAccount do - @moduledoc """ - Indexes aex9 balance account sorting by amount. - """ - - alias AeMdw.Collection - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.Db.WriteMutation - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - mutations = - state - |> Collection.stream( - Model.Aex9EventBalance, - :forward, - nil, - nil - ) - |> Stream.map(&State.fetch!(state, Model.Aex9EventBalance, &1)) - |> Enum.map(fn Model.aex9_event_balance( - index: {contract_pk, account_pk}, - txi: txi, - log_idx: log_idx, - amount: amount - ) -> - m_bal_acc = - Model.aex9_balance_account( - index: {contract_pk, amount, account_pk}, - txi: txi, - log_idx: log_idx - ) - - WriteMutation.new(Model.Aex9BalanceAccount, m_bal_acc) - end) - - _state = State.commit(state, mutations) - - {:ok, length(mutations)} - end -end diff --git a/priv/migrations/20230502090101_aexn_count.ex b/priv/migrations/20230502090101_aexn_count.ex deleted file mode 100644 index b5bd5ef0a..000000000 --- a/priv/migrations/20230502090101_aexn_count.ex +++ /dev/null @@ -1,52 +0,0 @@ -defmodule AeMdw.Migrations.AexnCount do - @moduledoc """ - Updates totalstats total supply incrementing with the lima contracts amount. - """ - - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.Db.WriteMutation - alias AeMdw.Collection - alias AeMdw.Util - alias AeMdw.Stats - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - aex9_count = - state - |> Collection.stream( - Model.AexnContract, - :forward, - {{:aex9, <<>>}, {:aex9, Util.max_256bit_bin()}}, - nil - ) - |> Enum.count() - - aex141_count = - state - |> Collection.stream( - Model.AexnContract, - :forward, - {{:aex141, <<>>}, {:aex141, Util.max_256bit_bin()}}, - nil - ) - |> Enum.count() - - mutations = [ - WriteMutation.new( - Model.Stat, - Model.stat(index: Stats.aexn_count_key(:aex9), payload: aex9_count) - ), - WriteMutation.new( - Model.Stat, - Model.stat(index: Stats.aexn_count_key(:aex141), payload: aex141_count) - ) - ] - - _state = State.commit(state, mutations) - - {:ok, 2} - end -end diff --git a/priv/migrations/20230503090101_reindex_int_transfers_with_new_format.ex b/priv/migrations/20230503090101_reindex_int_transfers_with_new_format.ex deleted file mode 100644 index e46702b18..000000000 --- a/priv/migrations/20230503090101_reindex_int_transfers_with_new_format.ex +++ /dev/null @@ -1,88 +0,0 @@ -defmodule AeMdw.Migrations.ReindexIntTransfersWithNewFormat do - @moduledoc """ - Reindexes internal transfers from `{height, txi | -1}` format to `{height, txi_idx | -1}`. - """ - - alias AeMdw.Collection - alias AeMdw.Db.DeleteKeysMutation - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - {_new_state, count} = - state - |> Collection.stream(Model.IntTransferTx, nil) - |> Stream.filter(fn - {{_height, -1}, _kind, _target_pk, -1} -> - false - - {{_height, txi}, _kind, _target_pk, -1} when is_integer(txi) -> - true - - {{_height, -1}, _kind, _target_pk, ref_txi} when is_integer(ref_txi) -> - true - - {{_height, txi}, _kind, _target_pk, ref_txi} - when is_integer(ref_txi) or is_integer(txi) -> - true - - _key -> - false - end) - |> Stream.map(fn {{height, opt_txi}, kind, target_pk, opt_ref_txi} = old_key -> - opt_txi_idx = opt_txi_to_opt_txi_idx(opt_txi) - opt_ref_txi_idx = opt_txi_to_opt_txi_idx(opt_ref_txi) - Model.int_transfer_tx(amount: amount) = State.fetch!(state, Model.IntTransferTx, old_key) - - int_transfer = - Model.int_transfer_tx( - index: {{height, opt_txi_idx}, kind, target_pk, opt_ref_txi_idx}, - amount: amount - ) - - kind_tx = - Model.kind_int_transfer_tx( - index: {kind, {height, opt_txi_idx}, target_pk, opt_ref_txi_idx} - ) - - target_kind_tx = - Model.target_kind_int_transfer_tx( - index: {target_pk, kind, {height, opt_txi_idx}, opt_ref_txi_idx} - ) - - {height, - [ - WriteMutation.new(Model.IntTransferTx, int_transfer), - WriteMutation.new(Model.KindIntTransferTx, kind_tx), - WriteMutation.new(Model.TargetKindIntTransferTx, target_kind_tx), - DeleteKeysMutation.new(%{ - Model.IntTransferTx => [old_key], - Model.KindIntTransferTx => [{kind, {height, opt_txi}, target_pk, opt_ref_txi}], - Model.TargetKindIntTransferTx => [{target_pk, kind, {height, opt_txi}, opt_ref_txi}] - }) - ]} - end) - |> Stream.chunk_every(10_000) - |> Enum.reduce({state, 0}, fn [{first_height, _mutations} | _rest] = height_mutations, - {state, count} -> - mutations = Enum.flat_map(height_mutations, &elem(&1, 1)) - IO.puts("Processed up to height #{first_height}..") - - new_state = State.commit(state, mutations) - - {new_state, count + length(mutations)} - end) - - {:ok, count} - end - - defp opt_txi_to_opt_txi_idx(-1), do: -1 - - defp opt_txi_to_opt_txi_idx({txi, idx}), do: {txi, idx} - - defp opt_txi_to_opt_txi_idx(txi), do: {txi, -1} -end diff --git a/priv/migrations/20230503120000_fix_inactive_name_owner_deactivation.ex b/priv/migrations/20230503120000_fix_inactive_name_owner_deactivation.ex deleted file mode 100644 index 30295343c..000000000 --- a/priv/migrations/20230503120000_fix_inactive_name_owner_deactivation.ex +++ /dev/null @@ -1,30 +0,0 @@ -defmodule AeMdw.Migrations.FixInactiveNameOwnerDeactivation do - @moduledoc """ - Deletes no longer valid InactiveNameOwnerDeactivation keys. - """ - - alias AeMdw.Collection - alias AeMdw.Db.DeleteKeysMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - invalid_deactivations = - state - |> Collection.stream(Model.InactiveNameOwnerDeactivation, nil) - |> Stream.filter(fn {_owner_pk, expire, plain_name} -> - not State.exists?(state, Model.InactiveNameExpiration, {expire, plain_name}) - end) - |> Enum.to_list() - - mutation = - DeleteKeysMutation.new(%{Model.InactiveNameOwnerDeactivation => invalid_deactivations}) - - _new_state = State.commit(state, [mutation]) - - {:ok, length(invalid_deactivations)} - end -end diff --git a/priv/migrations/20230506120000_dedup_balance_account.ex b/priv/migrations/20230506120000_dedup_balance_account.ex deleted file mode 100644 index 8126fb3c2..000000000 --- a/priv/migrations/20230506120000_dedup_balance_account.ex +++ /dev/null @@ -1,95 +0,0 @@ -defmodule AeMdw.Migrations.DedupBalanceAccount do - @moduledoc """ - Deduplicates balance accounts. - """ - - alias AeMdw.Collection - alias AeMdw.Db.AsyncStore - alias AeMdw.Db.DeleteKeysMutation - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.Sync.AsyncTasks.WealthRank - alias AeMdw.Sync.Transaction - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - all_keys = - state - |> Collection.stream(Model.BalanceAccount, :forward) - |> Enum.to_list() - - clear_mutation = DeleteKeysMutation.new(%{Model.BalanceAccount => all_keys}) - - state - |> Collection.stream(Model.Block, :forward, nil, {1, -1}) - |> Stream.filter(fn {kb, mbi} -> - if mbi != -1, do: IO.puts("height #{kb} mbi #{mbi}") - - mbi == -1 - end) - |> Stream.map(&State.fetch!(state, Model.Block, &1)) - |> Stream.flat_map(&AeMdw.Node.Db.get_micro_blocks(Model.block(&1, :hash))) - |> Stream.map(µ_block_balances/1) - |> Stream.with_index() - |> Enum.each(fn {balances, index} -> - _res = - if rem(index, 1000) == 0 do - WealthRank.prune_balance_ranking(AsyncStore.instance()) - end - - WealthRank.update_balances(balances) - end) - - {top_keys, _store} = WealthRank.prune_balance_ranking(AsyncStore.instance()) - - write_mutations = - Enum.map( - top_keys, - &WriteMutation.new(Model.BalanceAccount, Model.balance_account(index: &1)) - ) - - _new_state = State.commit(state, [clear_mutation | write_mutations]) - - {:ok, length(top_keys) + 1} - end - - defp micro_block_balances(micro_block) do - {:ok, mb_hash} = :aec_headers.hash_header(:aec_blocks.to_micro_header(micro_block)) - - with {:value, trees} <- :aec_db.find_block_state_partial(mb_hash, true, [:accounts]), - accounts_tree <- :aec_trees.accounts(trees), - account_set <- micro_block_accounts(micro_block) do - get_balances(accounts_tree, account_set) - end - end - - defp get_balances(accounts_tree, account_set) do - account_set - |> MapSet.to_list() - |> Enum.flat_map(fn pubkey -> - case :aec_accounts_trees.lookup(pubkey, accounts_tree) do - {:value, account} -> [{pubkey, :aec_accounts.balance(account)}] - :none -> [] - end - end) - end - - defp micro_block_accounts(micro_block) do - pubkeys = - micro_block - |> :aec_blocks.txs() - |> Enum.flat_map(fn signed_tx -> - signed_tx - |> Transaction.get_ids_from_tx() - |> Enum.flat_map(fn - {:id, :account, pubkey} -> [pubkey] - _other -> [] - end) - end) - - Enum.reduce(pubkeys, MapSet.new(), &MapSet.put(&2, &1)) - end -end diff --git a/priv/migrations/20230515090000_contract_event_logs.ex b/priv/migrations/20230515090000_contract_event_logs.ex deleted file mode 100644 index 443e2a61f..000000000 --- a/priv/migrations/20230515090000_contract_event_logs.ex +++ /dev/null @@ -1,29 +0,0 @@ -defmodule AeMdw.Migrations.ContractEventLogs do - # credo:disable-for-this-file - @moduledoc """ - Index logs by contract and event. - """ - - alias AeMdw.Collection - alias AeMdw.Db.Model - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - write_mutations = - state - |> Collection.stream(Model.EvtContractLog, :forward, nil, nil) - |> Stream.map(fn {event_hash, call_txi, create_txi, log_idx} -> - m_ctevt_log = Model.ctevt_contract_log(index: {event_hash, create_txi, call_txi, log_idx}) - WriteMutation.new(Model.CtEvtContractLog, m_ctevt_log) - end) - |> Enum.to_list() - - _state = State.commit(state, write_mutations) - - {:ok, length(write_mutations)} - end -end diff --git a/priv/migrations/20230515110000_index_oracle_queries_responses.ex b/priv/migrations/20230515110000_index_oracle_queries_responses.ex deleted file mode 100644 index 054adea42..000000000 --- a/priv/migrations/20230515110000_index_oracle_queries_responses.ex +++ /dev/null @@ -1,53 +0,0 @@ -defmodule AeMdw.Migrations.IndexOracleQueriesResponses do - @moduledoc """ - Indexes new Model.oracle_query response_txi_idx attribute. - """ - - alias :aeser_api_encoder, as: Enc - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.Db.Util, as: DbUtil - - @dialyzer {:nowarn_function, {:run, 2}} - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - mutations = - state - |> DbUtil.transactions_of_type(:oracle_response_tx, :forward, nil, nil) - |> Stream.map(fn response_txi_idx -> - oracle_response_tx = DbUtil.read_node_tx(state, response_txi_idx) - oracle_pk = :aeo_response_tx.oracle_pubkey(oracle_response_tx) - query_id = :aeo_response_tx.query_id(oracle_response_tx) - - case State.get(state, Model.OracleQuery, {oracle_pk, query_id}) do - {:ok, {:oracle_query, index, txi_idx}} -> - oracle_query = - Model.oracle_query( - index: index, - txi_idx: txi_idx, - response_txi_idx: response_txi_idx - ) - - WriteMutation.new(Model.OracleQuery, oracle_query) - - :not_found -> - IO.puts(""" - [OracleResponseMutation] Oracle response not found on txi #{inspect(response_txi_idx)} for #{Enc.encode(:oracle_pubkey, oracle_pk)} - (query #{Enc.encode(:oracle_query_id, query_id)}). - Probably due to ga_meta transaction not calculating nonce correcctly. - """) - - nil - end - end) - |> Enum.to_list() - - _new_state = State.commit(state, mutations) - - {:ok, length(mutations)} - end -end diff --git a/priv/migrations/20230515120000_fix_internal_oracle_responses_transfers.ex b/priv/migrations/20230515120000_fix_internal_oracle_responses_transfers.ex deleted file mode 100644 index ca147e660..000000000 --- a/priv/migrations/20230515120000_fix_internal_oracle_responses_transfers.ex +++ /dev/null @@ -1,109 +0,0 @@ -defmodule AeMdw.Migrations.FixInternalOracleResponsesTransfers do - @moduledoc """ - Reindexes internal oracle_reward transfers that are in the format `{height, {txi, -1}}`, - but should actually point to an internal call using `{height, {txi, idx}}`. - """ - - alias AeMdw.Collection - alias AeMdw.Db.DeleteKeysMutation - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.Util, as: DbUtil - alias AeMdw.Db.State - alias AeMdw.Util - - require Model - - @kind "reward_oracle" - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - scope = { - {@kind, {Util.min_int(), Util.min_int()}, nil, nil}, - {@kind, {Util.max_int(), Util.max_int()}, nil, nil} - } - - mutations = - state - |> Collection.stream(Model.KindIntTransferTx, :forward, scope, nil) - |> Stream.flat_map(fn {@kind, {height, old_response_txi_idx}, target_pk, old_query_txi_idx} = - old_kind_key -> - Model.int_transfer_tx(amount: amount) = - State.fetch!( - state, - Model.IntTransferTx, - {{height, old_response_txi_idx}, @kind, target_pk, old_query_txi_idx} - ) - - {:ok, response_txi_idx} = - correct_txi_idx(state, old_response_txi_idx, "Oracle.respond", :oracle_response_tx) - - query_txi_idx = - case correct_txi_idx(state, old_query_txi_idx, "Oracle.query", :oracle_query_tx) do - {:ok, query_txi_idx} -> - query_txi_idx - - :invalid -> - response_tx = DbUtil.read_node_tx(state, response_txi_idx) - oracle_pk = :aeo_response_tx.oracle_pubkey(response_tx) - query_id = :aeo_response_tx.query_id(response_tx) - - Model.oracle_query(txi_idx: txi_idx) = - State.fetch!(state, Model.OracleQuery, {oracle_pk, query_id}) - - txi_idx - end - - int_transfer = - Model.int_transfer_tx( - index: {{height, response_txi_idx}, @kind, target_pk, query_txi_idx}, - amount: amount - ) - - new_kind_key = {@kind, {height, response_txi_idx}, target_pk, query_txi_idx} - kind_tx = Model.kind_int_transfer_tx(index: new_kind_key) - - target_kind_tx = - Model.target_kind_int_transfer_tx( - index: {target_pk, @kind, {height, response_txi_idx}, query_txi_idx} - ) - - if old_kind_key != new_kind_key do - [ - WriteMutation.new(Model.IntTransferTx, int_transfer), - WriteMutation.new(Model.KindIntTransferTx, kind_tx), - WriteMutation.new(Model.TargetKindIntTransferTx, target_kind_tx), - DeleteKeysMutation.new(%{ - Model.IntTransferTx => [ - {{height, old_response_txi_idx}, @kind, target_pk, old_query_txi_idx} - ], - Model.KindIntTransferTx => [old_kind_key], - Model.TargetKindIntTransferTx => [ - {target_pk, @kind, {height, old_response_txi_idx}, old_query_txi_idx} - ] - }) - ] - else - [] - end - end) - |> Enum.to_list() - - _new_state = State.commit(state, mutations) - - {:ok, length(mutations)} - end - - defp correct_txi_idx(state, {txi, _idx} = old_txi_idx, fname, tx_type) do - case DbUtil.read_node_tx_details(state, old_txi_idx) do - {_tx, ^tx_type, _hash, _tx_type, _block_hash} -> - {:ok, old_txi_idx} - - _contract_call_details -> - case State.next(state, Model.FnameIntContractCall, {fname, txi, -1}) do - {:ok, {^fname, ^txi, idx}} -> {:ok, {txi, idx}} - _invalid_txi_idx -> :invalid - end - end - end -end diff --git a/priv/migrations/20230516120000_add_inner_tx_type_index.ex b/priv/migrations/20230516120000_add_inner_tx_type_index.ex deleted file mode 100644 index 51639b1c1..000000000 --- a/priv/migrations/20230516120000_add_inner_tx_type_index.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule AeMdw.Migrations.AddInnerTxTypeIndex do - @moduledoc """ - Indexes Model.InnerType table - """ - - alias AeMdw.Collection - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.Db.Sync.InnerTx - alias AeMdw.Node.Db - alias AeMdw.Util - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - mutations = - ~w(paying_for_tx ga_meta_tx)a - |> Enum.flat_map(fn tx_type -> - scope = {{tx_type, Util.min_int()}, {tx_type, Util.max_int()}} - - state - |> Collection.stream(Model.Type, :forward, scope, nil) - |> Stream.map(fn {tx_type, txi} -> - Model.tx(id: tx_hash) = State.fetch!(state, Model.Tx, txi) - {_block_hash, ^tx_type, _signed_tx, tx_rec} = Db.get_tx_data(tx_hash) - - {inner_type, _tx} = - tx_type - |> InnerTx.signed_tx(tx_rec) - |> :aetx_sign.tx() - |> :aetx.specialize_type() - - WriteMutation.new(Model.InnerType, Model.type(index: {inner_type, txi})) - end) - end) - - _new_state = State.commit(state, mutations) - - {:ok, length(mutations)} - end -end diff --git a/priv/migrations/20230519120000_add_inactive_channels_activation.ex b/priv/migrations/20230519120000_add_inactive_channels_activation.ex deleted file mode 100644 index 04c673700..000000000 --- a/priv/migrations/20230519120000_add_inactive_channels_activation.ex +++ /dev/null @@ -1,33 +0,0 @@ -defmodule AeMdw.Migrations.AddInactiveChannelsActivation do - @moduledoc """ - Indexes Model.InactiveChannelActivation table - """ - - alias AeMdw.Collection - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - mutations = - state - |> Collection.stream(Model.InactiveChannel, nil) - |> Stream.map(fn key -> - Model.channel(index: channel_pk, active: active_height) = - State.fetch!(state, Model.InactiveChannel, key) - - WriteMutation.new( - Model.InactiveChannelActivation, - Model.activation(index: {active_height, channel_pk}) - ) - end) - |> Enum.to_list() - - _new_state = State.commit(state, mutations) - - {:ok, length(mutations)} - end -end diff --git a/priv/migrations/20230612101501_aexn_count.ex b/priv/migrations/20230612101501_aexn_count.ex deleted file mode 100644 index 8251ceadc..000000000 --- a/priv/migrations/20230612101501_aexn_count.ex +++ /dev/null @@ -1,61 +0,0 @@ -defmodule AeMdw.Migrations.ReindexAexnCount do - @moduledoc """ - Indexes aexn contracts count disconsidering those with meta info error. - """ - - alias AeMdw.Collection - alias AeMdw.Db.WriteMutation - alias AeMdw.Db.Model - alias AeMdw.Db.State - alias AeMdw.AexnContracts - alias AeMdw.Stats - alias AeMdw.Util - - require Model - - @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} - def run(state, _from_start?) do - aex9_count = - state - |> Collection.stream( - Model.AexnContract, - :forward, - {{:aex9, <<>>}, {:aex9, Util.max_256bit_bin()}}, - nil - ) - |> Stream.map(&State.fetch!(state, Model.AexnContract, &1)) - |> Stream.filter(fn Model.aexn_contract(meta_info: meta_info) -> - AexnContracts.valid_meta_info?(meta_info) - end) - |> Enum.count() - - aex141_count = - state - |> Collection.stream( - Model.AexnContract, - :forward, - {{:aex141, <<>>}, {:aex141, Util.max_256bit_bin()}}, - nil - ) - |> Stream.map(&State.fetch!(state, Model.AexnContract, &1)) - |> Stream.filter(fn Model.aexn_contract(meta_info: meta_info) -> - AexnContracts.valid_meta_info?(meta_info) - end) - |> Enum.count() - - mutations = [ - WriteMutation.new( - Model.Stat, - Model.stat(index: Stats.aexn_count_key(:aex9), payload: aex9_count) - ), - WriteMutation.new( - Model.Stat, - Model.stat(index: Stats.aexn_count_key(:aex141), payload: aex141_count) - ) - ] - - _new_state = State.commit(state, mutations) - - {:ok, length(mutations)} - end -end diff --git a/scripts/test.sh b/scripts/test.sh index e5301e580..c72859abc 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -3,5 +3,6 @@ set -xe mix local.hex --force && mix local.rebar --force && mix deps.get +rm -rf test_data.db/ elixir --sname aeternity@localhost -S mix test test/ae_mdw exit 0 diff --git a/test/ae_mdw/contracts_test.exs b/test/ae_mdw/contracts_test.exs index 148bb5c3f..383363585 100644 --- a/test/ae_mdw/contracts_test.exs +++ b/test/ae_mdw/contracts_test.exs @@ -1,8 +1,9 @@ defmodule AeMdw.ContractsTest do use ExUnit.Case + alias AeMdw.Contract alias AeMdw.Contracts - alias AeMdw.Db.Contract + alias AeMdw.Db.Contract, as: DbContract alias AeMdw.Db.IntCallsMutation alias AeMdw.Db.Model alias AeMdw.Db.State @@ -11,6 +12,7 @@ defmodule AeMdw.ContractsTest do import AeMdw.Node.ContractCallFixtures, only: [call_rec: 5] import AeMdw.TestUtil, only: [empty_state: 0] import AeMdw.Util.Encoding + import Mock require Model @@ -43,7 +45,7 @@ defmodule AeMdw.ContractsTest do Model.rev_origin(index: {create_txi, :contract_create_tx, contract_pk}) ) |> State.cache_put(:ct_create_sync_cache, contract_pk, create_txi) - |> Contract.logs_write(create_txi, call_txi, call_rec) + |> DbContract.logs_write(create_txi, call_txi, call_rec) assert {:ok, {_prev, [log1, log2], _next}} = Contracts.fetch_logs(state, {:forward, false, 100, false}, nil, %{}, nil) @@ -107,30 +109,44 @@ defmodule AeMdw.ContractsTest do Model.tx(index: call_txi, id: <>, block_index: block_index) ) - state = - IntCallsMutation.execute( - IntCallsMutation.new(contract_pk, call_txi, int_calls), - state - ) - - assert {:ok, {_prev, [call1, call2], _next}} = - Contracts.fetch_calls(state, {:forward, false, 100, false}, nil, %{}, nil) - - contract_id = encode_contract(contract_pk) - - assert %{ - local_idx: 0, - call_txi: ^call_txi, - contract_txi: ^create_txi, - internal_tx: %{"contract_id" => ^contract_id} - } = call1 - - assert %{ - local_idx: 1, - call_txi: ^call_txi, - contract_txi: ^create_txi, - internal_tx: %{"contract_id" => ^contract_id} - } = call2 + with_mocks [ + {Contract, [:passthrough], + [ + get_init_call_details: fn tx_rec, ^block_hash -> + contract_id = + tx_rec + |> :aect_create_tx.contract_pubkey() + |> encode_contract() + + %{"contract_id" => contract_id} + end + ]} + ] do + state = + IntCallsMutation.execute( + IntCallsMutation.new(contract_pk, call_txi, int_calls), + state + ) + + assert {:ok, {_prev, [call1, call2], _next}} = + Contracts.fetch_calls(state, {:forward, false, 100, false}, nil, %{}, nil) + + contract_id = encode_contract(contract_pk) + + assert %{ + local_idx: 0, + call_txi: ^call_txi, + contract_txi: ^create_txi, + internal_tx: %{"contract_id" => ^contract_id} + } = call1 + + assert %{ + local_idx: 1, + call_txi: ^call_txi, + contract_txi: ^create_txi, + internal_tx: %{"contract_id" => ^contract_id} + } = call2 + end end end end diff --git a/test/test_helper.exs b/test/test_helper.exs index ea1215cd8..9a2861755 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -10,10 +10,11 @@ if Enum.all?(~w(integration iteration devmode)a, &(&1 not in included_tests)) do :ets.insert(:counters, {:kbi, 0}) # reset database - alias AeMdw.Db.RocksDb + :ok = AeMdw.Db.RocksDb.close() + :ok = AeMdw.Db.RocksDb.open(true) - :ok = RocksDb.close() - :ok = RocksDb.open(true) + # init for tests without sync + :persistent_term.put({:aec_db, :backend_module}, "rocksdb") end ExUnit.start()