diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e795a3c1e5..8e62267ed071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,29 @@ # Changelog +## 11.0.3 + +### 🐛 Bug Fixes + +- Insert AddressIdToAddressHash via safe_insert_all ([#14333](https://github.com/blockscout/blockscout/pull/14333)) +- Force search for contract creator if internal transactions module is disabled ([#14324](https://github.com/blockscout/blockscout/issues/14324)) +- Add transactions uniqueness before insert ([#14329](https://github.com/blockscout/blockscout/issues/14329)) + +### ⚙️ Miscellaneous Tasks + +- Don't lock tables if foreign keys are already dropped ([#14321](https://github.com/blockscout/blockscout/issues/14321)) +- Dev branch + CI, remove obsolete GA workflows ([#14317](https://github.com/blockscout/blockscout/issues/14317)) + + ## 11.0.2 ### 🐛 Bug Fixes -- Process empty list of changes on fetching contract codes ((#14312)[https://github.com/blockscout/blockscout/pull/14312]) +- Process empty list of changes on fetching contract codes ([#14312](https://github.com/blockscout/blockscout/pull/14312)) - Add fallback for empty "to" in Geth selfdestruct ([#14256](https://github.com/blockscout/blockscout/issues/14256)) - Trim contractaddresses in getcontractcreation ([#14306](https://github.com/blockscout/blockscout/issues/14306)) - Adapt maybe_reject_zero_value for empty blocks ([#14309](https://github.com/blockscout/blockscout/issues/14309)) - Add missing internal transactions address preload ([#14308](https://github.com/blockscout/blockscout/issues/14308)) -- Fix some web tests ([#14310][https://github.com/blockscout/blockscout/pull/14310]) +- Fix some web tests ([#14310](https://github.com/blockscout/blockscout/pull/14310)) ### ⚙️ Miscellaneous Tasks diff --git a/apps/block_scout_web/mix.exs b/apps/block_scout_web/mix.exs index adce29f0b6a8..e68c6fba48b3 100644 --- a/apps/block_scout_web/mix.exs +++ b/apps/block_scout_web/mix.exs @@ -19,7 +19,7 @@ defmodule BlockScoutWeb.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.0.2", + version: "11.0.3", xref: [ exclude: [ Explorer.Chain.Beacon.Reader, diff --git a/apps/ethereum_jsonrpc/mix.exs b/apps/ethereum_jsonrpc/mix.exs index 4e78bbc7d4a7..d4e9c47548db 100644 --- a/apps/ethereum_jsonrpc/mix.exs +++ b/apps/ethereum_jsonrpc/mix.exs @@ -19,7 +19,7 @@ defmodule EthereumJSONRPC.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.0.2" + version: "11.0.3" ] end diff --git a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex index d7a1a7e9f15f..8fe151942eb3 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/blocks.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/blocks.ex @@ -19,7 +19,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do BlockNumberHelper, DenormalizationHelper, Import, - PendingBlockOperation, PendingOperationsHelper, SmartContract, Token, @@ -106,16 +105,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do :blocks ) end) - |> Multi.run(:new_pending_block_operations, fn repo, %{blocks: blocks} -> - Instrumenter.block_import_stage_runner( - fn -> - new_pending_block_operations(repo, blocks, insert_options) - end, - :address_referencing, - :blocks, - :new_pending_block_operations - ) - end) |> Multi.run(:uncle_fetched_block_second_degree_relations, fn repo, _ -> Instrumenter.block_import_stage_runner( fn -> @@ -695,32 +684,6 @@ defmodule Explorer.Chain.Import.Runner.Blocks do lose_consensus(repo, blocks_changes, opts) end - defp new_pending_block_operations(repo, inserted_blocks, %{timeout: timeout, timestamps: timestamps}) do - case PendingOperationsHelper.pending_operations_type() do - "blocks" -> - sorted_pending_ops = - inserted_blocks - |> RangesHelper.filter_by_height_range(&RangesHelper.traceable_block_number?(&1.number)) - |> Enum.filter(& &1.consensus) - |> Enum.map(&%{block_hash: &1.hash, block_number: &1.number}) - |> Enum.sort() - - Import.insert_changes_list( - repo, - sorted_pending_ops, - conflict_target: :block_hash, - on_conflict: :nothing, - for: PendingBlockOperation, - returning: true, - timeout: timeout, - timestamps: timestamps - ) - - _other_type -> - {:ok, []} - end - end - defp delete_address_coin_balances(_repo, [], _options), do: {:ok, []} defp delete_address_coin_balances(repo, non_consensus_blocks, %{timeout: timeout}) do diff --git a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex index 117302585f64..66db3a1a90e9 100644 --- a/apps/explorer/lib/explorer/chain/import/runner/transactions.ex +++ b/apps/explorer/lib/explorer/chain/import/runner/transactions.ex @@ -10,7 +10,17 @@ defmodule Explorer.Chain.Import.Runner.Transactions do alias Ecto.{Multi, Repo} alias EthereumJSONRPC.Utility.RangesHelper - alias Explorer.Chain.{Block, Hash, Import, PendingOperationsHelper, PendingTransactionOperation, Transaction} + + alias Explorer.Chain.{ + Block, + Hash, + Import, + PendingBlockOperation, + PendingOperationsHelper, + PendingTransactionOperation, + Transaction + } + alias Explorer.Chain.Import.Runner.TokenTransfers alias Explorer.Prometheus.Instrumenter alias Explorer.Utility.MissingBlockRange @@ -66,14 +76,14 @@ defmodule Explorer.Chain.Import.Runner.Transactions do :transactions ) end) - |> Multi.run(:new_pending_transaction_operations, fn repo, %{transactions: transactions} -> + |> Multi.run(:new_pending_operations, fn repo, %{transactions: transactions} -> Instrumenter.block_import_stage_runner( fn -> - new_pending_transaction_operations(repo, transactions, insert_options) + new_pending_operations(repo, transactions, insert_options) end, :block_referencing, :transactions, - :new_pending_transaction_operations + :new_pending_operations ) end) end @@ -105,7 +115,10 @@ defmodule Explorer.Chain.Import.Runner.Transactions do on_conflict = Map.get_lazy(options, :on_conflict, &default_on_conflict/0) # Enforce Transaction ShareLocks order (see docs: sharelocks.md) - ordered_changes_list = Enum.sort_by(changes_list, & &1.hash) + ordered_changes_list = + changes_list + |> Enum.uniq_by(& &1.hash) + |> Enum.sort_by(& &1.hash) Import.insert_changes_list( repo, @@ -119,15 +132,21 @@ defmodule Explorer.Chain.Import.Runner.Transactions do ) end - defp new_pending_transaction_operations(repo, inserted_transactions, %{timeout: timeout, timestamps: timestamps}) do + defp new_pending_operations(repo, inserted_transactions, %{timeout: timeout, timestamps: timestamps}) do + traceable_consensus_transactions = + inserted_transactions + |> RangesHelper.filter_by_height_range(&RangesHelper.traceable_block_number?(&1.block_number)) + |> Transaction.filter_non_traceable_transactions() + + traceable_consensus_block_numbers = Enum.map(traceable_consensus_transactions, & &1.block_number) |> Enum.uniq() + block_numbers_with_priorities = MissingBlockRange.find_priority_by_numbers(traceable_consensus_block_numbers) + case PendingOperationsHelper.pending_operations_type() do "transactions" -> sorted_pending_ops = - inserted_transactions - |> RangesHelper.filter_by_height_range(&RangesHelper.traceable_block_number?(&1.block_number)) - |> Transaction.filter_non_traceable_transactions() + traceable_consensus_transactions |> Enum.reject(&is_nil(&1.block_number)) - |> Enum.map(&%{transaction_hash: &1.hash}) + |> Enum.map(&%{transaction_hash: &1.hash, priority: Map.get(block_numbers_with_priorities, &1.block_number)}) |> Enum.sort() Import.insert_changes_list( @@ -141,8 +160,29 @@ defmodule Explorer.Chain.Import.Runner.Transactions do timestamps: timestamps ) - _other_type -> - {:ok, []} + "blocks" -> + sorted_pending_ops = + traceable_consensus_transactions + |> Enum.reject(&is_nil(&1.block_number)) + |> Enum.map( + &%{ + block_hash: &1.block_hash, + block_number: &1.block_number, + priority: Map.get(block_numbers_with_priorities, &1.block_number) + } + ) + |> Enum.sort() + + Import.insert_changes_list( + repo, + sorted_pending_ops, + conflict_target: :block_hash, + on_conflict: :nothing, + for: PendingBlockOperation, + returning: true, + timeout: timeout, + timestamps: timestamps + ) end end diff --git a/apps/explorer/lib/explorer/chain/pending_block_operation.ex b/apps/explorer/lib/explorer/chain/pending_block_operation.ex index c58094cb2767..634dcb6bb03d 100644 --- a/apps/explorer/lib/explorer/chain/pending_block_operation.ex +++ b/apps/explorer/lib/explorer/chain/pending_block_operation.ex @@ -28,6 +28,8 @@ defmodule Explorer.Chain.PendingBlockOperation do type: Hash.Full, null: false ) + + field(:priority, :integer) end def changeset(%__MODULE__{} = pending_ops, attrs) do @@ -80,7 +82,7 @@ defmodule Explorer.Chain.PendingBlockOperation do limited? :: boolean() ) :: {:ok, accumulator} when accumulator: term() - def stream_blocks_with_unfetched_internal_transactions(initial, reducer, limited? \\ false) + def stream_blocks_with_unfetched_internal_transactions(initial, reducer, limited? \\ false, with_priority? \\ false) when is_function(reducer, 2) do direction = Application.get_env(:indexer, :internal_transactions_fetch_order) @@ -93,7 +95,16 @@ defmodule Explorer.Chain.PendingBlockOperation do ) query + |> maybe_add_priority_filter(with_priority?) |> add_fetcher_limit(limited?) |> Repo.stream_reduce(initial, reducer) end + + defp maybe_add_priority_filter(query, false), do: query + + defp maybe_add_priority_filter(query, true) do + from(pbo in query, + where: not is_nil(pbo.priority) + ) + end end diff --git a/apps/explorer/lib/explorer/chain/pending_operations_helper.ex b/apps/explorer/lib/explorer/chain/pending_operations_helper.ex index 6825c8a69fb7..cc6f9f0bf72c 100644 --- a/apps/explorer/lib/explorer/chain/pending_operations_helper.ex +++ b/apps/explorer/lib/explorer/chain/pending_operations_helper.ex @@ -3,7 +3,7 @@ defmodule Explorer.Chain.PendingOperationsHelper do import Ecto.Query - alias Explorer.Chain.{Hash, PendingBlockOperation, PendingTransactionOperation, Transaction} + alias Explorer.Chain.{Block, Hash, PendingBlockOperation, PendingTransactionOperation, Transaction} alias Explorer.{Helper, Repo} defp transactions_batch_size, @@ -81,7 +81,7 @@ defmodule Explorer.Chain.PendingOperationsHelper do from( pto in PendingTransactionOperation, join: t in assoc(pto, :transaction), - select: %{block_hash: t.block_hash, block_number: t.block_number}, + select: %{block_hash: t.block_hash, block_number: t.block_number, priority: pto.priority}, limit: ^batch_size ) @@ -246,4 +246,82 @@ defmodule Explorer.Chain.PendingOperationsHelper do |> block_range_in_query(max_block_number) |> Repo.exists?() end + + @doc """ + Inserts pending operations for the given block numbers. + """ + @spec insert_pending_operations([integer()], integer() | nil) :: {[integer()], [Explorer.Chain.Transaction.t()]} + def insert_pending_operations(block_numbers, priority \\ nil) do + case pending_operations_type() do + "transactions" -> + default_on_conflict = default_pto_on_conflict() + transactions = Transaction.get_transactions_of_block_numbers(block_numbers) + + pto_params = + transactions + |> Transaction.filter_non_traceable_transactions() + |> Enum.map(&%{transaction_hash: &1.hash, priority: priority}) + |> Helper.add_timestamps() + + Repo.insert_all(PendingTransactionOperation, pto_params, + on_conflict: default_on_conflict, + conflict_target: [:transaction_hash] + ) + + {[], transactions} + + "blocks" -> + default_on_conflict = default_pbo_on_conflict() + + pbo_params = + Block + |> where([b], b.number in ^block_numbers) + |> where([b], b.consensus == true) + |> select([b], %{block_hash: b.hash, block_number: b.number}) + |> Repo.all() + |> add_priority(priority) + |> Helper.add_timestamps() + + {_total, inserted} = + Repo.insert_all(PendingBlockOperation, pbo_params, + on_conflict: default_on_conflict, + conflict_target: [:block_hash], + returning: [:block_number] + ) + + {Enum.map(inserted, & &1.block_number), []} + end + end + + defp default_pbo_on_conflict do + from( + pending_block_operation in PendingBlockOperation, + update: [ + set: [ + priority: fragment("EXCLUDED.priority"), + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", pending_block_operation.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", pending_block_operation.updated_at) + ] + ], + where: is_nil(pending_block_operation.priority) and fragment("EXCLUDED.priority IS NOT NULL") + ) + end + + defp default_pto_on_conflict do + from( + pending_transaction_operation in PendingTransactionOperation, + update: [ + set: [ + priority: fragment("EXCLUDED.priority"), + inserted_at: fragment("LEAST(?, EXCLUDED.inserted_at)", pending_transaction_operation.inserted_at), + updated_at: fragment("GREATEST(?, EXCLUDED.updated_at)", pending_transaction_operation.updated_at) + ] + ], + where: is_nil(pending_transaction_operation.priority) and fragment("EXCLUDED.priority IS NOT NULL") + ) + end + + defp add_priority(params, priority) do + Enum.map(params, &Map.merge(&1, %{priority: priority})) + end end diff --git a/apps/explorer/lib/explorer/chain/pending_transaction_operation.ex b/apps/explorer/lib/explorer/chain/pending_transaction_operation.ex index 0f7d085377c9..e916810f9680 100644 --- a/apps/explorer/lib/explorer/chain/pending_transaction_operation.ex +++ b/apps/explorer/lib/explorer/chain/pending_transaction_operation.ex @@ -26,6 +26,8 @@ defmodule Explorer.Chain.PendingTransactionOperation do type: Hash.Full, null: false ) + + field(:priority, :integer) end def changeset(%__MODULE__{} = pending_ops, attrs) do @@ -69,10 +71,17 @@ defmodule Explorer.Chain.PendingTransactionOperation do """ @spec stream_transactions_with_unfetched_internal_transactions( initial :: accumulator, - reducer :: (entry :: term(), accumulator -> accumulator) + reducer :: (entry :: term(), accumulator -> accumulator), + limited? :: boolean(), + with_priority? :: boolean() ) :: {:ok, accumulator} when accumulator: term() - def stream_transactions_with_unfetched_internal_transactions(initial, reducer, limited? \\ false) + def stream_transactions_with_unfetched_internal_transactions( + initial, + reducer, + limited? \\ false, + with_priority? \\ false + ) when is_function(reducer, 2) do direction = Application.get_env(:indexer, :internal_transactions_fetch_order) @@ -85,7 +94,16 @@ defmodule Explorer.Chain.PendingTransactionOperation do ) query + |> maybe_add_priority_filter(with_priority?) |> add_fetcher_limit(limited?) |> Repo.stream_reduce(initial, reducer) end + + defp maybe_add_priority_filter(query, false), do: query + + defp maybe_add_priority_filter(query, true) do + from(pto in query, + where: not is_nil(pto.priority) + ) + end end diff --git a/apps/explorer/lib/explorer/migrator/heavy_db_index_operation/remove_internal_transactions_block_hash_transaction_hash_block_index_error.ex b/apps/explorer/lib/explorer/migrator/heavy_db_index_operation/remove_internal_transactions_block_hash_transaction_hash_block_index_error.ex index 503d54e4281f..3380136687ab 100644 --- a/apps/explorer/lib/explorer/migrator/heavy_db_index_operation/remove_internal_transactions_block_hash_transaction_hash_block_index_error.ex +++ b/apps/explorer/lib/explorer/migrator/heavy_db_index_operation/remove_internal_transactions_block_hash_transaction_hash_block_index_error.ex @@ -109,6 +109,15 @@ defmodule Explorer.Migrator.HeavyDbIndexOperation.RemoveInternalTransactionsBloc # sobelow_skip ["SQL"] defp drop_blocks_foreign_key do + if foreign_key_exists?("internal_transactions_block_hash_fkey") do + do_drop_blocks_foreign_key() + else + {:ok, :dropped} + end + end + + # sobelow_skip ["SQL"] + defp do_drop_blocks_foreign_key do Repo.transaction( fn -> with {:ok, _} <- Repo.query(lock_blocks_query_string(), [], timeout: :infinity), @@ -125,9 +134,19 @@ defmodule Explorer.Migrator.HeavyDbIndexOperation.RemoveInternalTransactionsBloc # sobelow_skip ["SQL"] defp drop_transactions_foreign_key do + if foreign_key_exists?("internal_transactions_transaction_hash_fkey") do + do_drop_transactions_foreign_key() + else + {:ok, :dropped} + end + end + + # sobelow_skip ["SQL"] + defp do_drop_transactions_foreign_key do Repo.transaction( fn -> - with {:ok, _} <- Repo.query(lock_transactions_query_string(), [], timeout: :infinity), + with {:ok, _} <- Repo.query(lock_internal_transactions_query_string(), [], timeout: :infinity), + {:ok, _} <- Repo.query(lock_transactions_query_string(), [], timeout: :infinity), {:ok, _} <- Repo.query(drop_transactions_foreign_key_query_string(), [], timeout: :infinity) do Logger.info( "Migration RemoveInternalTransactionsBlockHashTransactionHashBlockIndexError finished transactions" @@ -142,6 +161,20 @@ defmodule Explorer.Migrator.HeavyDbIndexOperation.RemoveInternalTransactionsBloc ) end + # sobelow_skip ["SQL"] + defp foreign_key_exists?(foreign_key_name) do + {:ok, %Postgrex.Result{rows: [[exists?]]}} = + Repo.query(""" + SELECT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = '#{foreign_key_name}' + ); + """) + + exists? + end + defp lock_blocks_query_string do """ LOCK TABLE blocks IN ACCESS EXCLUSIVE MODE; @@ -154,6 +187,12 @@ defmodule Explorer.Migrator.HeavyDbIndexOperation.RemoveInternalTransactionsBloc """ end + defp lock_internal_transactions_query_string do + """ + LOCK TABLE internal_transactions IN ACCESS EXCLUSIVE MODE; + """ + end + defp drop_blocks_foreign_key_query_string do """ ALTER TABLE #{@table_name} diff --git a/apps/explorer/lib/explorer/token/metadata_retriever.ex b/apps/explorer/lib/explorer/token/metadata_retriever.ex index 70e3f092b690..c0612f258315 100644 --- a/apps/explorer/lib/explorer/token/metadata_retriever.ex +++ b/apps/explorer/lib/explorer/token/metadata_retriever.ex @@ -14,7 +14,7 @@ defmodule Explorer.Token.MetadataRetriever do @vm_execution_error "VM execution error" @invalid_base64_data "invalid data:application/json;base64" @invalid_ipfs_path "invalid ipfs path" - @default_headers [{"User-Agent", "blockscout-11.0.2"}] + @default_headers [{"User-Agent", "blockscout-11.0.3"}] # https://eips.ethereum.org/EIPS/eip-1155#metadata @erc1155_token_id_placeholder "{id}" diff --git a/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex b/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex index d81ddda40043..cb6c5e726a93 100644 --- a/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex +++ b/apps/explorer/lib/explorer/utility/address_id_to_address_hash.ex @@ -76,7 +76,7 @@ defmodule Explorer.Utility.AddressIdToAddressHash do end) |> Enum.sort() - Repo.insert_all( + Repo.safe_insert_all( __MODULE__, Enum.map(filtered_address_hashes, &%{address_hash: &1}), on_conflict: :nothing diff --git a/apps/explorer/lib/explorer/utility/missing_block_range.ex b/apps/explorer/lib/explorer/utility/missing_block_range.ex index 021a99e70b06..0c17386ab7ea 100644 --- a/apps/explorer/lib/explorer/utility/missing_block_range.ex +++ b/apps/explorer/lib/explorer/utility/missing_block_range.ex @@ -711,4 +711,33 @@ defmodule Explorer.Utility.MissingBlockRange do fn range -> {:cont, range, nil} end ) end + + @doc """ + Finds the priority of missing block ranges for a list of block numbers. + + This function takes a list of block numbers and checks which missing block ranges + they fall into, returning a list of maps with the block number and its associated + priority (if any). + + ## Parameters + - `numbers`: A list of block numbers to check against the missing block ranges. + + ## Returns + - A list of tuples, each containing: + - `:number`: The block number from the input list. + - `:priority`: The priority of the range that includes the block number. + """ + @spec find_priority_by_numbers([Block.block_number()]) :: %{Block.block_number() => integer() | nil} + def find_priority_by_numbers(numbers) do + query = + from(i in fragment("SELECT unnest(?::int[]) AS number", ^numbers), + left_join: m in __MODULE__, + on: fragment("? BETWEEN ? AND ?", i.number, m.to_number, m.from_number), + select: {i.number, m.priority} + ) + + query + |> Repo.all() + |> Enum.into(%{}) + end end diff --git a/apps/explorer/mix.exs b/apps/explorer/mix.exs index bbabe942ea7a..92908370ca04 100644 --- a/apps/explorer/mix.exs +++ b/apps/explorer/mix.exs @@ -20,7 +20,7 @@ defmodule Explorer.Mixfile do lockfile: "../../mix.lock", package: package(), start_permanent: Mix.env() == :prod, - version: "11.0.2", + version: "11.0.3", xref: [exclude: [BlockScoutWeb.Routers.WebRouter.Helpers, Indexer.Helper, Indexer.Fetcher.InternalTransaction]] ] end diff --git a/apps/explorer/priv/repo/migrations/20260506124737_add_priority_to_pending_operations.exs b/apps/explorer/priv/repo/migrations/20260506124737_add_priority_to_pending_operations.exs new file mode 100644 index 000000000000..9027cf6d3750 --- /dev/null +++ b/apps/explorer/priv/repo/migrations/20260506124737_add_priority_to_pending_operations.exs @@ -0,0 +1,16 @@ +defmodule Explorer.Repo.Migrations.AddPriorityToPendingOperations do + use Ecto.Migration + + def change do + alter table(:pending_block_operations) do + add(:priority, :smallint) + end + + alter table(:pending_transaction_operations) do + add(:priority, :smallint) + end + + create(index(:pending_block_operations, :priority)) + create(index(:pending_transaction_operations, :priority)) + end +end diff --git a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs index eaa61c9ec23e..386fdf963c78 100644 --- a/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs +++ b/apps/explorer/test/explorer/chain/import/runner/blocks_test.exs @@ -539,50 +539,6 @@ defmodule Explorer.Chain.Import.Runner.BlocksTest do assert %{from_number: ^block_number, to_number: ^block_number} = Repo.one(MissingBlockRange) end - test "inserts pending_block_operations only for consensus blocks", - %{consensus_block: %{miner_hash: miner_hash}, options: options} do - %{number: number, hash: hash} = new_block = params_for(:block, miner_hash: miner_hash, consensus: true) - new_block1 = params_for(:block, miner_hash: miner_hash, consensus: false) - - %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, new_block) - %Ecto.Changeset{valid?: true, changes: block_changes1} = Block.changeset(%Block{}, new_block1) - - config = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth) - Application.put_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth, Keyword.put(config, :block_traceable?, true)) - - on_exit(fn -> Application.put_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth, config) end) - - Multi.new() - |> Blocks.run([block_changes, block_changes1], options) - |> Repo.transaction() - - assert %{block_number: ^number, block_hash: ^hash} = Repo.one(PendingBlockOperation) - end - - test "inserts pending_block_operations only for actually inserted blocks", - %{consensus_block: %{miner_hash: miner_hash}, options: options} do - %{number: number, hash: hash} = new_block = params_for(:block, miner_hash: miner_hash, consensus: true) - new_block1 = params_for(:block, miner_hash: miner_hash, consensus: true) - - miner = Repo.get_by(Address, hash: miner_hash) - - insert(:block, Map.put(new_block1, :miner, miner)) - - %Ecto.Changeset{valid?: true, changes: block_changes} = Block.changeset(%Block{}, new_block) - %Ecto.Changeset{valid?: true, changes: block_changes1} = Block.changeset(%Block{}, new_block1) - - config = Application.get_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth) - Application.put_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth, Keyword.put(config, :block_traceable?, true)) - - on_exit(fn -> Application.put_env(:ethereum_jsonrpc, EthereumJSONRPC.Geth, config) end) - - Multi.new() - |> Blocks.run([block_changes, block_changes1], options) - |> Repo.transaction() - - assert %{block_number: ^number, block_hash: ^hash} = Repo.one(PendingBlockOperation) - end - test "change instance owner if was token transfer in older blocks", %{consensus_block: %{hash: block_hash, miner_hash: miner_hash, number: block_number}, options: options} do block_number = block_number + 2 diff --git a/apps/indexer/lib/indexer/block/fetcher.ex b/apps/indexer/lib/indexer/block/fetcher.ex index f330de90f5f7..be62c3673ad4 100644 --- a/apps/indexer/lib/indexer/block/fetcher.ex +++ b/apps/indexer/lib/indexer/block/fetcher.ex @@ -585,10 +585,12 @@ defmodule Indexer.Block.Fetcher do def async_import_created_contract_codes(_, _), do: :ok + def async_import_internal_transactions(_imported, _realtime?) + def async_import_internal_transactions(%{blocks: blocks} = imported, realtime?) do blocks |> Enum.map(fn %Block{number: block_number} -> block_number end) - |> InternalTransaction.async_fetch(Map.get(imported, :transactions, []), realtime?, 10_000) + |> InternalTransaction.async_fetch(Map.get(imported, :transactions, []), realtime?, false, 10_000) end def async_import_internal_transactions(_, _), do: :ok diff --git a/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex b/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex index 84a4d18f4b8d..3c2bbf75e29a 100644 --- a/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex +++ b/apps/indexer/lib/indexer/fetcher/filecoin/address_info.ex @@ -118,7 +118,7 @@ defmodule Indexer.Fetcher.Filecoin.AddressInfo do @impl BufferedTask @decorate trace( name: "fetch", - resource: "Indexer.Fetcher.InternalTransaction.run/2", + resource: "Indexer.Fetcher.Filecoin.AddressInfo.run/2", service: :indexer, tracer: Tracer ) diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex index 0cffa66cb8e2..7b0d60365841 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction.ex @@ -25,7 +25,6 @@ defmodule Indexer.Fetcher.InternalTransaction do alias Explorer.Chain.{Block, Hash, PendingBlockOperation, PendingTransactionOperation, Transaction} alias Explorer.Chain.Cache.{Accounts, Blocks} alias Indexer.{BufferedTask, Tracer} - alias Indexer.Fetcher.InternalTransaction.Supervisor, as: InternalTransactionSupervisor alias Indexer.Transform.{AddressCoinBalances, Addresses, AddressTokenBalances} alias Indexer.Transform.Celo.TransactionTokenTransfers, as: CeloTransactionTokenTransfers @@ -50,9 +49,10 @@ defmodule Indexer.Fetcher.InternalTransaction do *Note*: The internal transactions for individual transactions cannot be paginated, so the total number of internal transactions that could be produced is unknown. """ - @spec async_fetch([Block.block_number()], [Transaction.t()], boolean()) :: :ok - def async_fetch(block_numbers, transactions, realtime?, timeout \\ 5000) when is_list(block_numbers) do - if InternalTransactionSupervisor.disabled?() do + @spec async_fetch([Block.block_number()], [Transaction.t()], boolean(), boolean(), integer()) :: :ok + def async_fetch(block_numbers, transactions, realtime?, for_contract_creator? \\ false, timeout \\ 5000) + when is_list(block_numbers) do + if disabled?() && !for_contract_creator? do :ok else data = @@ -95,16 +95,42 @@ defmodule Indexer.Fetcher.InternalTransaction do def init(initial, reducer, json_rpc_named_arguments) do stream_reducer = RangesHelper.stream_reducer_traceable(reducer) - {:ok, final} = - case queue_data_type(json_rpc_named_arguments) do - :block_number -> - PendingBlockOperation.stream_blocks_with_unfetched_internal_transactions(initial, stream_reducer) + if disabled?() do + {:ok, final} = + case queue_data_type(json_rpc_named_arguments) do + :block_number -> + PendingBlockOperation.stream_blocks_with_unfetched_internal_transactions( + initial, + stream_reducer, + false, + true + ) - :transaction_params -> - PendingTransactionOperation.stream_transactions_with_unfetched_internal_transactions(initial, stream_reducer) - end + :transaction_params -> + PendingTransactionOperation.stream_transactions_with_unfetched_internal_transactions( + initial, + stream_reducer, + false, + true + ) + end + + final + else + {:ok, final} = + case queue_data_type(json_rpc_named_arguments) do + :block_number -> + PendingBlockOperation.stream_blocks_with_unfetched_internal_transactions(initial, stream_reducer) + + :transaction_params -> + PendingTransactionOperation.stream_transactions_with_unfetched_internal_transactions( + initial, + stream_reducer + ) + end - final + final + end end defp params(%{block_number: block_number, hash: hash, index: index}) when is_integer(block_number) do @@ -449,7 +475,7 @@ defmodule Indexer.Fetcher.InternalTransaction do def defaults do [ - poll: false, + poll: true, flush_interval: :timer.seconds(3), max_concurrency: Application.get_env(:indexer, __MODULE__)[:concurrency] || @default_max_concurrency, max_batch_size: Application.get_env(:indexer, __MODULE__)[:batch_size] || @default_max_batch_size, @@ -483,4 +509,14 @@ defmodule Indexer.Fetcher.InternalTransaction do error -> Logger.error("Failed to cast string to hash: #{inspect(error)}") end end + + @doc """ + Returns whether the internal transaction fetcher is disabled. + + This can be used to conditionally disable fetching internal transactions, for example, in a staging environment where the load on the JSON-RPC should be minimized. + """ + @spec disabled? :: boolean() + def disabled? do + Application.get_env(:indexer, __MODULE__, [])[:disabled?] == true + end end diff --git a/apps/indexer/lib/indexer/fetcher/internal_transaction/delete_queue.ex b/apps/indexer/lib/indexer/fetcher/internal_transaction/delete_queue.ex index 3b79120e32a6..b20384d148da 100644 --- a/apps/indexer/lib/indexer/fetcher/internal_transaction/delete_queue.ex +++ b/apps/indexer/lib/indexer/fetcher/internal_transaction/delete_queue.ex @@ -11,17 +11,9 @@ defmodule Indexer.Fetcher.InternalTransaction.DeleteQueue do alias Explorer.Repo - alias Explorer.Chain.{ - Block, - InternalTransaction, - PendingBlockOperation, - PendingOperationsHelper, - PendingTransactionOperation, - Transaction - } + alias Explorer.Chain.{InternalTransaction, PendingOperationsHelper} alias Explorer.Chain.InternalTransaction.DeleteQueue - alias Explorer.Helper, as: ExplorerHelper alias Indexer.BufferedTask alias Indexer.Fetcher.InternalTransaction, as: InternalTransactionFetcher alias Indexer.Helper, as: IndexerHelper @@ -68,7 +60,7 @@ defmodule Indexer.Fetcher.InternalTransaction.DeleteQueue do |> where([it], it.block_number in ^block_numbers) |> Repo.delete_all(timeout: :infinity) - insert_pending_operations(block_numbers) + PendingOperationsHelper.insert_pending_operations(block_numbers) end) case result do @@ -85,36 +77,6 @@ defmodule Indexer.Fetcher.InternalTransaction.DeleteQueue do end end - defp insert_pending_operations(block_numbers) do - case PendingOperationsHelper.pending_operations_type() do - "transactions" -> - transactions = Transaction.get_transactions_of_block_numbers(block_numbers) - - pto_params = - transactions - |> Transaction.filter_non_traceable_transactions() - |> Enum.map(&%{transaction_hash: &1.hash}) - |> ExplorerHelper.add_timestamps() - - Repo.insert_all(PendingTransactionOperation, pto_params, on_conflict: :nothing) - {[], transactions} - - "blocks" -> - pbo_params = - Block - |> where([b], b.number in ^block_numbers) - |> where([b], b.consensus == true) - |> select([b], %{block_hash: b.hash, block_number: b.number}) - |> Repo.all() - |> ExplorerHelper.add_timestamps() - - {_total, inserted} = - Repo.insert_all(PendingBlockOperation, pbo_params, on_conflict: :nothing, returning: [:block_number]) - - {Enum.map(inserted, & &1.block_number), []} - end - end - defp defaults do [ flush_interval: :timer.seconds(10), diff --git a/apps/indexer/lib/indexer/fetcher/on_demand/contract_creator.ex b/apps/indexer/lib/indexer/fetcher/on_demand/contract_creator.ex index 2d900be6a9ca..9dda8e4a00a4 100644 --- a/apps/indexer/lib/indexer/fetcher/on_demand/contract_creator.ex +++ b/apps/indexer/lib/indexer/fetcher/on_demand/contract_creator.ex @@ -11,14 +11,11 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do import EthereumJSONRPC, only: [id_to_params: 1, integer_to_quantity: 1, json_rpc: 2] alias EthereumJSONRPC.Nonce - alias Explorer.Chain.{Address, Block} + alias EthereumJSONRPC.Utility.RangesHelper + alias Explorer.Chain.{Address, Block, PendingOperationsHelper} alias Explorer.Chain.Cache.BlockNumber alias Explorer.Utility.MissingBlockRange - - import Indexer.Block.Fetcher, - only: [ - async_import_internal_transactions: 2 - ] + alias Indexer.Fetcher.InternalTransaction @table_name :contract_creator_lookup @pending_blocks_cache_key "pending_blocks" @@ -176,7 +173,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do address_hash = address.hash :ets.insert(@table_name, {address_cache_name(address_hash), :in_progress}) - Task.Supervisor.start_child(Indexer.TaskSupervisor, fn -> + Task.start(fn -> result = fetch_contract_creator_address_hash(address_hash) GenServer.cast(__MODULE__, {:fetch_result, address_hash, result}) end) @@ -206,9 +203,25 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do :ets.insert(@table_name, {address_cache_name(address_hash), contract_creation_block_number}) :ets.insert(@table_name, {@pending_blocks_cache_key, updated_pending_blocks}) - unless Block.indexed?(contract_creation_block_number) do - # Change `1` to specific label when `priority` field becomes `Ecto.Enum`. - MissingBlockRange.add_ranges_by_block_numbers([contract_creation_block_number], 1) + # Change `1` to specific label when `priority` field becomes `Ecto.Enum`. + priority = 1 + + if InternalTransaction.disabled?() do + if Block.indexed?(contract_creation_block_number) do + # credo:disable-for-lines:2 Credo.Check.Refactor.Nesting + if RangesHelper.traceable_block_number?(contract_creation_block_number) do + {block_numbers, transactions} = + PendingOperationsHelper.insert_pending_operations([contract_creation_block_number], priority) + + InternalTransaction.async_fetch(block_numbers, transactions, false, true, 10_000) + end + else + MissingBlockRange.add_ranges_by_block_numbers([contract_creation_block_number], priority) + end + else + unless Block.indexed?(contract_creation_block_number) do + MissingBlockRange.add_ranges_by_block_numbers([contract_creation_block_number], priority) + end end {:noreply, state} @@ -232,7 +245,7 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do |> Map.get(:blocks, []) |> Enum.map(&Map.get(&1, :number)) - maybe_update_pending_contract_creator_cache(imported, imported_block_numbers) + maybe_update_pending_contract_creator_cache(imported_block_numbers) {:noreply, state} end @@ -273,30 +286,24 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do :ok end - defp maybe_update_pending_contract_creator_cache(_imported, []), do: :ok + defp maybe_update_pending_contract_creator_cache([]), do: :ok - defp maybe_update_pending_contract_creator_cache(imported, imported_block_numbers) do + defp maybe_update_pending_contract_creator_cache(imported_block_numbers) do case pending_blocks_cache() do [{@pending_blocks_cache_key, pending_blocks}] -> - update_pending_contract_creator_blocks(pending_blocks, imported_block_numbers, imported) + update_pending_contract_creator_blocks(pending_blocks, imported_block_numbers) [] -> :ok end end - defp update_pending_contract_creator_blocks([], _imported_block_numbers, _imported), do: [] + defp update_pending_contract_creator_blocks([], _imported_block_numbers), do: [] - defp update_pending_contract_creator_blocks(pending_blocks, imported_block_numbers, imported) do + defp update_pending_contract_creator_blocks(pending_blocks, imported_block_numbers) do updated_pending_block_numbers = Enum.filter(pending_blocks, fn pending_block -> if Enum.member?(imported_block_numbers, pending_block.block_number) do - contract_creation_block = - find_contract_creation_block_in_imported(imported, pending_block.block_number) - - internal_transactions_import_params = [%{blocks: [contract_creation_block]}] - async_import_internal_transactions(internal_transactions_import_params, true) - # todo: emit event that contract creator updated for the contract. This was the purpose keeping address_hash_string in that cache key. :ets.delete(@table_name, pending_block.address_hash_string) false @@ -310,10 +317,4 @@ defmodule Indexer.Fetcher.OnDemand.ContractCreator do {@pending_blocks_cache_key, updated_pending_block_numbers} ) end - - defp find_contract_creation_block_in_imported(imported, contract_creation_block_number) do - Enum.find(imported[:blocks], fn %Explorer.Chain.Block{number: block_number} -> - block_number == contract_creation_block_number - end) - end end diff --git a/apps/indexer/mix.exs b/apps/indexer/mix.exs index 36792750506a..6332a075c358 100644 --- a/apps/indexer/mix.exs +++ b/apps/indexer/mix.exs @@ -14,7 +14,7 @@ defmodule Indexer.MixProject do elixirc_paths: elixirc_paths(Mix.env()), lockfile: "../../mix.lock", start_permanent: Mix.env() == :prod, - version: "11.0.2", + version: "11.0.3", xref: [ exclude: [ Explorer.Chain.Optimism.Deposit, diff --git a/apps/indexer/test/indexer/block/fetcher_test.exs b/apps/indexer/test/indexer/block/fetcher_test.exs index 9ac37dd8457f..396960aa4856 100644 --- a/apps/indexer/test/indexer/block/fetcher_test.exs +++ b/apps/indexer/test/indexer/block/fetcher_test.exs @@ -298,6 +298,13 @@ defmodule Indexer.Block.FetcherTest do test "can import range with all synchronous imported schemas", %{ block_fetcher: %Fetcher{json_rpc_named_arguments: json_rpc_named_arguments} = block_fetcher } do + initial_env = Application.get_env(:indexer, Indexer.Fetcher.InternalTransaction) + + on_exit(fn -> + Application.put_env(:indexer, Indexer.Fetcher.InternalTransaction, initial_env) + end) + + Application.put_env(:indexer, Indexer.Fetcher.InternalTransaction, disabled?: false) block_number = @first_full_block_number if Application.get_env(:explorer, :chain_type) == :filecoin do diff --git a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs index 411ce5ba4506..2e973ce7b7dc 100644 --- a/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs +++ b/apps/indexer/test/indexer/fetcher/internal_transaction_test.exs @@ -134,9 +134,18 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end describe "init/2" do + setup do + initial_env = Application.get_env(:indexer, Indexer.Fetcher.InternalTransaction) + + on_exit(fn -> + Application.put_env(:indexer, Indexer.Fetcher.InternalTransaction, initial_env) + end) + end + test "buffers blocks with unfetched internal transactions", %{ json_rpc_named_arguments: json_rpc_named_arguments } do + Application.put_env(:indexer, Indexer.Fetcher.InternalTransaction, disabled?: false) block = insert(:block) insert(:pending_block_operation, block_hash: block.hash, block_number: block.number) @@ -332,41 +341,6 @@ defmodule Indexer.Fetcher.InternalTransactionTest do end end - test "doesn't delete pending block operations after block import if no async process was requested", %{ - json_rpc_named_arguments: json_rpc_named_arguments - } do - fetcher_options = - Keyword.merge([poll: true, json_rpc_named_arguments: json_rpc_named_arguments], InternalTransaction.defaults()) - - if fetcher_options[:poll] do - expect(EthereumJSONRPC.Mox, :json_rpc, fn [%{id: id}], _options -> - {:ok, [%{id: id, result: []}]} - end) - end - - InternalTransaction.Supervisor.Case.start_supervised!(fetcher_options) - - %Ecto.Changeset{valid?: true, changes: block_changes} = - Block.changeset(%Block{}, params_for(:block, miner_hash: insert(:address).hash, number: 1)) - - changes_list = [block_changes] - timestamp = DateTime.utc_now() - options = %{timestamps: %{inserted_at: timestamp, updated_at: timestamp}} - - assert [] = Repo.all(PendingBlockOperation) - - {:ok, %{blocks: [%{number: block_number, hash: block_hash}]}} = - Multi.new() - |> Blocks.run(changes_list, options) - |> Repo.transaction() - - assert %{block_number: ^block_number, block_hash: ^block_hash} = Repo.one(PendingBlockOperation) - - Process.sleep(4000) - - assert %{block_number: ^block_number, block_hash: ^block_hash} = Repo.one(PendingBlockOperation) - end - describe "filter_non_traceable_transactions/1" do test "does not raise when transaction params do not include type on zetachain" do chain_type = Application.get_env(:explorer, :chain_type) diff --git a/apps/nft_media_handler/mix.exs b/apps/nft_media_handler/mix.exs index 8dbe7ee8d3db..aae57c0f42f0 100644 --- a/apps/nft_media_handler/mix.exs +++ b/apps/nft_media_handler/mix.exs @@ -4,7 +4,7 @@ defmodule NFTMediaHandler.MixProject do def project do [ app: :nft_media_handler, - version: "11.0.2", + version: "11.0.3", build_path: "../../_build", config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/apps/utils/mix.exs b/apps/utils/mix.exs index 95b9ce17af94..b3abc85b878a 100644 --- a/apps/utils/mix.exs +++ b/apps/utils/mix.exs @@ -4,7 +4,7 @@ defmodule Utils.MixProject do def project do [ app: :utils, - version: "11.0.2", + version: "11.0.3", build_path: "../../_build", # config_path: "../../config/config.exs", deps_path: "../../deps", diff --git a/bin/install_chrome_headless.sh b/bin/install_chrome_headless.sh index 97d18d4115f8..1be34b17cc5c 100755 --- a/bin/install_chrome_headless.sh +++ b/bin/install_chrome_headless.sh @@ -1,8 +1,8 @@ export DISPLAY=:99.0 sh -e /etc/init.d/xvfb start -#export CHROMEDRIVER_VERSION=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json" | jq -r '.channels' | jq -r '.Stable' | jq -r '.version') -export CHROMEDRIVER_VERSION=146.0.7680.178 +export CHROMEDRIVER_VERSION=$(curl -s "https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions.json" | jq -r '.channels' | jq -r '.Stable' | jq -r '.version') +#export CHROMEDRIVER_VERSION=146.0.7680.178 curl -L -O "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/${CHROMEDRIVER_VERSION}/linux64/chromedriver-linux64.zip" unzip -j chromedriver-linux64.zip sudo chmod +x chromedriver diff --git a/config/runtime.exs b/config/runtime.exs index d0575056592e..5cad288fce93 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -1139,7 +1139,9 @@ config :indexer, Indexer.Fetcher.OnDemand.TokenInstanceMetadataRefetch, config :indexer, Indexer.Fetcher.BlockReward.Supervisor, disabled?: ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_BLOCK_REWARD_FETCHER") -config :indexer, Indexer.Fetcher.InternalTransaction.Supervisor, +config :indexer, Indexer.Fetcher.InternalTransaction.Supervisor, disabled?: trace_url_missing? + +config :indexer, Indexer.Fetcher.InternalTransaction, disabled?: trace_url_missing? or ConfigHelper.parse_bool_env_var("INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER") config :indexer, Indexer.Fetcher.OnDemand.InternalTransaction, diff --git a/cspell.json b/cspell.json index b30c5df00d0e..2c16e6a74a37 100644 --- a/cspell.json +++ b/cspell.json @@ -225,6 +225,7 @@ "disksup", "Docsify", "docstrings", + "doctest", "dparty", "dropzone", "dxgd", diff --git a/docker/Makefile b/docker/Makefile index 7e926cc62e9e..482cbbd02ee1 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -10,7 +10,7 @@ STATS_CONTAINER_NAME := stats STATS_DB_CONTAINER_NAME := stats-db PROXY_CONTAINER_NAME := proxy PG_CONTAINER_NAME := postgres -RELEASE_VERSION ?= '11.0.2' +RELEASE_VERSION ?= '11.0.3' TAG := $(RELEASE_VERSION)-commit-$(shell git log -1 --pretty=format:"%h") STABLE_TAG := $(RELEASE_VERSION) diff --git a/mix.exs b/mix.exs index f9af3f717175..f2bede6ad30f 100644 --- a/mix.exs +++ b/mix.exs @@ -7,7 +7,7 @@ defmodule BlockScout.Mixfile do [ # app: :block_scout, # aliases: aliases(config_env()), - version: "11.0.2", + version: "11.0.3", apps_path: "apps", deps: deps(), dialyzer: dialyzer(), diff --git a/rel/config.exs b/rel/config.exs index f7f6f2083017..ce0b8eecffef 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -71,7 +71,7 @@ end # will be used by default release :blockscout do - set version: "11.0.2" + set version: "11.0.3" set applications: [ :runtime_tools, block_scout_web: :permanent,