From ac36b2271a9cc9683d79fb10e6d249dbbb93e0df Mon Sep 17 00:00:00 2001 From: xiuy001 <97179309+xiuy001@users.noreply.github.com> Date: Tue, 9 May 2023 12:09:18 -0400 Subject: [PATCH 1/2] update (#61) --- models/silver/streamline/streamline__traces_realtime.sql | 2 +- models/silver/streamline/streamline__transactions_realtime.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/silver/streamline/streamline__traces_realtime.sql b/models/silver/streamline/streamline__traces_realtime.sql index 28dd1d6..06c7c25 100644 --- a/models/silver/streamline/streamline__traces_realtime.sql +++ b/models/silver/streamline/streamline__traces_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'exploded_key','[\"result\"]', 'method', 'trace_block', 'producer_batch_size',100, 'producer_limit_size', 2000000, 'worker_batch_size',10))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'traces', 'exploded_key','[\"result\"]', 'method', 'trace_block', 'producer_batch_size',200, 'producer_limit_size', 1000000, 'worker_batch_size',20))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/streamline__transactions_realtime.sql b/models/silver/streamline/streamline__transactions_realtime.sql index 4f22284..c76aafd 100644 --- a/models/silver/streamline/streamline__transactions_realtime.sql +++ b/models/silver/streamline/streamline__transactions_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 5000000, 'worker_batch_size',1000))", + func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',5000, 'producer_limit_size', 5000000, 'worker_batch_size',500))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} From f88c371430b06c6e152a1af1ec0a97976796a9e5 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 9 May 2023 14:06:57 -0400 Subject: [PATCH 2/2] release decoded event logs (#62) * release decoded event logs * workflows * update date * workflows2 * updated deprecation doc for clarity --------- Co-authored-by: drethereum --- .../workflows/dbt_run_streamline_decoder.yml | 3 +- .../dbt_run_streamline_decoder_history1.yml | 5 +- .../dbt_run_streamline_decoder_history2.yml | 5 +- .../dbt_run_streamline_decoder_history3.yml | 5 +- .../dbt_run_streamline_decoder_history4.yml | 5 +- .../dbt_run_streamline_decoder_history5.yml | 5 +- .../dbt_run_streamline_decoder_history6.yml | 5 +- .../dbt_run_temp_decoding_backfill.yml | 44 -------------- .../doc_descriptions/general/deprecation.md | 13 ++++ models/gold/core__ez_decoded_event_logs.sql | 27 +++++++++ models/gold/core__ez_decoded_event_logs.yml | 59 +++++++++++++++++++ models/gold/core__fact_decoded_event_logs.sql | 17 ++++++ models/gold/core__fact_decoded_event_logs.yml | 44 ++++++++++++++ models/gold/core__fact_event_logs.yml | 8 +-- models/gold/core__fact_transactions.yml | 2 +- models/silver/silver__decoded_logs.sql | 33 ++++------- 16 files changed, 195 insertions(+), 85 deletions(-) delete mode 100644 .github/workflows/dbt_run_temp_decoding_backfill.yml create mode 100644 models/doc_descriptions/general/deprecation.md create mode 100644 models/gold/core__ez_decoded_event_logs.sql create mode 100644 models/gold/core__ez_decoded_event_logs.yml create mode 100644 models/gold/core__fact_decoded_event_logs.sql create mode 100644 models/gold/core__fact_decoded_event_logs.yml diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml index d5b89f7..2e8b314 100644 --- a/.github/workflows/dbt_run_streamline_decoder.yml +++ b/.github/workflows/dbt_run_streamline_decoder.yml @@ -41,4 +41,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder --exclude models/silver/streamline/decoder/history \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder --exclude models/silver/streamline/decoder/history + dbt run -m models/silver/silver__decoded_logs.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history1.yml b/.github/workflows/dbt_run_streamline_decoder_history1.yml index 819ddd6..623785a 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history1.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history1.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history1 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 2:00 UTC PM" (see https://crontab.guru) + - cron: '0 14 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_decoder_history2.yml b/.github/workflows/dbt_run_streamline_decoder_history2.yml index 55f6898..f88781f 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history2.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history2.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history2 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 4:00 UTC PM" (see https://crontab.guru) + - cron: '0 16 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_decoder_history3.yml b/.github/workflows/dbt_run_streamline_decoder_history3.yml index 0bbcbfb..e9cef9c 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history3.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history3.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history3 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 6:00 UTC PM" (see https://crontab.guru) + - cron: '0 18 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_decoder_history4.yml b/.github/workflows/dbt_run_streamline_decoder_history4.yml index f5ea5b3..6b07be1 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history4.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history4.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history4 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 8:00 UTC PM" (see https://crontab.guru) + - cron: '0 20 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_decoder_history5.yml b/.github/workflows/dbt_run_streamline_decoder_history5.yml index f670604..0b93529 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history5.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history5.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history5 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 10:00 UTC PM" (see https://crontab.guru) + - cron: '0 22 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_decoder_history6.yml b/.github/workflows/dbt_run_streamline_decoder_history6.yml index 1d565fa..d199c3e 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history6.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history6.yml @@ -3,8 +3,9 @@ run-name: dbt_run_streamline_decoder_history6 on: workflow_dispatch: - branches: - - "main" + schedule: + # Runs "at 12:00 UTC PM" (see https://crontab.guru) + - cron: '0 24 * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_temp_decoding_backfill.yml b/.github/workflows/dbt_run_temp_decoding_backfill.yml deleted file mode 100644 index 55e617c..0000000 --- a/.github/workflows/dbt_run_temp_decoding_backfill.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_temp_decoding_backfill -run-name: dbt_run_temp_decoding_backfill - -on: - workflow_dispatch: - schedule: - # Runs "every 2 hours" (see https://crontab.guru) - - cron: '20 */2 * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m models/silver/silver__decoded_logs.sql diff --git a/models/doc_descriptions/general/deprecation.md b/models/doc_descriptions/general/deprecation.md new file mode 100644 index 0000000..7d77839 --- /dev/null +++ b/models/doc_descriptions/general/deprecation.md @@ -0,0 +1,13 @@ +{% docs deprecation %} + +Deprecating soon: This is a notice that we're only removing the below columns. Please migrate queries using these columns to `fact_decoded_event_logs`, `ez_decoded_event_logs` or use manual parsing of topics and data. The following columns will be deprecated on 6/6/23: + +`Fact_event_logs` Columns: +- `event_name` +- `event_inputs` +- `contract_name` + +`Fact_transactions` Columns: +- `tx_json` +{% enddocs %} + diff --git a/models/gold/core__ez_decoded_event_logs.sql b/models/gold/core__ez_decoded_event_logs.sql new file mode 100644 index 0000000..f6fa7ac --- /dev/null +++ b/models/gold/core__ez_decoded_event_logs.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + contract_address, + C.name AS contract_name, + event_name, + decoded_flat AS decoded_log, + decoded_data AS full_decoded_log, + origin_function_signature, + origin_from_address, + origin_to_address, + topics, + DATA, + event_removed, + tx_status +FROM + {{ ref('silver__decoded_logs') }} + LEFT JOIN {{ ref('core__dim_contracts') }} C + ON contract_address = C.address diff --git a/models/gold/core__ez_decoded_event_logs.yml b/models/gold/core__ez_decoded_event_logs.yml new file mode 100644 index 0000000..a24d89b --- /dev/null +++ b/models/gold/core__ez_decoded_event_logs.yml @@ -0,0 +1,59 @@ +version: 2 +models: + - name: core__ez_decoded_event_logs + description: > + 'For information on how to submit a contract for decoding, as well as how ABIs are sourced, please visit [here](https://science.flipsidecrypto.xyz/abi-requestor/). + This model contains decoded event logs for contracts that we have an ABI for. Please note, this table does not include all event logs, only those that we have an ABI for. + The `decoded_log` column is the easiest place to query decoded data. It is a JSON object, where the keys are the names of the event parameters, and the values are the values of the event parameters. + You can select from this column using the following sample format, `decoded_log:from::string` or more generally, `decoded_log:::datatype`. See below for a full sample query. + The `full_decoded_logs` column contains the same information, as well as additional fields such as the datatype of the decoded data. You may need to laterally flatten this column to query the data. + + Sample query for USDC Transfer events: + + ```sql + select + tx_hash, + block_number, + contract_address, + decoded_log:from::string as from_address, + decoded_log:to::string as to_address, + decoded_log:value::integer as value + from ethereum.core.fact_decoded_event_logs + where contract_address = lower('0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48') + and block_number between 16400000 and 16405000 + and event_name = 'Transfer' + limit 50```' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("bsc_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("bsc_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("bsc_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("bsc_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("bsc_logs_contract_address") }}' + - name: CONTRACT_NAME + description: 'The name of the contract, if the contract has a name() function.' + - name: EVENT_NAME + description: 'The name of the event, as defined in the contract ABI.' + - name: DECODED_LOG + description: 'The flattened decoded log, where the keys are the names of the event parameters, and the values are the values of the event parameters.' + - name: FULL_DECODED_LOG + description: 'The full decoded log, including the event name, the event parameters, and the data type of the event parameters.' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("bsc_tx_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("bsc_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("bsc_origin_to") }}' + - name: TOPICS + description: '{{ doc("bsc_topics") }}' + - name: DATA + description: '{{ doc("bsc_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("bsc_event_removed") }}' + - name: TX_STATUS + description: '{{ doc("bsc_tx_status") }}' \ No newline at end of file diff --git a/models/gold/core__fact_decoded_event_logs.sql b/models/gold/core__fact_decoded_event_logs.sql new file mode 100644 index 0000000..33f85a6 --- /dev/null +++ b/models/gold/core__fact_decoded_event_logs.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + event_index, + contract_address, + event_name, + decoded_flat AS decoded_log, + decoded_data AS full_decoded_log +FROM + {{ ref('silver__decoded_logs') }} diff --git a/models/gold/core__fact_decoded_event_logs.yml b/models/gold/core__fact_decoded_event_logs.yml new file mode 100644 index 0000000..a8b4dc7 --- /dev/null +++ b/models/gold/core__fact_decoded_event_logs.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core__fact_decoded_event_logs + description: > + 'For information on how to submit a contract for decoding, as well as how ABIs are sourced, please visit [here](https://science.flipsidecrypto.xyz/abi-requestor/). + This model contains decoded event logs for contracts that we have an ABI for. Please note, this table does not include all event logs, only those that we have an ABI for. + This table will perform better than the `core__ez_decoded_event_logs` table, but does not include as many columns. + The `decoded_log` column is the easiest place to query decoded data. It is a JSON object, where the keys are the names of the event parameters, and the values are the values of the event parameters. + You can select from this column using the following sample format, `decoded_log:from::string` or more generally, `decoded_log:::datatype`. See below for a full sample query. + The `full_decoded_logs` column contains the same information, as well as additional fields such as the datatype of the decoded data. You may need to laterally flatten this column to query the data. + + Sample query for USDC Transfer events: + + ```sql + select + tx_hash, + block_number, + contract_address, + decoded_log:from::string as from_address, + decoded_log:to::string as to_address, + decoded_log:value::integer as value + from ethereum.core.fact_decoded_event_logs + where contract_address = lower('0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48') + and block_number between 16400000 and 16405000 + and event_name = 'Transfer' + limit 50```' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("bsc_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("bsc_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("bsc_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("bsc_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("bsc_logs_contract_address") }}' + - name: EVENT_NAME + description: 'The name of the event, as defined in the contract ABI.' + - name: DECODED_LOG + description: 'The flattened decoded log, where the keys are the names of the event parameters, and the values are the values of the event parameters.' + - name: FULL_DECODED_LOG + description: 'The full decoded log, including the event name, the event parameters, and the data type of the event parameters.' diff --git a/models/gold/core__fact_event_logs.yml b/models/gold/core__fact_event_logs.yml index 45d138b..f17457a 100644 --- a/models/gold/core__fact_event_logs.yml +++ b/models/gold/core__fact_event_logs.yml @@ -1,7 +1,7 @@ version: 2 models: - name: core__fact_event_logs - description: '{{ doc("bsc_logs_table_doc") }}' + description: '{{ doc("deprecation") }}' columns: - name: BLOCK_NUMBER @@ -15,11 +15,11 @@ models: - name: CONTRACT_ADDRESS description: '{{ doc("bsc_logs_contract_address") }}' - name: CONTRACT_NAME - description: '{{ doc("bsc_logs_contract_name") }}' + description: '{{ doc("deprecation") }}' - name: EVENT_NAME - description: '{{ doc("bsc_event_name") }}' + description: '{{ doc("deprecation") }}' - name: EVENT_INPUTS - description: '{{ doc("bsc_event_inputs") }}' + description: '{{ doc("deprecation") }}' - name: TOPICS description: '{{ doc("bsc_topics") }}' - name: DATA diff --git a/models/gold/core__fact_transactions.yml b/models/gold/core__fact_transactions.yml index 79fcb18..f2518e2 100644 --- a/models/gold/core__fact_transactions.yml +++ b/models/gold/core__fact_transactions.yml @@ -35,7 +35,7 @@ models: - name: STATUS description: '{{ doc("bsc_tx_status") }}' - name: TX_JSON - description: '{{ doc("bsc_tx_json") }}' + description: '{{ doc("deprecation") }}' - name: INPUT_DATA description: '{{ doc("bsc_tx_input_data") }}' - name: ORIGIN_FUNCTION_SIGNATURE diff --git a/models/silver/silver__decoded_logs.sql b/models/silver/silver__decoded_logs.sql index fb9ee96..20004b0 100644 --- a/models/silver/silver__decoded_logs.sql +++ b/models/silver/silver__decoded_logs.sql @@ -9,7 +9,7 @@ WITH meta AS ( SELECT - registered_on AS job_created_time, + job_created_time, last_modified, TO_DATE( concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) @@ -18,29 +18,16 @@ WITH meta AS ( file_name FROM TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze_streamline", "decoded_logs") }}' + information_schema.external_table_file_registration_history( + table_name => '{{ source( "bronze_streamline", "decoded_logs") }}', + start_time => ( + SELECT + DATEADD('hour', -6, MAX(_INSERTED_TIMESTAMP)) + FROM + {{ this }} + ) ) - ) A - WHERE - 1 = 1 - -{% if is_incremental() %} -AND _partition_by_block_number BETWEEN ( - SELECT - MAX(block_number) - 50000 - FROM - {{ this }} -) -AND ( - SELECT - MAX(block_number) + 500000 - FROM - {{ this }} -) -{% else %} - AND _partition_by_block_number <= 8000000 -{% endif %} + ) ), decoded_logs AS ( SELECT