diff --git a/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml b/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml new file mode 100644 index 00000000..01f87802 --- /dev/null +++ b/.github/workflows/dbt_run_transactions_units_consumed_backfill.yml @@ -0,0 +1,45 @@ +name: dbt_run_transactions_units_consumed_backfill +run-name: dbt_run_transactions_units_consumed_backfill + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s "solana_models,tag:units_consumed_backfill" + dbt run -s "solana_models,tag:units_consumed_backfill" \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index bfd6f933..b53c598b 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -9,3 +9,4 @@ dbt_run_decode_logs_orchestrator,"*/15 * * * *" dbt_run_streamline_block_rewards,"*/15 * * * *" dbt_run_streamline_blocks,"*/5 * * * *" dbt_run_streamline_helius_cnft_metadata,"*/10 * * * *" +dbt_run_transactions_units_consumed_backfill,"2,7,12,17,22,27,32,37,42,47,52,57 * * * *" diff --git a/models/silver/backfill/silver__backfill_transactions_units_consumed.sql b/models/silver/backfill/silver__backfill_transactions_units_consumed.sql new file mode 100644 index 00000000..9f00ab7c --- /dev/null +++ b/models/silver/backfill/silver__backfill_transactions_units_consumed.sql @@ -0,0 +1,54 @@ +{{ + config( + materialized="incremental", + unique_key=['tx_id'], + tags=['units_consumed_backfill'] + ) +}} + +{% if execute %} + {% set max_partition = 120880 %} + + {% set next_partition_query %} + {% if is_incremental() %} + SELECT + LEAST(max(_partition_id) + 1, {{ max_partition }}), + LEAST(max(_partition_id) + 5, {{ max_partition }}) + FROM + {{ this }} + {% else %} + SELECT 24239, 24239 /* When computeUnitsConsumed first appears in node response */ + {% endif %} + {% endset %} + + {% set next_partition = run_query(next_partition_query)[0][0] %} + {% set next_partition_2 = run_query(next_partition_query)[0][1] %} + + -- list of partition IDs with >2m records to exclude + {% set excluded_partitions = [ + 116876, 116875, 116874, 116872, 116871, 116870, 116869, 116868, 116867, + 116863, 116862, 116850, 105987, 103411, 103410, 103409, 103406, 51215, + 45995, 31483, 31482, 27335, 27333, 27331, 27330, 27327, 27325 + ] %} +{% endif %} + +SELECT + t.tx_id, + t.data :meta :computeUnitsConsumed :: NUMBER AS units_consumed, + t._partition_id, + t._inserted_timestamp, + {{ dbt_utils.generate_surrogate_key(['tx_id']) }} AS transactions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze__transactions2') }} t +WHERE + tx_id IS NOT NULL + AND ( + COALESCE(t.data :transaction :message :instructions [0] :programId :: STRING, '') <> 'Vote111111111111111111111111111111111111111' + OR (array_size(t.data :transaction :message :instructions) > 1) + ) + AND _partition_id >= {{ next_partition }} + AND _partition_id <= {{ next_partition_2 }} + AND _partition_id NOT IN ({{ excluded_partitions | join(', ') }}) \ No newline at end of file