From 9fb3f67f01dcc3b77c47756a96b50cd69812a937 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 8 Nov 2023 17:22:25 -0500 Subject: [PATCH 1/7] coin info backfill --- .../workflows/dbt_run_coininfo_backfill.yml | 52 +++++++++++++++++ data/github_actions__workflows.csv | 1 + macros/dbt_helper.sql | 9 +++ .../bronze_api__aptoslabs_coin_info.sql | 57 +++++++++++++++++++ models/silver/core/silver__coin_info.sql | 42 ++++++++++++++ 5 files changed, 161 insertions(+) create mode 100644 .github/workflows/dbt_run_coininfo_backfill.yml create mode 100644 macros/dbt_helper.sql create mode 100644 models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql create mode 100644 models/silver/core/silver__coin_info.sql diff --git a/.github/workflows/dbt_run_coininfo_backfill.yml b/.github/workflows/dbt_run_coininfo_backfill.yml new file mode 100644 index 0000000..6aaf490 --- /dev/null +++ b/.github/workflows/dbt_run_coininfo_backfill.yml @@ -0,0 +1,52 @@ +name: dbt_run_coininfo_backfill +run-name: dbt_run_coininfo_backfill + +on: + workflow_dispatch: + + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 8ccd9a7..13ffaad 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1 +1,2 @@ workflow_name,workflow_schedule +dbt_run_coininfo_backfill,"17 * * * *" diff --git a/macros/dbt_helper.sql b/macros/dbt_helper.sql new file mode 100644 index 0000000..3a966e1 --- /dev/null +++ b/macros/dbt_helper.sql @@ -0,0 +1,9 @@ +{% macro get_last_transaction_version_created_coin_info() %} + {% if execute %} + {% set last_version = run_query("SELECT MAX(transaction_version_created) FROM bronze_api.aptoslabs_coin_info").columns [0] [0] %} + {% else %} + {% set last_version = -1 %} + {% endif %} + + {% do return(last_version) %} +{% endmacro %} diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql new file mode 100644 index 0000000..6efbc86 --- /dev/null +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql @@ -0,0 +1,57 @@ +{{ config( + materialized = 'incremental', + full_refresh = false +) }} + +{% if is_incremental() %} +{% set last_version = get_last_transaction_version_created_coin_info() %} +{% endif %} + +WITH params AS ( + + SELECT + 'query MyQuery { coin_infos( limit: 100 order_by: {transaction_version_created: asc}' || + +{% if is_incremental() %} +'where: {transaction_version_created: {_gte: "' || {{ last_version }} || '"}} ' || +{% endif %} + +') { coin_type coin_type_hash creator_address decimals name symbol transaction_created_timestamp transaction_version_created }} ' AS query +), +res AS ( + SELECT + live.udf_api( + 'post', + 'https://indexer.mainnet.aptoslabs.com/v1/graphql', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'query', + query, + 'variables',{} + ) + ) AS res, + query, + SYSDATE() AS _inserted_timestamp + FROM + params +) +SELECT + {# res, #} + query, + C.value :coin_type :: STRING AS coin_type, + C.value :coin_type_hash :: STRING AS coin_type_hash, + C.value :creator_address :: STRING AS creator_address, + C.value :decimals :: INT AS decimals, + C.value :name :: STRING AS NAME, + C.value :symbol :: STRING AS symbol, + C.value :transaction_created_timestamp :: datetime AS transaction_created_timestamp, + C.value :transaction_version_created :: INT AS transaction_version_created, + _inserted_timestamp +FROM + res, + LATERAL FLATTEN( + input => res :data :data :coin_infos + ) C diff --git a/models/silver/core/silver__coin_info.sql b/models/silver/core/silver__coin_info.sql new file mode 100644 index 0000000..1113e3c --- /dev/null +++ b/models/silver/core/silver__coin_info.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + unique_key = "coin_type", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['core'] +) }} + +SELECT + coin_type, + coin_type_hash, + creator_address, + decimals, + NAME, + symbol, + transaction_created_timestamp, + transaction_version_created, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze_api__aptoslabs_coin_info' + ) }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY coin_type +ORDER BY + _inserted_timestamp DESC)) = 1 From fa0173e0a3e88955cca3bba984803b7027836b3f Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 8 Nov 2023 17:23:09 -0500 Subject: [PATCH 2/7] gha models --- .../github_actions__current_task_status.sql | 6 ++++++ .../github_actions__current_task_status.yml | 17 +++++++++++++++++ .../github_actions__task_history.sql | 5 +++++ .../github_actions__task_performance.sql | 5 +++++ .../github_actions__task_schedule.sql | 5 +++++ .../github_actions/github_actions__tasks.sql | 5 +++++ 6 files changed, 43 insertions(+) create mode 100644 models/silver/core/github_actions/github_actions__current_task_status.sql create mode 100644 models/silver/core/github_actions/github_actions__current_task_status.yml create mode 100644 models/silver/core/github_actions/github_actions__task_history.sql create mode 100644 models/silver/core/github_actions/github_actions__task_performance.sql create mode 100644 models/silver/core/github_actions/github_actions__task_schedule.sql create mode 100644 models/silver/core/github_actions/github_actions__tasks.sql diff --git a/models/silver/core/github_actions/github_actions__current_task_status.sql b/models/silver/core/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/silver/core/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/silver/core/github_actions/github_actions__current_task_status.yml b/models/silver/core/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..e80ad12 --- /dev/null +++ b/models/silver/core/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/silver/core/github_actions/github_actions__task_history.sql b/models/silver/core/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/silver/core/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/silver/core/github_actions/github_actions__task_performance.sql b/models/silver/core/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/silver/core/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/silver/core/github_actions/github_actions__task_schedule.sql b/models/silver/core/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/silver/core/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/silver/core/github_actions/github_actions__tasks.sql b/models/silver/core/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/silver/core/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file From b8a548fde8b86a3b2c615922364f6cc522ce4084 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 8 Nov 2023 17:23:45 -0500 Subject: [PATCH 3/7] move gha --- .../github_actions/github_actions__current_task_status.sql | 0 .../github_actions/github_actions__current_task_status.yml | 0 .../core => }/github_actions/github_actions__task_history.sql | 0 .../core => }/github_actions/github_actions__task_performance.sql | 0 .../core => }/github_actions/github_actions__task_schedule.sql | 0 models/{silver/core => }/github_actions/github_actions__tasks.sql | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename models/{silver/core => }/github_actions/github_actions__current_task_status.sql (100%) rename models/{silver/core => }/github_actions/github_actions__current_task_status.yml (100%) rename models/{silver/core => }/github_actions/github_actions__task_history.sql (100%) rename models/{silver/core => }/github_actions/github_actions__task_performance.sql (100%) rename models/{silver/core => }/github_actions/github_actions__task_schedule.sql (100%) rename models/{silver/core => }/github_actions/github_actions__tasks.sql (100%) diff --git a/models/silver/core/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql similarity index 100% rename from models/silver/core/github_actions/github_actions__current_task_status.sql rename to models/github_actions/github_actions__current_task_status.sql diff --git a/models/silver/core/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml similarity index 100% rename from models/silver/core/github_actions/github_actions__current_task_status.yml rename to models/github_actions/github_actions__current_task_status.yml diff --git a/models/silver/core/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql similarity index 100% rename from models/silver/core/github_actions/github_actions__task_history.sql rename to models/github_actions/github_actions__task_history.sql diff --git a/models/silver/core/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql similarity index 100% rename from models/silver/core/github_actions/github_actions__task_performance.sql rename to models/github_actions/github_actions__task_performance.sql diff --git a/models/silver/core/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql similarity index 100% rename from models/silver/core/github_actions/github_actions__task_schedule.sql rename to models/github_actions/github_actions__task_schedule.sql diff --git a/models/silver/core/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql similarity index 100% rename from models/silver/core/github_actions/github_actions__tasks.sql rename to models/github_actions/github_actions__tasks.sql From ce07d0b19cce242148c59dc53567816f17f9c850 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 9 Nov 2023 11:25:18 -0500 Subject: [PATCH 4/7] test update --- models/silver/core/silver__blocks.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml index 79cb7c6..7c8d1f6 100644 --- a/models/silver/core/silver__blocks.yml +++ b/models/silver/core/silver__blocks.yml @@ -35,16 +35,9 @@ models: tests: - not_null - name: INSERTED_TIMESTAMP - tests: - - not_null - name: MODIFIED_TIMESTAMP - tests: - - not_null - name: _INSERTED_TIMESTAMP tests: - name: not_null_silver__blocks_INSERTED_TIMESTAMP_ test_name: not_null - name: _INVOCATION_ID - tests: - - name: not_null_silver__blocks_INVOCATION_ID - test_name: not_null From c7c9ba1008d63394764e57a53ad3036a93d64f27 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 9 Nov 2023 13:15:16 -0500 Subject: [PATCH 5/7] remove backfill --- .../workflows/dbt_run_coininfo_backfill.yml | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 .github/workflows/dbt_run_coininfo_backfill.yml diff --git a/.github/workflows/dbt_run_coininfo_backfill.yml b/.github/workflows/dbt_run_coininfo_backfill.yml deleted file mode 100644 index 6aaf490..0000000 --- a/.github/workflows/dbt_run_coininfo_backfill.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: dbt_run_coininfo_backfill -run-name: dbt_run_coininfo_backfill - -on: - workflow_dispatch: - - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: | - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql; - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit From 12cedf8f8349ae325ee601a76deec0b57e56ff00 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 9 Nov 2023 15:57:36 -0500 Subject: [PATCH 6/7] aptos names --- .../workflows/dbt_run_backfill_generic.yml | 52 ++++++++++++++++ data/github_actions__workflows.csv | 2 +- macros/dbt_helper.sql | 10 +++ .../bronze_api__aptoslabs_aptos_names.sql | 61 +++++++++++++++++++ models/silver/core/silver__aptos_names.sql | 49 +++++++++++++++ models/silver/core/silver__coin_info.sql | 3 + 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/dbt_run_backfill_generic.yml create mode 100644 models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql create mode 100644 models/silver/core/silver__aptos_names.sql diff --git a/.github/workflows/dbt_run_backfill_generic.yml b/.github/workflows/dbt_run_backfill_generic.yml new file mode 100644 index 0000000..1ca3d1f --- /dev/null +++ b/.github/workflows/dbt_run_backfill_generic.yml @@ -0,0 +1,52 @@ +name: dbt_run_backfill_generic +run-name: dbt_run_backfill_generic + +on: + workflow_dispatch: + + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 13ffaad..fd2ee77 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,2 +1,2 @@ workflow_name,workflow_schedule -dbt_run_coininfo_backfill,"17 * * * *" +dbt_run_backfill_generic,"17 * * * *" diff --git a/macros/dbt_helper.sql b/macros/dbt_helper.sql index 3a966e1..baba4c0 100644 --- a/macros/dbt_helper.sql +++ b/macros/dbt_helper.sql @@ -7,3 +7,13 @@ {% do return(last_version) %} {% endmacro %} + +{% macro get_last_transaction_version_aptos_names() %} + {% if execute %} + {% set last_version = run_query("SELECT MAX(last_transaction_version) FROM bronze_api.aptoslabs_aptos_names").columns [0] [0] %} + {% else %} + {% set last_version = -1 %} + {% endif %} + + {% do return(last_version) %} +{% endmacro %} diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql new file mode 100644 index 0000000..bc8fa06 --- /dev/null +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + full_refresh = false +) }} + +{% if is_incremental() %} +{% set last_version = get_last_transaction_version_aptos_names() %} +{% endif %} + +WITH params AS ( + + SELECT + 'query MyQuery { current_aptos_names( limit: 100 order_by: {last_transaction_version: asc}' || + +{% if is_incremental() %} +'where: {last_transaction_version: {_gte: "' || {{ last_version }} || '"}} ' || +{% endif %} + +' ) { domain domain_with_suffix expiration_timestamp is_active is_primary last_transaction_version owner_address registered_address subdomain token_name token_standard } } ' AS query +), +res AS ( + SELECT + live.udf_api( + 'post', + 'https://indexer.mainnet.aptoslabs.com/v1/graphql', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'query', + query, + 'variables',{} + ) + ) AS res, + query, + SYSDATE() AS _inserted_timestamp + FROM + params +) +SELECT + {# res, #} + query, + C.value :domain :: STRING AS domain, + C.value :domain_with_suffix :: STRING AS domain_with_suffix, + C.value :creator_address :: STRING AS creator_address, + C.value :expiration_timestamp :: datetime AS expiration_timestamp, + C.value :is_active :: BOOLEAN AS is_active, + C.value :is_primary :: BOOLEAN AS is_primary, + C.value :last_transaction_version :: INT AS last_transaction_version, + C.value :owner_address :: STRING AS owner_address, + C.value :registered_address :: STRING AS registered_address, + C.value :subdomain :: STRING AS subdomain, + C.value :token_name :: STRING AS token_name, + C.value :token_standard :: STRING AS token_standard, + _inserted_timestamp +FROM + res, + LATERAL FLATTEN( + input => res :data :data :current_aptos_names + ) C diff --git a/models/silver/core/silver__aptos_names.sql b/models/silver/core/silver__aptos_names.sql new file mode 100644 index 0000000..0ecebfd --- /dev/null +++ b/models/silver/core/silver__aptos_names.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + unique_key = "aptos_names_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['core'] +) }} + +SELECT + domain, + domain_with_suffix, + creator_address, + expiration_timestamp, + is_active, + is_primary, + last_transaction_version, + owner_address, + registered_address, + subdomain, + token_name, + token_standard, + {{ dbt_utils.generate_surrogate_key( + ['token_name'] + ) }} AS aptos_names_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze_api__aptoslabs_aptos_names' + ) }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY token_name +ORDER BY + last_transaction_version DESC, _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__coin_info.sql b/models/silver/core/silver__coin_info.sql index 1113e3c..4040c6c 100644 --- a/models/silver/core/silver__coin_info.sql +++ b/models/silver/core/silver__coin_info.sql @@ -16,6 +16,9 @@ SELECT symbol, transaction_created_timestamp, transaction_version_created, + {{ dbt_utils.generate_surrogate_key( + ['coin_type'] + ) }} AS coin_info_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, _inserted_timestamp, From d82d14b41a3e232b4cc28a39f9215ade78a46d78 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Tue, 14 Nov 2023 09:40:41 -0500 Subject: [PATCH 7/7] names remove creator --- .../bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql | 4 ++-- models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql index bc8fa06..73383e2 100644 --- a/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', - full_refresh = false + full_refresh = false, + tags = ['noncore'] ) }} {% if is_incremental() %} @@ -43,7 +44,6 @@ SELECT query, C.value :domain :: STRING AS domain, C.value :domain_with_suffix :: STRING AS domain_with_suffix, - C.value :creator_address :: STRING AS creator_address, C.value :expiration_timestamp :: datetime AS expiration_timestamp, C.value :is_active :: BOOLEAN AS is_active, C.value :is_primary :: BOOLEAN AS is_primary, diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql index 6efbc86..6551da7 100644 --- a/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', - full_refresh = false + full_refresh = false, + tags = ['noncore'] ) }} {% if is_incremental() %}