diff --git a/.github/workflows/dbt_run_backfill_generic.yml b/.github/workflows/dbt_run_backfill_generic.yml new file mode 100644 index 0000000..1ca3d1f --- /dev/null +++ b/.github/workflows/dbt_run_backfill_generic.yml @@ -0,0 +1,52 @@ +name: dbt_run_backfill_generic +run-name: dbt_run_backfill_generic + +on: + workflow_dispatch: + + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + dbt run -m models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql; + + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 0a05f06..e1ab895 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,2 +1,3 @@ workflow_name,workflow_schedule dbt_run_streamline_blocks_tx_realtime, "0 * * * *" +dbt_run_backfill_generic,"17 * * * *" diff --git a/macros/dbt_helper.sql b/macros/dbt_helper.sql new file mode 100644 index 0000000..baba4c0 --- /dev/null +++ b/macros/dbt_helper.sql @@ -0,0 +1,19 @@ +{% macro get_last_transaction_version_created_coin_info() %} + {% if execute %} + {% set last_version = run_query("SELECT MAX(transaction_version_created) FROM bronze_api.aptoslabs_coin_info").columns [0] [0] %} + {% else %} + {% set last_version = -1 %} + {% endif %} + + {% do return(last_version) %} +{% endmacro %} + +{% macro get_last_transaction_version_aptos_names() %} + {% if execute %} + {% set last_version = run_query("SELECT MAX(last_transaction_version) FROM bronze_api.aptoslabs_aptos_names").columns [0] [0] %} + {% else %} + {% set last_version = -1 %} + {% endif %} + + {% do return(last_version) %} +{% endmacro %} diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql new file mode 100644 index 0000000..73383e2 --- /dev/null +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_aptos_names.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + full_refresh = false, + tags = ['noncore'] +) }} + +{% if is_incremental() %} +{% set last_version = get_last_transaction_version_aptos_names() %} +{% endif %} + +WITH params AS ( + + SELECT + 'query MyQuery { current_aptos_names( limit: 100 order_by: {last_transaction_version: asc}' || + +{% if is_incremental() %} +'where: {last_transaction_version: {_gte: "' || {{ last_version }} || '"}} ' || +{% endif %} + +' ) { domain domain_with_suffix expiration_timestamp is_active is_primary last_transaction_version owner_address registered_address subdomain token_name token_standard } } ' AS query +), +res AS ( + SELECT + live.udf_api( + 'post', + 'https://indexer.mainnet.aptoslabs.com/v1/graphql', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'query', + query, + 'variables',{} + ) + ) AS res, + query, + SYSDATE() AS _inserted_timestamp + FROM + params +) +SELECT + {# res, #} + query, + C.value :domain :: STRING AS domain, + C.value :domain_with_suffix :: STRING AS domain_with_suffix, + C.value :expiration_timestamp :: datetime AS expiration_timestamp, + C.value :is_active :: BOOLEAN AS is_active, + C.value :is_primary :: BOOLEAN AS is_primary, + C.value :last_transaction_version :: INT AS last_transaction_version, + C.value :owner_address :: STRING AS owner_address, + C.value :registered_address :: STRING AS registered_address, + C.value :subdomain :: STRING AS subdomain, + C.value :token_name :: STRING AS token_name, + C.value :token_standard :: STRING AS token_standard, + _inserted_timestamp +FROM + res, + LATERAL FLATTEN( + input => res :data :data :current_aptos_names + ) C diff --git a/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql new file mode 100644 index 0000000..6551da7 --- /dev/null +++ b/models/bronze/bronze_api/bronze_api__aptoslabs_coin_info.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + full_refresh = false, + tags = ['noncore'] +) }} + +{% if is_incremental() %} +{% set last_version = get_last_transaction_version_created_coin_info() %} +{% endif %} + +WITH params AS ( + + SELECT + 'query MyQuery { coin_infos( limit: 100 order_by: {transaction_version_created: asc}' || + +{% if is_incremental() %} +'where: {transaction_version_created: {_gte: "' || {{ last_version }} || '"}} ' || +{% endif %} + +') { coin_type coin_type_hash creator_address decimals name symbol transaction_created_timestamp transaction_version_created }} ' AS query +), +res AS ( + SELECT + live.udf_api( + 'post', + 'https://indexer.mainnet.aptoslabs.com/v1/graphql', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json' + ), + OBJECT_CONSTRUCT( + 'query', + query, + 'variables',{} + ) + ) AS res, + query, + SYSDATE() AS _inserted_timestamp + FROM + params +) +SELECT + {# res, #} + query, + C.value :coin_type :: STRING AS coin_type, + C.value :coin_type_hash :: STRING AS coin_type_hash, + C.value :creator_address :: STRING AS creator_address, + C.value :decimals :: INT AS decimals, + C.value :name :: STRING AS NAME, + C.value :symbol :: STRING AS symbol, + C.value :transaction_created_timestamp :: datetime AS transaction_created_timestamp, + C.value :transaction_version_created :: INT AS transaction_version_created, + _inserted_timestamp +FROM + res, + LATERAL FLATTEN( + input => res :data :data :coin_infos + ) C diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..e80ad12 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/silver/core/silver__aptos_names.sql b/models/silver/core/silver__aptos_names.sql new file mode 100644 index 0000000..0ecebfd --- /dev/null +++ b/models/silver/core/silver__aptos_names.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + unique_key = "aptos_names_id", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['core'] +) }} + +SELECT + domain, + domain_with_suffix, + creator_address, + expiration_timestamp, + is_active, + is_primary, + last_transaction_version, + owner_address, + registered_address, + subdomain, + token_name, + token_standard, + {{ dbt_utils.generate_surrogate_key( + ['token_name'] + ) }} AS aptos_names_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze_api__aptoslabs_aptos_names' + ) }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY token_name +ORDER BY + last_transaction_version DESC, _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml index 79cb7c6..7c8d1f6 100644 --- a/models/silver/core/silver__blocks.yml +++ b/models/silver/core/silver__blocks.yml @@ -35,16 +35,9 @@ models: tests: - not_null - name: INSERTED_TIMESTAMP - tests: - - not_null - name: MODIFIED_TIMESTAMP - tests: - - not_null - name: _INSERTED_TIMESTAMP tests: - name: not_null_silver__blocks_INSERTED_TIMESTAMP_ test_name: not_null - name: _INVOCATION_ID - tests: - - name: not_null_silver__blocks_INVOCATION_ID - test_name: not_null diff --git a/models/silver/core/silver__coin_info.sql b/models/silver/core/silver__coin_info.sql new file mode 100644 index 0000000..4040c6c --- /dev/null +++ b/models/silver/core/silver__coin_info.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = "coin_type", + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['_inserted_timestamp::DATE'], + tags = ['core'] +) }} + +SELECT + coin_type, + coin_type_hash, + creator_address, + decimals, + NAME, + symbol, + transaction_created_timestamp, + transaction_version_created, + {{ dbt_utils.generate_surrogate_key( + ['coin_type'] + ) }} AS coin_info_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze_api__aptoslabs_coin_info' + ) }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over(PARTITION BY coin_type +ORDER BY + _inserted_timestamp DESC)) = 1