From 0decefd60d595ec50c2f16a012df3cbb74e80c36 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 7 Feb 2024 13:38:02 -0500 Subject: [PATCH] start pulling day counts --- .github/workflows/dbt_run_axelscan.yml | 45 +++++----- .../bronze_api__axelscan_day_counts_gmp.sql | 85 +++++++++++++++++++ ...nze_api__axelscan_day_counts_transfers.sql | 85 +++++++++++++++++++ 3 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql create mode 100644 models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql diff --git a/.github/workflows/dbt_run_axelscan.yml b/.github/workflows/dbt_run_axelscan.yml index 9b40a43..851aa28 100644 --- a/.github/workflows/dbt_run_axelscan.yml +++ b/.github/workflows/dbt_run_axelscan.yml @@ -43,23 +43,28 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_max_ids.sql models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql - dbt run -m models/bronze/bronze_api/bronze_api__axelscan_searchgmp.sql models/bronze/bronze_api/bronze_api__axelscan_searchtransfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql + dbt run -m models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql diff --git a/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql b/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql new file mode 100644 index 0000000..1f91a3c --- /dev/null +++ b/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_gmp.sql @@ -0,0 +1,85 @@ +{{ config( + materialized = 'incremental', + unique_key = 'date_day', + full_refresh = false +) }} + +WITH dates_hist AS ( + + SELECT + A.date_day, + DATE_PART( + epoch_second, + A.date_day + ) AS fromTime, + DATE_PART(epoch_second, DATEADD (DAY, 1, A.date_day)) -1 AS toTime + FROM + {{ source( + 'crosschain', + 'dim_dates' + ) }} A + +{% if is_incremental() %} +LEFT JOIN {{ this }} +b +ON A.date_day = b.date_day +WHERE + b.date_day IS NULL + AND A.date_day BETWEEN '2022-05-09' + AND SYSDATE() :: DATE - 2 +{% else %} +WHERE + A.date_day BETWEEN '2022-05-09' + AND SYSDATE() :: DATE - 2 +{% endif %} +), +dates_recent AS ( + SELECT + date_day, + DATE_PART( + epoch_second, + date_day + ) AS fromTime, + DATE_PART(epoch_second, DATEADD (DAY, 1, date_day)) -1 AS toTime + FROM + {{ source( + 'crosschain', + 'dim_dates' + ) }} + WHERE + date_day BETWEEN SYSDATE() :: DATE - 1 + AND SYSDATE() :: DATE +), +date_combo AS ( + SELECT + date_day, + fromTime, + toTime + FROM + dates_hist + UNION ALL + SELECT + date_day, + fromTime, + toTime + FROM + dates_recent +) +SELECT + date_day, + fromTime, + toTime, + {{ target.database }}.live.udf_api( + 'GET', + 'https://api.gmp.axelarscan.io',{},{ 'method': 'searchGMP', + 'fromTime': fromTime, + 'toTime': toTime, + 'size': 1 } + ) :data :total AS day_count, + SYSDATE() AS _inserted_timestamp +FROM + date_combo +ORDER BY + date_day +LIMIT + 20 diff --git a/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql b/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql new file mode 100644 index 0000000..1354267 --- /dev/null +++ b/models/bronze/bronze_api/axelscan/bronze_api__axelscan_day_counts_transfers.sql @@ -0,0 +1,85 @@ +{{ config( + materialized = 'incremental', + unique_key = 'date_day', + full_refresh = false +) }} + +WITH dates_hist AS ( + + SELECT + A.date_day, + DATE_PART( + epoch_second, + A.date_day + ) AS fromTime, + DATE_PART(epoch_second, DATEADD (DAY, 1, A.date_day)) -1 AS toTime + FROM + {{ source( + 'crosschain', + 'dim_dates' + ) }} A + +{% if is_incremental() %} +LEFT JOIN {{ this }} +b +ON A.date_day = b.date_day +WHERE + b.date_day IS NULL + AND A.date_day BETWEEN '2021-12-23' + AND SYSDATE() :: DATE - 2 +{% else %} +WHERE + A.date_day BETWEEN '2021-12-23' + AND SYSDATE() :: DATE - 2 +{% endif %} +), +dates_recent AS ( + SELECT + date_day, + DATE_PART( + epoch_second, + date_day + ) AS fromTime, + DATE_PART(epoch_second, DATEADD (DAY, 1, date_day)) -1 AS toTime + FROM + {{ source( + 'crosschain', + 'dim_dates' + ) }} + WHERE + date_day BETWEEN SYSDATE() :: DATE - 1 + AND SYSDATE() :: DATE +), +date_combo AS ( + SELECT + date_day, + fromTime, + toTime + FROM + dates_hist + UNION ALL + SELECT + date_day, + fromTime, + toTime + FROM + dates_recent +) +SELECT + date_day, + fromTime, + toTime, + {{ target.database }}.live.udf_api( + 'GET', + 'https://api.axelarscan.io',{},{ 'method': 'searchTransfers', + 'fromTime': fromTime, + 'toTime': toTime, + 'size': 1 } + ) :data :total AS day_count, + SYSDATE() AS _inserted_timestamp +FROM + date_combo +ORDER BY + date_day +LIMIT + 20