diff --git a/.github/workflows/dbt_activity_yearly.yml b/.github/workflows/dbt_activity_yearly.yml index a69e518..0e65a41 100644 --- a/.github/workflows/dbt_activity_yearly.yml +++ b/.github/workflows/dbt_activity_yearly.yml @@ -36,5 +36,5 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'GB_TOKEN', frequency: ['last_year']}" + dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['last_year']}" dbt run --model silver__github_activity diff --git a/.github/workflows/dbt_contributors_weekly.yml b/.github/workflows/dbt_contributors_weekly.yml index 67f698f..4c5f17c 100644 --- a/.github/workflows/dbt_contributors_weekly.yml +++ b/.github/workflows/dbt_contributors_weekly.yml @@ -36,5 +36,6 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'GB_TOKEN', frequency: ['contributors']}" + dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['contributors']}" dbt run --model silver__github_activity + dbt test -s github_data_null diff --git a/.github/workflows/dbt_general_weekly.yml b/.github/workflows/dbt_general_weekly.yml index 8f11b8d..24c7551 100644 --- a/.github/workflows/dbt_general_weekly.yml +++ b/.github/workflows/dbt_general_weekly.yml @@ -36,5 +36,6 @@ jobs: - name: Run DBT Jobs run: | - dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'GB_TOKEN', frequency: ['core']}" + dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['core']}" dbt run --model silver__github_activity + dbt test -s github_data_null diff --git a/macros/api_udf/request_repo_data.sql b/macros/api_udf/request_repo_data.sql index 136b993..494be6f 100644 --- a/macros/api_udf/request_repo_data.sql +++ b/macros/api_udf/request_repo_data.sql @@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS {{ target.database }}.bronze_api.github_repo_data( {% set event_table %} -CREATE TABLE IF NOT EXISTS {{ target.database }}.bronze_api.log_messages ( +CREATE OR REPLACE TABLE {{ target.database }}.bronze_api.log_messages ( timestamp TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP(), log_level STRING, message STRING diff --git a/tests/github__data_log.sql b/tests/github__data_log.sql new file mode 100644 index 0000000..57526af --- /dev/null +++ b/tests/github__data_log.sql @@ -0,0 +1,22 @@ +WITH base AS ( + SELECT + SPLIT_PART(MESSAGE, ',',2) AS status_code, + count(*) as counter + FROM + {{ target.database }}.bronze_api.log_messages + WHERE + timestamp BETWEEN DATEADD(HOUR, -1, CURRENT_TIMESTAMP()) + AND CURRENT_TIMESTAMP() + AND LOG_LEVEL = 'DEBUG' + GROUP BY 1 + +) +select + status_code, + counter +from + base +where + status_code not in (200, 202, 404, 451) + and counter > 0 + and counter < 100 \ No newline at end of file