From 910ac7df46f376cfee6c2f31fd43ceeed2cd3772 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Fri, 7 Mar 2025 14:43:32 -0700 Subject: [PATCH] added github actions workflows and cleaned up a few docs that were referencing Aptos --- .github/workflows/dbt_docs_update.yml | 71 +++++++++++++++++++ .github/workflows/dbt_run_adhoc.yml | 67 +++++++++++++++++ .github/workflows/dbt_run_core.yml | 42 +++++++++++ .github/workflows/dbt_run_dev_refresh.yml | 46 ++++++++++++ .../dbt_run_streamline_blocks_tx_realtime.yml | 42 +++++++++++ ...t_run_streamline_transactions_realtime.yml | 42 +++++++++++ models/descriptions/__overview__.md | 5 +- models/descriptions/epoch.md | 2 +- models/descriptions/signature.md | 2 +- .../tables/core__fact_transactions.md | 2 +- 10 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_docs_update.yml create mode 100644 .github/workflows/dbt_run_adhoc.yml create mode 100644 .github/workflows/dbt_run_core.yml create mode 100644 .github/workflows/dbt_run_dev_refresh.yml create mode 100644 .github/workflows/dbt_run_streamline_blocks_tx_realtime.yml create mode 100644 .github/workflows/dbt_run_streamline_transactions_realtime.yml diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 0000000..fcc9b86 --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,71 @@ +name: docs_update + +on: + push: + branches: + - "main" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + - name: generate dbt docs + run: | + dbt ls -t prod + dbt docs generate --no-compile -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..435a87d --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,67 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_core.yml b/.github/workflows/dbt_run_core.yml new file mode 100644 index 0000000..e539e80 --- /dev/null +++ b/.github/workflows/dbt_run_core.yml @@ -0,0 +1,42 @@ +name: dbt_run_incremental_core +run-name: dbt_run_incremental_core + +on: + workflow_dispatch: + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m movement_models,tag:core \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..51e9fd7 --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,46 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_streamline_blocks_tx_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_tx_realtime.yml new file mode 100644 index 0000000..4f9b780 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_blocks_tx_realtime.yml @@ -0,0 +1,42 @@ +name: dbt_run_streamline_blocks_tx_realtime +run-name: dbt_run_streamline_blocks_tx_realtime + +on: + workflow_dispatch: + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 2+models/streamline/core/realtime/streamline__blocks_tx_realtime.sql; diff --git a/.github/workflows/dbt_run_streamline_transactions_realtime.yml b/.github/workflows/dbt_run_streamline_transactions_realtime.yml new file mode 100644 index 0000000..606e01d --- /dev/null +++ b/.github/workflows/dbt_run_streamline_transactions_realtime.yml @@ -0,0 +1,42 @@ +name: dbt_run_streamline_transactions_realtime +run-name: dbt_run_streamline_transactions_realtime + +on: + workflow_dispatch: + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/core/realtime/streamline__transactions_realtime.sql models/silver/core/silver__blocks.sql; diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 35367d5..653298a 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -19,15 +19,12 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Fact Tables (`movement`.`CORE`.``) - [fact_blocks](#!/model/model.movement_models.core__fact_blocks) -- [fact_events](#!/model/model.movement_models.core__fact_events) - [fact_changes](#!/model/model.movement_models.core__fact_changes) +- [fact_events](#!/model/model.movement_models.core__fact_events) - [fact_transactions](#!/model/model.movement_models.core__fact_transactions) - [fact_transactions_block_metadata](#!/model/model.movement_models.core__fact_transactions_block_metadata) - [fact_transactions_state_checkpoint](#!/model/model.movement_models.core__fact_transactions_state_checkpoint) - - - ## **Data Model Overview** The movement models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** diff --git a/models/descriptions/epoch.md b/models/descriptions/epoch.md index 1befe29..6c5963d 100644 --- a/models/descriptions/epoch.md +++ b/models/descriptions/epoch.md @@ -1,5 +1,5 @@ {% docs epoch %} -An epoch in the Aptos blockchain is defined as a duration of time, in seconds, during which a number of blocks are voted on by the validators, the validator set is updated, and the rewards are distributed to the validators. The Aptos mainnet epoch is set as 7200 seconds (two hours). +An epoch in the Movement blockchain is defined as a duration of time, in seconds, during which a number of blocks are voted on by the validators, the validator set is updated, and the rewards are distributed to the validators. The Movement mainnet epoch is set as 7200 seconds (two hours). {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/signature.md b/models/descriptions/signature.md index 121914a..37ad219 100644 --- a/models/descriptions/signature.md +++ b/models/descriptions/signature.md @@ -1,5 +1,5 @@ {% docs signature %} -A signature is the result of hashing the signing message with the client's private key. By default Aptos uses the Ed25519 scheme to generate the signature of the raw transaction. +A signature is the result of hashing the signing message with the client's private key. By default Movement uses the Ed25519 scheme to generate the signature of the raw transaction. {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tables/core__fact_transactions.md b/models/descriptions/tables/core__fact_transactions.md index 4ee3daf..9cd37d2 100644 --- a/models/descriptions/tables/core__fact_transactions.md +++ b/models/descriptions/tables/core__fact_transactions.md @@ -1,6 +1,6 @@ {% docs core__fact_transactions %} -This table contains transaction level data for the Aptos blockchain. Each transaction will have a unique transaction hash and version. +This table contains transaction level data for the Movement blockchain. Each transaction will have a unique transaction hash and version. For more information see [docs.movementlabs.xyz docs] Each transaction will have a unique transaction hash {% enddocs %}