From d2168fcef28beee2a68e963d1c343f8ad31681db Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Mon, 11 Dec 2023 15:10:00 +0100 Subject: [PATCH] add: pk keys --- models/descriptions/pk_id.md | 5 +++++ models/gold/core/core__fact_blocks.sql | 6 +++++- models/gold/core/core__fact_blocks.yml | 8 ++++++++ models/gold/core/core__fact_logs.sql | 5 ++++- models/gold/core/core__fact_logs.yml | 6 ++++++ models/gold/core/core__fact_receipts.sql | 6 +++++- models/gold/core/core__fact_receipts.yml | 8 ++++++++ models/gold/core/core__fact_transactions.sql | 1 + models/gold/core/core__fact_transactions.yml | 8 ++++++++ models/silver/silver__blocks.sql | 8 +++++++- models/silver/silver__logs.sql | 3 +++ models/silver/silver__receipts.sql | 3 +++ models/silver/silver__relevant_contracts.sql | 5 ++++- models/silver/silver__transactions.sql | 5 ++++- 14 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 models/descriptions/pk_id.md diff --git a/models/descriptions/pk_id.md b/models/descriptions/pk_id.md new file mode 100644 index 0000000..8d21a8a --- /dev/null +++ b/models/descriptions/pk_id.md @@ -0,0 +1,5 @@ +{% docs pk_id %} + +A uniquely generated identifier assigned by a surrogate key + +{% enddocs %} diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index 5df19b5..d525747 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -6,6 +6,7 @@ ) }} SELECT + block_id, block_number, block_timestamp, 'mainnet' AS network, @@ -61,6 +62,9 @@ SELECT transactions_root, 'uncles', uncles - ) AS block_header_json + ) AS block_header_json, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index 742d1b9..d915f5a 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("blocks_table_doc") }}' columns: + - name: BLOCK_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -42,3 +44,9 @@ models: description: '{{ doc("uncle_blocks") }}' - name: BLOCK_HEADER_JSON description: '{{ doc("block_header_json") }}' + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_logs.sql b/models/gold/core/core__fact_logs.sql index 919e277..bbd4ebf 100644 --- a/models/gold/core/core__fact_logs.sql +++ b/models/gold/core/core__fact_logs.sql @@ -18,6 +18,9 @@ SELECT DATA, event_removed, tx_status, - _log_id + _log_id, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__logs') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_logs.yml b/models/gold/core/core__fact_logs.yml index 81d46fe..1e5a3fd 100644 --- a/models/gold/core/core__fact_logs.yml +++ b/models/gold/core/core__fact_logs.yml @@ -30,3 +30,9 @@ models: description: "The from address at the transaction level." - name: ORIGIN_TO_ADDRESS description: "The to address at the transaction level." + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_receipts.sql b/models/gold/core/core__fact_receipts.sql index acc9749..36dd073 100644 --- a/models/gold/core/core__fact_receipts.sql +++ b/models/gold/core/core__fact_receipts.sql @@ -6,6 +6,7 @@ ) }} SELECT + receipts_id, block_number, block_timestamp, block_hash, @@ -22,6 +23,9 @@ SELECT POSITION, TYPE, near_receipt_hash, - near_transaction_hash + near_transaction_hash, + inserted_timestamp, + modified_timestamp, + invocation_id FROM {{ ref('silver__receipts') }} \ No newline at end of file diff --git a/models/gold/core/core__fact_receipts.yml b/models/gold/core/core__fact_receipts.yml index 707d38e..bb095eb 100644 --- a/models/gold/core/core__fact_receipts.yml +++ b/models/gold/core/core__fact_receipts.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("receipts_table_doc") }}' columns: + - name: RECEIPTS_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -36,3 +38,9 @@ models: description: "Receipt hash on NEAR blockchain" - name: NEAR_TRANSACTION_HASH description: "Transaction hash on NEAR blockchain" + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 28b7205..4a0a33f 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -6,6 +6,7 @@ ) }} SELECT + tx_id, block_number, block_hash, block_timestamp, diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 068a675..08ad524 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -4,6 +4,8 @@ models: description: '{{ doc("tx_table_doc") }}' columns: + - name: TX_ID + description: '{{ doc("pk_id") }}' - name: BLOCK_NUMBER description: '{{ doc("block_number") }}' - name: BLOCK_TIMESTAMP @@ -50,3 +52,9 @@ models: description: The s value of the transaction signature. - name: v description: The v value of the transaction signature. + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + - name: INVOCACTION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql index 08993bf..ffd676a 100644 --- a/models/silver/silver__blocks.sql +++ b/models/silver/silver__blocks.sql @@ -8,6 +8,9 @@ ) }} SELECT + {{ dbt_utils.generate_surrogate_key( + ['block_number'] + ) }} AS block_id, block_number, utils.udf_hex_to_int( DATA :result :timestamp :: STRING @@ -47,7 +50,10 @@ SELECT DATA :result :stateRoot :: STRING AS state_root, DATA :result :transactionsRoot :: STRING AS transactions_root, _partition_by_block_id, - _inserted_timestamp + _inserted_timestamp, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM {% if is_incremental() %} diff --git a/models/silver/silver__logs.sql b/models/silver/silver__logs.sql index 0ccf606..c745550 100644 --- a/models/silver/silver__logs.sql +++ b/models/silver/silver__logs.sql @@ -175,6 +175,9 @@ FROM ) SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER','tx_hash', '_LOG_ID']) }} AS logs_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id, * FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY tx_hash, _log_id diff --git a/models/silver/silver__receipts.sql b/models/silver/silver__receipts.sql index b14f03c..cd47225 100644 --- a/models/silver/silver__receipts.sql +++ b/models/silver/silver__receipts.sql @@ -92,6 +92,9 @@ FINAL AS ( SELECT {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH']) }} AS receipts_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id, * FROM FINAL diff --git a/models/silver/silver__relevant_contracts.sql b/models/silver/silver__relevant_contracts.sql index 8130ff9..cdae72f 100644 --- a/models/silver/silver__relevant_contracts.sql +++ b/models/silver/silver__relevant_contracts.sql @@ -8,7 +8,10 @@ SELECT contract_address, 'Aurora' AS blockchain, COUNT(*) AS events, - MAX(block_number) AS latest_block + MAX(block_number) AS latest_block, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM {{ ref('silver__logs') }} GROUP BY diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql index 84c6094..8385331 100644 --- a/models/silver/silver__transactions.sql +++ b/models/silver/silver__transactions.sql @@ -287,7 +287,10 @@ FROM ) SELECT *, - {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} AS tx_id + {{ dbt_utils.generate_surrogate_key(['BLOCK_NUMBER', 'TX_HASH', 'POSITION']) }} AS tx_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id FROM FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_hash ORDER BY