From da564ddd0e5a108149bb0cf92d7b983e4dbcf4d1 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 23 May 2023 16:10:09 -0400 Subject: [PATCH] Update dnv (#29) --- .../bronze/bronze__dnv_collection_slugs.sql | 22 +++++++- .../bronze/bronze__dnv_collections.sql | 50 +++++++++++++++---- .../bronze__dnv_historical_valuations.sql | 8 +-- .../bronze/bronze__dnv_tokens.sql | 7 +-- .../gold/deepnftvalue__fact_tokens.sql | 1 + .../silver__dnv_historical_requests.sql | 2 +- .../requests/silver__dnv_token_requests.sql | 2 +- .../silver/silver__dnv_tokens.sql | 2 + 8 files changed, 74 insertions(+), 20 deletions(-) diff --git a/models/deepnftvalue/bronze/bronze__dnv_collection_slugs.sql b/models/deepnftvalue/bronze/bronze__dnv_collection_slugs.sql index 818969a..7d33360 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_collection_slugs.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_collection_slugs.sql @@ -17,11 +17,29 @@ UNION SELECT 'azuki', '0xed5af388653567af2f388e6224dc7c4b3241c544', - '2022-06-01' AS created_at, + '2022-06-01', 10000 UNION SELECT 'mutant-ape-yacht-club', '0x60E4d786628Fea6478F785A6d7e704777c86a7c6', - '2022-06-01' AS created_at, + '2022-06-01', 20000 +UNION +SELECT + 'nakamigos', + '0xd774557b647330c91bf44cfeab205095f7e6c367', + '2023-03-22', + 20000 +UNION +SELECT + 'pudgypenguins', + '0xbd3531da5cf5857e7cfaa92426877b022e612cf8', + '2022-04-21', + 8888 +UNION +SELECT + 'wrapped-cryptopunks', + '0xb7f7f6c52f2e2fdb1963eab30438024864c313f6', + '2021-01-01', + 1000 diff --git a/models/deepnftvalue/bronze/bronze__dnv_collections.sql b/models/deepnftvalue/bronze/bronze__dnv_collections.sql index 67b57ca..77d9c81 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_collections.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_collections.sql @@ -37,15 +37,45 @@ api_key AS ( ) }} WHERE api_name = 'deepnftvalue' -) -SELECT - ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, - SYSDATE() _inserted_timestamp, - CONCAT( +), +row_nos AS ( + SELECT + api_url, collection_slug, - '-', - _inserted_timestamp - ) AS _id + ROW_NUMBER() over ( + ORDER BY + api_url + ) AS row_no, + FLOOR( + row_no / 1 + ) - 1 AS batch_no, + header + FROM + api_url + CROSS JOIN api_key +), +batched AS ({% for item in range(10) %} +SELECT + ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, CONCAT(collection_slug, '-', _inserted_timestamp) AS _id FROM - api_url - CROSS JOIN api_key + row_nos rn +WHERE + batch_no = {{ item }} + AND EXISTS ( +SELECT + 1 +FROM + row_nos +WHERE + batch_no = {{ item }} +LIMIT + 1) {% if not loop.last %} + UNION ALL + {% endif %} +{% endfor %}) +SELECT + resp, + _inserted_timestamp, + _id +FROM + batched diff --git a/models/deepnftvalue/bronze/bronze__dnv_historical_valuations.sql b/models/deepnftvalue/bronze/bronze__dnv_historical_valuations.sql index bb183a0..a9eff2c 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_historical_valuations.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_historical_valuations.sql @@ -48,15 +48,15 @@ row_nos AS ( api_url ) AS row_no, FLOOR( - row_no / 2 - ) + 1 AS batch_no, + row_no/2 + ) AS batch_no, header FROM requests JOIN api_key ON 1 = 1 ), -batched AS ({% for item in range(15) %} +batched AS ({% for item in range(10) %} SELECT ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, api_url, SYSDATE() _inserted_timestamp FROM @@ -68,6 +68,8 @@ SELECT 1 FROM row_nos +WHERE + batch_no = {{ item }} LIMIT 1) {% if not loop.last %} UNION ALL diff --git a/models/deepnftvalue/bronze/bronze__dnv_tokens.sql b/models/deepnftvalue/bronze/bronze__dnv_tokens.sql index 7d73cb7..e40b03d 100644 --- a/models/deepnftvalue/bronze/bronze__dnv_tokens.sql +++ b/models/deepnftvalue/bronze/bronze__dnv_tokens.sql @@ -21,6 +21,7 @@ WHERE FROM {{ this }} ) + AND collection_slug <> 'cryptopunks' {% endif %} ), api_key AS ( @@ -48,15 +49,15 @@ row_nos AS ( api_url ) AS row_no, FLOOR( - row_no / 2 - ) + 1 AS batch_no, + row_no / 10 + ) AS batch_no, header FROM requests JOIN api_key ON 1 = 1 ), -batched AS ({% for item in range(15) %} +batched AS ({% for item in range(9) %} SELECT ethereum.streamline.udf_api(' GET ', api_url, PARSE_JSON(header),{}) AS resp, SYSDATE() _inserted_timestamp, collection_slug, _id FROM diff --git a/models/deepnftvalue/gold/deepnftvalue__fact_tokens.sql b/models/deepnftvalue/gold/deepnftvalue__fact_tokens.sql index aecb79c..4b7ad44 100644 --- a/models/deepnftvalue/gold/deepnftvalue__fact_tokens.sql +++ b/models/deepnftvalue/gold/deepnftvalue__fact_tokens.sql @@ -11,6 +11,7 @@ SELECT token_id :: INTEGER AS token_id, active_offer AS active_offer, attributes, + attributes_synthetic, image, CASE WHEN is_flagged :: STRING = 'true' THEN TRUE diff --git a/models/deepnftvalue/silver/requests/silver__dnv_historical_requests.sql b/models/deepnftvalue/silver/requests/silver__dnv_historical_requests.sql index 1dde190..afd4970 100644 --- a/models/deepnftvalue/silver/requests/silver__dnv_historical_requests.sql +++ b/models/deepnftvalue/silver/requests/silver__dnv_historical_requests.sql @@ -35,7 +35,7 @@ all_tokens AS ( ON created_at <= date_day AND date_day < SYSDATE() :: DATE JOIN generate_sequence - ON seq <= total_pages - 1 + ON seq <= CEIL(total_pages) - 1 ) SELECT collection_slug, diff --git a/models/deepnftvalue/silver/requests/silver__dnv_token_requests.sql b/models/deepnftvalue/silver/requests/silver__dnv_token_requests.sql index 15e87c8..c5abe61 100644 --- a/models/deepnftvalue/silver/requests/silver__dnv_token_requests.sql +++ b/models/deepnftvalue/silver/requests/silver__dnv_token_requests.sql @@ -12,7 +12,7 @@ WITH slugs AS ( ), offsets AS ( SELECT - 2000 AS limiter + 1000 AS limiter ), generate_sequence AS ( SELECT diff --git a/models/deepnftvalue/silver/silver__dnv_tokens.sql b/models/deepnftvalue/silver/silver__dnv_tokens.sql index 5af3622..655ebdf 100644 --- a/models/deepnftvalue/silver/silver__dnv_tokens.sql +++ b/models/deepnftvalue/silver/silver__dnv_tokens.sql @@ -31,6 +31,7 @@ FINAL AS ( VALUE :token_id AS token_id, VALUE :active_offer AS active_offer, VALUE :attributes AS attributes, + VALUE :attributes_synthetic AS attributes_synthetic, VALUE :collection AS collection, VALUE :image AS image, VALUE :is_flagged AS is_flagged, @@ -50,6 +51,7 @@ SELECT token_id, active_offer, attributes, + attributes_synthetic, collection :contract :: STRING AS contract_address, collection :name :: STRING AS collection_name, image,