diff --git a/.github/workflows/dbt_run_moments_metadata.yml b/.github/workflows/dbt_run_moments_metadata.yml index 2a4a726..d00bed6 100644 --- a/.github/workflows/dbt_run_moments_metadata.yml +++ b/.github/workflows/dbt_run_moments_metadata.yml @@ -31,6 +31,7 @@ jobs: dbt run -m silver__allday_moments_metadata_needed_s.sql; dbt run-operation get_allday_metadata; dbt run -m silver__nft_allday_metadata_s; + dbt run -s tag:topshot; environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/models/silver/nft/metadata/livequery/livequery__allday_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/allday/livequery__allday_moments_metadata_needed.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__allday_moments_metadata_needed.sql rename to models/silver/nft/metadata/livequery/allday/livequery__allday_moments_metadata_needed.sql diff --git a/models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql b/models/silver/nft/metadata/livequery/allday/silver__allday_moments_metadata_needed_s.sql similarity index 100% rename from models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql rename to models/silver/nft/metadata/livequery/allday/silver__allday_moments_metadata_needed_s.sql diff --git a/models/silver/nft/metadata/livequery/livequery__null_moments_metadata.sql b/models/silver/nft/metadata/livequery/topshot/livequery__null_moments_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__null_moments_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/livequery__null_moments_metadata.sql diff --git a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py similarity index 89% rename from models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py rename to models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py index aad8836..ebcc8d1 100644 --- a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py +++ b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py @@ -51,7 +51,11 @@ def model(dbt, session): # define params for UDF_API method = 'POST' headers = { - 'Content-Type': 'application/json' + 'Accept': 'application/json', + 'Accept-Encoding': 'gzip', + 'Connection': 'keep-alive', + 'Content-Type': 'application/json', + 'User-Agent': 'Flipside_Flow_metadata/0.1' } url = topshot_gql_params[0][0] @@ -63,7 +67,9 @@ def model(dbt, session): inputs = dbt.ref( 'livequery__topshot_moments_metadata_needed').select( "EVENT_CONTRACT", "MOMENT_ID" - ).limit(3500) + ).limit(600) + # Note prior limit of 3500 leads to 429 error / rate limit by system + # Per Dapper team, 50 reqs per 10 seconds. If exceeded, blocked for 30s. # register the udf_construct_data function udf_construct_data = register_udf_construct_data() diff --git a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.yml b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.yml similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.yml rename to models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.yml diff --git a/models/silver/nft/metadata/livequery/livequery__topshot_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__topshot_moments_metadata_needed.sql rename to models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql diff --git a/models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.sql b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql diff --git a/models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.yml b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.yml similarity index 100% rename from models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.yml rename to models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.yml diff --git a/models/silver/nft/metadata/livequery/streamline__null_moments_metadata.sql b/models/silver/nft/metadata/livequery/topshot/streamline__null_moments_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/streamline__null_moments_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/streamline__null_moments_metadata.sql