diff --git a/r/shroomDK/DESCRIPTION b/r/shroomDK/DESCRIPTION index d5e7f9c..5caa861 100644 --- a/r/shroomDK/DESCRIPTION +++ b/r/shroomDK/DESCRIPTION @@ -1,10 +1,10 @@ Package: shroomDK Type: Package -Title: Accessing the Flipside Crypto ShroomDK REST API -Version: 0.1.1 +Title: Accessing the Flipside Crypto ShroomDK API +Version: 0.2.0 Author: Carlos Mercado Maintainer: Carlos Mercado -Description: Programmatic access to Flipside Crypto data via the REST API: . As simple as auto_paginate_query() but with core functions as needed for troubleshooting. +Description: Programmatic access to Flipside Crypto data via the Compass RPC API: . As simple as auto_paginate_query() but with core functions as needed for troubleshooting. Imports: jsonlite, httr License: MIT + file LICENSE Encoding: UTF-8 diff --git a/r/shroomDK/NAMESPACE b/r/shroomDK/NAMESPACE index 3238d8a..55e8b1d 100644 --- a/r/shroomDK/NAMESPACE +++ b/r/shroomDK/NAMESPACE @@ -1,6 +1,7 @@ # Generated by roxygen2: do not edit by hand export(auto_paginate_query) +export(cancel_query) export(clean_query) export(create_query_token) export(get_query_from_token) diff --git a/r/shroomDK/R/cancel_query.R b/r/shroomDK/R/cancel_query.R new file mode 100644 index 0000000..26e714a --- /dev/null +++ b/r/shroomDK/R/cancel_query.R @@ -0,0 +1,60 @@ +library(jsonlite) +library(httr) + +#' Cancel Query +#' +#' Uses Flipside ShroomDK to CANCEL a query run id from `create_query_token()`, as the new API uses warehouse-seconds to charge users above the free tier, +#' the ability to cancel is critical for cost management. +#' @param query_run_id queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId` +#' @param api_key Flipside Crypto ShroomDK API Key +#' @param api_url default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user. +#' @return returns a list of the status_canceled (TRUE or FALSE) and the cancel object (which includes related details). +#' @import jsonlite httr +#' @export +#' +#' @examples +#' \dontrun{ +#' query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000000", api_key) +#' query_status <- get_query_status(query$result$queryRequest$queryRunId, api_key) +#' canceled <- cancel_query(query$result$queryRequest$queryRunId, api_key) +#' } +cancel_query <- function(query_run_id, api_key, + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc"){ + + headers = c( + "Content-Type" = 'application/json', + "x-api-key" = api_key + ) + + # get status of a run id + request_cancel_body <- as.character( + jsonlite::toJSON(pretty = TRUE, + list( + "jsonrpc" = "2.0", + "method" = "cancelQueryRun", + "params" = list( + list( "queryRunId" = query_run_id + ) + ), + "id" = 1 + ), + auto_unbox = TRUE + ) + ) + + canceled <- content( + httr::POST( + api_url, + config = httr::add_headers(.headers = headers), + body = request_cancel_body) + ) + + statecheck = canceled$result$canceledQueryRun$state == "QUERY_STATE_CANCELED" + + return( + list( + status_canceled = statecheck, + cancellation_details = canceled + ) + ) +} diff --git a/r/shroomDK/R/clean_query.R b/r/shroomDK/R/clean_query.R index dd8dcd9..9fd6d45 100644 --- a/r/shroomDK/R/clean_query.R +++ b/r/shroomDK/R/clean_query.R @@ -9,7 +9,7 @@ #' @param try_simplify because requests can return JSON and may not have the same length #' across values, they may not be data frame compliant (all columns having the same number of rows). #' A key example would be TX_JSON in EVM FACT_TRANSACTION tables which include 50+ -#' extra details from transaction logs. But other examples like NULLs TO_ADDRESS can have similar +#' extra details from transaction logs. But other examples like NULLs in TO_ADDRESS can have similar #' issues. Default TRUE. #' #' @return A data frame. If `try_simplify` is FALSE OR if `try_simplify` TRUE fails: @@ -20,9 +20,10 @@ #' #' @examples #' \dontrun{ -#' query = create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) -#' request = get_query_from_token(query$result$queryRequest$queryRunId, api_key) -#' clean_query(request, try_simplify = FALSE) +#' query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) +#' request <- get_query_from_token(query$result$queryRequest$queryRunId, api_key) +#' df1 <- clean_query(request, try_simplify = TRUE) # warning b/c of tx_json +#' df2 <- clean_query(request, try_simplify = FALSE) # silently returns columns of lists #' } clean_query <- function(request, try_simplify = TRUE){ diff --git a/r/shroomDK/R/get_query_from_token.R b/r/shroomDK/R/get_query_from_token.R index 854121e..dfa8477 100644 --- a/r/shroomDK/R/get_query_from_token.R +++ b/r/shroomDK/R/get_query_from_token.R @@ -3,10 +3,8 @@ library(httr) #' Get Query From Token #' -#' Uses Flipside ShroomDK to access a Query Token. Query tokens are cached up to `ttl` minutes -#' for each `query`. This function is for pagination and multiple requests -#' while . Note: To reduce payload it returns -#' a list of outputs (separating column names from rows). +#' Uses Flipside ShroomDK to access a Query Token (Run ID). This function is for pagination and multiple requests. +#' Note: To reduce payload it returns a list of outputs (separating column names from rows). Use `clean_query()` to #' #' @param query_run_id queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId` #' @param api_key Flipside Crypto ShroomDK API Key @@ -14,15 +12,16 @@ library(httr) #' @param page_size Default 1000. Paginate via page_number. May return error if page_size causes data to exceed 30MB. #' @param result_format Default to csv. Options: csv and json. #' @param api_url default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user. -#' @return returns a request of length 8: `results`, `columnLabels`, -#' `columnTypes`, `startedAt`, `endedAt`, `pageNumber`, `pageSize`, `status` +#' @return returns a list of jsonrpc, id, and result. Within result are: +#' columnNames, columnTypes, rows, page, sql, format, originalQueryRun, redirectedToQueryRun +#' use `clean_query()` to transform this into a data frame. #' @import jsonlite httr #' @export #' #' @examples #' \dontrun{ -#' query = create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) -#' get_query_from_token(query$result$queryRequest$queryRunId, api_key, 1, 1000) +#' query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) +#' fact_transactions <- get_query_from_token(query$result$queryRequest$queryRunId, api_key, 1, 1000) #' } get_query_from_token <- function(query_run_id, api_key, page_number = 1, diff --git a/r/shroomDK/R/get_query_status.R b/r/shroomDK/R/get_query_status.R index c6147bf..fe0fedd 100644 --- a/r/shroomDK/R/get_query_status.R +++ b/r/shroomDK/R/get_query_status.R @@ -7,7 +7,8 @@ library(httr) #' @param query_run_id queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId` #' @param api_key Flipside Crypto ShroomDK API Key #' @param api_url default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user. -#' @return returns request content; for content `x`, use `x$result$queryRun$state` and `x$result$queryRun$errorMessage` +#' @return returns request content; for content `x`, use `x$result$queryRun$state` and `x$result$queryRun$errorMessage`. Expect one of +#' QUERY_STATE_READY, QUERY_STATE_RUNNING, QUERY_STATE_STREAMING_RESULTS, QUERY_STATE_SUCCESS, QUERY_STATE_FAILED, QUERY_STATE_CANCELED #' @import jsonlite httr #' @export #' diff --git a/r/shroomDK/man/auto_paginate_query.Rd b/r/shroomDK/man/auto_paginate_query.Rd index 9941b76..15b0579 100644 --- a/r/shroomDK/man/auto_paginate_query.Rd +++ b/r/shroomDK/man/auto_paginate_query.Rd @@ -4,25 +4,36 @@ \alias{auto_paginate_query} \title{Auto Paginate Queries} \usage{ -auto_paginate_query(query, api_key, maxrows = 1e+06) +auto_paginate_query( + query, + api_key, + page_size = 1000, + page_count = 1, + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc" +) } \arguments{ \item{query}{The SQL query to pass to ShroomDK} \item{api_key}{ShroomDK API key.} -\item{maxrows}{Max rows allowed in ShroomDK, 1M at time of writing.} +\item{page_size}{Default 1000. May return error if page_size is tool large and data to exceed 30MB.} + +\item{page_count}{Default 1. How many pages, of page_size rows each, to read.} + +\item{api_url}{default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user.} } \value{ -data frame of up to 1M rows, see ?clean_query for more details on column classes. +data frame of up to `page_size * page_count` rows, see ?clean_query for more details on column classes. } \description{ -Grabs up to maxrows in a query by going through each page 100k rows at a time. +Grabs up to maxrows in a query by going through each page to download one at a time. } \examples{ \dontrun{ pull_data <- auto_paginate_query(" -SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 10000", -api_key = readLines("api_key.txt")) +SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 10001", +api_key = readLines("api_key.txt"), +page_count = 10) } } diff --git a/r/shroomDK/man/cancel_query.Rd b/r/shroomDK/man/cancel_query.Rd new file mode 100644 index 0000000..641f2d8 --- /dev/null +++ b/r/shroomDK/man/cancel_query.Rd @@ -0,0 +1,33 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/cancel_query.R +\name{cancel_query} +\alias{cancel_query} +\title{Cancel Query} +\usage{ +cancel_query( + query_run_id, + api_key, + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc" +) +} +\arguments{ +\item{query_run_id}{queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId`} + +\item{api_key}{Flipside Crypto ShroomDK API Key} + +\item{api_url}{default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user.} +} +\value{ +returns a list of the status_canceled (TRUE or FALSE) and the cancel object (which includes related details). +} +\description{ +Uses Flipside ShroomDK to CANCEL a query run id from `create_query_token()`, as the new API uses warehouse-seconds to charge users above the free tier, +the ability to cancel is critical for cost management. +} +\examples{ +\dontrun{ +query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000000", api_key) +query_status <- get_query_status(query$result$queryRequest$queryRunId, api_key) +canceled <- cancel_query(query$result$queryRequest$queryRunId, api_key) +} +} diff --git a/r/shroomDK/man/clean_query.Rd b/r/shroomDK/man/clean_query.Rd index 5fcd52a..045ecb8 100644 --- a/r/shroomDK/man/clean_query.Rd +++ b/r/shroomDK/man/clean_query.Rd @@ -12,7 +12,7 @@ clean_query(request, try_simplify = TRUE) \item{try_simplify}{because requests can return JSON and may not have the same length across values, they may not be data frame compliant (all columns having the same number of rows). A key example would be TX_JSON in EVM FACT_TRANSACTION tables which include 50+ -extra details from transaction logs. But other examples like NULLs TO_ADDRESS can have similar +extra details from transaction logs. But other examples like NULLs in TO_ADDRESS can have similar issues. Default TRUE.} } \value{ @@ -26,8 +26,9 @@ intelligently. } \examples{ \dontrun{ -query = create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 10000", api_key) -request = get_query_from_token(query$token, api_key, 1, 10000) -clean_query(request, try_simplify = FALSE) +query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) +request <- get_query_from_token(query$result$queryRequest$queryRunId, api_key) +df1 <- clean_query(request, try_simplify = TRUE) # warning b/c of tx_json +df2 <- clean_query(request, try_simplify = FALSE) # silently returns columns of lists } } diff --git a/r/shroomDK/man/create_query_token.Rd b/r/shroomDK/man/create_query_token.Rd index 0adab84..900e331 100644 --- a/r/shroomDK/man/create_query_token.Rd +++ b/r/shroomDK/man/create_query_token.Rd @@ -4,16 +4,24 @@ \alias{create_query_token} \title{Create Query Token} \usage{ -create_query_token(query, api_key, ttl = 10, cache = TRUE) +create_query_token( + query, + api_key, + ttl = 1, + mam = 10, + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc" +) } \arguments{ \item{query}{Flipside Crypto Snowflake SQL compatible query as a string.} \item{api_key}{Flipside Crypto ShroomDK API Key} -\item{ttl}{time (in minutes) to keep query in cache.} +\item{ttl}{time-to-live (in hours) to keep query results available. Default 1 hour.} -\item{cache}{Use cached results; set as FALSE to re-execute.} +\item{mam}{max-age-minutes, lifespan of cache. set to 0 to always re-execute. Default 10 minutes.} + +\item{api_url}{default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user.} } \value{ list of `token` and `cached` use `token` in `get_query_from_token()` @@ -28,7 +36,7 @@ allowing for pagination and multiple requests before expending more daily reques create_query_token( query = "SELECT * FROM ethereum.core.fact_transactions LIMIT 1", api_key = readLines("api_key.txt"), -ttl = 15, -cache = TRUE) +ttl = 1, +mam = 5) } } diff --git a/r/shroomDK/man/get_query_from_token.Rd b/r/shroomDK/man/get_query_from_token.Rd index b2d47d1..2fff968 100644 --- a/r/shroomDK/man/get_query_from_token.Rd +++ b/r/shroomDK/man/get_query_from_token.Rd @@ -4,30 +4,40 @@ \alias{get_query_from_token} \title{Get Query From Token} \usage{ -get_query_from_token(query_token, api_key, page_number = 1, page_size = 1e+05) +get_query_from_token( + query_run_id, + api_key, + page_number = 1, + page_size = 1000, + result_format = "csv", + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc" +) } \arguments{ -\item{query_token}{token from `create_query_token()`} +\item{query_run_id}{queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId`} \item{api_key}{Flipside Crypto ShroomDK API Key} -\item{page_number}{Query tokens are cached and 100k rows max. Get up to 1M rows by going through pages.} +\item{page_number}{Results are cached, max 30MB of data per page.} -\item{page_size}{Default 100,000. Paginate via page_number.} +\item{page_size}{Default 1000. Paginate via page_number. May return error if page_size causes data to exceed 30MB.} + +\item{result_format}{Default to csv. Options: csv and json.} + +\item{api_url}{default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user.} } \value{ -returns a request of length 8: `results`, `columnLabels`, - `columnTypes`, `startedAt`, `endedAt`, `pageNumber`, `pageSize`, `status` +returns a list of jsonrpc, id, and result. Within result are: +columnNames, columnTypes, rows, page, sql, format, originalQueryRun, redirectedToQueryRun +use `clean_query()` to transform this into a data frame. } \description{ -Uses Flipside ShroomDK to access a Query Token. Query tokens are cached up to `ttl` minutes -for each `query`. This function is for pagination and multiple requests -while . Note: To reduce payload it returns -a list of outputs (separating column names from rows). +Uses Flipside ShroomDK to access a Query Token (Run ID). This function is for pagination and multiple requests. +Note: To reduce payload it returns a list of outputs (separating column names from rows). Use `clean_query()` to } \examples{ \dontrun{ -query = create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 10000", api_key) -get_query_from_token(query$token, api_key, 1, 10000) +query <- create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 1000", api_key) +fact_transactions <- get_query_from_token(query$result$queryRequest$queryRunId, api_key, 1, 1000) } } diff --git a/r/shroomDK/man/get_query_status.Rd b/r/shroomDK/man/get_query_status.Rd new file mode 100644 index 0000000..14e87e6 --- /dev/null +++ b/r/shroomDK/man/get_query_status.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_query_status.R +\name{get_query_status} +\alias{get_query_status} +\title{Get Query ID Status} +\usage{ +get_query_status( + query_run_id, + api_key, + api_url = "https://api-v2.flipsidecrypto.xyz/json-rpc" +) +} +\arguments{ +\item{query_run_id}{queryRunId from `create_query_token()`, for token stored as `x`, use `x$result$queryRequest$queryRunId`} + +\item{api_key}{Flipside Crypto ShroomDK API Key} + +\item{api_url}{default to https://api-v2.flipsidecrypto.xyz/json-rpc but upgradeable for user.} +} +\value{ +returns request content; for content `x`, use `x$result$queryRun$state` and `x$result$queryRun$errorMessage`. Expect one of +QUERY_STATE_READY, QUERY_STATE_RUNNING, QUERY_STATE_STREAMING_RESULTS, QUERY_STATE_SUCCESS, QUERY_STATE_FAILED, QUERY_STATE_CANCELED +} +\description{ +Uses Flipside ShroomDK to access the status of a query run id from `create_query_token()` +} +\examples{ +\dontrun{ +query = create_query_token("SELECT * FROM ETHEREUM.CORE.FACT_TRANSACTIONS LIMIT 10000", api_key) +get_query_status(query$result$queryRequest$queryRunId, api_key) +} +} diff --git a/r/shroomDK_0.2.0.tar.gz b/r/shroomDK_0.2.0.tar.gz new file mode 100644 index 0000000..ba8b349 Binary files /dev/null and b/r/shroomDK_0.2.0.tar.gz differ