From cf619eec91e466d3e18440414d43c39721fed010 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 28 Dec 2025 14:46:43 +0100 Subject: [PATCH] system cache namespaces WIP --- .../src/main/scala/code/api/cache/Redis.scala | 47 ++++++++ .../scala/code/api/constant/constant.scala | 17 ++- .../main/scala/code/api/util/ApiRole.scala | 9 ++ .../scala/code/api/v6_0_0/APIMethods600.scala | 106 +++++++++++++++++- .../code/api/v6_0_0/JSONFactory6.0.0.scala | 35 ++++++ 5 files changed, 212 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index 18fb9e9a5..bf9622929 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -197,4 +197,51 @@ object Redis extends MdcLoggable { memoize(ttl)(f) } + + /** + * Scan Redis keys matching a pattern using KEYS command + * Note: In production with large datasets, consider using SCAN instead + * + * @param pattern Redis pattern (e.g., "rl_counter_*", "rd_*") + * @return List of matching keys + */ + def scanKeys(pattern: String): List[String] = { + var jedisConnection: Option[Jedis] = None + try { + jedisConnection = Some(jedisPool.getResource()) + val jedis = jedisConnection.get + + import scala.collection.JavaConverters._ + val keys = jedis.keys(pattern) + keys.asScala.toList + + } catch { + case e: Throwable => + logger.error(s"Error scanning Redis keys with pattern $pattern: ${e.getMessage}") + List.empty + } finally { + if (jedisConnection.isDefined && jedisConnection.get != null) + jedisConnection.foreach(_.close()) + } + } + + /** + * Count keys matching a pattern + * + * @param pattern Redis pattern (e.g., "rl_counter_*") + * @return Number of matching keys + */ + def countKeys(pattern: String): Int = { + scanKeys(pattern).size + } + + /** + * Get a sample key matching a pattern (first found) + * + * @param pattern Redis pattern + * @return Option of a sample key + */ + def getSampleKey(pattern: String): Option[String] = { + scanKeys(pattern).headOption + } } diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 128f7b209..f8c70ed9d 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -127,6 +127,21 @@ object Constant extends MdcLoggable { final val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt final val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt final val SHOW_USED_CONNECTOR_METHODS: Boolean = APIUtil.getPropsAsBoolValue(s"show_used_connector_methods", false) + + // Rate Limiting Cache Prefixes + final val RATE_LIMIT_COUNTER_PREFIX = "rl_counter_" + final val RATE_LIMIT_ACTIVE_PREFIX = "rl_active_" + final val RATE_LIMIT_ACTIVE_CACHE_TTL: Int = APIUtil.getPropsValue("rateLimitActive.cache.ttl.seconds", "3600").toInt + + // Connector Cache Prefixes + final val CONNECTOR_PREFIX = "connector_" + + // Metrics Cache Prefixes + final val METRICS_STABLE_PREFIX = "metrics_stable_" + final val METRICS_RECENT_PREFIX = "metrics_recent_" + + // ABAC Cache Prefixes + final val ABAC_RULE_PREFIX = "abac_rule_" final val CAN_SEE_TRANSACTION_OTHER_BANK_ACCOUNT = "can_see_transaction_other_bank_account" final val CAN_SEE_TRANSACTION_METADATA = "can_see_transaction_metadata" @@ -517,7 +532,7 @@ object PrivateKeyConstants { object JedisMethod extends Enumeration { type JedisMethod = Value - val GET, SET, EXISTS, DELETE, TTL, INCR, FLUSHDB= Value + val GET, SET, EXISTS, DELETE, TTL, INCR, FLUSHDB, SCAN = Value } diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index 445288078..defdd4db8 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -412,6 +412,15 @@ object ApiRole extends MdcLoggable{ lazy val canGetMetricsAtOneBank = CanGetMetricsAtOneBank() case class CanGetConfig(requiresBankId: Boolean = false) extends ApiRole + + case class CanGetCacheNamespaces(requiresBankId: Boolean = false) extends ApiRole + lazy val canGetCacheNamespaces = CanGetCacheNamespaces() + + case class CanDeleteCacheNamespace(requiresBankId: Boolean = false) extends ApiRole + lazy val canDeleteCacheNamespace = CanDeleteCacheNamespace() + + case class CanDeleteCacheKey(requiresBankId: Boolean = false) extends ApiRole + lazy val canDeleteCacheKey = CanDeleteCacheKey() lazy val canGetConfig = CanGetConfig() case class CanGetAdapterInfo(requiresBankId: Boolean = false) extends ApiRole diff --git a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala index 8e1fae385..9ce5774d2 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/APIMethods600.scala @@ -4,7 +4,7 @@ import code.accountattribute.AccountAttributeX import code.api.Constant import code.api.{DirectLogin, ObpApiFailure} import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._ -import code.api.cache.Caching +import code.api.cache.{Caching, Redis} import code.api.util.APIUtil._ import code.api.util.ApiRole import code.api.util.ApiRole._ @@ -1028,6 +1028,110 @@ trait APIMethods600 { } } + staticResourceDocs += ResourceDoc( + getCacheNamespaces, + implementedInApiVersion, + nameOf(getCacheNamespaces), + "GET", + "/system/cache/namespaces", + "Get Cache Namespaces", + """Returns information about all cache namespaces in the system. + | + |This endpoint provides visibility into: + |* Cache namespace prefixes and their purposes + |* Number of keys in each namespace + |* TTL configurations + |* Example keys for each namespace + | + |This is useful for: + |* Monitoring cache usage + |* Understanding cache structure + |* Debugging cache-related issues + |* Planning cache management operations + | + |""", + EmptyBody, + CacheNamespacesJsonV600( + namespaces = List( + CacheNamespaceJsonV600( + prefix = "rl_counter_", + description = "Rate limiting counters per consumer and time period", + ttl_seconds = "varies", + category = "Rate Limiting", + key_count = 42, + example_key = "rl_counter_consumer123_PER_MINUTE" + ), + CacheNamespaceJsonV600( + prefix = "rl_active_", + description = "Active rate limit configurations", + ttl_seconds = "3600", + category = "Rate Limiting", + key_count = 15, + example_key = "rl_active_consumer123_2024-12-27-14" + ), + CacheNamespaceJsonV600( + prefix = "rd_localised_", + description = "Localized resource documentation", + ttl_seconds = "3600", + category = "Resource Documentation", + key_count = 128, + example_key = "rd_localised_operationId:getBanks-locale:en" + ) + ) + ), + List( + $UserNotLoggedIn, + UserHasMissingRoles, + UnknownError + ), + List(apiTagSystem, apiTagApi), + Some(List(canGetCacheNamespaces)) + ) + + lazy val getCacheNamespaces: OBPEndpoint = { + case "system" :: "cache" :: "namespaces" :: Nil JsonGet _ => { + cc => implicit val ec = EndpointContext(Some(cc)) + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, canGetCacheNamespaces, callContext) + } yield { + // Define known cache namespaces with their metadata + val namespaces = List( + // Rate Limiting + (Constant.RATE_LIMIT_COUNTER_PREFIX, "Rate limiting counters per consumer and time period", "varies", "Rate Limiting"), + (Constant.RATE_LIMIT_ACTIVE_PREFIX, "Active rate limit configurations", Constant.RATE_LIMIT_ACTIVE_CACHE_TTL.toString, "Rate Limiting"), + // Resource Documentation + (Constant.LOCALISED_RESOURCE_DOC_PREFIX, "Localized resource documentation", Constant.CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL.toString, "Resource Documentation"), + (Constant.DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX, "Dynamic resource documentation", Constant.GET_DYNAMIC_RESOURCE_DOCS_TTL.toString, "Resource Documentation"), + (Constant.STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX, "Static resource documentation", Constant.GET_STATIC_RESOURCE_DOCS_TTL.toString, "Resource Documentation"), + (Constant.ALL_RESOURCE_DOC_CACHE_KEY_PREFIX, "All resource documentation", Constant.GET_STATIC_RESOURCE_DOCS_TTL.toString, "Resource Documentation"), + (Constant.STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX, "Swagger documentation", Constant.GET_STATIC_RESOURCE_DOCS_TTL.toString, "Resource Documentation"), + // Connector + (Constant.CONNECTOR_PREFIX, "Connector method names and metadata", "3600", "Connector"), + // Metrics + (Constant.METRICS_STABLE_PREFIX, "Stable metrics (historical)", "86400", "Metrics"), + (Constant.METRICS_RECENT_PREFIX, "Recent metrics", "7", "Metrics"), + // ABAC + (Constant.ABAC_RULE_PREFIX, "ABAC rule cache", "indefinite", "ABAC") + ).map { case (prefix, description, ttl, category) => + // Get actual key count and example from Redis + val keyCount = Redis.countKeys(s"${prefix}*") + val exampleKey = Redis.getSampleKey(s"${prefix}*") + JSONFactory600.createCacheNamespaceJsonV600( + prefix = prefix, + description = description, + ttlSeconds = ttl, + category = category, + keyCount = keyCount, + exampleKey = exampleKey + ) + } + + (JSONFactory600.createCacheNamespacesJsonV600(namespaces), HttpCode.`200`(callContext)) + } + } + } + staticResourceDocs += ResourceDoc( createTransactionRequestCardano, implementedInApiVersion, diff --git a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala index 3fda74ce9..3ae2d70e6 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala @@ -246,6 +246,17 @@ case class ProvidersJsonV600(providers: List[String]) case class ConnectorMethodNamesJsonV600(connector_method_names: List[String]) +case class CacheNamespaceJsonV600( + prefix: String, + description: String, + ttl_seconds: String, + category: String, + key_count: Int, + example_key: String +) + +case class CacheNamespacesJsonV600(namespaces: List[CacheNamespaceJsonV600]) + case class PostCustomerJsonV600( legal_name: String, customer_number: Option[String] = None, @@ -1030,4 +1041,28 @@ object JSONFactory600 extends CustomJsonFormats with MdcLoggable { ): AbacRulesJsonV600 = { AbacRulesJsonV600(rules.map(createAbacRuleJsonV600)) } + + def createCacheNamespaceJsonV600( + prefix: String, + description: String, + ttlSeconds: String, + category: String, + keyCount: Int, + exampleKey: Option[String] + ): CacheNamespaceJsonV600 = { + CacheNamespaceJsonV600( + prefix = prefix, + description = description, + ttl_seconds = ttlSeconds, + category = category, + key_count = keyCount, + example_key = exampleKey.getOrElse("") + ) + } + + def createCacheNamespacesJsonV600( + namespaces: List[CacheNamespaceJsonV600] + ): CacheNamespacesJsonV600 = { + CacheNamespacesJsonV600(namespaces) + } }