From 9842ed896a52803a0c06a244abdb5645c4300e47 Mon Sep 17 00:00:00 2001 From: hongwei Date: Thu, 9 Nov 2023 13:52:34 +0100 Subject: [PATCH] feature/added the jedisPool and revert the resourceDocs ttl props --- .../resources/props/sample.props.template | 6 ++++ .../ResourceDocsAPIMethods.scala | 8 ++++- .../main/scala/code/api/cache/Caching.scala | 34 +++++++++---------- .../src/main/scala/code/api/cache/Redis.scala | 18 ++++++++-- .../scala/code/api/constant/constant.scala | 4 +++ .../code/api/v1_4_0/JSONFactory1_4_0.scala | 2 +- .../scala/code/api/v4_0_0/APIMethods400.scala | 10 +++++- release_notes.md | 4 +++ 8 files changed, 64 insertions(+), 22 deletions(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 24859597c..e79ea0032 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -91,6 +91,12 @@ authTypeValidation.cache.ttl.seconds=36 ## no 0 value will cause new ConnectorMethod will works after that seconds connectorMethod.cache.ttl.seconds=40 +## swagger file should not generated for every request, this is a time-to-live in seconds for the generated swagger of OBP api, +## this value also represent how many seconds before the new endpoints will be shown after upload a new DynamicEntity. +## So if you want the new endpoints shown timely, set this value to a small number. +dynamicResourceDocsObp.cache.ttl.seconds=3600 +staticResourceDocsObp.cache.ttl.seconds=3600 +createLocalisedResourceDocJson.cache.ttl.seconds=3600 ## This can change the behavior of `Get Resource Docs`/`Get API Glossary`. If we set it to `true`, OBP will check the authentication and CanReadResourceDoc/CanReadGlossary Role # the default value is false, so the `Get Resource Docs`/`Get API Glossary` is anonymous as default. diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index c0eb2b03e..988184bfc 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -1,6 +1,6 @@ package code.api.ResourceDocs1_4_0 -import code.api.Constant.PARAM_LOCALE +import code.api.Constant.{GET_DYNAMIC_RESOURCE_DOCS_TTL, GET_STATIC_RESOURCE_DOCS_TTL, PARAM_LOCALE} import java.util.UUID.randomUUID import code.api.OBPRestHelper @@ -402,6 +402,10 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth | |See the Resource Doc endpoint for more information. | + |Note: Dynamic Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds + | Static Resource Docs are cached, TTL is ${GET_STATIC_RESOURCE_DOCS_TTL} seconds + | + | |Following are more examples: |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp |${getObpApiRoot}/v4.0.0$endpointBankIdPath/resource-docs/v4.0.0/obp?tags=Account,Bank @@ -674,6 +678,8 @@ trait ResourceDocsAPIMethods extends MdcLoggable with APIMethods220 with APIMeth | |See the Resource Doc endpoint for more information. | + | Note: Resource Docs are cached, TTL is ${GET_DYNAMIC_RESOURCE_DOCS_TTL} seconds + | |Following are more examples: |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger |${getObpApiRoot}/v3.1.0/resource-docs/v3.1.0/swagger?tags=Account,Bank diff --git a/obp-api/src/main/scala/code/api/cache/Caching.scala b/obp-api/src/main/scala/code/api/cache/Caching.scala index b6d1b0c11..c8fd59128 100644 --- a/obp-api/src/main/scala/code/api/cache/Caching.scala +++ b/obp-api/src/main/scala/code/api/cache/Caching.scala @@ -78,71 +78,71 @@ object Caching extends MdcLoggable { } def getLocalisedResourceDocCache(key: String) = { - if(Redis.isRedisAvailable()) - jedis.get(LOCALISED_RESOURCE_DOC_PREFIX + key) + if(CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL!=0 && Redis.isRedisAvailable()) + jedis.get(LOCALISED_RESOURCE_DOC_PREFIX + key) // if the key is not existing, jedis will return null else null } def setLocalisedResourceDocCache(key:String, value: String)={ - if (Redis.isRedisAvailable()) - jedis.set(LOCALISED_RESOURCE_DOC_PREFIX+key,value) + if (CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL!=0 && Redis.isRedisAvailable()) + jedis.setex(LOCALISED_RESOURCE_DOC_PREFIX+key, CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL, value) else null } def getDynamicResourceDocCache(key: String) = { - if (Redis.isRedisAvailable()) + if (GET_DYNAMIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) jedis.get(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key) else null } def setDynamicResourceDocCache(key:String, value: String)={ - if (Redis.isRedisAvailable()) - jedis.set(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value) + if (GET_DYNAMIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) + jedis.setex(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,GET_DYNAMIC_RESOURCE_DOCS_TTL,value) else null } def getStaticResourceDocCache(key: String) = { - if (Redis.isRedisAvailable()) + if (GET_STATIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) jedis.get(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX + key) else null } def setStaticResourceDocCache(key:String, value: String)={ - if (Redis.isRedisAvailable()) - jedis.set(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value) + if (GET_STATIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) + jedis.setex(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX+key,GET_STATIC_RESOURCE_DOCS_TTL,value) else null } def getAllResourceDocCache(key: String) = { - if (Redis.isRedisAvailable()) - jedis.get(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX + key) + if (GET_DYNAMIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) + jedis.get(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX + key) // null else null } def setAllResourceDocCache(key:String, value: String)={ - if (Redis.isRedisAvailable()) - jedis.set(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX+key,value) + if (GET_DYNAMIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) + jedis.setex(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX+key,GET_DYNAMIC_RESOURCE_DOCS_TTL,value) else null } def getStaticSwaggerDocCache(key: String) = { - if (Redis.isRedisAvailable()) + if (GET_STATIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) jedis.get(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX + key) else null } def setStaticSwaggerDocCache(key:String, value: String)={ - if (Redis.isRedisAvailable()) - jedis.set(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX+key,value) + if (GET_STATIC_RESOURCE_DOCS_TTL!=0 && Redis.isRedisAvailable()) + jedis.setex(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX+key,GET_STATIC_RESOURCE_DOCS_TTL,value) else null } diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index d4e4a5faf..3644608c3 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -3,7 +3,7 @@ package code.api.cache import code.api.util.APIUtil import code.util.Helper.MdcLoggable import com.openbankproject.commons.ExecutionContext.Implicits.global -import redis.clients.jedis.Jedis +import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig} import scalacache.memoization.{cacheKeyExclude, memoize, memoizeSync} import scalacache.{Flags, ScalaCache} import scalacache.redis.RedisCache @@ -18,7 +18,21 @@ object Redis extends MdcLoggable { val url = APIUtil.getPropsValue("cache.redis.url", "127.0.0.1") val port = APIUtil.getPropsAsIntValue("cache.redis.port", 6379) - lazy val jedis = new Jedis(url, port) + final val poolConfig = new JedisPoolConfig() + poolConfig.setMaxTotal(128) + poolConfig.setMaxIdle(128) + poolConfig.setMinIdle(16) + poolConfig.setTestOnBorrow(true) + poolConfig.setTestOnReturn(true) + poolConfig.setTestWhileIdle(true) + poolConfig.setMinEvictableIdleTimeMillis(30*60*1000) + poolConfig.setTimeBetweenEvictionRunsMillis(30*60*1000) + poolConfig.setNumTestsPerEvictionRun(3) + poolConfig.setBlockWhenExhausted(true) + + val jedisPool = new JedisPool(poolConfig,url, port, 4000) + + lazy val jedis = jedisPool.getResource() def isRedisAvailable() = { try { diff --git a/obp-api/src/main/scala/code/api/constant/constant.scala b/obp-api/src/main/scala/code/api/constant/constant.scala index 15d715913..3a71d201c 100644 --- a/obp-api/src/main/scala/code/api/constant/constant.scala +++ b/obp-api/src/main/scala/code/api/constant/constant.scala @@ -94,6 +94,10 @@ object Constant extends MdcLoggable { final val STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_static_" final val ALL_RESOURCE_DOC_CACHE_KEY_PREFIX = "rd_all_" final val STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX = "swagger_static_" + val CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL: Int = APIUtil.getPropsValue(s"createLocalisedResourceDocJson.cache.ttl.seconds", "3600").toInt + val GET_DYNAMIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"dynamicResourceDocsObp.cache.ttl.seconds", "3600").toInt + val GET_STATIC_RESOURCE_DOCS_TTL: Int = APIUtil.getPropsValue(s"staticResourceDocsObp.cache.ttl.seconds", "3600").toInt + } diff --git a/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala b/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala index 3b899513b..9e9ea7132 100644 --- a/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala +++ b/obp-api/src/main/scala/code/api/v1_4_0/JSONFactory1_4_0.scala @@ -593,7 +593,7 @@ object JSONFactory1_4_0 extends MdcLoggable{ def createLocalisedResourceDocJson(rd: ResourceDoc, isVersion4OrHigher:Boolean, locale: Option[String], urlParametersI18n:String ,jsonRequestBodyFieldsI18n:String, jsonResponseBodyFieldsI18n:String) : ResourceDocJson = { - // We MUST recompute all resource doc values due to translation via Web UI props --> now need to wait $createLocalisedResourceDocJsonTTL seconds + // We MUST recompute all resource doc values due to translation via Web UI props --> now need to wait $CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL seconds val userDefinedEndpointTags = getAllEndpointTagsBox(rd.operationId).map(endpointTag =>ResourceDocTag(endpointTag.tagName)) val resourceDocWithUserDefinedEndpointTags: ResourceDoc = rd.copy(tags = userDefinedEndpointTags++ rd.tags) diff --git a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala index e892021ba..0f812e84a 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala @@ -7,7 +7,7 @@ import java.util.{Calendar, Date} import code.DynamicData.{DynamicData, DynamicDataProvider} import code.DynamicEndpoint.DynamicEndpointSwagger import code.accountattribute.AccountAttributeX -import code.api.Constant.{PARAM_LOCALE, PARAM_TIMESTAMP, SYSTEM_OWNER_VIEW_ID, localIdentityProvider} +import code.api.Constant.{CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL, PARAM_LOCALE, PARAM_TIMESTAMP, SYSTEM_OWNER_VIEW_ID, localIdentityProvider} import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.{jsonDynamicResourceDoc, _} import code.api.UKOpenBanking.v2_0_0.OBP_UKOpenBanking_200 @@ -11828,6 +11828,8 @@ trait APIMethods400 extends MdcLoggable { "/management/endpoints/OPERATION_ID/tags", "Create System Level Endpoint Tag", s"""Create System Level Endpoint Tag + | + |Note: Resource Docs are cached, TTL is ${CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL} seconds | |""".stripMargin, endpointTagJson400, @@ -11870,6 +11872,8 @@ trait APIMethods400 extends MdcLoggable { "/management/endpoints/OPERATION_ID/tags/ENDPOINT_TAG_ID", "Update System Level Endpoint Tag", s"""Update System Level Endpoint Tag, you can only update the tag_name here, operation_id can not be updated. + | + |Note: Resource Docs are cached, TTL is ${CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL} seconds | |""".stripMargin, endpointTagJson400, @@ -11976,6 +11980,8 @@ trait APIMethods400 extends MdcLoggable { "/management/banks/BANK_ID/endpoints/OPERATION_ID/tags", "Create Bank Level Endpoint Tag", s"""Create Bank Level Endpoint Tag + | + |Note: Resource Docs are cached, TTL is ${CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL} seconds | | |""".stripMargin, @@ -12021,6 +12027,8 @@ trait APIMethods400 extends MdcLoggable { "/management/banks/BANK_ID/endpoints/OPERATION_ID/tags/ENDPOINT_TAG_ID", "Update Bank Level Endpoint Tag", s"""Update Endpoint Tag, you can only update the tag_name here, operation_id can not be updated. + | + |Note: Resource Docs are cached, TTL is ${CREATE_LOCALISED_RESOURCE_DOC_JSON_TTL} seconds | |""".stripMargin, endpointTagJson400, diff --git a/release_notes.md b/release_notes.md index 689973341..45db4d457 100644 --- a/release_notes.md +++ b/release_notes.md @@ -3,6 +3,7 @@ ### Most recent changes at top of file ``` Date Commit Action +30/10/2023 4e82c66c Added createLocalisedResourceDocJson.cache.ttl.seconds, default is 3600 13/10/2023 d87c99d8 Added props hikari.connectionTimeout, default is from hikari. Added props hikari.maximumPoolSize, default is from hikari. Added props hikari.idleTimeout, default is from hikari. @@ -41,6 +42,9 @@ Date Commit Action 18/08/2021 83c3b4fa Added props: webui_favicon_link_url, default is /favicon.ico 18/08/2021 5924c820 Added props: webui_api_documentation_bottom_url, default is https://github.com/OpenBankProject/OBP-API/wiki Added props: webui_privacy_policy_url, default is https://openbankproject.com/privacy-policy +30/06/2021 cf2dd987 Changed props, static will cache 24 hours, dynamic only 1 hour as default. + dynamicResourceDocsObp.cache.ttl.seconds=3600 + staticResourceDocsObp.cache.ttl.seconds=3600 30/06/2021 cf2dd987 Added props: email_domain_to_entitlement_mappings, default is empty We can automatically grant the Entitlements required to the User has access to via their validated email domain. Entitlements are generated /refreshed both following manual locin and Direct Login token generation (POST).