feature/Log Cache Endpoints - Add OBP API for tail (last 500 lines) of log file WIP

This commit is contained in:
Marko Milić 2025-08-26 09:58:43 +02:00
parent bd43d9347f
commit 9130b3a3af
8 changed files with 328 additions and 52 deletions

View File

@ -1511,3 +1511,15 @@ regulated_entities = []
# Note: For secure and http only settings for cookies see resources/web.xml which is mentioned in the README.md
## SIX different FIFO redis queues. Each queue have a maximum of 1000 entries.
## with 6 Props for the number of messages in each queue.
## 0 is the default and we don't write to the redis cache
# keep_n_trace_level_logs_in_cache = 0
# keep_n_debug_level_logs_in_cache = 0
# keep_n_info_level_logs_in_cache = 0
# keep_n_warning_level_logs_in_cache = 0
# keep_n_error_level_logs_in_cache = 0
# keep_n_all_level_logs_in_cache = 0

View File

@ -57,6 +57,22 @@ object Redis extends MdcLoggable {
def jedisPoolDestroy: Unit = jedisPool.destroy()
def isRedisReady: Boolean = {
var jedisConnection: Option[Jedis] = None
try {
jedisConnection = Some(jedisPool.getResource)
val pong = jedisConnection.get.ping() // sends PING command
pong == "PONG"
} catch {
case e: Throwable =>
logger.error(s"Redis is not ready: ${e.getMessage}")
false
} finally {
jedisConnection.foreach(_.close())
}
}
private def configureSslContext(): SSLContext = {
// Load the CA certificate

View File

@ -0,0 +1,124 @@
package code.api.cache
import code.api.util.APIUtil
import redis.clients.jedis.Pipeline
import scala.collection.JavaConverters._
/**
* Redis queue configuration per log level.
*/
case class RedisLogConfig(
queueName: String,
keepInCache: Int
)
/**
* Simple Redis FIFO log writer.
*/
object RedisLogger {
/**
* Redis-backed logging utilities for OBP.
*/
object LogLevel extends Enumeration {
type LogLevel = Value
val TRACE, DEBUG, INFO, WARNING, ERROR, ALL = Value
/** Parse a string into LogLevel, defaulting to INFO if unknown */
def valueOf(str: String): LogLevel = str.toUpperCase match {
case "TRACE" => TRACE
case "DEBUG" => DEBUG
case "INFO" => INFO
case "WARN" | "WARNING" => WARNING
case "ERROR" => ERROR
case "ALL" => ALL
case other =>
// fallback
INFO
}
}
// Define FIFO queues with max 1000 entries, configurable keepInCache
val configs = Map(
LogLevel.TRACE -> RedisLogConfig("trace_logs", APIUtil.getPropsAsIntValue("keep_n_trace_level_logs_in_cache", 0)),
LogLevel.DEBUG -> RedisLogConfig("debug_logs", APIUtil.getPropsAsIntValue("keep_n_debug_level_logs_in_cache", 0)),
LogLevel.INFO -> RedisLogConfig("info_logs", APIUtil.getPropsAsIntValue("keep_n_info_level_logs_in_cache", 0)),
LogLevel.WARNING -> RedisLogConfig("warning_logs", APIUtil.getPropsAsIntValue("keep_n_warning_level_logs_in_cache", 0)),
LogLevel.ERROR -> RedisLogConfig("error_logs", APIUtil.getPropsAsIntValue("keep_n_error_level_logs_in_cache", 0)),
LogLevel.ALL -> RedisLogConfig("all_logs", APIUtil.getPropsAsIntValue("keep_n_all_level_logs_in_cache", 0))
)
/**
* Write a log line to Redis FIFO queue.
*/
def log(level: LogLevel.LogLevel, message: String): Unit = {
if (Redis.jedisPool != null && configs != null) {
val jedis = Redis.jedisPool.getResource
try {
val pipeline: Pipeline = jedis.pipelined()
// Always log to the given level
val levelConfig = configs(level)
if (levelConfig.keepInCache > 0) {
pipeline.lpush(levelConfig.queueName, message)
pipeline.ltrim(levelConfig.queueName, 0, levelConfig.keepInCache - 1)
}
// Also log to ALL
val allConfig = configs(LogLevel.ALL)
if (allConfig.keepInCache > 0) {
pipeline.lpush(allConfig.queueName, s"[$level] $message")
pipeline.ltrim(allConfig.queueName, 0, allConfig.keepInCache - 1)
}
pipeline.sync()
} finally {
jedis.close()
}
}
}
case class LogEntry(level: String, message: String)
case class LogTail(entries: List[LogEntry])
/**
* Read latest messages from Redis FIFO queue.
*/
def tail(level: LogLevel.LogLevel): LogTail = {
val config = configs(level)
val jedis = Redis.jedisPool.getResource
try {
val rawLogs = jedis.lrange(config.queueName, 0, -1).asScala.toList.reverse
// define regex once
val pattern = """\[(\w+)\]\s+(.*)""".r
val entries: List[LogEntry] = level match {
case LogLevel.ALL =>
rawLogs.flatMap {
case pattern(lvl, msg) =>
Some(LogEntry(lvl, msg)) // lvl is string like "DEBUG"
case _ =>
None
}
case other =>
rawLogs.map(msg => LogEntry(other.toString, msg))
}
LogTail(entries)
} finally {
jedis.close()
}
}
}

View File

@ -2937,8 +2937,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
val errorResponse = getFilteredOrFullErrorMessage(e)
Full(reply.apply(errorResponse))
case Failure(msg, e, _) =>
surroundErrorMessage(msg)
e.foreach(logger.debug("", _))
e.foreach(logger.error(msg, _))
extractAPIFailureNewStyle(msg) match {
case Some(af) =>
val callContextLight = af.ccl.map(_.copy(httpCode = Some(af.failCode)))
@ -3013,8 +3012,8 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
val xRequestId: Option[String] =
reqHeaders.find(_.name.toLowerCase() == RequestHeader.`X-Request-ID`.toLowerCase())
.map(_.values.mkString(","))
val title = s"Request Headers for verb: $verb, URL: $url"
surroundDebugMessage(reqHeaders.map(h => h.name + ": " + h.values.mkString(",")).mkString, title)
logger.debug(s"Request Headers for verb: $verb, URL: $url")
logger.debug(reqHeaders.map(h => h.name + ": " + h.values.mkString(",")).mkString)
val remoteIpAddress = getRemoteIpAddress()
val authHeaders = AuthorisationUtil.getAuthorisationHeaders(reqHeaders)

View File

@ -107,6 +107,38 @@ object ApiRole extends MdcLoggable{
case class CanCreateCustomer(requiresBankId: Boolean = true) extends ApiRole
lazy val canCreateCustomer = CanCreateCustomer()
// TRACE
case class CanGetTraceLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetTraceLevelLogsAtOneBank = CanGetTraceLevelLogsAtOneBank()
case class CanGetTraceLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetTraceLevelLogsAtAllBanks = CanGetTraceLevelLogsAtAllBanks()
// DEBUG
case class CanGetDebugLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetDebugLevelLogsAtOneBank = CanGetDebugLevelLogsAtOneBank()
case class CanGetDebugLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetDebugLevelLogsAtAllBanks = CanGetDebugLevelLogsAtAllBanks()
// INFO
case class CanGetInfoLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetInfoLevelLogsAtOneBank = CanGetInfoLevelLogsAtOneBank()
case class CanGetInfoLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetInfoLevelLogsAtAllBanks = CanGetInfoLevelLogsAtAllBanks()
// WARNING
case class CanGetWarningLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetWarningLevelLogsAtOneBank = CanGetWarningLevelLogsAtOneBank()
case class CanGetWarningLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetWarningLevelLogsAtAllBanks = CanGetWarningLevelLogsAtAllBanks()
// ERROR
case class CanGetErrorLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetErrorLevelLogsAtOneBank = CanGetErrorLevelLogsAtOneBank()
case class CanGetErrorLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetErrorLevelLogsAtAllBanks = CanGetErrorLevelLogsAtAllBanks()
// ALL
case class CanGetAllLevelLogsAtOneBank(requiresBankId: Boolean = true) extends ApiRole
lazy val canGetAllLevelLogsAtOneBank = CanGetAllLevelLogsAtOneBank()
case class CanGetAllLevelLogsAtAllBanks(requiresBankId: Boolean = false) extends ApiRole
lazy val canGetAllLevelLogsAtAllBanks = CanGetAllLevelLogsAtAllBanks()
case class CanUpdateAgentStatusAtAnyBank(requiresBankId: Boolean = false) extends ApiRole
lazy val canUpdateAgentStatusAtAnyBank = CanUpdateAgentStatusAtAnyBank()

View File

@ -6,6 +6,7 @@ import code.api.Constant._
import code.api.OAuth2Login.{Keycloak, OBPOIDC}
import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._
import code.api.berlin.group.v1_3.JSONFactory_BERLIN_GROUP_1_3.{ConsentAccessAccountsJson, ConsentAccessJson}
import code.api.cache.RedisLogger
import code.api.util.APIUtil._
import code.api.util.ApiRole._
import code.api.util.ApiTag._
@ -203,6 +204,33 @@ trait APIMethods510 {
}
}
staticResourceDocs += ResourceDoc(
logCacheEndpoint,
implementedInApiVersion,
nameOf(logCacheEndpoint),
"GET",
"/log-cache/LOG_LEVEL",
"Get Log Cache",
"""Returns information about:
|
|* Log Cache
""",
EmptyBody,
EmptyBody,
List($UserNotLoggedIn, UnknownError),
apiTagApi :: Nil,
Some(List(canGetAllLevelLogsAtAllBanks)))
lazy val logCacheEndpoint: OBPEndpoint = {
case "log-cache" :: logLevel :: Nil JsonGet _ =>
cc => implicit val ec = EndpointContext(Some(cc))
for {
logs <- Future(RedisLogger.tail(RedisLogger.LogLevel.valueOf(logLevel)))
} yield {
(logs, HttpCode.`200`(cc.callContext))
}
}
staticResourceDocs += ResourceDoc(
getRegulatedEntityById,
implementedInApiVersion,

View File

@ -128,6 +128,9 @@ case class RegulatedEntityPostJsonV510(
)
case class RegulatedEntitiesJsonV510(entities: List[RegulatedEntityJsonV510])
case class LogCacheJsonV510(level: String, message: String)
case class LogsCacheJsonV510(logs: List[String])
case class WaitingForGodotJsonV510(sleep_in_milliseconds: Long)
case class CertificateInfoJsonV510(

View File

@ -1,5 +1,7 @@
package code.util
import code.api.cache.{Redis, RedisLogger}
import java.net.{Socket, SocketException, URL}
import java.util.UUID.randomUUID
import java.util.{Date, GregorianCalendar}
@ -171,36 +173,36 @@ object Helper extends Loggable {
/**
*
*
* @param redirectUrl eg: http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
* @return http://localhost:8082/oauthcallback
*/
def getStaticPortionOfRedirectURL(redirectUrl: String): Box[String] = {
tryo(redirectUrl.split("\\?")(0)) //return everything before the "?"
}
/**
* extract clean redirect url from input value, because input may have some parameters, such as the following examples <br/>
* eg1: http://localhost:8082/oauthcallback?....--> http://localhost:8082 <br/>
* extract clean redirect url from input value, because input may have some parameters, such as the following examples <br/>
* eg1: http://localhost:8082/oauthcallback?....--> http://localhost:8082 <br/>
* eg2: http://localhost:8016?oautallback?=3NLMGV ...--> http://localhost:8016
*
* @param redirectUrl -> http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
* @return hostOnlyOfRedirectURL-> http://localhost:8082
*/
@deprecated("We can not only use hostname as the redirectUrl, now add new method `getStaticPortionOfRedirectURL` ","05.12.2023")
@deprecated("We can not only use hostname as the redirectUrl, now add new method `getStaticPortionOfRedirectURL` ","05.12.2023")
def getHostOnlyOfRedirectURL(redirectUrl: String): Box[String] = {
val url = new URL(redirectUrl) //eg: http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
val protocol = url.getProtocol() // http
val authority = url.getAuthority()// localhost:8082, this will contain the port.
tryo(s"$protocol://$authority") // http://localhost:8082
tryo(s"$protocol://$authority") // http://localhost:8082
}
/**
* extract Oauth Token String from input value, because input may have some parameters, such as the following examples <br/>
* http://localhost:8082/oauthcallback?oauth_token=DKR242MB3IRCUVG35UZ0QQOK3MBS1G2HL2ZIKK2O&oauth_verifier=64465
* extract Oauth Token String from input value, because input may have some parameters, such as the following examples <br/>
* http://localhost:8082/oauthcallback?oauth_token=DKR242MB3IRCUVG35UZ0QQOK3MBS1G2HL2ZIKK2O&oauth_verifier=64465
* --> DKR242MB3IRCUVG35UZ0QQOK3MBS1G2HL2ZIKK2O
*
* @param input a long url with parameters
*
* @param input a long url with parameters
* @return Oauth Token String
*/
def extractOauthToken(input: String): Box[String] = {
@ -236,7 +238,7 @@ object Helper extends Loggable {
* Used for version extraction from props string
*/
val matchAnyStoredProcedure = "stored_procedure.*|star".r
/**
* change the TimeZone to the current TimeZOne
* reference the following trait
@ -246,25 +248,25 @@ object Helper extends Loggable {
*/
//TODO need clean this format, we have set the TimeZone in boot.scala
val DateFormatWithCurrentTimeZone = new Formats {
import java.text.{ParseException, SimpleDateFormat}
val dateFormat = new DateFormat {
def parse(s: String) = try {
Some(formatter.parse(s))
} catch {
case e: ParseException => None
}
def format(d: Date) = formatter.format(d)
private def formatter = {
val f = dateFormatter
f.setTimeZone(new GregorianCalendar().getTimeZone)
f
}
}
protected def dateFormatter = APIUtil.DateWithMsFormat
}
@ -316,32 +318,92 @@ object Helper extends Loggable {
}
trait MdcLoggable extends Loggable {
protected def initiate(): Unit = () // The type is Unit and the only value this type can take is the literal ()
protected def surroundWarnMessage(msg: String, title: String = ""): Unit = {
logger.warn(s"+-${title}${StringUtils.repeat("-", msg.length - title.length)}-+")
logger.warn(s"| $msg |")
logger.warn(s"+-${StringUtils.repeat("-", msg.length)}-+")
}
protected def surroundInfoMessage(msg: String, title: String = ""): Unit = {
logger.info(s"+-${title}${StringUtils.repeat("-", msg.length - title.length)}-+")
logger.info(s"| $msg |")
logger.info(s"+-${StringUtils.repeat("-", msg.length)}-+")
}
protected def surroundErrorMessage(msg: String, title: String = ""): Unit = {
logger.error(s"+-${title}${StringUtils.repeat("-", msg.length - title.length)}-+")
logger.error(s"| $msg |")
logger.error(s"+-${StringUtils.repeat("-", msg.length)}-+")
}
protected def surroundDebugMessage(msg: String, title: String = ""): Unit = {
logger.debug(s"+-${title}${StringUtils.repeat("-", msg.length - title.length)}-+")
logger.debug(s"| $msg |")
logger.debug(s"+-${StringUtils.repeat("-", msg.length)}-+")
override protected val logger: net.liftweb.common.Logger = {
val loggerName = this.getClass.getName
new net.liftweb.common.Logger {
private val underlyingLogger = net.liftweb.common.Logger(loggerName)
// INFO
override def info(msg: => AnyRef): Unit = {
val m = msg.toString
underlyingLogger.info(m)
RedisLogger.log(RedisLogger.LogLevel.INFO, m)
}
override def info(msg: => AnyRef, t: => Throwable): Unit = {
val m = msg.toString
underlyingLogger.info(m, t)
RedisLogger.log(RedisLogger.LogLevel.INFO, m)
}
// WARN
override def warn(msg: => AnyRef): Unit = {
val m = msg.toString
underlyingLogger.warn(m)
RedisLogger.log(RedisLogger.LogLevel.WARNING, m)
}
override def warn(msg: => AnyRef, t: Throwable): Unit = {
val m = msg.toString
underlyingLogger.warn(m, t)
RedisLogger.log(RedisLogger.LogLevel.WARNING, m)
}
// ERROR
override def error(msg: => AnyRef): Unit = {
val m = msg.toString
underlyingLogger.error(m)
RedisLogger.log(RedisLogger.LogLevel.ERROR, m)
}
override def error(msg: => AnyRef, t: Throwable): Unit = {
val m = msg.toString
underlyingLogger.error(m, t)
RedisLogger.log(RedisLogger.LogLevel.ERROR, m)
}
// DEBUG
override def debug(msg: => AnyRef): Unit = {
val m = msg.toString
underlyingLogger.debug(m)
RedisLogger.log(RedisLogger.LogLevel.DEBUG, m)
}
override def debug(msg: => AnyRef, t: Throwable): Unit = {
val m = msg.toString
underlyingLogger.debug(m, t)
RedisLogger.log(RedisLogger.LogLevel.DEBUG, m)
}
// TRACE
override def trace(msg: => AnyRef): Unit = {
val m = msg.toString
underlyingLogger.trace(m)
RedisLogger.log(RedisLogger.LogLevel.TRACE, m)
}
// Delegate enabled checks
override def isDebugEnabled: Boolean = underlyingLogger.isDebugEnabled
override def isErrorEnabled: Boolean = underlyingLogger.isErrorEnabled
override def isInfoEnabled: Boolean = underlyingLogger.isInfoEnabled
override def isTraceEnabled: Boolean = underlyingLogger.isTraceEnabled
override def isWarnEnabled: Boolean = underlyingLogger.isWarnEnabled
}
}
protected def initiate(): Unit = ()
initiate()
MDC.put("host" -> getHostname)
}
/*
Return true for Y, YES and true etc.
*/
@ -393,7 +455,7 @@ object Helper extends Loggable {
case _ => Nil
}
default.getOrElse(words.mkString(" ") + ".")
} else
} else
S.?(message)
} else {
logger.error(s"i18n(message($message), default${default}: Attempted to use resource bundles outside of an initialized S scope. " +
@ -411,8 +473,8 @@ object Helper extends Loggable {
* @return modified instance
*/
private def convertId[T](
obj: T,
customerIdConverter: String=> String,
obj: T,
customerIdConverter: String=> String,
accountIdConverter: String=> String,
transactionIdConverter: String=> String
): T = {
@ -433,7 +495,7 @@ object Helper extends Loggable {
(ownerType <:< typeOf[AccountBalances] && fieldName.equalsIgnoreCase("id") && fieldType =:= typeOf[String])||
(ownerType <:< typeOf[AccountHeld] && fieldName.equalsIgnoreCase("id") && fieldType =:= typeOf[String])
}
def isTransactionId(fieldName: String, fieldType: Type, fieldValue: Any, ownerType: Type) = {
ownerType <:< typeOf[TransactionId] ||
(fieldName.equalsIgnoreCase("transactionId") && fieldType =:= typeOf[String])||
@ -502,10 +564,10 @@ object Helper extends Loggable {
lazy val result = method.invoke(net.liftweb.http.S, args: _*)
val methodName = method.getName
if (methodName.equals("param")&&result.isInstanceOf[Box[String]]&&result.asInstanceOf[Box[String]].isDefined) {
//we provide the basic check for all the parameters
val resultAfterChecked =
val resultAfterChecked =
if((args.length>0) && args.apply(0).toString.equalsIgnoreCase("username")) {
result.asInstanceOf[Box[String]].filter(APIUtil.checkUsernameString(_)==SILENCE_IS_GOLDEN)
}else if((args.length>0) && args.apply(0).toString.equalsIgnoreCase("password")){
@ -517,7 +579,7 @@ object Helper extends Loggable {
} else{
result.asInstanceOf[Box[String]].filter(APIUtil.checkMediumString(_)==SILENCE_IS_GOLDEN)
}
if(resultAfterChecked.isEmpty) {
if(resultAfterChecked.isEmpty) {
logger.debug(s"ObpS.${methodName} validation failed. (resultAfterChecked.isEmpty A) The input key is: ${if (args.length>0)args.apply(0) else ""}, value is:$result")
}
resultAfterChecked
@ -532,7 +594,7 @@ object Helper extends Loggable {
} else if (methodName.equals("uriAndQueryString") && result.isInstanceOf[Box[String]] && result.asInstanceOf[Box[String]].isDefined ||
methodName.equals("queryString") && result.isInstanceOf[Box[String]]&&result.asInstanceOf[Box[String]].isDefined){
val resultAfterChecked = result.asInstanceOf[Box[String]].filter(APIUtil.basicUriAndQueryStringValidation(_))
if(resultAfterChecked.isEmpty) {
if(resultAfterChecked.isEmpty) {
logger.debug(s"ObpS.${methodName} validation failed. (resultAfterChecked.isEmpty B) The value is:$result")
}
resultAfterChecked
@ -540,7 +602,7 @@ object Helper extends Loggable {
result
}
}
val enhancer: Enhancer = new Enhancer()
enhancer.setSuperclass(classOf[S])
enhancer.setCallback(intercept)
@ -602,4 +664,4 @@ object Helper extends Loggable {
}
}