diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index a79b5687e..d9091a2ee 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -996,6 +996,9 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ case "exclude_app_names" => Full(OBPExcludeAppNames(values)) //This will return a string list. case "exclude_url_patterns" => Full(OBPExcludeUrlPatterns(values))//This will return a string list. case "exclude_implemented_by_partial_functions" => Full(OBPExcludeImplementedByPartialFunctions(values)) //This will return a string list. + case "include_app_names" => Full(OBPIncludeAppNames(values)) //This will return a string list. + case "include_url_patterns" => Full(OBPIncludeUrlPatterns(values))//This will return a string list. + case "include_implemented_by_partial_functions" => Full(OBPIncludeImplementedByPartialFunctions(values)) //This will return a string list. case "function_name" => Full(OBPFunctionName(values.head)) case "connector_name" => Full(OBPConnectorName(values.head)) case "customer_id" => Full(OBPCustomerId(values.head)) @@ -1033,8 +1036,11 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ correlationId <- getHttpParamValuesByName(httpParams, "correlation_id") duration <- getHttpParamValuesByName(httpParams, "duration") excludeAppNames <- getHttpParamValuesByName(httpParams, "exclude_app_names") + includeAppNames <- getHttpParamValuesByName(httpParams, "include_app_names") excludeUrlPattern <- getHttpParamValuesByName(httpParams, "exclude_url_patterns") + includeUrlPattern <- getHttpParamValuesByName(httpParams, "include_url_patterns") excludeImplementedByPartialfunctions <- getHttpParamValuesByName(httpParams, "exclude_implemented_by_partial_functions") + includeImplementedByPartialfunctions <- getHttpParamValuesByName(httpParams, "include_implemented_by_partial_functions") connectorName <- getHttpParamValuesByName(httpParams, "connector_name") functionName <- getHttpParamValuesByName(httpParams, "function_name") customerId <- getHttpParamValuesByName(httpParams, "customer_id") @@ -1058,6 +1064,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ List(limit, offset, ordering, fromDate, toDate, anon, consumerId, userId, url, appName, implementedByPartialFunction, implementedInVersion, verb, correlationId, duration, excludeAppNames, excludeUrlPattern, excludeImplementedByPartialfunctions, + includeAppNames, includeUrlPattern, includeImplementedByPartialfunctions, connectorName,functionName, bankId, accountId, customerId, lockedStatus, deletedStatus ).filter(_ != OBPEmpty()) } @@ -1111,6 +1118,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ val excludeAppNames = getHttpRequestUrlParam(httpRequestUrl, "exclude_app_names").split(",").toList val excludeUrlPattern = getHttpRequestUrlParam(httpRequestUrl, "exclude_url_patterns").split(",").toList val excludeImplementedByPartialfunctions = getHttpRequestUrlParam(httpRequestUrl, "exclude_implemented_by_partial_functions").split(",").toList + + val includeAppNames = getHttpRequestUrlParam(httpRequestUrl, "include_app_names").split(",").toList + val includeUrlPattern = getHttpRequestUrlParam(httpRequestUrl, "include_url_patterns").split(",").toList + val includeImplementedByPartialfunctions = getHttpRequestUrlParam(httpRequestUrl, "include_implemented_by_partial_functions").split(",").toList val functionName = getHttpRequestUrlParam(httpRequestUrl, "function_name") val connectorName = getHttpRequestUrlParam(httpRequestUrl, "connector_name") @@ -1120,7 +1131,10 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ HTTPParam("anon", anon), HTTPParam("consumer_id", consumerId), HTTPParam("user_id", userId), HTTPParam("url", url), HTTPParam("app_name", appName), HTTPParam("implemented_by_partial_function",implementedByPartialFunction), HTTPParam("implemented_in_version",implementedInVersion), HTTPParam("verb", verb), HTTPParam("correlation_id", correlationId), HTTPParam("duration", duration), HTTPParam("exclude_app_names", excludeAppNames), - HTTPParam("exclude_url_patterns", excludeUrlPattern),HTTPParam("exclude_implemented_by_partial_functions", excludeImplementedByPartialfunctions), + HTTPParam("exclude_url_patterns", excludeUrlPattern),HTTPParam("exclude_implemented_by_partial_functions", excludeImplementedByPartialfunctions), + HTTPParam("include_app_names", includeAppNames), + HTTPParam("include_url_patterns", includeUrlPattern), + HTTPParam("include_implemented_by_partial_functions", includeImplementedByPartialfunctions), HTTPParam("function_name", functionName), HTTPParam("currency", currency), HTTPParam("amount", amount), diff --git a/obp-api/src/main/scala/code/api/util/OBPParam.scala b/obp-api/src/main/scala/code/api/util/OBPParam.scala index 6c01526b9..56e65ff42 100644 --- a/obp-api/src/main/scala/code/api/util/OBPParam.scala +++ b/obp-api/src/main/scala/code/api/util/OBPParam.scala @@ -32,6 +32,7 @@ case class OBPAccountId(value: String) extends OBPQueryParam case class OBPUrl(value: String) extends OBPQueryParam case class OBPAppName(value: String) extends OBPQueryParam case class OBPExcludeAppNames(values: List[String]) extends OBPQueryParam +case class OBPIncludeAppNames(values: List[String]) extends OBPQueryParam case class OBPImplementedByPartialFunction(value: String) extends OBPQueryParam case class OBPImplementedInVersion(value: String) extends OBPQueryParam case class OBPVerb(value: String) extends OBPQueryParam @@ -39,7 +40,9 @@ case class OBPAnon(value: Boolean) extends OBPQueryParam case class OBPCorrelationId(value: String) extends OBPQueryParam case class OBPDuration(value: Long) extends OBPQueryParam case class OBPExcludeUrlPatterns(values: List[String]) extends OBPQueryParam +case class OBPIncludeUrlPatterns(values: List[String]) extends OBPQueryParam case class OBPExcludeImplementedByPartialFunctions(value: List[String]) extends OBPQueryParam +case class OBPIncludeImplementedByPartialFunctions(value: List[String]) extends OBPQueryParam case class OBPFunctionName(value: String) extends OBPQueryParam case class OBPConnectorName(value: String) extends OBPQueryParam case class OBPEmpty() extends OBPQueryParam diff --git a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala index 7c2dbe2c0..d20b90cc2 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala @@ -2164,7 +2164,7 @@ trait APIMethods300 { _ <- NewStyle.function.hasEntitlement("", u.userId, ApiRole.canReadAggregateMetrics, callContext) httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url) (obpQueryParams, callContext) <- createQueriesByHttpParamsFuture(httpParams, callContext) - aggregateMetrics <- APIMetrics.apiMetrics.vend.getAllAggregateMetricsFuture(obpQueryParams) map { + aggregateMetrics <- APIMetrics.apiMetrics.vend.getAllAggregateMetricsFuture(obpQueryParams, false) map { x => unboxFullOrFail(x, callContext, GetAggregateMetricsError) } } yield { diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala index d076d4cc1..16fb9e044 100644 --- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala +++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala @@ -9,11 +9,13 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _} import code.api.util.{ApiRole, NewStyle} import code.api.util.NewStyle.HttpCode +import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson import code.api.v3_1_0.ConsentJsonV310 import code.api.v3_1_0.JSONFactory310.createBadLoginStatusJson import code.api.v4_0_0.{JSONFactory400, PostApiCollectionJson400} import code.consent.Consents import code.loginattempts.LoginAttempt +import code.metrics.APIMetrics import code.transactionrequests.TransactionRequests.TransactionRequestTypes.{apply => _} import code.userlocks.UserLocksProvider import code.users.Users @@ -360,6 +362,85 @@ trait APIMethods510 { } } + resourceDocs += ResourceDoc( + getAggregateMetrics, + implementedInApiVersion, + nameOf(getAggregateMetrics), + "GET", + "/management/aggregate-metrics", + "Get Aggregate Metrics", + s"""Returns aggregate metrics on api usage eg. total count, response time (in ms), etc. + | + |Should be able to filter on the following fields + | + |eg: /management/aggregate-metrics?from_date=$DateWithMsExampleString&to_date=$DateWithMsExampleString&consumer_id=5 + |&user_id=66214b8e-259e-44ad-8868-3eb47be70646&implemented_by_partial_function=getTransactionsForBankAccount + |&implemented_in_version=v3.0.0&url=/obp/v3.0.0/banks/gh.29.uk/accounts/8ca8a7e4-6d02-48e3-a029-0b2bf89de9f0/owner/transactions + |&verb=GET&anon=false&app_name=MapperPostman + |&exclude_app_names=API-EXPLORER,API-Manager,SOFI,null + | + |1 from_date (defaults to the day before the current date): eg:from_date=$DateWithMsExampleString + | + |2 to_date (defaults to the current date) eg:to_date=$DateWithMsExampleString + | + |3 consumer_id (if null ignore) + | + |4 user_id (if null ignore) + | + |5 anon (if null ignore) only support two value : true (return where user_id is null.) or false (return where user_id is not null.) + | + |6 url (if null ignore), note: can not contain '&'. + | + |7 app_name (if null ignore) + | + |8 implemented_by_partial_function (if null ignore), + | + |9 implemented_in_version (if null ignore) + | + |10 verb (if null ignore) + | + |11 correlation_id (if null ignore) + | + |12 duration (if null ignore) non digit chars will be silently omitted + | + |13 include_app_names (if null ignore).eg: &include_app_names=API-EXPLORER,API-Manager,SOFI,null + | + |14 include_url_patterns (if null ignore).you can design you own SQL LIKE pattern. eg: &include_url_patterns=%management/metrics%,%management/aggregate-metrics% + | + |15 include_implemented_by_partial_functions (if null ignore).eg: &include_implemented_by_partial_functions=getMetrics,getConnectorMetrics,getAggregateMetrics + | + |${authenticationRequiredMessage(true)} + | + """.stripMargin, + EmptyBody, + aggregateMetricsJSONV300, + List( + UserNotLoggedIn, + UserHasMissingRoles, + UnknownError + ), + List(apiTagMetric, apiTagAggregateMetrics, apiTagNewStyle), + Some(List(canReadAggregateMetrics))) + + lazy val getAggregateMetrics: OBPEndpoint = { + case "management" :: "aggregate-metrics" :: Nil JsonGet _ => { + cc => { + for { + (Full(u), callContext) <- authenticatedAccess(cc) + _ <- NewStyle.function.hasEntitlement("", u.userId, ApiRole.canReadAggregateMetrics, callContext) + httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url) + (obpQueryParams, callContext) <- createQueriesByHttpParamsFuture(httpParams, callContext) + aggregateMetrics <- APIMetrics.apiMetrics.vend.getAllAggregateMetricsFuture(obpQueryParams,true) map { + x => unboxFullOrFail(x, callContext, GetAggregateMetricsError) + } + } yield { + (createAggregateMetricJson(aggregateMetrics), HttpCode.`200`(callContext)) + } + } + + } + } + } } diff --git a/obp-api/src/main/scala/code/metrics/APIMetrics.scala b/obp-api/src/main/scala/code/metrics/APIMetrics.scala index d34abb534..9e113f57e 100644 --- a/obp-api/src/main/scala/code/metrics/APIMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/APIMetrics.scala @@ -1,9 +1,9 @@ package code.metrics import java.util.{Calendar, Date} - import code.api.util.{APIUtil, OBPQueryParam} import code.remotedata.RemotedataMetrics +import com.openbankproject.commons.util.ApiVersion import net.liftweb.common.Box import net.liftweb.util.SimpleInjector @@ -84,8 +84,14 @@ trait APIMetrics { // def getAllGroupedByUserId() : Map[String, List[APIMetric]] def getAllMetrics(queryParams: List[OBPQueryParam]): List[APIMetric] - - def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]): Future[Box[List[AggregateMetrics]]] + + /** + * + * @param queryParams + * @param isNewVersion from V510, we change the queryParams, use includeXxx instead of excludeXxxx, so add this flag + * @return + */ + def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Future[Box[List[AggregateMetrics]]] def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] @@ -102,7 +108,7 @@ class RemotedataMetricsCaseClasses { // case class getAllGroupedByDay() // case class getAllGroupedByUserId() case class getAllMetrics(queryParams: List[OBPQueryParam]) - case class getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]) + case class getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean) case class getTopApisFuture(queryParams: List[OBPQueryParam]) case class getTopConsumersFuture(queryParams: List[OBPQueryParam]) case class bulkDeleteMetrics() diff --git a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala index 0b9e55d9b..5a86442ef 100644 --- a/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ElasticsearchMetrics.scala @@ -1,9 +1,9 @@ package code.metrics import java.util.Date - import code.api.util._ import code.search.elasticsearchMetrics +import com.openbankproject.commons.util.ApiVersion import net.liftweb.common.Box import net.liftweb.mapper._ @@ -55,7 +55,7 @@ object ElasticsearchMetrics extends APIMetrics { MappedMetric.findAll(optionalParams: _*) } - override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]): Future[Box[List[AggregateMetrics]]] = ??? + override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Future[Box[List[AggregateMetrics]]] = ??? override def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] = ??? diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index 54fd2b45d..a4b70c9fe 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -11,6 +11,7 @@ import code.model.MappedConsumersProvider import code.util.Helper.MdcLoggable import code.util.{MappedUUID, UUIDString} import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.util.ApiVersion import com.tesobe.CacheKeyFromArguments import net.liftweb.common.Box import net.liftweb.mapper.{Index, _} @@ -205,24 +206,28 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } - private def extendCurrentQuery (length: Int) ={ + private def extendCurrentQuery (params: List[String]) ={ // --> "?,?," - val a = for(i <- 1 to (length-1) ) yield {"?,"} + val a = for(i <- 0 to (params.length-2) ) yield {s"${params(i)},"} //"?,?,--> "?,?,?" - a.mkString("").concat("?") + a.mkString("").concat(s"${params.last}") } - private def extendNotLikeQuery(length: Int) = { - if (length == 1) - "? ))" + private def extendLikeQuery(params: List[String], isLike: Boolean) = { + val isLikeQuery = if (isLike) "" else "NOT" + + if (params.length == 1) + s"${params.head}" else { - val a = for (i <- 1 to (length - 2)) yield + val a = for (i <- 1 to (params.length - 2)) yield { - " and url NOT LIKE (?)" + s" and url ${isLikeQuery} LIKE ('${params(i)}')" } - "? )" + a.mkString("").concat(" and url NOT LIKE (?))") +// OR (url NOT LIKE ( '%management/metrics%' ) AND url NOT LIKE ( '%management/metrics%' ) '))) +// or (url NOT LIKE ($excludeUrlPatternsQueries))) + s"${params.head}')" + a.mkString("").concat(s" and url ${isLikeQuery} LIKE ('${params.last} ") } } @@ -259,7 +264,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ } // TODO Cache this as long as fromDate and toDate are in the past (before now) - def getAllAggregateMetricsBox(queryParams: List[OBPQueryParam]): Box[List[AggregateMetrics]] = { + def getAllAggregateMetricsBox(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Box[List[AggregateMetrics]] = { /** * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)" * is just a temporary value field with UUID values in order to prevent any ambiguity. @@ -275,6 +280,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val url = queryParams.collect { case OBPUrl(value) => value }.headOption val appName = queryParams.collect { case OBPAppName(value) => value }.headOption val excludeAppNames = queryParams.collect { case OBPExcludeAppNames(value) => value }.headOption + val includeAppNames = queryParams.collect { case OBPIncludeAppNames(value) => value }.headOption val implementedByPartialFunction = queryParams.collect { case OBPImplementedByPartialFunction(value) => value }.headOption val implementedInVersion = queryParams.collect { case OBPImplementedInVersion(value) => value }.headOption val verb = queryParams.collect { case OBPVerb(value) => value }.headOption @@ -282,36 +288,65 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val correlationId = queryParams.collect { case OBPCorrelationId(value) => value }.headOption val duration = queryParams.collect { case OBPDuration(value) => value }.headOption val excludeUrlPatterns = queryParams.collect { case OBPExcludeUrlPatterns(value) => value }.headOption + val includeUrlPatterns = queryParams.collect { case OBPIncludeUrlPatterns(value) => value }.headOption val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption + val includeImplementedByPartialFunctions = queryParams.collect { case OBPIncludeImplementedByPartialFunctions(value) => value }.headOption - val excludeUrlPatternsSet= excludeUrlPatterns.getOrElse(List("")).toSet - val excludeAppNamesNumberSet = excludeAppNames.getOrElse(List("")).toSet - val excludeImplementedByPartialFunctionsNumberSet = excludeImplementedByPartialFunctions.getOrElse(List("")).toSet + val excludeUrlPatternsList= excludeUrlPatterns.getOrElse(List("")) + val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")) + val excludeImplementedByPartialFunctionsNumberList = excludeImplementedByPartialFunctions.getOrElse(List("")) - val excludeUrlPatternsQueries = extendNotLikeQuery(excludeUrlPatternsSet.size) - val extendedExcludeAppNameQueries = extendCurrentQuery(excludeAppNamesNumberSet.size) - val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberSet.size) + val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) + val extendedExcludeAppNameQueries = extendCurrentQuery(excludeAppNamesNumberList) + val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberList) + + val includeUrlPatternsList= includeUrlPatterns.getOrElse(List("")) + val includeAppNamesNumberList = includeAppNames.getOrElse(List("")) + val includeImplementedByPartialFunctionsNumberList = includeImplementedByPartialFunctions.getOrElse(List("")) + + val includeUrlPatternsQueries = extendLikeQuery(includeUrlPatternsList, true) + val extendedIncludeAppNameQueries = extendCurrentQuery(includeAppNamesNumberList) + val extendedIncludeImplementedByPartialFunctionsQueries = extendCurrentQuery(includeImplementedByPartialFunctionsNumberList) val result = scalikeDB readOnly { implicit session => - val sqlResult = + val sqlQuery = if(isNewVersion) // in the version, we use includeXxx instead of excludeXxx, the performance should be better. sql"""SELECT count(*), avg(duration), min(duration), max(duration) - FROM metric - WHERE date_c >= ${new Timestamp(fromDate.get.getTime)} - AND date_c <= ${new Timestamp(toDate.get.getTime)} - AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) - AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) - AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) - AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) - AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) - AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) - AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') - AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) - AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($extendedExcludeAppNameQueries)) - AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($extendedExcludeImplementedByPartialFunctionsQueries)) - """.stripMargin - .map( + FROM metric + WHERE date_c >= ${new Timestamp(fromDate.get.getTime)} + AND date_c <= ${new Timestamp(toDate.get.getTime)} + AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) + AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') + AND (${trueOrFalse(includeUrlPatterns.isEmpty) } or (url LIKE ($includeUrlPatternsQueries))) + AND (${trueOrFalse(includeAppNames.isEmpty) } or appname in ($extendedIncludeAppNameQueries)) + AND (${trueOrFalse(includeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction in ($extendedIncludeImplementedByPartialFunctionsQueries)) + """.stripMargin + else + sql"""SELECT count(*), avg(duration), min(duration), max(duration) + FROM metric + WHERE date_c >= ${new Timestamp(fromDate.get.getTime)} + AND date_c <= ${new Timestamp(toDate.get.getTime)} + AND (${trueOrFalse(consumerId.isEmpty)} or consumerid = ${consumerId.getOrElse("")}) + AND (${trueOrFalse(userId.isEmpty)} or userid = ${userId.getOrElse("")}) + AND (${trueOrFalse(implementedByPartialFunction.isEmpty)} or implementedbypartialfunction = ${implementedByPartialFunction.getOrElse("")}) + AND (${trueOrFalse(implementedInVersion.isEmpty)} or implementedinversion = ${implementedInVersion.getOrElse("")}) + AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("")}) + AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("")}) + AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("")}) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = 'null') + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != 'null') + AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) + AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($extendedExcludeAppNameQueries)) + AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($extendedExcludeImplementedByPartialFunctionsQueries)) + """.stripMargin + logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlQuery --: " +sqlQuery.statement) + val sqlResult = sqlQuery.map( rs => // Map result to case class AggregateMetrics( rs.stringOpt(1).map(_.toInt).getOrElse(0), @@ -320,14 +355,15 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ rs.stringOpt(4).map(_.toDouble).getOrElse(0) ) ).list().apply() + logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlResult --: "+sqlResult.toString) sqlResult } tryo(result) }} } - override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]): Future[Box[List[AggregateMetrics]]] = Future{ - getAllAggregateMetricsBox(queryParams: List[OBPQueryParam]) + override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Future[Box[List[AggregateMetrics]]] = Future{ + getAllAggregateMetricsBox(queryParams: List[OBPQueryParam], isNewVersion) } override def bulkDeleteMetrics(): Boolean = { @@ -362,13 +398,13 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption val limit = queryParams.collect { case OBPLimit(value) => value }.headOption.getOrElse(10) - val excludeUrlPatternsSet= excludeUrlPatterns.getOrElse(List("")).toSet - val excludeAppNamesNumberSet = excludeAppNames.getOrElse(List("")).toSet - val excludeImplementedByPartialFunctionsNumberSet = excludeImplementedByPartialFunctions.getOrElse(List("")).toSet + val excludeUrlPatternsList= excludeUrlPatterns.getOrElse(List("")) + val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")) + val excludeImplementedByPartialFunctionsNumberList = excludeImplementedByPartialFunctions.getOrElse(List("")) - val excludeUrlPatternsQueries = extendNotLikeQuery(excludeUrlPatternsSet.size) - val extendedExclueAppNameQueries = extendCurrentQuery(excludeAppNamesNumberSet.size) - val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberSet.size) + val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) + val extendedExclueAppNameQueries = extendCurrentQuery(excludeAppNamesNumberList) + val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberList) val (dbUrl, _, _) = getDbConnectionParameters @@ -440,13 +476,13 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val excludeImplementedByPartialFunctions = queryParams.collect { case OBPExcludeImplementedByPartialFunctions(value) => value }.headOption val limit = queryParams.collect { case OBPLimit(value) => value }.headOption - val excludeUrlPatternsSet = excludeUrlPatterns.getOrElse(List("")).toSet - val excludeAppNamesNumberSet = excludeAppNames.getOrElse(List("")).toSet - val excludeImplementedByPartialFunctionsNumberSet = excludeImplementedByPartialFunctions.getOrElse(List("")).toSet + val excludeUrlPatternsList = excludeUrlPatterns.getOrElse(List("")) + val excludeAppNamesNumberList = excludeAppNames.getOrElse(List("")) + val excludeImplementedByPartialFunctionsNumberList = excludeImplementedByPartialFunctions.getOrElse(List("")) - val excludeUrlPatternsQueries = extendNotLikeQuery(excludeUrlPatternsSet.size) - val extendedExclueAppNameQueries = extendCurrentQuery(excludeAppNamesNumberSet.size) - val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberSet.size) + val excludeUrlPatternsQueries = extendLikeQuery(excludeUrlPatternsList, false) + val extendedExclueAppNameQueries = extendCurrentQuery(excludeAppNamesNumberList) + val extendedExcludeImplementedByPartialFunctionsQueries = extendCurrentQuery(excludeImplementedByPartialFunctionsNumberList) val (dbUrl, _, _) = getDbConnectionParameters diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataMetrics.scala b/obp-api/src/main/scala/code/remotedata/RemotedataMetrics.scala index 354e7264e..19c3d4e4e 100644 --- a/obp-api/src/main/scala/code/remotedata/RemotedataMetrics.scala +++ b/obp-api/src/main/scala/code/remotedata/RemotedataMetrics.scala @@ -6,6 +6,7 @@ import akka.pattern.ask import code.actorsystem.ObpActorInit import code.api.util.OBPQueryParam import code.metrics._ +import com.openbankproject.commons.util.ApiVersion import net.liftweb.common.Box import scala.concurrent.Future @@ -36,9 +37,9 @@ object RemotedataMetrics extends ObpActorInit with APIMetrics { (actor ? cc.getAllMetrics(queryParams)).mapTo[List[APIMetric]] ) - override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]): Future[Box[List[AggregateMetrics]]] ={ - logger.debug(s"RemotedataMetrics.getAllAggregateMetrics($queryParams)") - (actor ? cc.getAllAggregateMetricsFuture(queryParams)).mapTo[Box[List[AggregateMetrics]]] + override def getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean): Future[Box[List[AggregateMetrics]]] ={ + logger.debug(s"RemotedataMetrics.getAllAggregateMetrics($queryParams, $isNewVersion)") + (actor ? cc.getAllAggregateMetricsFuture(queryParams,isNewVersion)).mapTo[Box[List[AggregateMetrics]]] } override def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] = { diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataMetricsActor.scala b/obp-api/src/main/scala/code/remotedata/RemotedataMetricsActor.scala index c5b40f252..df8365cf4 100644 --- a/obp-api/src/main/scala/code/remotedata/RemotedataMetricsActor.scala +++ b/obp-api/src/main/scala/code/remotedata/RemotedataMetricsActor.scala @@ -1,15 +1,14 @@ package code.remotedata import java.util.Date - import akka.actor.Actor import code.actorsystem.ObpActorHelper import code.metrics.{MappedMetrics, RemotedataMetricsCaseClasses} import code.util.Helper.MdcLoggable import akka.pattern.pipe import code.api.util.OBPQueryParam - import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.util.ApiVersion class RemotedataMetricsActor extends Actor with ObpActorHelper with MdcLoggable { @@ -41,9 +40,9 @@ class RemotedataMetricsActor extends Actor with ObpActorHelper with MdcLoggable logger.debug("getAllMetrics()") sender ! (mapper.getAllMetrics(queryParams)) - case cc.getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam]) => - logger.debug(s"RemotedataMetricsActor.getAllAggregateMetricsFuture($queryParams)") - sender ! (mapper.getAllAggregateMetricsBox(queryParams)) + case cc.getAllAggregateMetricsFuture(queryParams: List[OBPQueryParam], isNewVersion: Boolean) => + logger.debug(s"RemotedataMetricsActor.getAllAggregateMetricsFuture($queryParams, $isNewVersion)") + sender ! (mapper.getAllAggregateMetricsBox(queryParams, isNewVersion)) case cc.getTopApisFuture(queryParams: List[OBPQueryParam]) => logger.debug(s"getTopApisFuture($queryParams)") diff --git a/obp-api/src/test/scala/code/util/APIUtilTest.scala b/obp-api/src/test/scala/code/util/APIUtilTest.scala index b4741f400..3cf517b50 100644 --- a/obp-api/src/test/scala/code/util/APIUtilTest.scala +++ b/obp-api/src/test/scala/code/util/APIUtilTest.scala @@ -492,7 +492,9 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop OBPAppName("PlaneApp"), OBPImplementedByPartialFunction("getBanks"), OBPImplementedInVersion("v1.2.1"), OBPVerb("GET"), OBPCorrelationId("123"), OBPDuration(1000), OBPExcludeAppNames(List("TrainApp", "BusApp")), OBPExcludeUrlPatterns(List("%/obp/v1.2.1%")), - OBPExcludeImplementedByPartialFunctions(List("getBank", "getAccounts")))) + OBPExcludeImplementedByPartialFunctions(List("getBank", "getAccounts")), + OBPIncludeAppNames(List("TrainApp", "BusApp")), OBPIncludeUrlPatterns(List("%/obp/v1.2.1%")), + OBPIncludeImplementedByPartialFunctions(List("getBank", "getAccounts")))) val httpParams: List[HTTPParam] = List( HTTPParam("from_date",List(s"$DefaultFromDateString")), HTTPParam("to_date",List(s"$DefaultToDateString")), @@ -508,7 +510,10 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop HTTPParam("duration","1000"), HTTPParam("exclude_app_names",List("TrainApp","BusApp")), HTTPParam("exclude_url_patterns","%/obp/v1.2.1%"), - HTTPParam("exclude_implemented_by_partial_functions",List("getBank","getAccounts")) + HTTPParam("exclude_implemented_by_partial_functions",List("getBank","getAccounts")), + HTTPParam("include_app_names",List("TrainApp","BusApp")), + HTTPParam("include_url_patterns","%/obp/v1.2.1%"), + HTTPParam("include_implemented_by_partial_functions",List("getBank","getAccounts")) ) val returnValue = createQueriesByHttpParams(httpParams) returnValue should be (ExpectResult) @@ -588,7 +593,10 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop HTTPParam("duration",List("100")), HTTPParam("exclude_app_names",List("API-EXPLORER","API-Manager","SOFI","null","SOFIT")), HTTPParam("exclude_url_patterns",List("%25management/metrics%25","%management/aggregate-metrics%")), - HTTPParam("exclude_implemented_by_partial_functions",List("getMetrics","getConnectorMetrics","getAggregateMetrics")))) + HTTPParam("exclude_implemented_by_partial_functions",List("getMetrics","getConnectorMetrics","getAggregateMetrics")), + HTTPParam("include_app_names",List("API-EXPLORER","API-Manager","SOFI","SOFIT")), + HTTPParam("include_url_patterns",List("%25management/metrics%25","%management/aggregate-metrics%")), + HTTPParam("include_implemented_by_partial_functions",List("getMetrics","getConnectorMetrics","getAggregateMetrics")))) val httpRequestUrl = "/obp/v3.0.0/management/aggregate-metrics?" + s"offset=3&" + @@ -606,7 +614,11 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop "exclude_app_names=API-EXPLORER,API-Manager,SOFI,null,SOFIT&" + "exclude_url_patterns=%25management/metrics%25,%management/aggregate-metrics%&" + "exclude_implemented_by_partial_functions=getMetrics,getConnectorMetrics,getAggregateMetrics&"+ - "correlation_id=123&duration=100" + "correlation_id=123&duration=100&"+ + "include_app_names=API-EXPLORER,API-Manager,SOFI,SOFIT&" + + "include_url_patterns=%25management/metrics%25,%management/aggregate-metrics%&" + + "include_implemented_by_partial_functions=getMetrics,getConnectorMetrics,getAggregateMetrics&" + val returnValue = createHttpParamsByUrl(httpRequestUrl) returnValue should be (ExpectResult) } @@ -640,6 +652,17 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop val returnValue = createHttpParamsByUrl(httpRequestUrl) returnValue should be (ExpectResult) } + + scenario(s"test the correct case4: include_app_names,include_url_patterns,include_implemented_by_partial_functions") + { + val ExpectResult = Full(List( + HTTPParam("include_app_names",List("API-EXPLORER","API-Manager")), + HTTPParam("include_url_patterns", List("%25management/metrics%25", "%management/aggregate-metrics%")), + HTTPParam("include_implemented_by_partial_functions", List("getMetrics")))) + val httpRequestUrl = s"/obp/v3.0.0/management/aggregate-metrics?include_app_names=API-EXPLORER,API-Manager&include_url_patterns=%25management/metrics%25,%management/aggregate-metrics%&include_implemented_by_partial_functions=getMetrics" + val returnValue = createHttpParamsByUrl(httpRequestUrl) + returnValue should be (ExpectResult) + } } feature("test APIUtil.firstCharToLowerCase method") {