From 04361d37e0c3b2355278b153a2d3df119e028395 Mon Sep 17 00:00:00 2001 From: hongwei Date: Fri, 4 Oct 2019 19:02:21 +0200 Subject: [PATCH 1/2] added the size guard to 100000 for dataWarehouseSearch and dataWarehouseStatistics apis --- .../scala/code/api/v3_0_0/APIMethods300.scala | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala index dcb1c5be0..53d2d2585 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala @@ -40,8 +40,8 @@ import scala.collection.immutable.{List, Nil} import scala.collection.mutable.ArrayBuffer import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future - import code.api.v2_0_0.AccountsHelper._ +import net.liftweb.json.JsonAST.JField trait APIMethods300 { @@ -747,6 +747,18 @@ trait APIMethods300 { _ <- Helper.booleanToFuture(failMsg = ElasticSearchDisabled) { esw.isEnabled() } + + //This is for performance issue, we can not support query more than 10000 records in one call. + // If it contains the size and if it over 10000, we will throw the error back. + _ <- Helper.booleanToFuture(failMsg = maximumLimitExceeded.replace("Maximum number is 10000.","Please check query body, the maximum size is 10000.")) { + // find all the size field. + val allSizeFields = json filterField { + case JField(key, _) => key.equals("size") + } + //loop all the items and if find any value is over 10000, then throw the proper error ! + allSizeFields.map(_.value.values.toString.toInt).find(_ > 10000).isEmpty + } + indexPart <- Future { esw.getElasticSearchUri(index) } map { x => unboxFullOrFail(x, callContext, ElasticSearchIndexNotFound) } @@ -816,6 +828,18 @@ trait APIMethods300 { _ <- Helper.booleanToFuture(failMsg = ElasticSearchDisabled) { esw.isEnabled() } + + //This is for performance issue, we can not support query more than 10000 records in one call. + // If it contains the size and if it over 10000, we will throw the error back. + _ <- Helper.booleanToFuture(failMsg = maximumLimitExceeded.replace("Maximum number is 10000.","Please check query body, the maximum size is 10000.")) { + // find all the size field. + val allSizeFields = json filterField { + case JField(key, _) => key.equals("size") + } + //loop all the items and if find any value is over 10000, then throw the proper error ! + allSizeFields.map(_.value.values.toString.toInt).find(_ > 10000).isEmpty + } + indexPart <- Future { esw.getElasticSearchUri(index) } map { x => unboxFullOrFail(x, callContext, ElasticSearchIndexNotFound) } From 1501ad8a209e674fd3b77dd77b1cc68a6fc21e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Fri, 4 Oct 2019 22:42:09 +0200 Subject: [PATCH 2/2] Tweaked endpoint dataWarehouseStatistics v3.0.0 --- obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala index 53d2d2585..8a306048e 100644 --- a/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala +++ b/obp-api/src/main/scala/code/api/v3_0_0/APIMethods300.scala @@ -848,7 +848,7 @@ trait APIMethods300 { } result <- esw.searchProxyStatsAsyncV300(u.userId, indexPart, bodyPart, field) } yield { - (esw.parseResponse(result), HttpCode.`201`(callContext)) + (esw.parseResponse(result, true), HttpCode.`201`(callContext)) } } }