From 0d65aef0f51d2cde92035a9513c8cf2edca0d50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 27 Feb 2023 10:30:30 +0100 Subject: [PATCH 1/4] feature/Log clutter 1 --- .../src/main/scala/code/model/dataAccess/AuthUser.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/obp-api/src/main/scala/code/model/dataAccess/AuthUser.scala b/obp-api/src/main/scala/code/model/dataAccess/AuthUser.scala index 21a4ef904..fa5a9375e 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/AuthUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/AuthUser.scala @@ -473,9 +473,13 @@ import net.liftweb.util.Helpers._ //AuthUser.currentUser.get.user.foreign // this will be issue when the resource user is in remote side { val user = AuthUser.currentUser.openOrThrowException(ErrorMessages.attemptedToOpenAnEmptyBox) // In case that the provider is empty field we default to "local_identity_provider" or "hostname" - val provider = if(user.provider.get.isEmpty) Constant.localIdentityProvider else user.provider.get + val provider = + if(user.provider.get == null || user.provider.get.isEmpty) + Constant.localIdentityProvider + else + user.provider.get Users.users.vend.getUserByUserName(provider, user.username.get) - }else if (directLogin.isDefined) // Direct Login + } else if (directLogin.isDefined) // Direct Login DirectLogin.getUser else if (hasDirectLoginHeader(authorization)) // Direct Login Deprecated DirectLogin.getUser From db101d9a00afbd933f8224fb50a48af854462ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 27 Feb 2023 10:33:05 +0100 Subject: [PATCH 2/4] feature/Add props: retain_metrics_move_limit and retain_metrics_scheduler_interval_in_seconds --- obp-api/src/main/resources/props/sample.props.template | 7 +++++-- obp-api/src/main/scala/bootstrap/liftweb/Boot.scala | 4 +++- .../scala/code/scheduler/MetricsArchiveScheduler.scala | 7 ++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index c09fe9900..9c53c18f5 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1207,12 +1207,15 @@ user_account_validated_redirect_url = # In case is not defined default value is false user_account_is_validated = false -# Defines the number of days we keep rows in the table "MetricsArchive" +# Defines the number of days we keep rows in the table "MetricArchive" # default value is 3 years retain_archive_metrics_days = 1095 - # Defines the number of days we keep rows in the table "Metric" former "MappedMetric" retain_metrics_days = 367 +# Defines the number of rows we can process at once +retain_metrics_move_limit = 50000 +# Defines the interval of the scheduler +retain_metrics_scheduler_interval_in_seconds = 3600 #if same session used for different ip address, we can show this warning, default is false. show_ip_address_change_warning=false \ No newline at end of file diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 6f0a01708..96587ee81 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -684,7 +684,9 @@ class Boot extends MdcLoggable { case Full(i) => DatabaseDriverScheduler.start(i) case _ => // Do not start it } - MetricsArchiveScheduler.start(intervalInSeconds = 86400) + val interval = + APIUtil.getPropsAsIntValue("retain_metrics_scheduler_interval_in_seconds", 3600) + MetricsArchiveScheduler.start(intervalInSeconds = interval) object UsernameLockedChecker { diff --git a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala index 636281791..37893f4c7 100644 --- a/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala +++ b/obp-api/src/main/scala/code/scheduler/MetricsArchiveScheduler.scala @@ -4,7 +4,7 @@ import java.util.concurrent.TimeUnit import java.util.{Calendar, Date} import code.actorsystem.ObpLookupSystem -import code.api.util.{APIUtil, OBPToDate} +import code.api.util.{APIUtil, OBPLimit, OBPToDate} import code.metrics.{APIMetric, APIMetrics, MappedMetric, MetricArchive} import code.util.Helper.MdcLoggable import net.liftweb.common.Full @@ -23,7 +23,7 @@ object MetricsArchiveScheduler extends MdcLoggable { def start(intervalInSeconds: Long): Unit = { logger.info("Hello from MetricsArchiveScheduler.start") scheduler.schedule( - initialDelay = Duration(getMillisTillMidnight(), TimeUnit.MILLISECONDS), + initialDelay = Duration(intervalInSeconds, TimeUnit.SECONDS), interval = Duration(intervalInSeconds, TimeUnit.SECONDS), runnable = new Runnable { def run(): Unit = { @@ -57,9 +57,10 @@ object MetricsArchiveScheduler extends MdcLoggable { case _ => 60 } val someDaysAgo: Date = new Date(currentTime.getTime - (oneDayInMillis * days)) + val limit = APIUtil.getPropsAsIntValue("retain_metrics_move_limit", 50000) // Get the data from the table "Metric" older than specified by retain_metrics_days logger.info("MetricsArchiveScheduler.conditionalDeleteMetricsRow says before candidateMetricRowsToMove val") - val candidateMetricRowsToMove = APIMetrics.apiMetrics.vend.getAllMetrics(List(OBPToDate(someDaysAgo))) + val candidateMetricRowsToMove = APIMetrics.apiMetrics.vend.getAllMetrics(List(OBPToDate(someDaysAgo), OBPLimit(limit))) logger.info("MetricsArchiveScheduler.conditionalDeleteMetricsRow says after candidateMetricRowsToMove val") logger.info(s"Number of rows: ${candidateMetricRowsToMove.length}") candidateMetricRowsToMove map { i => From 157cd0e5763fc9ffeeb84a8328d2fee81968d22d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Mon, 27 Feb 2023 15:21:37 +0100 Subject: [PATCH 3/4] feature/Add props disable_metrics_scheduler --- .../src/main/resources/props/sample.props.template | 2 ++ obp-api/src/main/scala/bootstrap/liftweb/Boot.scala | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 9c53c18f5..3560162fe 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1207,6 +1207,8 @@ user_account_validated_redirect_url = # In case is not defined default value is false user_account_is_validated = false +# Disable/Enable Metric Scheduler +disable_metrics_scheduler = false # Defines the number of days we keep rows in the table "MetricArchive" # default value is 3 years retain_archive_metrics_days = 1095 diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 96587ee81..a837f39c0 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -684,10 +684,14 @@ class Boot extends MdcLoggable { case Full(i) => DatabaseDriverScheduler.start(i) case _ => // Do not start it } - val interval = - APIUtil.getPropsAsIntValue("retain_metrics_scheduler_interval_in_seconds", 3600) - MetricsArchiveScheduler.start(intervalInSeconds = interval) - + + APIUtil.getPropsAsBoolValue("disable_metrics_scheduler", false) match { + case false => + val interval = + APIUtil.getPropsAsIntValue("retain_metrics_scheduler_interval_in_seconds", 3600) + MetricsArchiveScheduler.start(intervalInSeconds = interval) + case true => // Do not start it + } object UsernameLockedChecker { def beginServicing(session: LiftSession, req: Req){ From 6bcecd1b07f3a80d93a8a5aa5730b3ab784f1237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 28 Feb 2023 12:21:42 +0100 Subject: [PATCH 4/4] feature/Tweak props name disable_metrics_scheduler = false -> enable_metrics_scheduler = true --- obp-api/src/main/resources/props/sample.props.template | 2 +- obp-api/src/main/scala/bootstrap/liftweb/Boot.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 3560162fe..55ccceeae 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1208,7 +1208,7 @@ user_account_validated_redirect_url = user_account_is_validated = false # Disable/Enable Metric Scheduler -disable_metrics_scheduler = false +enable_metrics_scheduler = true # Defines the number of days we keep rows in the table "MetricArchive" # default value is 3 years retain_archive_metrics_days = 1095 diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index a837f39c0..4fd63c46e 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -685,12 +685,12 @@ class Boot extends MdcLoggable { case _ => // Do not start it } - APIUtil.getPropsAsBoolValue("disable_metrics_scheduler", false) match { - case false => + APIUtil.getPropsAsBoolValue("enable_metrics_scheduler", true) match { + case true => val interval = APIUtil.getPropsAsIntValue("retain_metrics_scheduler_interval_in_seconds", 3600) MetricsArchiveScheduler.start(intervalInSeconds = interval) - case true => // Do not start it + case false => // Do not start it } object UsernameLockedChecker {