From 7130bd108a6be0b30e388d995461649ee313b7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mili=C4=87?= Date: Tue, 24 Sep 2019 16:22:41 +0200 Subject: [PATCH] Rate Limiting - WIP --- .../main/scala/bootstrap/liftweb/Boot.scala | 4 +- .../scala/code/api/v3_1_0/APIMethods310.scala | 4 +- .../code/api/v3_1_0/JSONFactory3.1.0.scala | 13 +++ .../ratelimiting/MappedRateLimiting.scala | 110 ++++++++++++++++++ .../code/ratelimiting/RateLimiting.scala | 55 +++++++++ .../code/remotedata/RemotedataActors.scala | 3 +- .../remotedata/RemotedataRateLimiting.scala | 30 +++++ .../RemotedataRateLimitingActor.scala | 32 +++++ 8 files changed, 246 insertions(+), 5 deletions(-) create mode 100644 obp-api/src/main/scala/code/ratelimiting/MappedRateLimiting.scala create mode 100644 obp-api/src/main/scala/code/ratelimiting/RateLimiting.scala create mode 100644 obp-api/src/main/scala/code/remotedata/RemotedataRateLimiting.scala create mode 100644 obp-api/src/main/scala/code/remotedata/RemotedataRateLimitingActor.scala diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index e3ca699f7..960d6a993 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -81,6 +81,7 @@ import code.productAttributeattribute.MappedProductAttribute import code.productcollection.MappedProductCollection import code.productcollectionitem.MappedProductCollectionItem import code.products.MappedProduct +import code.ratelimiting.RateLimiting import code.remotedata.RemotedataActors import code.scheduler.DatabaseDriverScheduler import code.scope.{MappedScope, MappedUserScope} @@ -618,7 +619,8 @@ object ToSchemify { MappedProductCollection, MappedProductCollectionItem, MappedAccountAttribute, - MappedCardAttribute + MappedCardAttribute, + RateLimiting ) // The following tables are accessed directly via Mapper / JDBC diff --git a/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala b/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala index adb84f086..9ee85fa48 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala @@ -20,7 +20,6 @@ import code.api.v2_2_0.{CreateAccountJSONV220, JSONFactory220} import code.api.v3_0_0.JSONFactory300 import code.api.v3_0_0.JSONFactory300.createAdapterInfoJson import code.api.v3_1_0.JSONFactory310._ -import com.openbankproject.commons.util.ReflectUtils import code.bankconnectors.rest.RestConnector_vMar2019 import code.bankconnectors.{Connector, LocalMappedConnector} import code.consent.{ConsentStatus, Consents} @@ -33,7 +32,6 @@ import code.methodrouting.{MethodRouting, MethodRoutingCommons, MethodRoutingPar import code.metrics.APIMetrics import code.model._ import code.model.dataAccess.{AuthUser, BankAccountCreation} -import com.openbankproject.commons.model.Product import code.users.Users import code.util.Helper import code.views.Views @@ -44,6 +42,7 @@ import com.nexmo.client.NexmoClient import com.nexmo.client.sms.messages.TextMessage import com.openbankproject.commons.model.enums.{AccountAttributeType, CardAttributeType, ProductAttributeType, StrongCustomerAuthentication} import com.openbankproject.commons.model.{CreditLimit, Product, _} +import com.openbankproject.commons.util.ReflectUtils import net.liftweb.common.{Box, Empty, Full} import net.liftweb.http.S import net.liftweb.http.provider.HTTPParam @@ -54,7 +53,6 @@ import net.liftweb.util.Mailer.{From, PlainMailBodyType, Subject, To} import net.liftweb.util.{Helpers, Mailer} import org.apache.commons.lang3.{StringUtils, Validate} -import scala.collection.immutable import scala.collection.immutable.{List, Nil} import scala.collection.mutable.ArrayBuffer import scala.concurrent.ExecutionContext.Implicits.global diff --git a/obp-api/src/main/scala/code/api/v3_1_0/JSONFactory3.1.0.scala b/obp-api/src/main/scala/code/api/v3_1_0/JSONFactory3.1.0.scala index c9d1540e7..6ceabb00b 100644 --- a/obp-api/src/main/scala/code/api/v3_1_0/JSONFactory3.1.0.scala +++ b/obp-api/src/main/scala/code/api/v3_1_0/JSONFactory3.1.0.scala @@ -49,6 +49,7 @@ import code.entitlement.Entitlement import code.loginattempts.BadLoginAttempt import code.metrics.{TopApi, TopConsumer} import code.model.{Consumer, ModeratedBankAccount, UserX} +import code.ratelimiting import code.webhook.AccountWebhook import com.openbankproject.commons.model.{AccountApplication, AmountOfMoneyJsonV121, Product, ProductCollection, ProductCollectionItem, TaxResidence, User, UserAuthContextUpdate, _} import net.liftweb.common.{Box, Full} @@ -783,6 +784,18 @@ object JSONFactory310{ redisRateLimit ) + } + def createCallsLimitJson(rateLimiting: ratelimiting.RateLimiting) : CallLimitJson = { + CallLimitJson( + rateLimiting.perSecondCallLimit.toString, + rateLimiting.perMinuteCallLimit.toString, + rateLimiting.perHourCallLimit.toString, + rateLimiting.perDayCallLimit.toString, + rateLimiting.perWeekCallLimit.toString, + rateLimiting.perMonthCallLimit.toString, + None + ) + } def createCheckFundsAvailableJson(fundsAvailable : String, availableFundsRequestId: String) : CheckFundsAvailableJson = { CheckFundsAvailableJson(fundsAvailable,new Date(), availableFundsRequestId) diff --git a/obp-api/src/main/scala/code/ratelimiting/MappedRateLimiting.scala b/obp-api/src/main/scala/code/ratelimiting/MappedRateLimiting.scala new file mode 100644 index 000000000..a34aa0cd1 --- /dev/null +++ b/obp-api/src/main/scala/code/ratelimiting/MappedRateLimiting.scala @@ -0,0 +1,110 @@ +package code.ratelimiting + +import code.util.{MappedUUID, UUIDString} +import net.liftweb.common.{Box, Full} +import net.liftweb.mapper._ +import net.liftweb.util.Helpers.tryo + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +object MappedRateLimitingProvider extends RateLimitingProviderTrait { + def getAll(): Future[List[RateLimiting]] = Future(RateLimiting.findAll()) + def createOrUpdateConsumerCallLimits(consumerId: String, + perSecond: Option[String], + perMinute: Option[String], + perHour: Option[String], + perDay: Option[String], + perWeek: Option[String], + perMonth: Option[String]): Future[Box[RateLimiting]] = Future { + + def createRateLimit(c: RateLimiting): Box[RateLimiting] = { + tryo { + perSecond match { + case Some(v) => c.PerSecondCallLimit(v.toLong) + case None => + } + perMinute match { + case Some(v) => c.PerMinuteCallLimit(v.toLong) + case None => + } + perHour match { + case Some(v) => c.PerHourCallLimit(v.toLong) + case None => + } + perDay match { + case Some(v) => c.PerDayCallLimit(v.toLong) + case None => + } + perWeek match { + case Some(v) => c.PerWeekCallLimit(v.toLong) + case None => + } + perMonth match { + case Some(v) => c.PerMonthCallLimit(v.toLong) + case None => + } + c.BankId(null) + c.ApiName(null) + c.ApiVersion(null) + c.ConsumerId(consumerId) + c.saveMe() + } + } + + val rateLimit = RateLimiting.find( + By(RateLimiting.ConsumerId, consumerId), + NullRef(RateLimiting.BankId), + NullRef(RateLimiting.ApiVersion), + NullRef(RateLimiting.ApiName) + ) + rateLimit match { + case Full(limit) => createRateLimit(limit) + case _ => createRateLimit(RateLimiting.create) + } + } +} + +class RateLimiting extends RateLimitingTrait with LongKeyedMapper[RateLimiting] with IdPK with CreatedUpdated { + override def getSingleton = RateLimiting + object RateLimitingId extends MappedUUID(this) + object ApiVersion extends MappedString(this, 250) + object ApiName extends MappedString(this, 250) + object ConsumerId extends MappedString(this, 250) + object BankId extends UUIDString(this) + object PerSecondCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + object PerMinuteCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + object PerHourCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + object PerDayCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + object PerWeekCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + object PerMonthCallLimit extends MappedLong(this) { + override def defaultValue = -1 + } + + def rateLimitingId: String = RateLimitingId.get + def apiName: String = ApiName.get + def apiVersion: String = ApiVersion.get + def consumerId: String = ConsumerId.get + def bankId: String = BankId.get + def perSecondCallLimit: Long = PerSecondCallLimit.get + def perMinuteCallLimit: Long = PerMinuteCallLimit.get + def perHourCallLimit: Long = PerHourCallLimit.get + def perDayCallLimit: Long = PerDayCallLimit.get + def perWeekCallLimit: Long = PerWeekCallLimit.get + def perMonthCallLimit: Long = PerMonthCallLimit.get + +} + +object RateLimiting extends RateLimiting with LongKeyedMetaMapper[RateLimiting] { + override def dbIndexes = UniqueIndex(RateLimitingId) :: super.dbIndexes +} diff --git a/obp-api/src/main/scala/code/ratelimiting/RateLimiting.scala b/obp-api/src/main/scala/code/ratelimiting/RateLimiting.scala new file mode 100644 index 000000000..e107ad850 --- /dev/null +++ b/obp-api/src/main/scala/code/ratelimiting/RateLimiting.scala @@ -0,0 +1,55 @@ +package code.ratelimiting + +import code.api.util.APIUtil +import net.liftweb.util.SimpleInjector +import code.remotedata.RemotedataRateLimiting +import net.liftweb.common.Box + +import scala.concurrent.Future + +object RateLimitingDI extends SimpleInjector { + val rateLimiting = new Inject(buildOne _) {} + def buildOne: RateLimitingProviderTrait = APIUtil.getPropsAsBoolValue("use_akka", false) match { + case false => MappedRateLimitingProvider + case true => RemotedataRateLimiting // We will use Akka as a middleware + } +} + +trait RateLimitingProviderTrait { + def getAll(): Future[List[RateLimiting]] + def createOrUpdateConsumerCallLimits(consumerId: String, + perSecond: Option[String], + perMinute: Option[String], + perHour: Option[String], + perDay: Option[String], + perWeek: Option[String], + perMonth: Option[String]): Future[Box[RateLimiting]] +} + +trait RateLimitingTrait { + def rateLimitingId: String + def apiVersion: String + def apiName: String + def consumerId: String + def bankId: String + def perSecondCallLimit: Long + def perMinuteCallLimit: Long + def perHourCallLimit: Long + def perDayCallLimit: Long + def perWeekCallLimit: Long + def perMonthCallLimit: Long +} + + +class RemotedataRateLimitingCaseClasses { + case class getAll() + case class createOrUpdateConsumerCallLimits(consumerId: String, + perSecond: Option[String], + perMinute: Option[String], + perHour: Option[String], + perDay: Option[String], + perWeek: Option[String], + perMonth: Option[String]) +} + +object RemotedataRateLimitingCaseClasses extends RemotedataRateLimitingCaseClasses diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataActors.scala b/obp-api/src/main/scala/code/remotedata/RemotedataActors.scala index c0daacd74..589b57744 100644 --- a/obp-api/src/main/scala/code/remotedata/RemotedataActors.scala +++ b/obp-api/src/main/scala/code/remotedata/RemotedataActors.scala @@ -57,7 +57,8 @@ object RemotedataActors extends MdcLoggable { ActorProps[RemotedataProductCollectionItemActor]-> RemotedataProductCollectionItem.actorName, ActorProps[RemotedataAccountAttributeActor] -> RemotedataAccountAttribute.actorName, ActorProps[RemotedataCardAttributeActor] -> RemotedataCardAttribute.actorName, - ActorProps[RemotedataProductCollectionActor] -> RemotedataProductCollection.actorName + ActorProps[RemotedataProductCollectionActor] -> RemotedataProductCollection.actorName, + ActorProps[RemotedataRateLimitingActor] -> RemotedataRateLimiting.actorName ) actorsRemotedata.foreach { a => logger.info(actorSystem.actorOf(a._1, name = a._2)) } diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataRateLimiting.scala b/obp-api/src/main/scala/code/remotedata/RemotedataRateLimiting.scala new file mode 100644 index 000000000..b6255c2ec --- /dev/null +++ b/obp-api/src/main/scala/code/remotedata/RemotedataRateLimiting.scala @@ -0,0 +1,30 @@ +package code.remotedata + +import akka.pattern.ask +import code.actorsystem.ObpActorInit +import code.ratelimiting.{RateLimiting, RateLimitingProviderTrait, RemotedataRateLimitingCaseClasses} +import net.liftweb.common.Box + +import scala.collection.immutable.List +import scala.concurrent.Future + + +object RemotedataRateLimiting extends ObpActorInit with RateLimitingProviderTrait { + + val cc = RemotedataRateLimitingCaseClasses + override def getAll(): Future[List[RateLimiting]] = { + (actor ? cc.getAll()).mapTo[List[RateLimiting]] + } + + def createOrUpdateConsumerCallLimits(id: String, + perSecond: Option[String], + perMinute: Option[String], + perHour: Option[String], + perDay: Option[String], + perWeek: Option[String], + perMonth: Option[String]): Future[Box[RateLimiting]] = + (actor ? cc.createOrUpdateConsumerCallLimits(id, perSecond, perMinute, perHour, perDay, perWeek, perMonth)).mapTo[Box[RateLimiting]] + + + +} diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataRateLimitingActor.scala b/obp-api/src/main/scala/code/remotedata/RemotedataRateLimitingActor.scala new file mode 100644 index 000000000..e7d920278 --- /dev/null +++ b/obp-api/src/main/scala/code/remotedata/RemotedataRateLimitingActor.scala @@ -0,0 +1,32 @@ +package code.remotedata + +import akka.actor.Actor +import akka.pattern.pipe +import code.actorsystem.ObpActorHelper +import code.ratelimiting.{MappedRateLimitingProvider, RemotedataRateLimitingCaseClasses} +import code.util.Helper.MdcLoggable + +import scala.concurrent.ExecutionContext.Implicits.global + +class RemotedataRateLimitingActor extends Actor with ObpActorHelper with MdcLoggable { + + val mapper = MappedRateLimitingProvider + val cc = RemotedataRateLimitingCaseClasses + + def receive: PartialFunction[Any, Unit] = { + + case cc.getAll() => + logger.debug("getAll()") + mapper.getAll() pipeTo sender + + case cc.createOrUpdateConsumerCallLimits(id: String, perSecond: Option[String], perMinute: Option[String], perHour: Option[String], perDay: Option[String], perWeek: Option[String], perMonth: Option[String]) => + logger.debug("createOrUpdateConsumerCallLimits(" + id + ", " + perSecond.getOrElse("None")+ ", " + perMinute.getOrElse("None") + ", " + perHour.getOrElse("None") + ", " + perDay.getOrElse("None") + ", " + perWeek.getOrElse("None") + ", " + perMonth.getOrElse("None") + ")") + mapper.createOrUpdateConsumerCallLimits(id, perSecond, perMinute, perHour, perDay, perWeek, perMonth) pipeTo sender + + case message => logger.warn("[AKKA ACTOR ERROR - REQUEST NOT RECOGNIZED] " + message) + + } + +} + +