Rate Limiting - WIP 2

This commit is contained in:
Marko Milić 2019-09-25 13:35:15 +02:00
parent 7130bd108a
commit 64b24e9966
8 changed files with 137 additions and 73 deletions

View File

@ -42,9 +42,9 @@ import code.api.oauth1a.OauthParams._
import code.api.sandbox.SandboxApiCalls
import code.api.util.ApiTag.{ResourceDocTag, apiTagBank, apiTagNewStyle}
import code.api.util.Glossary.GlossaryItem
import com.openbankproject.commons.model.enums.StrongCustomerAuthentication.SCA
import code.api.util.RateLimitingJson.CallLimit
import code.api.v1_2.ErrorMessage
import code.api.{DirectLogin, util, _}
import code.api.{DirectLogin, _}
import code.bankconnectors.Connector
import code.consumer.Consumers
import code.customer.CustomerX
@ -52,11 +52,13 @@ import code.entitlement.Entitlement
import code.metrics._
import code.model._
import code.model.dataAccess.AuthUser
import code.ratelimiting.{RateLimiting, RateLimitingDI}
import code.sanitycheck.SanityCheck
import code.scope.Scope
import code.usercustomerlinks.UserCustomerLink
import code.util.Helper
import code.util.Helper.{MdcLoggable, SILENCE_IS_GOLDEN}
import com.openbankproject.commons.model.enums.StrongCustomerAuthentication.SCA
import com.openbankproject.commons.model.enums.{PemCertificateRole, StrongCustomerAuthentication}
import com.openbankproject.commons.model.{Customer, _}
import dispatch.url
@ -1955,8 +1957,51 @@ Returns a string showed to the developer
else {
Future { (Empty, None) }
}
// Update Call Context
res map {
/******************************************************************************************************************
* This block of code needs to update Call Context with Rate Limiting
* Please note that first source is the table RateLimiting and second the table Consumer
*/
def getRateLimiting(consumerId: String): Future[Box[RateLimiting]] = {
RateLimitingUtil.useConsumerLimits match {
case true => RateLimitingDI.rateLimiting.vend.getByConsumerId(consumerId)
case false => Future(Empty)
}
}
val resultWithRateLimiting: Future[(Box[User], Option[CallContext])] = for {
(user, cc) <- res
consumer = cc.flatMap(_.consumer)
rateLimiting <- getRateLimiting(consumer.map(_.consumerId.get).getOrElse(""))
} yield {
val limit: Option[CallLimit] = rateLimiting match {
case Full(rl) => Some(CallLimit(
rl.consumerId,
rl.perSecondCallLimit,
rl.perMinuteCallLimit,
rl.perHourCallLimit,
rl.perDayCallLimit,
rl.perWeekCallLimit,
rl.perMonthCallLimit))
case Empty =>
Some(CallLimit(
consumer.map(_.consumerId.get).getOrElse(""),
consumer.map(_.perSecondCallLimit.get).getOrElse(-1),
consumer.map(_.perMinuteCallLimit.get).getOrElse(-1),
consumer.map(_.perHourCallLimit.get).getOrElse(-1),
consumer.map(_.perDayCallLimit.get).getOrElse(-1),
consumer.map(_.perWeekCallLimit.get).getOrElse(-1),
consumer.map(_.perMonthCallLimit.get).getOrElse(-1)
))
case _ => None
}
(user, cc.map(_.copy(rateLimiting = limit)))
}
/*************************************************************************************************************** */
// Update Call Context
resultWithRateLimiting map {
x => (x._1, ApiSession.updateCallContext(Spelling(spelling), x._2))
} map {
x => (x._1, x._2.map(_.copy(implementedInVersion = implementedInVersion)))

View File

@ -6,6 +6,7 @@ import code.api.JSONFactoryGateway.PayloadOfJwtJSON
import code.api.oauth1a.OauthParams._
import code.api.util.APIUtil._
import code.api.util.ErrorMessages.BankAccountNotFound
import code.api.util.RateLimitingJson.CallLimit
import code.context.UserAuthContextProvider
import code.customer.CustomerX
import code.model.{Consumer, _}
@ -40,6 +41,7 @@ case class CallContext(
httpCode: Option[Int] = None,
httpBody: Option[String] = None,
requestHeaders: List[HTTPParam] = Nil,
rateLimiting: Option[CallLimit] = None,
`X-Rate-Limit-Limit` : Long = -1,
`X-Rate-Limit-Remaining` : Long = -1,
`X-Rate-Limit-Reset` : Long = -1

View File

@ -1,13 +1,12 @@
package code.api.util
import code.api.{APIFailureNewStyle, util}
import code.api.APIFailureNewStyle
import code.api.util.APIUtil.fullBoxOrException
import code.api.util.ErrorMessages.TooManyRequests
import code.api.util.RateLimitingPeriod.{LimitCallPeriod, PER_DAY, PER_HOUR, PER_MINUTE, PER_MONTH, PER_SECOND, PER_WEEK, PER_YEAR}
import code.model.Consumer
import code.api.util.RateLimitingJson.CallLimit
import code.util.Helper.MdcLoggable
import com.openbankproject.commons.model.User
import net.liftweb.common.{Box, Empty, Full}
import net.liftweb.common.{Box, Empty}
import net.liftweb.util.Props
import redis.clients.jedis.Jedis
@ -55,8 +54,21 @@ object RateLimitingPeriod extends Enumeration {
}
}
object RateLimitingUtil extends MdcLoggable {
object RateLimitingJson {
case class CallLimit(
consumer_id : String,
per_second : Long,
per_minute : Long,
per_hour : Long,
per_day : Long,
per_week : Long,
per_month : Long
)
}
object RateLimitingUtil extends MdcLoggable {
import code.api.util.RateLimitingPeriod._
val useConsumerLimits = APIUtil.getPropsAsBoolValue("use_consumer_limits", false)
val inMemoryMode = APIUtil.getPropsAsBoolValue("use_consumer_limits_in_memory_mode", false)
@ -231,22 +243,15 @@ object RateLimitingUtil extends MdcLoggable {
def composeMsgAuthorizedAccess(period: LimitCallPeriod, limit: Long): String = TooManyRequests + s" We only allow $limit requests ${RateLimitingPeriod.humanReadable(period)} for this Consumer."
def composeMsgAnonymousAccess(period: LimitCallPeriod, limit: Long): String = TooManyRequests + s" We only allow $limit requests ${RateLimitingPeriod.humanReadable(period)} for anonymous access."
def setXRateLimits(c: Consumer, z: (Long, Long), period: LimitCallPeriod): Option[CallContext] = {
def setXRateLimits(c: CallLimit, z: (Long, Long), period: LimitCallPeriod): Option[CallContext] = {
val limit = period match {
case PER_SECOND =>
c.perSecondCallLimit.get
case PER_MINUTE =>
c.perMinuteCallLimit.get
case PER_HOUR =>
c.perHourCallLimit.get
case PER_DAY =>
c.perDayCallLimit.get
case PER_WEEK =>
c.perWeekCallLimit.get
case PER_MONTH =>
c.perMonthCallLimit.get
case PER_YEAR =>
-1
case PER_SECOND => c.per_second
case PER_MINUTE => c.per_minute
case PER_HOUR => c.per_hour
case PER_DAY => c.per_day
case PER_WEEK => c.per_week
case PER_MONTH => c.per_month
case PER_YEAR => -1
}
userAndCallContext._2.map(_.copy(`X-Rate-Limit-Limit` = limit))
.map(_.copy(`X-Rate-Limit-Reset` = z._1))
@ -262,23 +267,16 @@ object RateLimitingUtil extends MdcLoggable {
.map(_.copy(`X-Rate-Limit-Remaining` = limit - z._2))
}
def exceededRateLimit(c: Consumer, period: LimitCallPeriod): Option[CallContextLight] = {
val remain = ttl(c.key.get, period)
def exceededRateLimit(c: CallLimit, period: LimitCallPeriod): Option[CallContextLight] = {
val remain = ttl(c.consumer_id, period)
val limit = period match {
case PER_SECOND =>
c.perSecondCallLimit.get
case PER_MINUTE =>
c.perMinuteCallLimit.get
case PER_HOUR =>
c.perHourCallLimit.get
case PER_DAY =>
c.perDayCallLimit.get
case PER_WEEK =>
c.perWeekCallLimit.get
case PER_MONTH =>
c.perMonthCallLimit.get
case PER_YEAR =>
-1
case PER_SECOND => c.per_second
case PER_MINUTE => c.per_minute
case PER_HOUR => c.per_hour
case PER_DAY => c.per_day
case PER_WEEK => c.per_week
case PER_MONTH => c.per_month
case PER_YEAR => -1
}
userAndCallContext._2.map(_.copy(`X-Rate-Limit-Limit` = limit))
.map(_.copy(`X-Rate-Limit-Reset` = remain))
@ -298,56 +296,56 @@ object RateLimitingUtil extends MdcLoggable {
userAndCallContext._2 match {
case Some(cc) =>
cc.consumer match {
case Full(c) => // Authorized access
cc.rateLimiting match {
case Some(rl) => // Authorized access
val checkLimits = List(
underConsumerLimits(c.key.get, PER_SECOND, c.perSecondCallLimit.get),
underConsumerLimits(c.key.get, PER_MINUTE, c.perMinuteCallLimit.get),
underConsumerLimits(c.key.get, PER_HOUR, c.perHourCallLimit.get),
underConsumerLimits(c.key.get, PER_DAY, c.perDayCallLimit.get),
underConsumerLimits(c.key.get, PER_WEEK, c.perWeekCallLimit.get),
underConsumerLimits(c.key.get, PER_MONTH, c.perMonthCallLimit.get)
underConsumerLimits(rl.consumer_id, PER_SECOND, rl.per_second),
underConsumerLimits(rl.consumer_id, PER_MINUTE, rl.per_minute),
underConsumerLimits(rl.consumer_id, PER_HOUR, rl.per_hour),
underConsumerLimits(rl.consumer_id, PER_DAY, rl.per_day),
underConsumerLimits(rl.consumer_id, PER_WEEK, rl.per_week),
underConsumerLimits(rl.consumer_id, PER_MONTH, rl.per_month)
)
checkLimits match {
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x1 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_SECOND, c.perSecondCallLimit.get), 429, exceededRateLimit(c, PER_SECOND))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_SECOND, rl.per_second), 429, exceededRateLimit(rl, PER_SECOND))), userAndCallContext._2)
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x2 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_MINUTE, c.perMinuteCallLimit.get), 429, exceededRateLimit(c, PER_MINUTE))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_MINUTE, rl.per_minute), 429, exceededRateLimit(rl, PER_MINUTE))), userAndCallContext._2)
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x3 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_HOUR, c.perHourCallLimit.get), 429, exceededRateLimit(c, PER_HOUR))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_HOUR, rl.per_hour), 429, exceededRateLimit(rl, PER_HOUR))), userAndCallContext._2)
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x4 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_DAY, c.perDayCallLimit.get), 429, exceededRateLimit(c, PER_DAY))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_DAY, rl.per_day), 429, exceededRateLimit(rl, PER_DAY))), userAndCallContext._2)
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x5 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_WEEK, c.perWeekCallLimit.get), 429, exceededRateLimit(c, PER_WEEK))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_WEEK, rl.per_week), 429, exceededRateLimit(rl, PER_WEEK))), userAndCallContext._2)
case x1 :: x2 :: x3 :: x4 :: x5 :: x6 :: Nil if x6 == false =>
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_MONTH, c.perMonthCallLimit.get), 429, exceededRateLimit(c, PER_MONTH))), userAndCallContext._2)
(fullBoxOrException(Empty ~> APIFailureNewStyle(composeMsgAuthorizedAccess(PER_MONTH, rl.per_month), 429, exceededRateLimit(rl, PER_MONTH))), userAndCallContext._2)
case _ =>
val incrementCounters = List (
incrementConsumerCounters(c.key.get, PER_SECOND, c.perSecondCallLimit.get), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(c.key.get, PER_MINUTE, c.perMinuteCallLimit.get), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(c.key.get, PER_HOUR, c.perHourCallLimit.get), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(c.key.get, PER_DAY, c.perDayCallLimit.get), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(c.key.get, PER_WEEK, c.perWeekCallLimit.get), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(c.key.get, PER_MONTH, c.perMonthCallLimit.get) // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_SECOND, rl.per_second), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_MINUTE, rl.per_minute), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_HOUR, rl.per_hour), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_DAY, rl.per_day), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_WEEK, rl.per_week), // Responses other than the 429 status code MUST be stored by a cache.
incrementConsumerCounters(rl.consumer_id, PER_MONTH, rl.per_month) // Responses other than the 429 status code MUST be stored by a cache.
)
incrementCounters match {
case first :: _ :: _ :: _ :: _ :: _ :: Nil if first._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, first, PER_SECOND))
(userAndCallContext._1, setXRateLimits(rl, first, PER_SECOND))
case _ :: second :: _ :: _ :: _ :: _ :: Nil if second._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, second, PER_MINUTE))
(userAndCallContext._1, setXRateLimits(rl, second, PER_MINUTE))
case _ :: _ :: third :: _ :: _ :: _ :: Nil if third._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, third, PER_HOUR))
(userAndCallContext._1, setXRateLimits(rl, third, PER_HOUR))
case _ :: _ :: _ :: fourth :: _ :: _ :: Nil if fourth._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, fourth, PER_DAY))
(userAndCallContext._1, setXRateLimits(rl, fourth, PER_DAY))
case _ :: _ :: _ :: _ :: fifth :: _ :: Nil if fifth._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, fifth, PER_WEEK))
(userAndCallContext._1, setXRateLimits(rl, fifth, PER_WEEK))
case _ :: _ :: _ :: _ :: _ :: sixth :: Nil if sixth._1 > 0 =>
(userAndCallContext._1, setXRateLimits(c, sixth, PER_MONTH))
(userAndCallContext._1, setXRateLimits(rl, sixth, PER_MONTH))
case _ =>
(userAndCallContext._1, userAndCallContext._2)
}
}
case Empty => // Anonymous access
case None => // Anonymous access
val consumerId = cc.ipAddress
val checkLimits = List(
underConsumerLimits(consumerId, PER_HOUR, perHourLimitAnonymous)
@ -366,7 +364,6 @@ object RateLimitingUtil extends MdcLoggable {
(userAndCallContext._1, userAndCallContext._2)
}
}
case _ => (userAndCallContext._1, userAndCallContext._2)
}
case _ => (userAndCallContext._1, userAndCallContext._2)
}

View File

@ -32,6 +32,7 @@ import code.methodrouting.{MethodRouting, MethodRoutingCommons, MethodRoutingPar
import code.metrics.APIMetrics
import code.model._
import code.model.dataAccess.{AuthUser, BankAccountCreation}
import code.ratelimiting.RateLimitingDI
import code.users.Users
import code.util.Helper
import code.views.Views
@ -609,9 +610,9 @@ trait APIMethods310 {
postJson <- NewStyle.function.tryons(s"$InvalidJsonFormat The Json body should be the $CallLimitPostJson ", 400, callContext) {
json.extract[CallLimitPostJson]
}
consumer <- NewStyle.function.getConsumerByConsumerId(consumerId, callContext)
updatedConsumer <- Consumers.consumers.vend.updateConsumerCallLimits(
consumer.id.get,
_ <- NewStyle.function.getConsumerByConsumerId(consumerId, callContext)
rateLimiting <- RateLimitingDI.rateLimiting.vend.createOrUpdateConsumerCallLimits(
consumerId,
Some(postJson.per_second_call_limit),
Some(postJson.per_minute_call_limit),
Some(postJson.per_hour_call_limit),
@ -621,7 +622,7 @@ trait APIMethods310 {
unboxFullOrFail(_, callContext, UpdateConsumerError)
}
} yield {
(createCallLimitJson(updatedConsumer, Nil), HttpCode.`200`(callContext))
(createCallsLimitJson(rateLimiting), HttpCode.`200`(callContext))
}
}
}

View File

@ -10,6 +10,14 @@ import scala.concurrent.Future
object MappedRateLimitingProvider extends RateLimitingProviderTrait {
def getAll(): Future[List[RateLimiting]] = Future(RateLimiting.findAll())
def getByConsumerId(consumerId: String): Future[Box[RateLimiting]] = Future {
RateLimiting.find(
By(RateLimiting.ConsumerId, consumerId),
NullRef(RateLimiting.BankId),
NullRef(RateLimiting.ApiVersion),
NullRef(RateLimiting.ApiName)
)
}
def createOrUpdateConsumerCallLimits(consumerId: String,
perSecond: Option[String],
perMinute: Option[String],

View File

@ -17,6 +17,7 @@ object RateLimitingDI extends SimpleInjector {
trait RateLimitingProviderTrait {
def getAll(): Future[List[RateLimiting]]
def getByConsumerId(consumerId: String): Future[Box[RateLimiting]]
def createOrUpdateConsumerCallLimits(consumerId: String,
perSecond: Option[String],
perMinute: Option[String],
@ -43,6 +44,7 @@ trait RateLimitingTrait {
class RemotedataRateLimitingCaseClasses {
case class getAll()
case class getByConsumerId(consumerId: String)
case class createOrUpdateConsumerCallLimits(consumerId: String,
perSecond: Option[String],
perMinute: Option[String],

View File

@ -12,10 +12,15 @@ import scala.concurrent.Future
object RemotedataRateLimiting extends ObpActorInit with RateLimitingProviderTrait {
val cc = RemotedataRateLimitingCaseClasses
override def getAll(): Future[List[RateLimiting]] = {
def getAll(): Future[List[RateLimiting]] = {
(actor ? cc.getAll()).mapTo[List[RateLimiting]]
}
def getByConsumerId(consumerId: String): Future[Box[RateLimiting]] = {
(actor ? cc.getByConsumerId(consumerId)).mapTo[Box[RateLimiting]]
}
def createOrUpdateConsumerCallLimits(id: String,
perSecond: Option[String],
perMinute: Option[String],

View File

@ -18,6 +18,10 @@ class RemotedataRateLimitingActor extends Actor with ObpActorHelper with MdcLogg
case cc.getAll() =>
logger.debug("getAll()")
mapper.getAll() pipeTo sender
case cc.getByConsumerId(consumerId: String) =>
logger.debug("getByConsumerId(" + consumerId + ")")
mapper.getByConsumerId(consumerId) pipeTo sender
case cc.createOrUpdateConsumerCallLimits(id: String, perSecond: Option[String], perMinute: Option[String], perHour: Option[String], perDay: Option[String], perWeek: Option[String], perMonth: Option[String]) =>
logger.debug("createOrUpdateConsumerCallLimits(" + id + ", " + perSecond.getOrElse("None")+ ", " + perMinute.getOrElse("None") + ", " + perHour.getOrElse("None") + ", " + perDay.getOrElse("None") + ", " + perWeek.getOrElse("None") + ", " + perMonth.getOrElse("None") + ")")