From 38909e2a87c7b5423d5da29bad4b4a218fdc55f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Mon, 27 Nov 2023 14:19:46 +0100
Subject: [PATCH 01/36] feature/Add regulated entities endpoint

---
 .../resources/props/sample.props.template     |  5 +++-
 .../scala/code/api/v5_1_0/APIMethods510.scala | 27 +++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index 401c2fe1a..47780cdb2 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -1275,4 +1275,7 @@ validate_iban=false
 
 # Show all dependent connector methods for each endpoint. The default value is false.
 # If set to true, it may consume a significant amount of heap memory.
-#show_used_connector_methods=false
\ No newline at end of file
+#show_used_connector_methods=false
+
+# This returns Regulated Entities
+regulated_entities = []
\ No newline at end of file
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index ef355f8ec..369581562 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -120,6 +120,33 @@ trait APIMethods510 {
             (SuggestedSessionTimeoutV510(timeoutInSeconds.toString), HttpCode.`200`(cc.callContext))
           }
     }
+
+
+    staticResourceDocs += ResourceDoc(
+      regulatedEntities,
+      implementedInApiVersion,
+      nameOf(regulatedEntities),
+      "GET",
+      "/regulated-entities",
+      "Get Regulated Entities",
+      """Returns information about:
+        |
+        |* Regulated Entities
+        """,
+      EmptyBody,
+      EmptyBody,
+      List(UnknownError),
+      apiTagApi  :: Nil)
+
+    lazy val regulatedEntities: OBPEndpoint = {
+      case "regulated-entities" :: Nil JsonGet _ =>
+        cc => implicit val ec = EndpointContext(Some(cc))
+          APIUtil.scalaFutureToBoxedJsonResponse(for {
+            regulatedEntities <- Future(APIUtil.getPropsValue("regulated_entities", "[]"))
+          } yield {
+            (parse(regulatedEntities), HttpCode.`200`(cc.callContext))
+          })
+    }
     
     staticResourceDocs += ResourceDoc(
       waitingForGodot,

From de3bd6a215e9b362c73fa938727c230807fcfd90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 28 Nov 2023 11:04:13 +0100
Subject: [PATCH 02/36] feature/Tweak props regulated_entities

---
 obp-api/src/main/resources/props/sample.props.template | 1 +
 1 file changed, 1 insertion(+)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index 47780cdb2..e1726d9f4 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -1278,4 +1278,5 @@ validate_iban=false
 #show_used_connector_methods=false
 
 # This returns Regulated Entities
+# sample props regulated_entities = [{"certificate_authority_CA_OwnerID":"CY_CBC","entity_certificate_public_key":"-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----","entity_code":"PSD_PICY_CBC!12345","entity_type":"PSD_PI","entity_address":"EXAMPLE COMPANY LTD, 5 SOME STREET","entity_town_city":"SOME CITY","entity_post_code":"1060","entity_country":"CY","entity_web_site":"www.example.com","services":[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]}]
 regulated_entities = []
\ No newline at end of file

From e642b5d55874932947c4c77b1c1817ee2e004065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 28 Nov 2023 11:06:11 +0100
Subject: [PATCH 03/36] feature/Tweak props regulated_entities 2

---
 obp-api/src/main/resources/props/sample.props.template | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index e1726d9f4..3b2fcf9b6 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -1278,5 +1278,5 @@ validate_iban=false
 #show_used_connector_methods=false
 
 # This returns Regulated Entities
-# sample props regulated_entities = [{"certificate_authority_CA_OwnerID":"CY_CBC","entity_certificate_public_key":"-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----","entity_code":"PSD_PICY_CBC!12345","entity_type":"PSD_PI","entity_address":"EXAMPLE COMPANY LTD, 5 SOME STREET","entity_town_city":"SOME CITY","entity_post_code":"1060","entity_country":"CY","entity_web_site":"www.example.com","services":[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]}]
+# sample props regulated_entities = [{"certificate_authority_ca_owner_id":"CY_CBC","entity_certificate_public_key":"-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----","entity_code":"PSD_PICY_CBC!12345","entity_type":"PSD_PI","entity_address":"EXAMPLE COMPANY LTD, 5 SOME STREET","entity_town_city":"SOME CITY","entity_post_code":"1060","entity_country":"CY","entity_web_site":"www.example.com","services":[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]}]
 regulated_entities = []
\ No newline at end of file

From f2c2d348c8d3f3f854105419c7900551e81fd078 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 29 Nov 2023 10:03:21 +0100
Subject: [PATCH 04/36] feature/Add Tag Directory for regulated entities

---
 obp-api/src/main/scala/code/api/util/ApiTag.scala          | 3 +++
 obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/obp-api/src/main/scala/code/api/util/ApiTag.scala b/obp-api/src/main/scala/code/api/util/ApiTag.scala
index 58b153bf8..59fc4b65c 100644
--- a/obp-api/src/main/scala/code/api/util/ApiTag.scala
+++ b/obp-api/src/main/scala/code/api/util/ApiTag.scala
@@ -100,6 +100,9 @@ object ApiTag {
   val apiTagPSD2PIIS=ResourceDocTag("Confirmation of Funds Service (PIIS)")
   val apiTagPSD2PIS=ResourceDocTag("Payment Initiation Service (PIS)")
 
+
+  val apiTagDirectory = ResourceDocTag("Directory")
+
   
   //Note: the followings are for the code generator -- UKOpenBankingV3.1.0
   val apiTagUkAccountAccess = ResourceDocTag("UK-AccountAccess")
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 369581562..7c306f597 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -136,7 +136,7 @@ trait APIMethods510 {
       EmptyBody,
       EmptyBody,
       List(UnknownError),
-      apiTagApi  :: Nil)
+      apiTagDirectory :: apiTagApi  :: Nil)
 
     lazy val regulatedEntities: OBPEndpoint = {
       case "regulated-entities" :: Nil JsonGet _ =>

From 9653a676bfe7243474660df3dad9ff09d92f2029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 29 Nov 2023 22:37:12 +0100
Subject: [PATCH 05/36] feature/Regulated entities CRUD - WIP

---
 .../main/scala/bootstrap/liftweb/Boot.scala   |   4 +-
 .../SwaggerDefinitionsJSON.scala              |  27 ++++
 .../main/scala/code/api/util/ApiRole.scala    |   5 +
 .../api/util/newstyle/RegulatedEntity.scala   |  62 +++++++++
 .../scala/code/api/v5_1_0/APIMethods510.scala | 119 +++++++++++++++--
 .../code/api/v5_1_0/JSONFactory5.1.0.scala    |  52 +++++++-
 .../MappedRegulatedEntitiyProvider.scala      | 120 ++++++++++++++++++
 .../regulatedentities/RegulatedEntity.scala   |  33 +++++
 .../commons/model/CommonModelTrait.scala      |  15 +++
 9 files changed, 422 insertions(+), 15 deletions(-)
 create mode 100644 obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
 create mode 100644 obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
 create mode 100644 obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala

diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
index 9ba6c1f1f..1526b7e9a 100644
--- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
+++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
@@ -29,7 +29,6 @@ package bootstrap.liftweb
 import java.io.{File, FileInputStream}
 import java.util.stream.Collectors
 import java.util.{Locale, TimeZone}
-
 import code.CustomerDependants.MappedCustomerDependant
 import code.DynamicData.DynamicData
 import code.DynamicEndpoint.DynamicEndpoint
@@ -107,6 +106,7 @@ import code.productcollectionitem.MappedProductCollectionItem
 import code.productfee.ProductFee
 import code.products.MappedProduct
 import code.ratelimiting.RateLimiting
+import code.regulatedentities.MappedRegulatedEntity
 import code.remotedata.RemotedataActors
 import code.scheduler.{DataBaseCleanerScheduler, DatabaseDriverScheduler, JobScheduler, MetricsArchiveScheduler}
 import code.scope.{MappedScope, MappedUserScope}
@@ -135,6 +135,7 @@ import code.webuiprops.WebUiProps
 import com.openbankproject.commons.model.ErrorMessage
 import com.openbankproject.commons.util.Functions.Implicits._
 import com.openbankproject.commons.util.{ApiVersion, Functions}
+
 import javax.mail.internet.MimeMessage
 import net.liftweb.common._
 import net.liftweb.db.{DB, DBLogEntry}
@@ -1034,6 +1035,7 @@ object ToSchemify {
     AuthUser,
     JobScheduler,
     MappedETag,
+    MappedRegulatedEntity,
     AtmAttribute,
     Admin,
     MappedBank,
diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
index cfd720dd6..0aaa6dfa3 100644
--- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
+++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
@@ -52,6 +52,33 @@ import scala.collection.immutable.List
 object SwaggerDefinitionsJSON {
 
 
+  lazy val regulatedEntitiesJsonV510: RegulatedEntitiesJsonV510 = RegulatedEntitiesJsonV510(List(regulatedEntityJsonV510))
+  lazy val regulatedEntityJsonV510: RegulatedEntityJsonV510 = RegulatedEntityJsonV510(
+    entity_id = "0af807d7-3c39-43ef-9712-82bcfde1b9ca",
+    certificate_authority_ca_owner_id = "CY_CBC",
+    entity_certificate_public_key = "-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----",
+    entity_code = "PSD_PICY_CBC!12345",
+    entity_type = "PSD_PI",
+    entity_address = "EXAMPLE COMPANY LTD, 5 SOME STREET",
+    entity_town_city = "SOME CITY",
+    entity_post_code = "1060",
+    entity_country = "CY",
+    entity_web_site = "www.example.com",
+    services = json.parse("""[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]""")
+  )
+
+  lazy val regulatedEntityPostJsonV510: RegulatedEntityPostJsonV510 = RegulatedEntityPostJsonV510(
+    certificate_authority_ca_owner_id = "CY_CBC",
+    entity_certificate_public_key = "-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----",
+    entity_code = "PSD_PICY_CBC!12345",
+    entity_type = "PSD_PI",
+    entity_address = "EXAMPLE COMPANY LTD, 5 SOME STREET",
+    entity_town_city = "SOME CITY",
+    entity_post_code = "1060",
+    entity_country = "CY",
+    entity_web_site = "www.example.com",
+    services = """[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]"""
+  )
 
   val license =  License(
     id = licenseIdExample.value,
diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala
index d6a40c152..1263c2519 100644
--- a/obp-api/src/main/scala/code/api/util/ApiRole.scala
+++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala
@@ -66,6 +66,11 @@ object RoleCombination {
 
 object ApiRole extends MdcLoggable{
 
+  case class CanCreateRegulatedEntity(requiresBankId: Boolean = false) extends ApiRole
+  lazy val canCreateRegulatedEntity = CanCreateRegulatedEntity()
+  case class CanDeleteRegulatedEntity(requiresBankId: Boolean = false) extends ApiRole
+  lazy val canDeleteRegulatedEntity = CanDeleteRegulatedEntity()
+
   case class CanSearchWarehouse(requiresBankId: Boolean = false) extends ApiRole
   lazy val canSearchWarehouse = CanSearchWarehouse()
 
diff --git a/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
new file mode 100644
index 000000000..6318c516c
--- /dev/null
+++ b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
@@ -0,0 +1,62 @@
+package code.api.util.newstyle
+
+import code.api.util.APIUtil.{OBPReturnType, unboxFull}
+import code.api.util.{APIUtil, CallContext}
+import code.consumer.Consumers
+import code.model.{AppType, Consumer}
+import code.regulatedentities.MappedRegulatedEntityProvider
+import com.openbankproject.commons.model.RegulatedEntityTrait
+
+import scala.concurrent.Future
+
+object RegulatedEntityNewStyle {
+
+  import com.openbankproject.commons.ExecutionContext.Implicits.global
+
+  def createRegulatedEntityNewStyle(certificateAuthorityCaOwnerId: Option[String],
+                                    entityCertificatePublicKey: Option[String],
+                                    entityCode: Option[String],
+                                    entityType: Option[String],
+                                    entityAddress: Option[String],
+                                    entityTownCity: Option[String],
+                                    entityPostCode: Option[String],
+                                    entityCountry: Option[String],
+                                    entityWebSite: Option[String],
+                                    services: Option[String],
+                                    callContext: Option[CallContext]): OBPReturnType[RegulatedEntityTrait] = {
+    Future {
+      MappedRegulatedEntityProvider.createRegulatedEntity(
+        certificateAuthorityCaOwnerId: Option[String],
+        entityCertificatePublicKey: Option[String],
+        entityCode: Option[String],
+        entityType: Option[String],
+        entityAddress: Option[String],
+        entityTownCity: Option[String],
+        entityPostCode: Option[String],
+        entityCountry: Option[String],
+        entityWebSite: Option[String],
+        services: Option[String]
+      ) map {
+        (_, callContext)
+      }
+    } map {
+      unboxFull(_)
+    }
+  }
+
+  def deleteRegulatedEntityNewStyle(id: String,
+                                callContext: Option[CallContext]
+                               ): OBPReturnType[Boolean] = {
+    Future {
+      MappedRegulatedEntityProvider.deleteRegulatedEntity(
+        id
+      ) map {
+        (_, callContext)
+      }
+    } map {
+      unboxFull(_)
+    }
+  }
+
+
+}
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 7c306f597..74dc3b7cb 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -2,7 +2,7 @@ package code.api.v5_1_0
 
 
 import code.api.Constant
-import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.{apiCollectionJson400, apiCollectionsJson400, apiInfoJson400, postApiCollectionJson400, revokedConsentJsonV310, _}
+import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON._
 import code.api.util.APIUtil._
 import code.api.util.ApiRole._
 import code.api.util.ApiTag._
@@ -10,19 +10,20 @@ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFo
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
 import code.api.util._
-import code.api.v2_0_0.{EntitlementJSONs, JSONFactory200}
+import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle}
 import code.api.v3_0_0.JSONFactory300
 import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson
 import code.api.v3_1_0.ConsentJsonV310
 import code.api.v3_1_0.JSONFactory310.createBadLoginStatusJson
 import code.api.v4_0_0.{JSONFactory400, PostApiCollectionJson400}
-import code.api.v5_0_0.ConsentJsonV500
+import code.api.v5_1_0.JSONFactory510.{createRegulatedEntitiesJson, createRegulatedEntityJson}
 import code.atmattribute.AtmAttribute
 import code.bankconnectors.Connector
 import code.consent.Consents
 import code.loginattempts.LoginAttempt
 import code.metrics.APIMetrics
 import code.model.dataAccess.MappedBankAccount
+import code.regulatedentities.MappedRegulatedEntityProvider
 import code.transactionrequests.TransactionRequests.TransactionRequestTypes.{apply => _}
 import code.userlocks.UserLocksProvider
 import code.users.Users
@@ -32,12 +33,10 @@ import code.views.Views
 import code.views.system.{AccountAccess, ViewDefinition}
 import com.github.dwickern.macros.NameOf.nameOf
 import com.openbankproject.commons.ExecutionContext.Implicits.global
-import com.openbankproject.commons.dto.CustomerAndAttribute
 import com.openbankproject.commons.model.enums.{AtmAttributeType, UserAttributeType}
-import com.openbankproject.commons.model.{AtmId, AtmT, BankId, Permission}
+import com.openbankproject.commons.model._
 import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion}
-import net.liftweb.common.{Box, Full}
-import net.liftweb.http.S
+import net.liftweb.common.Full
 import net.liftweb.http.rest.RestHelper
 import net.liftweb.json.parse
 import net.liftweb.mapper.By
@@ -134,19 +133,115 @@ trait APIMethods510 {
         |* Regulated Entities
         """,
       EmptyBody,
-      EmptyBody,
+      regulatedEntitiesJsonV510,
       List(UnknownError),
       apiTagDirectory :: apiTagApi  :: Nil)
 
     lazy val regulatedEntities: OBPEndpoint = {
       case "regulated-entities" :: Nil JsonGet _ =>
         cc => implicit val ec = EndpointContext(Some(cc))
-          APIUtil.scalaFutureToBoxedJsonResponse(for {
-            regulatedEntities <- Future(APIUtil.getPropsValue("regulated_entities", "[]"))
+          for {
+            entities: List[RegulatedEntityTrait] <- Future(MappedRegulatedEntityProvider.getRegulatedEntities())
           } yield {
-            (parse(regulatedEntities), HttpCode.`200`(cc.callContext))
-          })
+            (createRegulatedEntitiesJson(entities), HttpCode.`200`(cc.callContext))
+          }
     }
+
+
+    staticResourceDocs += ResourceDoc(
+      createRegulatedEntity,
+      implementedInApiVersion,
+      nameOf(createRegulatedEntity),
+      "POST",
+      "/regulated-entities",
+      "Create Regulated Entity",
+      s""" Create Regulated Entity
+         |
+         |${authenticationRequiredMessage(true)}
+         |
+         |""",
+      List(regulatedEntityPostJsonV510),
+      regulatedEntitiesJsonV510,
+      List(
+        $UserNotLoggedIn,
+        UserHasMissingRoles,
+        InvalidJsonFormat,
+        UnknownError
+      ),
+      List(apiTagDirectory, apiTagApi),
+      Some(List(canCreateRegulatedEntity))
+    )
+
+    lazy val createRegulatedEntity: OBPEndpoint = {
+      case "regulated-entities" :: Nil JsonPost json -> _ => {
+        cc =>
+          implicit val ec = EndpointContext(Some(cc))
+          val failMsg = s"$InvalidJsonFormat The Json body should be the $RegulatedEntityPostJsonV510 "
+          for {
+            postedData <- NewStyle.function.tryons(failMsg, 400, cc.callContext) {
+              json.extract[RegulatedEntityPostJsonV510]
+            }
+            failMsg = s"$InvalidJsonFormat The `services` field is not valid JSON"
+            _ <- NewStyle.function.tryons(failMsg, 400, cc.callContext) {
+              parse(postedData.services)
+            }
+            (entity, callContext) <- createRegulatedEntityNewStyle(
+              certificateAuthorityCaOwnerId = Some(postedData.certificate_authority_ca_owner_id),
+              entityCertificatePublicKey = Some(postedData.entity_certificate_public_key),
+              entityCode = Some(postedData.entity_code),
+              entityType = Some(postedData.entity_type),
+              entityAddress = Some(postedData.entity_address),
+              entityTownCity = Some(postedData.entity_town_city),
+              entityPostCode = Some(postedData.entity_post_code),
+              entityCountry = Some(postedData.entity_country),
+              entityWebSite = Some(postedData.entity_web_site),
+              services = Some(postedData.services),
+              cc.callContext
+            )
+          } yield {
+            (createRegulatedEntityJson(entity), HttpCode.`201`(callContext))
+          }
+      }
+    }
+
+    resourceDocs += ResourceDoc(
+      deleteRegulatedEntity,
+      implementedInApiVersion,
+      nameOf(deleteRegulatedEntity),
+      "DELETE",
+      "/regulated-entities/REGULATED_ENTITY_ID",
+      "Delete Regulated Entity",
+      s"""Delete Regulated Entity specified by REGULATED_ENTITY_ID
+         |
+         |${authenticationRequiredMessage(true)}
+         |""".stripMargin,
+      EmptyBody,
+      EmptyBody,
+      List(
+        $UserNotLoggedIn,
+        UserHasMissingRoles,
+        InvalidConnectorResponse,
+        UnknownError
+      ),
+      List(apiTagDirectory, apiTagApi),
+      Some(List(canDeleteRegulatedEntity)))
+
+    lazy val deleteRegulatedEntity: OBPEndpoint = {
+      case "regulated-entities" :: regulatedEntityId :: Nil JsonDelete _ => {
+        cc =>
+          implicit val ec = EndpointContext(Some(cc))
+          for {
+            (deleted, callContext) <- deleteRegulatedEntityNewStyle(
+              regulatedEntityId: String,
+              cc.callContext: Option[CallContext]
+            )
+          } yield {
+            org.scalameta.logger.elem(deleted)
+            (Full(deleted), HttpCode.`204`(callContext))
+          }
+      }
+    }
+
     
     staticResourceDocs += ResourceDoc(
       waitingForGodot,
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 80763b884..9f18abd19 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -37,14 +37,15 @@ import code.atmattribute.AtmAttribute
 import code.atms.Atms.Atm
 import code.users.UserAttribute
 import code.views.system.{AccountAccess, ViewDefinition}
-import com.openbankproject.commons.model.{Address, AtmId, AtmT, BankId, BankIdAccountId, Customer, Location, Meta}
+import com.openbankproject.commons.model.{Address, AtmId, AtmT, BankId, BankIdAccountId, Customer, Location, Meta, RegulatedEntityTrait}
 import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion}
 
 import java.util.Date
 import code.consent.MappedConsent
 import code.metrics.APIMetric
 import net.liftweb.common.Box
-import net.liftweb.json.parse
+import net.liftweb.json
+import net.liftweb.json.{JValue, parse}
 
 import scala.collection.immutable.List
 import scala.util.Try
@@ -64,6 +65,34 @@ case class APIInfoJsonV510(
                            energy_source : EnergySource400,
                            resource_docs_requires_role: Boolean
                          )
+
+case class RegulatedEntityJsonV510(
+                                    entity_id: String,
+                                    certificate_authority_ca_owner_id: String,
+                                    entity_certificate_public_key: String,
+                                    entity_code: String,
+                                    entity_type: String,
+                                    entity_address: String,
+                                    entity_town_city: String,
+                                    entity_post_code: String,
+                                    entity_country: String,
+                                    entity_web_site: String,
+                                    services: JValue
+                                  )
+case class RegulatedEntityPostJsonV510(
+                                      certificate_authority_ca_owner_id: String,
+                                      entity_certificate_public_key: String,
+                                      entity_code: String,
+                                      entity_type: String,
+                                      entity_address: String,
+                                      entity_town_city: String,
+                                      entity_post_code: String,
+                                      entity_country: String,
+                                      entity_web_site: String,
+                                      services: String
+                                    )
+case class RegulatedEntitiesJsonV510(entities: List[RegulatedEntityJsonV510])
+
 case class WaitingForGodotJsonV510(sleep_in_milliseconds: Long)
 
 case class CertificateInfoJsonV510(
@@ -534,6 +563,25 @@ object JSONFactory510 extends CustomJsonFormats {
     UserAttributesResponseJsonV510(userAttribute.map(createUserAttributeJson))
   }
 
+  def createRegulatedEntityJson(entity: RegulatedEntityTrait): RegulatedEntityJsonV510 = {
+    RegulatedEntityJsonV510(
+      entity_id = entity.entityId,
+      certificate_authority_ca_owner_id = entity.certificateAuthorityCaOwnerId,
+      entity_certificate_public_key = entity.entityCertificatePublicKey,
+      entity_code = entity.entityCode,
+      entity_type = entity.entityType,
+      entity_address = entity.entityAddress,
+      entity_town_city = entity.entityTownCity,
+      entity_post_code = entity.entityPostCode,
+      entity_country = entity.entityCountry,
+      entity_web_site = entity.entityWebSite,
+      services = json.parse(entity.services)
+    )
+  }
+  def createRegulatedEntitiesJson(entities: List[RegulatedEntityTrait]): RegulatedEntitiesJsonV510 = {
+    RegulatedEntitiesJsonV510(entities.map(createRegulatedEntityJson))
+  }
+
   def createMetricJson(metric: APIMetric): MetricJsonV510 = {
     MetricJsonV510(
       user_id = metric.getUserId(),
diff --git a/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
new file mode 100644
index 000000000..e91e6e3a4
--- /dev/null
+++ b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
@@ -0,0 +1,120 @@
+package code.regulatedentities
+
+import code.util.MappedUUID
+import com.openbankproject.commons.model.RegulatedEntityTrait
+import net.liftweb.common.Box
+import net.liftweb.common.Box.tryo
+import net.liftweb.mapper._
+
+import scala.concurrent.Future
+
+object MappedRegulatedEntityProvider extends RegulatedEntityProvider {
+  def getRegulatedEntities(): List[RegulatedEntityTrait] = {
+    MappedRegulatedEntity.findAll()
+  }
+
+  override def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
+                                     entityCertificatePublicKey: Option[String],
+                                     entityCode: Option[String],
+                                     entityType: Option[String],
+                                     entityAddress: Option[String],
+                                     entityTownCity: Option[String],
+                                     entityPostCode: Option[String],
+                                     entityCountry: Option[String],
+                                     entityWebSite: Option[String],
+                                     services: Option[String]
+                                    ): Box[RegulatedEntityTrait] = {
+    tryo {
+      val entity = MappedRegulatedEntity.create
+      certificateAuthorityCaOwnerId match {
+        case Some(v) => entity.CertificateAuthorityCaOwnerId(v)
+        case None =>
+      }
+      entityCertificatePublicKey match {
+        case Some(v) => entity.EntityCertificatePublicKey(v)
+        case None =>
+      }
+      entityCode match {
+        case Some(v) => entity.EntityCode(v)
+        case None =>
+      }
+      entityType match {
+        case Some(v) => entity.EntityType(v)
+        case None =>
+      }
+      entityAddress match {
+        case Some(v) => entity.EntityAddress(v)
+        case None =>
+      }
+      entityTownCity match {
+        case Some(v) => entity.EntityTownCity(v)
+        case None =>
+      }
+      entityPostCode match {
+        case Some(v) => entity.EntityPostCode(v)
+        case None =>
+      }
+      entityCountry match {
+        case Some(v) => entity.EntityCountry(v)
+        case None =>
+      }
+      entityWebSite match {
+        case Some(v) => entity.EntityWebSite(v)
+        case None =>
+      }
+      services match {
+        case Some(v) => entity.Services(v)
+        case None =>
+      }
+
+      if (entity.validate.isEmpty) {
+        entity.saveMe()
+      } else {
+        throw new Error(entity.validate.map(_.msg.toString()).mkString(";"))
+      }
+    }
+  }
+
+  override def deleteRegulatedEntity(id: String): Box[Boolean] = {
+    tryo(
+      MappedRegulatedEntity.bulkDelete_!!(By(MappedRegulatedEntity.EntityId, id))
+    )
+  }
+
+}
+
+class MappedRegulatedEntity extends RegulatedEntityTrait with LongKeyedMapper[MappedRegulatedEntity] with IdPK {
+  override def getSingleton = MappedRegulatedEntity
+  object EntityId extends MappedUUID(this)
+  object CertificateAuthorityCaOwnerId extends MappedString(this, 50)
+  object EntityCode extends MappedString(this, 50)
+  object EntityCertificatePublicKey extends MappedText(this)
+  object EntityType extends MappedString(this, 50)
+  object EntityAddress extends MappedString(this, 50)
+  object EntityTownCity extends MappedString(this, 50)
+  object EntityPostCode extends MappedString(this, 50)
+  object EntityCountry extends MappedString(this, 50)
+  object EntityWebSite extends MappedString(this, 50)
+  object Services extends MappedText(this)
+
+
+  override def entityId: String = EntityId.get
+  override def certificateAuthorityCaOwnerId: String = CertificateAuthorityCaOwnerId.get
+  override def entityCode: String = EntityCode.get
+  override def entityCertificatePublicKey: String = EntityCertificatePublicKey.get
+  override def entityType: String = EntityType.get
+  override def entityAddress: String = EntityAddress.get
+  override def entityTownCity: String = EntityTownCity.get
+  override def entityPostCode: String = EntityPostCode.get
+  override def entityCountry: String = EntityCountry.get
+  override def entityWebSite: String = EntityWebSite.get
+  override def services: String = Services.get
+
+
+}
+
+object MappedRegulatedEntity extends MappedRegulatedEntity with LongKeyedMetaMapper[MappedRegulatedEntity]  {
+  override def dbTableName = "RegulatedEntity" // define the DB table name
+  override def dbIndexes = Index(CertificateAuthorityCaOwnerId) :: super.dbIndexes
+}
+
diff --git a/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
new file mode 100644
index 000000000..46a679fbf
--- /dev/null
+++ b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
@@ -0,0 +1,33 @@
+package code.regulatedentities
+
+
+import com.openbankproject.commons.model.RegulatedEntityTrait
+import net.liftweb.common.{Box, Logger}
+import net.liftweb.util.SimpleInjector
+
+object RegulatedEntityX extends SimpleInjector {
+  val regulatedEntityProvider = new Inject(buildOne _) {}
+  def buildOne: RegulatedEntityProvider = MappedRegulatedEntityProvider
+}
+/* For ProductFee */
+trait RegulatedEntityProvider {
+
+  private val logger = Logger(classOf[RegulatedEntityProvider])
+
+  def getRegulatedEntities(): List[RegulatedEntityTrait]
+
+  def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
+                            entityCertificatePublicKey: Option[String],
+                            entityCode: Option[String],
+                            entityType: Option[String],
+                            entityAddress: Option[String],
+                            entityTownCity: Option[String],
+                            entityPostCode: Option[String],
+                            entityCountry: Option[String],
+                            entityWebSite: Option[String],
+                            services: Option[String]
+                           ): Box[RegulatedEntityTrait]
+
+  def deleteRegulatedEntity(id: String): Box[Boolean]
+
+}
\ No newline at end of file
diff --git a/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala b/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
index cd0a28455..54c4cac53 100644
--- a/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
+++ b/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
@@ -96,6 +96,21 @@ trait AccountApplication {
   def status: String
 }
 
+
+trait RegulatedEntityTrait {
+  def entityId: String
+  def certificateAuthorityCaOwnerId: String
+  def entityCode: String
+  def entityCertificatePublicKey: String
+  def entityType: String
+  def entityAddress: String
+  def entityTownCity: String
+  def entityPostCode: String
+  def entityCountry: String
+  def entityWebSite: String
+  def services: String
+}
+
 trait UserAttributeTrait {
   def userAttributeId: String
   def userId: String

From 52d2e4ffb1e04dc8cf570702fee3bab485b7706e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Thu, 30 Nov 2023 11:44:36 +0100
Subject: [PATCH 06/36] feature/Regulated entities CRUD - WIP 2

---
 .../scala/code/api/util/ErrorMessages.scala   |  5 ++-
 .../api/util/newstyle/RegulatedEntity.scala   | 38 ++++++++++++++-----
 .../scala/code/api/v5_1_0/APIMethods510.scala | 36 +++++++++++++++---
 .../MappedRegulatedEntitiyProvider.scala      |  4 ++
 .../regulatedentities/RegulatedEntity.scala   |  2 +
 5 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
index e96757dbc..611e437e9 100644
--- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
+++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
@@ -516,7 +516,10 @@ object ErrorMessages {
   // Bank related messages
   val bankIdAlreadyExists = "OBP-34000: Bank Id already exists. Please specify a different value."
   val updateBankError = "OBP-34001: Could not update the Bank"
-  
+
+  val RegulatedEntityNotFound = "OBP-34100: Regulated Entity not found. Please specify a valid value for REGULATED_ENTITY_ID."
+  val RegulatedEntityNotDeleted = "OBP-34101: Regulated Entity cannot be deleted. Please specify a valid value for REGULATED_ENTITY_ID."
+
   // Consents
   val ConsentNotFound = "OBP-35001: Consent not found by CONSENT_ID. "
   val ConsentNotBeforeIssue = "OBP-35002: The time Consent-ID token was issued is set in the future. "
diff --git a/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
index 6318c516c..c4d79bbd2 100644
--- a/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
+++ b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
@@ -1,11 +1,13 @@
 package code.api.util.newstyle
 
-import code.api.util.APIUtil.{OBPReturnType, unboxFull}
+import code.api.util.APIUtil.{OBPReturnType, unboxFull, unboxFullOrFail}
+import code.api.util.ErrorMessages.{RegulatedEntityNotDeleted, RegulatedEntityNotFound}
 import code.api.util.{APIUtil, CallContext}
 import code.consumer.Consumers
 import code.model.{AppType, Consumer}
 import code.regulatedentities.MappedRegulatedEntityProvider
 import com.openbankproject.commons.model.RegulatedEntityTrait
+import net.liftweb.common.Box
 
 import scala.concurrent.Future
 
@@ -44,17 +46,33 @@ object RegulatedEntityNewStyle {
     }
   }
 
-  def deleteRegulatedEntityNewStyle(id: String,
-                                callContext: Option[CallContext]
-                               ): OBPReturnType[Boolean] = {
+  def getRegulatedEntitiesNewStyle(callContext: Option[CallContext]): OBPReturnType[List[RegulatedEntityTrait]] = {
     Future {
-      MappedRegulatedEntityProvider.deleteRegulatedEntity(
-        id
-      ) map {
-        (_, callContext)
-      }
+      MappedRegulatedEntityProvider.getRegulatedEntities()
     } map {
-      unboxFull(_)
+      (_, callContext)
+    }
+  }
+  def getRegulatedEntityByEntityIdNewStyle(id: String,
+                                           callContext: Option[CallContext]
+                                          ): OBPReturnType[RegulatedEntityTrait] = {
+    Future {
+      MappedRegulatedEntityProvider.getRegulatedEntityByEntityId(id)
+    }  map {
+      (_, callContext)
+    } map {
+      x => (unboxFullOrFail(x._1, callContext, RegulatedEntityNotFound, 404), x._2)
+    }
+  }
+  def deleteRegulatedEntityNewStyle(id: String,
+                                    callContext: Option[CallContext]
+                                   ): OBPReturnType[Boolean] = {
+    Future {
+      MappedRegulatedEntityProvider.deleteRegulatedEntity(id)
+    } map {
+      (_, callContext)
+    } map {
+      x => (unboxFullOrFail(x._1, callContext, RegulatedEntityNotDeleted, 400), x._2)
     }
   }
 
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 74dc3b7cb..34d2ba056 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -10,7 +10,7 @@ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFo
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
 import code.api.util._
-import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle}
+import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle}
 import code.api.v3_0_0.JSONFactory300
 import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson
 import code.api.v3_1_0.ConsentJsonV310
@@ -141,9 +141,33 @@ trait APIMethods510 {
       case "regulated-entities" :: Nil JsonGet _ =>
         cc => implicit val ec = EndpointContext(Some(cc))
           for {
-            entities: List[RegulatedEntityTrait] <- Future(MappedRegulatedEntityProvider.getRegulatedEntities())
+            (entities, callContext) <- getRegulatedEntitiesNewStyle(cc.callContext)
           } yield {
-            (createRegulatedEntitiesJson(entities), HttpCode.`200`(cc.callContext))
+            (createRegulatedEntitiesJson(entities), HttpCode.`200`(callContext))
+          }
+    }
+
+    staticResourceDocs += ResourceDoc(
+      getRegulatedEntityById,
+      implementedInApiVersion,
+      nameOf(getRegulatedEntityById),
+      "GET",
+      "/regulated-entities/REGULATED_ENTITY_ID",
+      "Get Regulated Entity",
+      """Get Regulated Entity By REGULATED_ENTITY_ID
+        """,
+      EmptyBody,
+      regulatedEntitiesJsonV510,
+      List(UnknownError),
+      apiTagDirectory :: apiTagApi  :: Nil)
+
+    lazy val getRegulatedEntityById: OBPEndpoint = {
+      case "regulated-entities" :: regulatedEntityId :: Nil JsonGet _ =>
+        cc => implicit val ec = EndpointContext(Some(cc))
+          for {
+            (entity, callContext)  <- getRegulatedEntityByEntityIdNewStyle(regulatedEntityId, cc.callContext)
+          } yield {
+            (createRegulatedEntityJson(entity), HttpCode.`200`(callContext))
           }
     }
 
@@ -155,12 +179,12 @@ trait APIMethods510 {
       "POST",
       "/regulated-entities",
       "Create Regulated Entity",
-      s""" Create Regulated Entity
+      s"""Create Regulated Entity
          |
          |${authenticationRequiredMessage(true)}
          |
          |""",
-      List(regulatedEntityPostJsonV510),
+      regulatedEntityPostJsonV510,
       regulatedEntitiesJsonV510,
       List(
         $UserNotLoggedIn,
@@ -237,7 +261,7 @@ trait APIMethods510 {
             )
           } yield {
             org.scalameta.logger.elem(deleted)
-            (Full(deleted), HttpCode.`204`(callContext))
+            (Full(deleted), HttpCode.`200`(callContext))
           }
       }
     }
diff --git a/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
index e91e6e3a4..b1295e0f5 100644
--- a/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
+++ b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
@@ -13,6 +13,10 @@ object MappedRegulatedEntityProvider extends RegulatedEntityProvider {
     MappedRegulatedEntity.findAll()
   }
 
+  override def getRegulatedEntityByEntityId(entityId: String): Box[RegulatedEntityTrait] = {
+    MappedRegulatedEntity.find(By(MappedRegulatedEntity.EntityId, entityId))
+  }
+
   override def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
                                      entityCertificatePublicKey: Option[String],
                                      entityCode: Option[String],
diff --git a/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
index 46a679fbf..25af53275 100644
--- a/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
+++ b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
@@ -16,6 +16,8 @@ trait RegulatedEntityProvider {
 
   def getRegulatedEntities(): List[RegulatedEntityTrait]
 
+  def getRegulatedEntityByEntityId(entityId: String): Box[RegulatedEntityTrait]
+
   def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
                             entityCertificatePublicKey: Option[String],
                             entityCode: Option[String],

From 760eb0b62361d0dff8e318b180bf383642966d51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Fri, 1 Dec 2023 11:07:49 +0100
Subject: [PATCH 07/36] test/Add test case in case of Regulated entities

---
 .../code/api/v5_1_0/RegulatedEntityTest.scala | 120 ++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 obp-api/src/test/scala/code/api/v5_1_0/RegulatedEntityTest.scala

diff --git a/obp-api/src/test/scala/code/api/v5_1_0/RegulatedEntityTest.scala b/obp-api/src/test/scala/code/api/v5_1_0/RegulatedEntityTest.scala
new file mode 100644
index 000000000..5d7696209
--- /dev/null
+++ b/obp-api/src/test/scala/code/api/v5_1_0/RegulatedEntityTest.scala
@@ -0,0 +1,120 @@
+package code.api.v5_1_0
+
+import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON.regulatedEntityPostJsonV510
+import code.api.util.APIUtil.OAuth._
+import code.api.util.ApiRole.{CanCreateRegulatedEntity, CanDeleteRegulatedEntity, CanGetSystemIntegrity}
+import code.api.util.ErrorMessages.{UserHasMissingRoles, UserNotLoggedIn}
+import code.api.v5_1_0.OBPAPI5_1_0.Implementations5_1_0
+import code.entitlement.Entitlement
+import com.github.dwickern.macros.NameOf.nameOf
+import com.openbankproject.commons.model.ErrorMessage
+import com.openbankproject.commons.util.ApiVersion
+import net.liftweb.json.Serialization._
+import org.scalatest.Tag
+
+class RegulatedEntityTest extends V510ServerSetup {
+  /**
+    * Test tags
+    * Example: To run tests with tag "getPermissions":
+    * 	mvn test -D tagsToInclude
+    *
+    *  This is made possible by the scalatest maven plugin
+    */
+  object VersionOfApi extends Tag(ApiVersion.v5_1_0.toString)
+  object ApiEndpoint1 extends Tag(nameOf(Implementations5_1_0.createRegulatedEntity))
+  object ApiEndpoint2 extends Tag(nameOf(Implementations5_1_0.getRegulatedEntityById))
+  object ApiEndpoint3 extends Tag(nameOf(Implementations5_1_0.getRegulatedEntityById))
+  object ApiEndpoint4 extends Tag(nameOf(Implementations5_1_0.deleteRegulatedEntity))
+  
+  feature(s"test $ApiEndpoint1 version $VersionOfApi - Unauthorized access") {
+    scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) {
+      When("We make a request v5.1.0")
+      val request510 = (v5_1_0_Request / "regulated-entities").POST
+      val response510 = makePostRequest(request510, write(regulatedEntityPostJsonV510))
+      Then("We should get a 401")
+      response510.code should equal(401)
+      response510.body.extract[ErrorMessage].message should equal(UserNotLoggedIn)
+    }
+  }
+  
+  feature(s"test $ApiEndpoint1 version $VersionOfApi - Authorized access") {
+    scenario("We will call the endpoint with user credentials but without a proper entitlement", ApiEndpoint1, VersionOfApi) {
+      When("We make a request v5.1.0")
+      val request510 = (v5_1_0_Request / "regulated-entities").POST <@(user1)
+      val response510 = makePostRequest(request510, write(regulatedEntityPostJsonV510))
+      Then("error should be " + UserHasMissingRoles + CanCreateRegulatedEntity)
+      response510.code should equal(403)
+      response510.body.extract[ErrorMessage].message should be (UserHasMissingRoles + CanCreateRegulatedEntity)
+    }
+  }
+  
+  feature(s"test $ApiEndpoint1 version $VersionOfApi - Authorized access") {
+    scenario("We will call the endpoint with user credentials and a proper entitlement", ApiEndpoint1, VersionOfApi) {
+      Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateRegulatedEntity.toString)
+      When("We make a request v5.1.0")
+      val request510 = (v5_1_0_Request / "regulated-entities").POST <@ (user1)
+      val response510 = makePostRequest(request510, write(regulatedEntityPostJsonV510))
+      Then("We get successful response")
+      response510.code should equal(201)
+      response510.body.extract[RegulatedEntityJsonV510]
+    }
+  }
+
+  // ApiEndpoint4 - deleteRegulatedEntity
+  feature(s"test $ApiEndpoint4 version $VersionOfApi - Unauthorized access") {
+    scenario("We will call the endpoint without user credentials", ApiEndpoint1, VersionOfApi) {
+      When("We make a request v5.1.0")
+      val request510 = (v5_1_0_Request / "regulated-entities" / "some id").DELETE
+      val response510 = makeDeleteRequest(request510)
+      Then("We should get a 401")
+      response510.code should equal(401)
+      response510.body.extract[ErrorMessage].message should equal(UserNotLoggedIn)
+    }
+  }
+  feature(s"test $ApiEndpoint4 version $VersionOfApi - Authorized access") {
+    scenario("We will call the endpoint with user credentials but without a proper entitlement", ApiEndpoint1, VersionOfApi) {
+      When("We make a request v5.1.0")
+      val request510 = (v5_1_0_Request / "regulated-entities" / "some id").DELETE <@ (user1)
+      val response510 = makeDeleteRequest(request510)
+      Then("error should be " + UserHasMissingRoles + CanDeleteRegulatedEntity)
+      response510.code should equal(403)
+      response510.body.extract[ErrorMessage].message should be(UserHasMissingRoles + CanDeleteRegulatedEntity)
+    }
+  }
+
+
+  feature(s"test $ApiEndpoint1, $ApiEndpoint2, $ApiEndpoint3, $ApiEndpoint4 version $VersionOfApi - CRUD") {
+    scenario("We will call the endpoint with user credentials but without a proper entitlement", ApiEndpoint1, VersionOfApi) {
+      // Create a row
+      Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanCreateRegulatedEntity.toString)
+      val request510 = (v5_1_0_Request / "regulated-entities").POST <@ (user1)
+      val response510 = makePostRequest(request510, write(regulatedEntityPostJsonV510))
+      Then("We get successful response")
+      response510.code should equal(201)
+      val createdRow: RegulatedEntityJsonV510 = response510.body.extract[RegulatedEntityJsonV510]
+
+      // Get the row by id
+      val getRequest510 = (v5_1_0_Request / "regulated-entities" / createdRow.entity_id).GET
+      val getResponse510 = makeGetRequest(getRequest510)
+      getResponse510.code should equal(200)
+      val gottenRow: RegulatedEntityJsonV510 = getResponse510.body.extract[RegulatedEntityJsonV510]
+
+      // TRy to match responses
+      createdRow should equal(gottenRow)
+
+      // Delete the row
+      Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanDeleteRegulatedEntity.toString)
+      val deleteRequest510 = (v5_1_0_Request / "regulated-entities" / gottenRow.entity_id).DELETE <@ (user1)
+      val deleteResponse510 = makeDeleteRequest(deleteRequest510)
+      deleteResponse510.code should equal(200)
+
+
+      // Get all rows
+      val getAllRequest510 = (v5_1_0_Request / "regulated-entities").GET
+      val getAllResponse510 = makeGetRequest(getAllRequest510)
+      getAllResponse510.code should equal(200)
+      val allRows = getResponse510.body.extract[RegulatedEntitiesJsonV510]
+      allRows.entities.length should equal(0)
+    }
+  }
+}

From 6145af48ac0e99c4ed6be079c71a7f052e28e2e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Fri, 1 Dec 2023 12:28:57 +0100
Subject: [PATCH 08/36] feature/Add endpoint updateConsumerRedirectUrl v5.1.0

---
 .../scala/code/api/v5_1_0/APIMethods510.scala | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 34d2ba056..ea4270e28 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -11,6 +11,7 @@ import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
 import code.api.util._
 import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle}
+import code.api.v2_1_0.{ConsumerRedirectUrlJSON, JSONFactory210}
 import code.api.v3_0_0.JSONFactory300
 import code.api.v3_0_0.JSONFactory300.createAggregateMetricJson
 import code.api.v3_1_0.ConsentJsonV310
@@ -1767,6 +1768,60 @@ trait APIMethods510 {
       }
     }
 
+
+    staticResourceDocs += ResourceDoc(
+      updateConsumerRedirectUrl,
+      implementedInApiVersion,
+      "updateConsumerRedirectUrl",
+      "PUT",
+      "/management/consumers/CONSUMER_ID/consumer/redirect_url",
+      "Update Consumer RedirectUrl",
+      s"""Update an existing redirectUrl for a Consumer specified by CONSUMER_ID.
+         |
+         | CONSUMER_ID can be obtained after you register the application.
+         |
+         | Or use the endpoint 'Get Consumers' to get it
+         |
+       """.stripMargin,
+      consumerRedirectUrlJSON,
+      consumerJSON,
+      List(
+        UserNotLoggedIn,
+        UserHasMissingRoles,
+        UnknownError
+      ),
+      List(apiTagConsumer),
+      Some(List(canUpdateConsumerRedirectUrl))
+    )
+
+    lazy val updateConsumerRedirectUrl: OBPEndpoint = {
+      case "management" :: "consumers" :: consumerId :: "consumer" :: "redirect_url" :: Nil JsonPut json -> _ => {
+        cc =>
+          implicit val ec = EndpointContext(Some(cc))
+          for {
+            (Full(u), callContext) <- authenticatedAccess(cc)
+            _ <- APIUtil.getPropsAsBoolValue("consumers_enabled_by_default", false) match {
+              case true => Future(Full(Unit))
+              case false => NewStyle.function.hasEntitlement("", u.userId, ApiRole.canUpdateConsumerRedirectUrl, callContext)
+            }
+            postJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, callContext) {
+              json.extract[ConsumerRedirectUrlJSON]
+            }
+            consumer <- NewStyle.function.getConsumerByConsumerId(consumerId, callContext)
+            //only the developer that created the Consumer should be able to edit it
+            _ <- Helper.booleanToFuture(UserNoPermissionUpdateConsumer, 400, callContext) {
+              consumer.createdByUserId.equals(u.userId)
+            }
+            //update the redirectURL and isactive (set to false when change redirectUrl) field in consumer table
+            updatedConsumer <- NewStyle.function.updateConsumer(consumer.id.get, None, None, Some(APIUtil.getPropsAsBoolValue("consumers_enabled_by_default", false)), None, None, None, None, Some(postJson.redirect_url), None, callContext)
+          } yield {
+            val json = JSONFactory210.createConsumerJSON(updatedConsumer)
+            (json, HttpCode.`200`(callContext))
+          }
+      }
+    }
+
+
   }
 }
 

From f9e5cc61486334722689f35c907c07cd3e358dce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Fri, 1 Dec 2023 12:35:32 +0100
Subject: [PATCH 09/36] feature/Tweak endpoint updateConsumerRedirectUrl v5.1.0

---
 .../scala/code/api/v5_1_0/APIMethods510.scala |  2 +-
 .../code/api/v5_1_0/JSONFactory5.1.0.scala    | 44 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index ea4270e28..41545ead0 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -1815,7 +1815,7 @@ trait APIMethods510 {
             //update the redirectURL and isactive (set to false when change redirectUrl) field in consumer table
             updatedConsumer <- NewStyle.function.updateConsumer(consumer.id.get, None, None, Some(APIUtil.getPropsAsBoolValue("consumers_enabled_by_default", false)), None, None, None, None, Some(postJson.redirect_url), None, callContext)
           } yield {
-            val json = JSONFactory210.createConsumerJSON(updatedConsumer)
+            val json = JSONFactory510.createConsumerJSON(updatedConsumer)
             (json, HttpCode.`200`(callContext))
           }
       }
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 9f18abd19..ccd8eb5d8 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -30,12 +30,13 @@ import code.api.Constant
 import code.api.util.{APIUtil, ConsentJWT, CustomJsonFormats, JwtUtil, Role}
 import code.api.util.APIUtil.gitCommit
 import code.api.v1_4_0.JSONFactory1_4_0.{LocationJsonV140, MetaJsonV140, transformToLocationFromV140, transformToMetaFromV140}
+import code.api.v2_1_0.ResourceUserJSON
 import code.api.v3_0_0.JSONFactory300.{createLocationJson, createMetaJson, transformToAddressFromV300}
 import code.api.v3_0_0.{AccountIdJson, AccountsIdsJsonV300, AddressJsonV300, OpeningTimesV300}
 import code.api.v4_0_0.{EnergySource400, HostedAt400, HostedBy400}
 import code.atmattribute.AtmAttribute
 import code.atms.Atms.Atm
-import code.users.UserAttribute
+import code.users.{UserAttribute, Users}
 import code.views.system.{AccountAccess, ViewDefinition}
 import com.openbankproject.commons.model.{Address, AtmId, AtmT, BankId, BankIdAccountId, Customer, Location, Meta, RegulatedEntityTrait}
 import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion}
@@ -43,7 +44,8 @@ import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion}
 import java.util.Date
 import code.consent.MappedConsent
 import code.metrics.APIMetric
-import net.liftweb.common.Box
+import code.model.Consumer
+import net.liftweb.common.{Box, Full}
 import net.liftweb.json
 import net.liftweb.json.{JValue, parse}
 
@@ -279,6 +281,18 @@ case class MetricJsonV510(
                      )
 case class MetricsJsonV510(metrics: List[MetricJsonV510])
 
+case class ConsumerJsonV510(consumer_id: String,
+                            app_name: String,
+                            app_type: String,
+                            description: String,
+                            developer_email: String,
+                            redirect_url: String,
+                            created_by_user_id: String,
+                            created_by_user: ResourceUserJSON,
+                            enabled: Boolean,
+                            created: Date
+                           )
+
 object JSONFactory510 extends CustomJsonFormats {
 
   def createCustomersIds(customers :  List[Customer]): CustomersIdsJsonV510 =
@@ -606,6 +620,32 @@ object JSONFactory510 extends CustomJsonFormats {
     MetricsJsonV510(metrics.map(createMetricJson))
   }
 
+  def createConsumerJSON(c: Consumer): ConsumerJsonV510 = {
+
+    val resourceUserJSON = Users.users.vend.getUserByUserId(c.createdByUserId.toString()) match {
+      case Full(resourceUser) => ResourceUserJSON(
+        user_id = resourceUser.userId,
+        email = resourceUser.emailAddress,
+        provider_id = resourceUser.idGivenByProvider,
+        provider = resourceUser.provider,
+        username = resourceUser.name
+      )
+      case _ => null
+    }
+
+    ConsumerJsonV510(consumer_id = c.consumerId.get,
+      app_name = c.name.get,
+      app_type = c.appType.toString(),
+      description = c.description.get,
+      developer_email = c.developerEmail.get,
+      redirect_url = c.redirectURL.get,
+      created_by_user_id = c.createdByUserId.get,
+      created_by_user = resourceUserJSON,
+      enabled = c.isActive.get,
+      created = c.createdAt.get
+    )
+  }
+
 
 }
 

From 59802b0d84841a5f8abd5aa374d3492baa0c8c8d Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Fri, 1 Dec 2023 13:10:24 +0100
Subject: [PATCH 10/36] bugfix/tweaked the extractCleanRedirectURL method

---
 obp-api/src/main/scala/code/util/Helper.scala | 21 +--------
 .../src/test/scala/code/util/HelperTest.scala | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+), 20 deletions(-)
 create mode 100644 obp-api/src/test/scala/code/util/HelperTest.scala

diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index cb09e79c7..15c12c894 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -167,27 +167,8 @@ object Helper extends Loggable {
     prettyRender(decompose(input))
   }
 
-  /**
-    * extract clean redirect url from input value, because input may have some parameters, such as the following examples  <br/> 
-    * eg1: http://localhost:8082/oauthcallback?....--> http://localhost:8082 <br/> 
-    * eg2: http://localhost:8016?oautallback?=3NLMGV ...--> http://localhost:8016
-    *
-    * @param input a long url with parameters 
-    * @return clean redirect url
-    */
   def extractCleanRedirectURL(input: String): Box[String] = {
-    /**
-      * pattern eg1: http://xxxxxx?oautxxxx  -->http://xxxxxx
-      * pattern eg2: https://xxxxxx/oautxxxx -->http://xxxxxx
-      */
-    //Note: the pattern should be : val  pattern = "(https?):\\/\\/(.*)(?=((\\/)|(\\?))oauthcallback*)".r, but the OAuthTest is different, so add the following logic
-    val pattern = "([A-Za-z][A-Za-z0-9+.-]*):\\/\\/(.*)(?=((\\/)|(\\?))oauth*)".r
-    val validRedirectURL = pattern findFirstIn input
-    // Now for the OAuthTest, the redirect format is : http://localhost:8016?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
-    // It is not the normal case: http://localhost:8082/oauthcallback?oauth_token=LUDKELGJXRDOC1AK1X1TOYIXM5W1AORFJT5KE43B&oauth_verifier=14062
-    // So add the split function to select the first value; eg: Array(http://localhost:8082, thcallback) --> http://localhost:8082
-    val extractCleanURL = validRedirectURL.getOrElse("").split("/oauth")(0) 
-    Full(extractCleanURL)
+    Full(input.split("\\?oauth_token=")(0))
   }
   
   /**
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
new file mode 100644
index 000000000..eaad59c97
--- /dev/null
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -0,0 +1,47 @@
+/**
+ * Open Bank Project - API
+ * Copyright (C) 2011-2019, TESOBE GmbH.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Email: contact@tesobe.com
+ * TESOBE GmbH.
+ * Osloer Strasse 16/17
+ * Berlin 13359, Germany
+ *
+ * This product includes software developed at
+ * TESOBE (http://www.tesobe.com/)
+ *
+ */
+
+package code.util
+
+
+import code.api.util._
+import code.setup.PropsReset
+import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers}
+
+
+class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with PropsReset {
+
+  feature("test APIUtil.basicUrlValidation method") {
+    val testString1 = "http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+    val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+
+    Helper.extractCleanRedirectURL(testString1).head should be("http://localhost:8082/oauthcallback")
+    Helper.extractCleanRedirectURL(testString2).head should be("http://localhost:8082")
+
+  }
+
+}
\ No newline at end of file

From 46a423d83837fae4cced097635bb745f3a9e3bfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Fri, 1 Dec 2023 15:57:16 +0100
Subject: [PATCH 11/36] test/Fix failed test

---
 obp-api/src/test/scala/code/util/MappedClassNameTest.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/obp-api/src/test/scala/code/util/MappedClassNameTest.scala b/obp-api/src/test/scala/code/util/MappedClassNameTest.scala
index 5cced4867..06dc3040f 100644
--- a/obp-api/src/test/scala/code/util/MappedClassNameTest.scala
+++ b/obp-api/src/test/scala/code/util/MappedClassNameTest.scala
@@ -115,6 +115,7 @@ class MappedClassNameTest extends FeatureSpec {
       "code.model.dataAccess.MappedBank",
       "code.UserRefreshes.MappedUserRefreshes",
       "code.DynamicEndpoint.DynamicEndpoint",
+      "code.regulatedentities.MappedRegulatedEntity",
       "code.CustomerDependants.MappedCustomerDependant")
 
   val newMappedTypes = ClassScanUtils.findTypes{ info =>

From da6bd064c87dbd3be73d219f68589b4084d5844d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Mon, 4 Dec 2023 11:35:37 +0100
Subject: [PATCH 12/36] docfix/Tweak endpoint updateConsumerRedirectUrl v5.1.0

---
 .../src/main/scala/code/api/v5_1_0/APIMethods510.scala | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 41545ead0..438562cc6 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -1769,6 +1769,14 @@ trait APIMethods510 {
     }
 
 
+    private def consumerDisabledText() = {
+      if(APIUtil.getPropsAsBoolValue("consumers_enabled_by_default", false) == false) {
+        "Please note: Your consumer may be disabled as a result of this action."
+      } else {
+        ""
+      }
+    }
+
     staticResourceDocs += ResourceDoc(
       updateConsumerRedirectUrl,
       implementedInApiVersion,
@@ -1777,6 +1785,8 @@ trait APIMethods510 {
       "/management/consumers/CONSUMER_ID/consumer/redirect_url",
       "Update Consumer RedirectUrl",
       s"""Update an existing redirectUrl for a Consumer specified by CONSUMER_ID.
+         |
+         | ${consumerDisabledText()}
          |
          | CONSUMER_ID can be obtained after you register the application.
          |

From 2d422db5de491bcd08cfa061c3f46023e93bdb38 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Mon, 4 Dec 2023 12:45:15 +0100
Subject: [PATCH 13/36] refactor/remove the basicUrlValidation, use
 basicUriAndQueryStringValidation stead

---
 obp-api/src/main/scala/code/api/util/APIUtil.scala | 14 --------------
 obp-api/src/main/scala/code/util/Helper.scala      |  2 +-
 obp-api/src/test/scala/code/util/APIUtilTest.scala | 10 +++++++---
 obp-api/src/test/scala/code/util/HelperTest.scala  |  4 ++++
 4 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala
index 4bf30c681..239e2fb3a 100644
--- a/obp-api/src/main/scala/code/api/util/APIUtil.scala
+++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala
@@ -777,20 +777,6 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
     }
   }
   
-  def basicUrlValidation(urlString: String): Boolean = {
-    //in scala test - org.scalatest.FeatureSpecLike.scenario: 
-    // redirectUrl = http%3A%2F%2Flocalhost%3A8016%3Foauth_token%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461
-    // URLDecoder.decode(urlString,"UTF-8")-->http://localhost:8016?oauth_token=EBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK&oauth_verifier=63461
-    val regex =
-      """((([A-Za-z]{3,9}:(?:\/\/)?)(?:[-;:&=\+\$,\w]+@)?[A-Za-z0-9.-]+(:[0-9]+)?|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w-_]*)?\??(?:[-\+=&;%@.\w_\/]*)#?(?:[\w]*))?)""".r
-    val decodeUrlValue = URLDecoder.decode(urlString, "UTF-8").trim()
-    decodeUrlValue match {
-      case regex(_*) if (decodeUrlValue.length <= 2048) => true
-      case _ => false
-    }
-  }
-  
-  
   /** only  A-Z, a-z, 0-9,-,_,. =, & and max length <= 2048  */
   def basicUriAndQueryStringValidation(urlString: String): Boolean = {
     val regex =
diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index 15c12c894..ddc6bb13a 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -461,7 +461,7 @@ object Helper extends Loggable {
           }else if((args.length>0) && args.apply(0).toString.equalsIgnoreCase("consumer_key")){
             result.asInstanceOf[Box[String]].filter(APIUtil.basicConsumerKeyValidation(_)==SILENCE_IS_GOLDEN)
           }else if((args.length>0) && args.apply(0).toString.equalsIgnoreCase("redirectUrl")){
-            result.asInstanceOf[Box[String]].filter(APIUtil.basicUrlValidation(_))
+            result.asInstanceOf[Box[String]].filter(APIUtil.basicUriAndQueryStringValidation(_))
           } else{
             result.asInstanceOf[Box[String]].filter(APIUtil.checkMediumString(_)==SILENCE_IS_GOLDEN)
           }
diff --git a/obp-api/src/test/scala/code/util/APIUtilTest.scala b/obp-api/src/test/scala/code/util/APIUtilTest.scala
index 44b354495..fbe2d0359 100644
--- a/obp-api/src/test/scala/code/util/APIUtilTest.scala
+++ b/obp-api/src/test/scala/code/util/APIUtilTest.scala
@@ -698,12 +698,16 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop
     APIUtil.getObpFormatOperationId("xxx") should be ("xxx")
   }
   
-  feature("test APIUtil.basicUrlValidation method") {
+  feature("test APIUtil.basicUriAndQueryStringValidation method") {
     val testString1 = "https%3A%2F%2Fapisandbox.openbankproject.com%2Foauth%2Fauthorize%3Fnext%3D%2Fen%2Fusers%2Fmyuser%26oauth_token%3DWTOBT2YRCTMI1BCCF4XAIKRXPLLZDZPFAIL5K03Z%26oauth_verifier%3D45381"
     val testString2 = "http%3A%2F%2Flocalhost%3A8016%3Foauth_token%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
+    val testString3 = "myapp://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
+    val testString4 = "fb00000000:://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     
-    APIUtil.basicUrlValidation(testString1) should be (true)
-    APIUtil.basicUrlValidation(testString2) should be (true)
+    APIUtil.basicUriAndQueryStringValidation(testString1) should be (true)
+    APIUtil.basicUriAndQueryStringValidation(testString2) should be (true)
+    APIUtil.basicUriAndQueryStringValidation(testString3) should be (true)
+    APIUtil.basicUriAndQueryStringValidation(testString4) should be (true)
     
   }
 
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
index eaad59c97..452d9a59c 100644
--- a/obp-api/src/test/scala/code/util/HelperTest.scala
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -38,9 +38,13 @@ class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with Props
   feature("test APIUtil.basicUrlValidation method") {
     val testString1 = "http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
     val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+    val testString3 = "myapp://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
+    val testString4 = "fb00000000:://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
 
     Helper.extractCleanRedirectURL(testString1).head should be("http://localhost:8082/oauthcallback")
     Helper.extractCleanRedirectURL(testString2).head should be("http://localhost:8082")
+    Helper.extractCleanRedirectURL(testString3).head should be("myapp://callback")
+    Helper.extractCleanRedirectURL(testString4).head should be("fb00000000:://callback")
 
   }
 

From f6b83f130a6a533995ea14cd75251cd612a4cbe5 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Mon, 4 Dec 2023 13:12:01 +0100
Subject: [PATCH 14/36] refactor/remove the basicUrlValidation, use
 basicUriAndQueryStringValidation stead - step2

---
 obp-api/src/main/scala/code/util/Helper.scala      | 2 +-
 obp-api/src/test/scala/code/util/APIUtilTest.scala | 2 ++
 obp-api/src/test/scala/code/util/HelperTest.scala  | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index ddc6bb13a..f6492a545 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -168,7 +168,7 @@ object Helper extends Loggable {
   }
 
   def extractCleanRedirectURL(input: String): Box[String] = {
-    Full(input.split("\\?oauth_token=")(0))
+    Full(input.split("\\?")(0))
   }
   
   /**
diff --git a/obp-api/src/test/scala/code/util/APIUtilTest.scala b/obp-api/src/test/scala/code/util/APIUtilTest.scala
index fbe2d0359..450def8dc 100644
--- a/obp-api/src/test/scala/code/util/APIUtilTest.scala
+++ b/obp-api/src/test/scala/code/util/APIUtilTest.scala
@@ -703,11 +703,13 @@ class APIUtilTest extends FeatureSpec with Matchers with GivenWhenThen with Prop
     val testString2 = "http%3A%2F%2Flocalhost%3A8016%3Foauth_token%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     val testString3 = "myapp://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     val testString4 = "fb00000000:://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
+    val testString5 = "http://127.0.0.1:8000/oauth/authorize?next=/en/metrics/api/&oauth_token=TN0124OCPRCL4KUJRF5LNLVMRNHTVZPJDBS2PNWU&oauth_verifier=10470"
     
     APIUtil.basicUriAndQueryStringValidation(testString1) should be (true)
     APIUtil.basicUriAndQueryStringValidation(testString2) should be (true)
     APIUtil.basicUriAndQueryStringValidation(testString3) should be (true)
     APIUtil.basicUriAndQueryStringValidation(testString4) should be (true)
+    APIUtil.basicUriAndQueryStringValidation(testString5) should be (true)
     
   }
 
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
index 452d9a59c..d3f7472ce 100644
--- a/obp-api/src/test/scala/code/util/HelperTest.scala
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -40,12 +40,13 @@ class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with Props
     val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
     val testString3 = "myapp://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     val testString4 = "fb00000000:://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
+    val testString5 = "http://127.0.0.1:8000/oauth/authorize?next=/en/metrics/api/&oauth_token=TN0124OCPRCL4KUJRF5LNLVMRNHTVZPJDBS2PNWU&oauth_verifier=10470"
 
     Helper.extractCleanRedirectURL(testString1).head should be("http://localhost:8082/oauthcallback")
     Helper.extractCleanRedirectURL(testString2).head should be("http://localhost:8082")
     Helper.extractCleanRedirectURL(testString3).head should be("myapp://callback")
     Helper.extractCleanRedirectURL(testString4).head should be("fb00000000:://callback")
-
+    Helper.extractCleanRedirectURL(testString5).head should be("http://127.0.0.1:8000/oauth/authorize")
   }
 
 }
\ No newline at end of file

From ffee2c118f67ba3dcb01a0d0b6d778ebda19e8ed Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Mon, 4 Dec 2023 17:30:36 +0100
Subject: [PATCH 15/36] refactor/rename extractCleanRedirectURL to
 getStaticPortionOfRedirectURL

---
 .../main/scala/code/snippet/OAuthWorkedThanks.scala |  2 +-
 obp-api/src/main/scala/code/util/Helper.scala       | 10 ++++++++--
 obp-api/src/test/scala/code/util/HelperTest.scala   | 13 +++++++------
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
index 982b4a4dc..e6a5e307b 100644
--- a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
+++ b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
@@ -48,7 +48,7 @@ class OAuthWorkedThanks extends MdcLoggable {
     val redirectUrl = ObpS.param("redirectUrl").map(urlDecode(_))
     logger.debug(s"OAuthWorkedThanks.thanks.redirectUrl $redirectUrl")
     //extract the clean(omit the parameters) redirect url from request url
-    val requestedRedirectURL = Helper.extractCleanRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
+    val requestedRedirectURL = Helper.getStaticPortionOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
     logger.debug(s"OAuthWorkedThanks.thanks.requestedRedirectURL $requestedRedirectURL")
     
     val requestedOauthToken = Helper.extractOauthToken(redirectUrl.openOr("No Oauth Token here")) openOr("No Oauth Token here")
diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index f6492a545..41f7a1a77 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -167,8 +167,14 @@ object Helper extends Loggable {
     prettyRender(decompose(input))
   }
 
-  def extractCleanRedirectURL(input: String): Box[String] = {
-    Full(input.split("\\?")(0))
+
+  /**
+   * 
+   * @param redirectUrl eg: http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
+   * @return http://localhost:8082/oauthcallback
+   */
+  def getStaticPortionOfRedirectURL(redirectUrl: String): Box[String] = {
+    Full(redirectUrl.split("\\?")(0)) //return everything before the "?"
   }
   
   /**
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
index d3f7472ce..76a265f7d 100644
--- a/obp-api/src/test/scala/code/util/HelperTest.scala
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -35,18 +35,19 @@ import org.scalatest.{FeatureSpec, GivenWhenThen, Matchers}
 
 class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with PropsReset {
 
-  feature("test APIUtil.basicUrlValidation method") {
+  feature("test APIUtil.getStaticPortionOfRedirectURL method") {
+    // The redirectURl is `http://localhost:8082/oauthcallback`
     val testString1 = "http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
     val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
     val testString3 = "myapp://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     val testString4 = "fb00000000:://callback?oauth_token=%3DEBRZBMOPDXEUGGJP421FPFGK01IY2DGM5O3TLVSK%26oauth_verifier%3D63461"
     val testString5 = "http://127.0.0.1:8000/oauth/authorize?next=/en/metrics/api/&oauth_token=TN0124OCPRCL4KUJRF5LNLVMRNHTVZPJDBS2PNWU&oauth_verifier=10470"
 
-    Helper.extractCleanRedirectURL(testString1).head should be("http://localhost:8082/oauthcallback")
-    Helper.extractCleanRedirectURL(testString2).head should be("http://localhost:8082")
-    Helper.extractCleanRedirectURL(testString3).head should be("myapp://callback")
-    Helper.extractCleanRedirectURL(testString4).head should be("fb00000000:://callback")
-    Helper.extractCleanRedirectURL(testString5).head should be("http://127.0.0.1:8000/oauth/authorize")
+    Helper.getStaticPortionOfRedirectURL(testString1).head should be("http://localhost:8082/oauthcallback")
+    Helper.getStaticPortionOfRedirectURL(testString2).head should be("http://localhost:8082")
+    Helper.getStaticPortionOfRedirectURL(testString3).head should be("myapp://callback")
+    Helper.getStaticPortionOfRedirectURL(testString4).head should be("fb00000000:://callback")
+    Helper.getStaticPortionOfRedirectURL(testString5).head should be("http://127.0.0.1:8000/oauth/authorize")
   }
 
 }
\ No newline at end of file

From 5f58e7c6ef897d30adca322e203e2afacfc3891a Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Tue, 5 Dec 2023 10:19:05 +0100
Subject: [PATCH 16/36] bugfix/supported both staticPortionOfRedirectUrl and
 hostOnlyOfRedirectUrlLegacy

---
 .../code/snippet/OAuthWorkedThanks.scala      | 15 ++++++------
 obp-api/src/main/scala/code/util/Helper.scala | 24 +++++++++++++++++++
 .../src/test/scala/code/util/HelperTest.scala |  9 +++++++
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
index e6a5e307b..59b017efd 100644
--- a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
+++ b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
@@ -48,8 +48,10 @@ class OAuthWorkedThanks extends MdcLoggable {
     val redirectUrl = ObpS.param("redirectUrl").map(urlDecode(_))
     logger.debug(s"OAuthWorkedThanks.thanks.redirectUrl $redirectUrl")
     //extract the clean(omit the parameters) redirect url from request url
-    val requestedRedirectURL = Helper.getStaticPortionOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
-    logger.debug(s"OAuthWorkedThanks.thanks.requestedRedirectURL $requestedRedirectURL")
+    val staticPortionOfRedirectUrl = Helper.getStaticPortionOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
+    val hostOnlyOfRedirectUrlLegacy = Helper.getHostOnlyOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
+    logger.debug(s"OAuthWorkedThanks.thanks.staticPortionOfRedirectUrl $staticPortionOfRedirectUrl")
+    logger.debug(s"OAuthWorkedThanks.thanks.hostOnlyOfRedirectUrlLegacy $hostOnlyOfRedirectUrlLegacy")
     
     val requestedOauthToken = Helper.extractOauthToken(redirectUrl.openOr("No Oauth Token here")) openOr("No Oauth Token here")
     logger.debug(s"OAuthWorkedThanks.thanks.requestedOauthToken $requestedOauthToken")
@@ -62,13 +64,10 @@ class OAuthWorkedThanks extends MdcLoggable {
     
     redirectUrl match {
       case Full(url) =>
-        //this redirect url is checked by following, no open redirect issue.
-        // TODO maybe handle case of extra trailing / on the url ?
+        val incorrectRedirectUrlMessage =  s"The validRedirectURL is $validRedirectURL but the staticPortionOfRedirectUrl was $staticPortionOfRedirectUrl" 
 
-        val incorrectRedirectUrlMessage =  s"The validRedirectURL is $validRedirectURL but the requestedRedirectURL was $requestedRedirectURL"
-
-
-        if(validRedirectURL.equals(requestedRedirectURL)) {
+        //hostOnlyOfRedirectUrlLegacy is deprecated now, we use the staticPortionOfRedirectUrl stead,
+        if(validRedirectURL.equals(staticPortionOfRedirectUrl)|| validRedirectURL.equals(hostOnlyOfRedirectUrlLegacy)) {
           "#redirect-link [href]" #> url &
           ".app-name"#> appName //there may be several places to be modified in html, so here use the class, not the id.
         }else{
diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index 41f7a1a77..a9889d3b2 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -177,6 +177,30 @@ object Helper extends Loggable {
     Full(redirectUrl.split("\\?")(0)) //return everything before the "?"
   }
   
+  /**
+   * extract clean redirect url from input value, because input may have some parameters, such as the following examples  <br/> 
+   * eg1: http://localhost:8082/oauthcallback?....--> http://localhost:8082 <br/> 
+   * eg2: http://localhost:8016?oautallback?=3NLMGV ...--> http://localhost:8016
+   *
+   * @param input a long url with parameters 
+   * @return clean redirect url
+   */
+  @deprecated("We can not only use hostname as the redirectUrl, now add new method `getStaticPortionOfRedirectURL` ","05.12.2023")  
+  def getHostOnlyOfRedirectURL(redirectUrl: String): Box[String] = {
+    /**
+     * pattern eg1: http://xxxxxx?oautxxxx  -->http://xxxxxx
+     * pattern eg2: https://xxxxxx/oautxxxx -->http://xxxxxx
+     */
+    //Note: the pattern should be : val  pattern = "(https?):\\/\\/(.*)(?=((\\/)|(\\?))oauthcallback*)".r, but the OAuthTest is different, so add the following logic
+    val pattern = "([A-Za-z][A-Za-z0-9+.-]*):\\/\\/(.*)(?=((\\/)|(\\?))oauth*)".r
+    val validRedirectURL = pattern findFirstIn redirectUrl
+    // Now for the OAuthTest, the redirect format is : http://localhost:8016?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
+    // It is not the normal case: http://localhost:8082/oauthcallback?oauth_token=LUDKELGJXRDOC1AK1X1TOYIXM5W1AORFJT5KE43B&oauth_verifier=14062
+    // So add the split function to select the first value; eg: Array(http://localhost:8082, thcallback) --> http://localhost:8082
+    val extractCleanURL = validRedirectURL.getOrElse("").split("/oauth")(0)
+    Full(extractCleanURL)
+  }
+  
   /**
     * extract Oauth Token String from input value, because input may have some parameters, such as the following examples  <br/> 
     * http://localhost:8082/oauthcallback?oauth_token=DKR242MB3IRCUVG35UZ0QQOK3MBS1G2HL2ZIKK2O&oauth_verifier=64465 
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
index 76a265f7d..deb6cff59 100644
--- a/obp-api/src/test/scala/code/util/HelperTest.scala
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -49,5 +49,14 @@ class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with Props
     Helper.getStaticPortionOfRedirectURL(testString4).head should be("fb00000000:://callback")
     Helper.getStaticPortionOfRedirectURL(testString5).head should be("http://127.0.0.1:8000/oauth/authorize")
   }
+  
+  feature("test APIUtil.getHostOnlyOfRedirectURL method") {
+    // The redirectURl is `http://localhost:8082/oauthcallback`
+    val testString1 = "http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+    val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+
+    Helper.getHostOnlyOfRedirectURL(testString1).head should be("http://localhost:8082")
+    Helper.getHostOnlyOfRedirectURL(testString2).head should be("http://localhost:8082")
+  }
 
 }
\ No newline at end of file

From 6f8f40e5e6f598ce05c4efdf5e12797258597ba1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 5 Dec 2023 13:03:10 +0100
Subject: [PATCH 17/36] feature/Dynamic Registration of Consumer - WIP

---
 .../SwaggerDefinitionsJSON.scala              | 33 ++++++---
 .../scala/code/api/util/ErrorMessages.scala   |  1 +
 .../scala/code/api/v5_1_0/APIMethods510.scala | 69 +++++++++++++++++++
 .../code/api/v5_1_0/JSONFactory5.1.0.scala    | 22 ++++--
 4 files changed, 112 insertions(+), 13 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
index 0aaa6dfa3..a91347362 100644
--- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
+++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
@@ -2667,6 +2667,30 @@ object SwaggerDefinitionsJSON {
     enabled = true,
     created = DateWithDayExampleObject
   )
+  lazy val pem = "-----BEGIN CERTIFICATE-----\nMIIFIjCCBAqgAwIBAgIIX3qsz7QQxngwDQYJKoZIhvcNAQELBQAwgZ8xCzAJBgNV\r\nBAYTAkRFMQ8wDQYDVQQIEwZCZXJsaW4xDzANBgNVBAcTBkJlcmxpbjEPMA0GA1UE\r\nChMGVEVTT0JFMRowGAYDVQQLExFURVNPQkUgT3BlcmF0aW9uczESMBAGA1UEAxMJ\r\nVEVTT0JFIENBMR8wHQYJKoZIhvcNAQkBFhBhZG1pbkB0ZXNvYmUuY29tMQwwCgYD\r\nVQQpEwNWUE4wHhcNMjMwNzE3MDg0MDAwWhcNMjQwNzE3MDg0MDAwWjCBizELMAkG\r\nA1UEBhMCREUxDzANBgNVBAgTBkJlcmxpbjEPMA0GA1UEBxMGQmVybGluMRQwEgYD\r\nVQQKEwtUZXNvYmUgR21iSDEPMA0GA1UECxMGc3lzb3BzMRIwEAYDVQQDEwlsb2Nh\r\nbGhvc3QxHzAdBgkqhkiG9w0BCQEWEGFkbWluQHRlc29iZS5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCwxGuWUN1H0d0IeYPYWdLA0I/5BXx4DLO6\r\nzfi1GGJlF8BIXRN0VTJckIY9C3J1RnXDs6p6ufA01iHe1PQdL6VzfcaC3j+jUSgV\r\n1z9ybEUPyUwq3PCCxqoVI9n8yh+O6FDn3dvu/9Q2NtBpJHUBDCLf7OO9TgsFU2sE\r\nMys+Hw5DuuX5n5OQ2VIwH+qlMTQnd+yw5y8FKHqAZT5hE60lF/x6sQnwi58hLGRW\r\nSqo/548c2ZpoeWtnyY1I6PyR7zUYGuhruLY8gVFfLE+610u/lj2wYTXMxntpV+tV\r\nralLFRMhvbqZXW/EpuDb/pEbCnLDNDxq5NarLVDzcHs7VhT9MPChAgMBAAGjggFy\r\nMIIBbjATBgNVHSUEDDAKBggrBgEFBQcDAjAaBgNVHREEEzARgglsb2NhbGhvc3SH\r\nBH8AAAEwggEGBggrBgEFBQcBAwSB+TCB9jAIBgYEAI5GAQEwOAYGBACORgEFMC4w\r\nLBYhaHR0cHM6Ly9leGFtcGxlLm9yZy9wa2lkaXNjbG9zdXJlEwdleGFtcGxlMIGI\r\nBgYEAIGYJwIwfjBMMBEGBwQAgZgnAQMMBlBTUF9BSTARBgcEAIGYJwEBDAZQU1Bf\r\nQVMwEQYHBACBmCcBAgwGUFNQX1BJMBEGBwQAgZgnAQQMBlBTUF9JQwwlRHVtbXkg\r\nRmluYW5jaWFsIFN1cGVydmlzaW9uIEF1dGhvcml0eQwHWFgtREZTQTAlBgYEAI5G\r\nAQYwGwYHBACORgEGAQYHBACORgEGAgYHBACORgEGAzARBglghkgBhvhCAQEEBAMC\r\nB4AwHgYJYIZIAYb4QgENBBEWD3hjYSBjZXJ0aWZpY2F0ZTANBgkqhkiG9w0BAQsF\r\nAAOCAQEAKTS7exS9A7rWJLRzWrlHoTu68Avm5g9Dz1GKjgt8rnvj3D21SE14Rf5p\r\n0JWHYH4SiCdnh8Tx+IA7o0TmPJ1JRfAXR3i/5R7TJi/HrnqL+V7SIx2Cuq/hkZEU\r\nAhVs07nnvHURcrlQGwcfn4TbgpCURpCPpYZlNsYySb6BS6I4qFaadHGqMTyEkphV\r\nwfXyB3brmzxj9V4Qgp0t+s/uFuFirWyIayRc9nSSC7vuNVYvib2Kim4y8kvuWpA4\r\nZ51+fFOmBqCqpmwfAADNgDsLJiA/741eBflVd/ZUeAzgOjMCMIaDGlwiwZlePKT7\r\n553GtfsGxZMf05oqfUrQEQfJaU+/+Q==\n-----END CERTIFICATE-----\n"
+  lazy val certificateInfoJsonV510 = CertificateInfoJsonV510(
+    subject_domain_name = "OID.2.5.4.41=VPN, EMAILADDRESS=admin@tesobe.com, CN=TESOBE CA, OU=TESOBE Operations, O=TESOBE, L=Berlin, ST=Berlin, C=DE",
+    issuer_domain_name = "CN=localhost, O=TESOBE GmbH, ST=Berlin, C=DE",
+    not_before = "2022-04-01T10:13:00.000Z",
+    not_after = "2032-04-01T10:13:00.000Z",
+    roles = None,
+    roles_info = Some("PEM Encoded Certificate does not contain PSD2 roles.")
+  )
+  lazy val consumerJsonV510: ConsumerJsonV510 = ConsumerJsonV510(
+    consumer_id = "d0d7b08c-f0ec-4e57-ac99-7d9eafe99225",
+    consumer_key = "d0d7b08c-f0ec-4e57-ac99-7d9eafe99225",
+    consumer_secret = "d0d7b08c-f0ec-4e57-ac99-7d9eafe99225",
+    app_name = "SOFI",
+    app_type = "Web",
+    description = "Account Management",
+    developer_email = ExampleValue.emailExample.value,
+    redirect_url = "www.openbankproject.com",
+    certificate_pem = pem,
+    certificate_info = Some(certificateInfoJsonV510),
+    created_by_user = resourceUserJSON,
+    enabled = true,
+    created = DateWithDayExampleObject
+  )
 
   val consumersJson = ConsumersJson(
     list = List(consumerJSON)
@@ -4200,15 +4224,6 @@ object SwaggerDefinitionsJSON {
   val oAuth2ServerJWKURIJson = OAuth2ServerJWKURIJson("https://www.googleapis.com/oauth2/v3/certs")
   
   val oAuth2ServerJwksUrisJson = OAuth2ServerJwksUrisJson(List(oAuth2ServerJWKURIJson))
-
-  val certificateInfoJsonV510 = CertificateInfoJsonV510(
-    subject_domain_name = "OID.2.5.4.41=VPN, EMAILADDRESS=admin@tesobe.com, CN=TESOBE CA, OU=TESOBE Operations, O=TESOBE, L=Berlin, ST=Berlin, C=DE",
-    issuer_domain_name = "CN=localhost, O=TESOBE GmbH, ST=Berlin, C=DE",
-    not_before = "2022-04-01T10:13:00.000Z",
-    not_after = "2032-04-01T10:13:00.000Z",
-    roles = None,
-    roles_info = Some("PEM Encoded Certificate does not contain PSD2 roles.")
-  )
   
   val updateAccountRequestJsonV310 = UpdateAccountRequestJsonV310(
     label = "Label",
diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
index 611e437e9..934207029 100644
--- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
+++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
@@ -519,6 +519,7 @@ object ErrorMessages {
 
   val RegulatedEntityNotFound = "OBP-34100: Regulated Entity not found. Please specify a valid value for REGULATED_ENTITY_ID."
   val RegulatedEntityNotDeleted = "OBP-34101: Regulated Entity cannot be deleted. Please specify a valid value for REGULATED_ENTITY_ID."
+  val RegulatedEntityNotFoundByCertificate = "OBP-34102: Regulated Entity cannot be found by provided certificate."
 
   // Consents
   val ConsentNotFound = "OBP-35001: Consent not found by CONSENT_ID. "
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 438562cc6..0630947f8 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -10,6 +10,7 @@ import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFo
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
 import code.api.util._
+import code.api.util.newstyle.Consumer.createConsumerNewStyle
 import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle}
 import code.api.v2_1_0.{ConsumerRedirectUrlJSON, JSONFactory210}
 import code.api.v3_0_0.JSONFactory300
@@ -23,6 +24,7 @@ import code.bankconnectors.Connector
 import code.consent.Consents
 import code.loginattempts.LoginAttempt
 import code.metrics.APIMetrics
+import code.model.AppType
 import code.model.dataAccess.MappedBankAccount
 import code.regulatedentities.MappedRegulatedEntityProvider
 import code.transactionrequests.TransactionRequests.TransactionRequestTypes.{apply => _}
@@ -41,6 +43,7 @@ import net.liftweb.common.Full
 import net.liftweb.http.rest.RestHelper
 import net.liftweb.json.parse
 import net.liftweb.mapper.By
+import net.liftweb.util.Helpers
 import net.liftweb.util.Helpers.tryo
 
 import scala.collection.immutable.{List, Nil}
@@ -1769,6 +1772,72 @@ trait APIMethods510 {
     }
 
 
+    staticResourceDocs += ResourceDoc(
+      createConsumer,
+      implementedInApiVersion,
+      "createConsumer",
+      "POST",
+      "/dynamic-registration/consumers",
+      "Create a Consumer",
+      s"""Create a Consumer (mTLS access).
+         |
+         |""",
+      ConsumerPostJsonV510(
+        "TESOBE GmbH",
+        "Test",
+        "Web",
+        "Description",
+        "some@email.com",
+        "redirecturl",
+      ),
+      consumerJsonV510,
+      List(
+        InvalidJsonFormat,
+        UnknownError
+      ),
+      List(apiTagConsumer),
+      Some(Nil))
+
+
+    lazy val createConsumer: OBPEndpoint = {
+      case "dynamic-registration" :: "consumers" :: Nil JsonPost json -> _ => {
+        cc =>
+          implicit val ec = EndpointContext(Some(cc))
+          for {
+            postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, cc.callContext) {
+              json.extract[ConsumerPostJsonV510]
+            }
+            pem = APIUtil.`getPSD2-CERT`(cc.requestHeaders)
+            certificateInfo: CertificateInfoJsonV510 <- Future(X509.getCertificateInfo(pem)) map {
+              unboxFullOrFail(_, cc.callContext, X509GeneralError)
+            }
+            _ <- Helper.booleanToFuture(RegulatedEntityNotFoundByCertificate, 400, cc.callContext) {
+              MappedRegulatedEntityProvider.getRegulatedEntities()
+                .exists(_.entityCertificatePublicKey.replace("""\n""", "") == pem.getOrElse("").replace("""\n""", ""))
+            }
+            (consumer, callContext) <- createConsumerNewStyle(
+              key = Some(Helpers.randomString(40).toLowerCase),
+              secret = Some(Helpers.randomString(40).toLowerCase),
+              isActive = Some(true),
+              name = Some(postedJson.app_name),
+              appType = Some(AppType.valueOf(postedJson.app_type)),
+              description = Some(postedJson.description),
+              developerEmail = Some(postedJson.developer_email),
+              redirectURL = Some(postedJson.redirect_url),
+              createdByUserId = None,
+              clientCertificate = pem,
+              cc.callContext
+            )
+          } yield {
+            // Format the data as json
+            val json = JSONFactory510.createConsumerJSON(consumer, Some(certificateInfo))
+            // Return
+            (json, HttpCode.`201`(callContext))
+          }
+      }
+    }
+
+
     private def consumerDisabledText() = {
       if(APIUtil.getPropsAsBoolValue("consumers_enabled_by_default", false) == false) {
         "Please note: Your consumer may be disabled as a result of this action."
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index ccd8eb5d8..9ec054fed 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -281,13 +281,23 @@ case class MetricJsonV510(
                      )
 case class MetricsJsonV510(metrics: List[MetricJsonV510])
 
+case class ConsumerPostJsonV510(entity_name: String,
+                                app_name: String,
+                                app_type: String,
+                                description: String,
+                                developer_email: String,
+                                redirect_url: String,
+                               )
 case class ConsumerJsonV510(consumer_id: String,
+                            consumer_key: String,
+                            consumer_secret: String,
                             app_name: String,
                             app_type: String,
                             description: String,
                             developer_email: String,
                             redirect_url: String,
-                            created_by_user_id: String,
+                            certificate_pem: String,
+                            certificate_info: Option[CertificateInfoJsonV510],
                             created_by_user: ResourceUserJSON,
                             enabled: Boolean,
                             created: Date
@@ -620,7 +630,7 @@ object JSONFactory510 extends CustomJsonFormats {
     MetricsJsonV510(metrics.map(createMetricJson))
   }
 
-  def createConsumerJSON(c: Consumer): ConsumerJsonV510 = {
+  def createConsumerJSON(c: Consumer, certificateInfo: Option[CertificateInfoJsonV510] = None): ConsumerJsonV510 = {
 
     val resourceUserJSON = Users.users.vend.getUserByUserId(c.createdByUserId.toString()) match {
       case Full(resourceUser) => ResourceUserJSON(
@@ -633,13 +643,17 @@ object JSONFactory510 extends CustomJsonFormats {
       case _ => null
     }
 
-    ConsumerJsonV510(consumer_id = c.consumerId.get,
+    ConsumerJsonV510(
+      consumer_id = c.consumerId.get,
+      consumer_key = c.key.get,
+      consumer_secret = c.secret.get,
       app_name = c.name.get,
       app_type = c.appType.toString(),
       description = c.description.get,
       developer_email = c.developerEmail.get,
       redirect_url = c.redirectURL.get,
-      created_by_user_id = c.createdByUserId.get,
+      certificate_pem = c.clientCertificate.get,
+      certificate_info = certificateInfo,
       created_by_user = resourceUserJSON,
       enabled = c.isActive.get,
       created = c.createdAt.get

From 76a1544c0af33f90a9bc97b540f17116fe14218f Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Tue, 5 Dec 2023 13:23:12 +0100
Subject: [PATCH 18/36] refactor/tweaked the getHostOnlyOfRedirectURL code

---
 .../scala/code/snippet/OAuthWorkedThanks.scala     |  2 +-
 obp-api/src/main/scala/code/util/Helper.scala      | 14 +++++++-------
 obp-api/src/test/scala/code/util/HelperTest.scala  |  6 +++++-
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
index 59b017efd..8b315dbbf 100644
--- a/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
+++ b/obp-api/src/main/scala/code/snippet/OAuthWorkedThanks.scala
@@ -49,7 +49,7 @@ class OAuthWorkedThanks extends MdcLoggable {
     logger.debug(s"OAuthWorkedThanks.thanks.redirectUrl $redirectUrl")
     //extract the clean(omit the parameters) redirect url from request url
     val staticPortionOfRedirectUrl = Helper.getStaticPortionOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
-    val hostOnlyOfRedirectUrlLegacy = Helper.getHostOnlyOfRedirectURL(redirectUrl.openOr("invalidRequestedRedirectURL")) openOr("invalidRequestedRedirectURL")
+    val hostOnlyOfRedirectUrlLegacy = Helper.getHostOnlyOfRedirectURL(staticPortionOfRedirectUrl) openOr("invalidRequestedRedirectURL")
     logger.debug(s"OAuthWorkedThanks.thanks.staticPortionOfRedirectUrl $staticPortionOfRedirectUrl")
     logger.debug(s"OAuthWorkedThanks.thanks.hostOnlyOfRedirectUrlLegacy $hostOnlyOfRedirectUrlLegacy")
     
diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index a9889d3b2..f167f4847 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -3,7 +3,6 @@ package code.util
 import java.net.{Socket, SocketException}
 import java.util.UUID.randomUUID
 import java.util.{Date, GregorianCalendar}
-
 import code.api.util.{APIUtil, CallContext, CallContextLight, CustomJsonFormats}
 import code.api.{APIFailureNewStyle, Constant}
 import code.api.util.APIUtil.fullBoxOrException
@@ -20,6 +19,7 @@ import com.openbankproject.commons.util.{ReflectUtils, RequiredFieldValidation,
 import com.tesobe.CacheKeyFromArguments
 import net.liftweb.http.S
 import net.liftweb.util.Helpers
+import net.liftweb.util.Helpers.tryo
 import net.sf.cglib.proxy.{Enhancer, MethodInterceptor, MethodProxy}
 import java.lang.reflect.Method
 import scala.concurrent.Future
@@ -174,7 +174,7 @@ object Helper extends Loggable {
    * @return http://localhost:8082/oauthcallback
    */
   def getStaticPortionOfRedirectURL(redirectUrl: String): Box[String] = {
-    Full(redirectUrl.split("\\?")(0)) //return everything before the "?"
+    tryo(redirectUrl.split("\\?")(0)) //return everything before the "?"
   }
   
   /**
@@ -182,23 +182,23 @@ object Helper extends Loggable {
    * eg1: http://localhost:8082/oauthcallback?....--> http://localhost:8082 <br/> 
    * eg2: http://localhost:8016?oautallback?=3NLMGV ...--> http://localhost:8016
    *
-   * @param input a long url with parameters 
-   * @return clean redirect url
+   * @param redirectUrl -> http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
+   * @return hostOnlyOfRedirectURL-> http://localhost:8082
    */
   @deprecated("We can not only use hostname as the redirectUrl, now add new method `getStaticPortionOfRedirectURL` ","05.12.2023")  
   def getHostOnlyOfRedirectURL(redirectUrl: String): Box[String] = {
+    val staticPortionOfRedirectURL = getStaticPortionOfRedirectURL(redirectUrl).getOrElse(redirectUrl)
     /**
      * pattern eg1: http://xxxxxx?oautxxxx  -->http://xxxxxx
      * pattern eg2: https://xxxxxx/oautxxxx -->http://xxxxxx
      */
     //Note: the pattern should be : val  pattern = "(https?):\\/\\/(.*)(?=((\\/)|(\\?))oauthcallback*)".r, but the OAuthTest is different, so add the following logic
     val pattern = "([A-Za-z][A-Za-z0-9+.-]*):\\/\\/(.*)(?=((\\/)|(\\?))oauth*)".r
-    val validRedirectURL = pattern findFirstIn redirectUrl
+    val validRedirectURL = pattern findFirstIn staticPortionOfRedirectURL
     // Now for the OAuthTest, the redirect format is : http://localhost:8016?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
     // It is not the normal case: http://localhost:8082/oauthcallback?oauth_token=LUDKELGJXRDOC1AK1X1TOYIXM5W1AORFJT5KE43B&oauth_verifier=14062
     // So add the split function to select the first value; eg: Array(http://localhost:8082, thcallback) --> http://localhost:8082
-    val extractCleanURL = validRedirectURL.getOrElse("").split("/oauth")(0)
-    Full(extractCleanURL)
+    tryo(validRedirectURL.getOrElse("").split("/oauth")(0))
   }
   
   /**
diff --git a/obp-api/src/test/scala/code/util/HelperTest.scala b/obp-api/src/test/scala/code/util/HelperTest.scala
index deb6cff59..2e766e37d 100644
--- a/obp-api/src/test/scala/code/util/HelperTest.scala
+++ b/obp-api/src/test/scala/code/util/HelperTest.scala
@@ -53,10 +53,14 @@ class HelperTest extends FeatureSpec with Matchers with GivenWhenThen with Props
   feature("test APIUtil.getHostOnlyOfRedirectURL method") {
     // The redirectURl is `http://localhost:8082/oauthcallback`
     val testString1 = "http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
-    val testString2 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+    val testString2 = "http://localhost:8082/oauthcallback"
+    val testString3 = "http://localhost:8082?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018"
+    val testString4 = "http://localhost:8082"
 
     Helper.getHostOnlyOfRedirectURL(testString1).head should be("http://localhost:8082")
     Helper.getHostOnlyOfRedirectURL(testString2).head should be("http://localhost:8082")
+    Helper.getHostOnlyOfRedirectURL(testString3).head should be("http://localhost:8082")
+    Helper.getHostOnlyOfRedirectURL(testString4).head should be("http://localhost:8082")
   }
 
 }
\ No newline at end of file

From b8c6e46f0ec7075e5f4386168830956757a21432 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Tue, 5 Dec 2023 13:56:58 +0100
Subject: [PATCH 19/36] refactor/use Java URL instead of regular expression

---
 obp-api/src/main/scala/code/util/Helper.scala | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala
index f167f4847..fa3cd2aa9 100644
--- a/obp-api/src/main/scala/code/util/Helper.scala
+++ b/obp-api/src/main/scala/code/util/Helper.scala
@@ -1,6 +1,6 @@
 package code.util
 
-import java.net.{Socket, SocketException}
+import java.net.{Socket, SocketException, URL}
 import java.util.UUID.randomUUID
 import java.util.{Date, GregorianCalendar}
 import code.api.util.{APIUtil, CallContext, CallContextLight, CustomJsonFormats}
@@ -21,6 +21,7 @@ import net.liftweb.http.S
 import net.liftweb.util.Helpers
 import net.liftweb.util.Helpers.tryo
 import net.sf.cglib.proxy.{Enhancer, MethodInterceptor, MethodProxy}
+
 import java.lang.reflect.Method
 import scala.concurrent.Future
 import scala.util.Random
@@ -187,18 +188,10 @@ object Helper extends Loggable {
    */
   @deprecated("We can not only use hostname as the redirectUrl, now add new method `getStaticPortionOfRedirectURL` ","05.12.2023")  
   def getHostOnlyOfRedirectURL(redirectUrl: String): Box[String] = {
-    val staticPortionOfRedirectURL = getStaticPortionOfRedirectURL(redirectUrl).getOrElse(redirectUrl)
-    /**
-     * pattern eg1: http://xxxxxx?oautxxxx  -->http://xxxxxx
-     * pattern eg2: https://xxxxxx/oautxxxx -->http://xxxxxx
-     */
-    //Note: the pattern should be : val  pattern = "(https?):\\/\\/(.*)(?=((\\/)|(\\?))oauthcallback*)".r, but the OAuthTest is different, so add the following logic
-    val pattern = "([A-Za-z][A-Za-z0-9+.-]*):\\/\\/(.*)(?=((\\/)|(\\?))oauth*)".r
-    val validRedirectURL = pattern findFirstIn staticPortionOfRedirectURL
-    // Now for the OAuthTest, the redirect format is : http://localhost:8016?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
-    // It is not the normal case: http://localhost:8082/oauthcallback?oauth_token=LUDKELGJXRDOC1AK1X1TOYIXM5W1AORFJT5KE43B&oauth_verifier=14062
-    // So add the split function to select the first value; eg: Array(http://localhost:8082, thcallback) --> http://localhost:8082
-    tryo(validRedirectURL.getOrElse("").split("/oauth")(0))
+    val url = new URL(redirectUrl) //eg: http://localhost:8082/oauthcallback?oauth_token=G5AEA2U1WG404EGHTIGBHKRR4YJZAPPHWKOMNEEV&oauth_verifier=53018
+    val protocol = url.getProtocol() // http
+    val authority = url.getAuthority()// localhost:8082, this will contain the port.
+    tryo(s"$protocol://$authority") // http://localhost:8082 
   }
   
   /**

From d5eb84deef2b622d3c706836f614e3e8ab3e0ec8 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Wed, 6 Dec 2023 11:24:10 +0100
Subject: [PATCH 20/36] refactor/remove the EmbeddedKafka

---
 obp-api/pom.xml                               |  10 -
 ...toredProcedureConnector_vDec2019Test.scala | 101 -----
 .../KafkaMappedConnector_vMay2019Test.scala   | 157 -------
 .../src/test/scala/code/kafka/KafkaTest.scala | 401 ------------------
 .../test/scala/code/setup/KafkaSetup.scala    |  76 ----
 5 files changed, 745 deletions(-)
 delete mode 100644 obp-api/src/test/scala/code/bankconnectors/StoredProcedureConnector_vDec2019Test.scala
 delete mode 100644 obp-api/src/test/scala/code/bankconnectors/vMay2019/KafkaMappedConnector_vMay2019Test.scala
 delete mode 100644 obp-api/src/test/scala/code/kafka/KafkaTest.scala
 delete mode 100644 obp-api/src/test/scala/code/setup/KafkaSetup.scala

diff --git a/obp-api/pom.xml b/obp-api/pom.xml
index 714d8986c..211c527f2 100644
--- a/obp-api/pom.xml
+++ b/obp-api/pom.xml
@@ -360,16 +360,6 @@
       <artifactId>geocalc</artifactId>
       <version>0.5.7</version>
     </dependency>
-    <!-- embeded kafka for unit test start -->
-    <!-- https://mvnrepository.com/artifact/io.github.embeddedkafka/embedded-kafka -->
-    <dependency>
-      <groupId>io.github.embeddedkafka</groupId>
-      <artifactId>embedded-kafka_2.12</artifactId>
-      <version>2.4.1.1</version>
-      <scope>test</scope>
-    </dependency>
-    <!-- embeded kafka for unit test end -->
-    <!-- https://mvnrepository.com/artifact/com.twilio.sdk/twilio -->
     <dependency>
       <groupId>com.twilio.sdk</groupId>
       <artifactId>twilio</artifactId>
diff --git a/obp-api/src/test/scala/code/bankconnectors/StoredProcedureConnector_vDec2019Test.scala b/obp-api/src/test/scala/code/bankconnectors/StoredProcedureConnector_vDec2019Test.scala
deleted file mode 100644
index c60464054..000000000
--- a/obp-api/src/test/scala/code/bankconnectors/StoredProcedureConnector_vDec2019Test.scala
+++ /dev/null
@@ -1,101 +0,0 @@
-package code.bankconnectors.vMay2019
-
-/*
-Open Bank Project - API
-Copyright (C) 2011-2019, TESOBE GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-
-Email: contact@tesobe.com
-TESOBE GmbH
-Osloerstrasse 16/17
-Berlin 13359, Germany
-*/
-
-
-import code.actorsystem.ObpActorInit
-import code.api.JSONFactoryGateway.PayloadOfJwtJSON
-import code.api.util.{APIUtil, CallContext, CustomJsonFormats}
-import code.bankconnectors.Connector
-import code.bankconnectors.storedprocedure.StoredProcedureConnector_vDec2019
-import code.bankconnectors.vSept2018._
-import code.kafka.KafkaHelper
-import code.setup.{DefaultUsers, KafkaSetup, ServerSetupWithTestData}
-import com.openbankproject.commons.dto.InBoundGetBanks
-import com.openbankproject.commons.model._
-import net.liftweb.common.{Box, Failure, Full}
-import org.scalatest.Tag
-
-class StoredProcedureConnector_vDec2019Test extends ServerSetupWithTestData with DefaultUsers with ObpActorInit{
-
-  override implicit val formats = CustomJsonFormats.formats
-
-  object StoredProcedureConnector_vDec2019Test extends Tag("StoredProcedureConnector_vDec2019")
-
-  val callContext = Some(
-    CallContext(
-      gatewayLoginRequestPayload = Some(PayloadOfJwtJSON(
-        login_user_name = "",
-        is_first = false,
-        app_id = "",
-        app_name = "",
-        time_stamp = "",
-        cbs_token = Some(""),
-        cbs_id = "",
-        session_id = Some(""))),
-      user = Full(resourceUser1)
-    )
-  )
-
-  val PropsConnectorVersion = APIUtil.getPropsValue("connector").openOrThrowException("connector props filed `connector` not set")
-
-
-  feature("Test all stored_procedure methods") {
-    if (PropsConnectorVersion == "stored_procedure_vDec2019") {
-//      scenario("test `checkBankAccountExists` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-//        val checkBankAccountExists = StoredProcedureConnector_vDec2019.checkBankAccountExists(testBankId1,testAccountId1,callContext)
-//        getValueFromFuture(checkBankAccountExists)._1.isDefined equals (true)
-//      }
-//
-//      scenario("test `getBankAccounts` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-//        val checkBankAccountExists = StoredProcedureConnector_vDec2019.checkBankAccountExists(testBankId1,testAccountId1,callContext)
-//        getValueFromFuture(checkBankAccountExists)._1.isDefined equals (true)
-//      }
-//      scenario("test `getBank` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-//        val transantionRequests210 = StoredProcedureConnector_vDec2019.getBank(testBankId1, callContext)
-//        getValueFromFuture(transantionRequests210).isDefined equals (true)
-//      }
-//
-//
-//      scenario("test `getBanks` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-//        val transantionRequests210 = StoredProcedureConnector_vDec2019.getBanks(callContext)
-//        getValueFromFuture(transantionRequests210).isDefined equals (true)
-//      }
-//      
-//      scenario("test `getTransactionRequests210` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-//        val transantionRequests210: Box[(List[TransactionRequest], Option[CallContext])] = StoredProcedureConnector_vDec2019.getTransactionRequests210(resourceUser1, null, callContext)
-//        transantionRequests210.isDefined equals(true)
-//      }
-
-      scenario("test `getTransactions` method, there no need Adapter message for this method!", StoredProcedureConnector_vDec2019Test) {
-        val transactions = StoredProcedureConnector_vDec2019.getTransactions(testBankId1, testAccountId1, callContext, Nil)
-        val trans = getValueFromFuture(transactions)._1.openOrThrowException("Should not be empty!")
-        trans.head.description.isDefined equals (true)
-      }
-      
-    } else {
-      ignore("ignore test getObpConnectorLoopback, if it is mapped connector", StoredProcedureConnector_vDec2019Test) {}
-    }
-  }
-}
\ No newline at end of file
diff --git a/obp-api/src/test/scala/code/bankconnectors/vMay2019/KafkaMappedConnector_vMay2019Test.scala b/obp-api/src/test/scala/code/bankconnectors/vMay2019/KafkaMappedConnector_vMay2019Test.scala
deleted file mode 100644
index e3883ac1b..000000000
--- a/obp-api/src/test/scala/code/bankconnectors/vMay2019/KafkaMappedConnector_vMay2019Test.scala
+++ /dev/null
@@ -1,157 +0,0 @@
-package code.bankconnectors.vMay2019
-
-/*
-Open Bank Project - API
-Copyright (C) 2011-2019, TESOBE GmbH
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program. If not, see http://www.gnu.org/licenses/.
-
-Email: contact@tesobe.com
-TESOBE GmbH
-Osloerstrasse 16/17
-Berlin 13359, Germany
-*/
-
-
-import code.api.JSONFactoryGateway.PayloadOfJwtJSON
-import code.api.util.{APIUtil, CallContext, CustomJsonFormats}
-import code.bankconnectors.Connector
-import code.bankconnectors.vSept2018._
-import code.kafka.KafkaHelper
-import code.setup.{KafkaSetup, ServerSetupWithTestData}
-import com.openbankproject.commons.dto.InBoundGetBanks
-import com.openbankproject.commons.model._
-import net.liftweb.common.{Box, Failure, Full}
-import org.scalatest.Tag
-
-class KafkaMappedConnector_vMay2019Test extends KafkaSetup with ServerSetupWithTestData {
-
-  override implicit val formats = CustomJsonFormats.formats
-
-  object kafkaTest extends Tag("kafkaTest")
-
-  val callContext = Some(
-    CallContext(
-      gatewayLoginRequestPayload = Some(PayloadOfJwtJSON(
-        login_user_name = "",
-        is_first = false,
-        app_id = "",
-        app_name = "",
-        time_stamp = "",
-        cbs_token = Some(""),
-        cbs_id = "",
-        session_id = Some(""))),
-      user = Full(resourceUser1)
-    )
-  )
-
-  val PropsConnectorVersion = APIUtil.getPropsValue("connector").openOrThrowException("connector props filed `connector` not set")
-
-
-  feature("Send and retrieve message") {
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star") {
-      ignore("ignore test getObpConnectorLoopback, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("1st test `getObpConnectorLoopback` method, there no need Adapter message for this method!", kafkaTest) {
-        //This method is only used for `kafka` connector, should first set `connector=kafka_vSept2018` in test.default.props. 
-        //and also need to set up `api_instance_id` and `remotedata.timeout` field for it.
-        val propsApiInstanceId = code.api.Constant.ApiInstanceId
-        val propsRemotedataTimeout = APIUtil.getPropsValue("remotedata.timeout").openOrThrowException("connector props filed `remotedata.timeout` not set")
-
-        PropsConnectorVersion contains ("kafka") should be(true)
-        propsApiInstanceId should be("1")
-        propsRemotedataTimeout should be("10")
-
-        When("We call this method, and get the response. ")
-        val future = KafkaHelper.echoKafkaServer
-        val result = future.getContent
-
-        Then("If it return value successfully, that mean api <--> kafka is working well. We only need check one filed of response.")
-        val connectorVersion = result.connectorVersion
-        connectorVersion should be(PropsConnectorVersion)
-
-        Then("For KafkaMappedConnector_vSept2018 connector, we need to make these two methods work `getAuthInfoFirstCbsCall` and `getAuthInfo`")
-
-        val firstAuthInfo: Box[AuthInfo] = for {
-          firstGetAuthInfo <- KafkaMappedConnector_vSept2018.getAuthInfoFirstCbsCall("","", callContext)
-        } yield {
-          (firstGetAuthInfo)
-        }
-        firstAuthInfo.openOrThrowException("firstAuthInfo Can not be empty here. ")
-
-        val authInfo: Box[AuthInfo] = for {
-          getAuthInfo <- KafkaMappedConnector_vSept2018.getAuthInfo(callContext)
-        } yield {
-          getAuthInfo
-        }
-        authInfo.openOrThrowException("firstAuthInfo Can not be empty here. ")
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star") {
-      ignore("ignore test processRequest, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("Send and retrieve message directly to and from kafka", kafkaTest) {
-        val emptyStatusMessage = InboundStatusMessage("", "", "", "")
-        val inBound = InboundGetBanks(InboundAuthInfo("", ""), Status("", List(emptyStatusMessage)), List(InboundBank("1", "2", "3", "4")))
-        When("send a OutboundGetBanks message")
-
-        dispathResponse(inBound)
-        val req = OutboundGetBanks(AuthInfo())
-
-        val future = processRequest[InboundGetBanks](req)
-        val result: Box[InboundGetBanks] = future.getContent
-
-        result should be(Full(inBound))
-      }
-
-  }
-
-  feature("Test the getBank error cases") {
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star") {
-      ignore("ignore test getBanks, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBanksFuture -- status.hasError", kafkaTest) {
-        val inbound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InBoundGetBanks]).map(_.exampleInboundMessage).head.asInstanceOf[InBoundGetBanks]
-        //This inBound.status.errorCode != "", so it will throw the error back.
-        val expectedValue = Failure("INTERNAL-"+ inbound.status.errorCode+". + CoreBank-Status:" + inbound.status.backendMessages)
-        dispathResponse(inbound)
-        val future = Connector.connector.vend.getBanks(callContext)
-
-        dispathResponse(inbound)
-        val result = future.getContent
-        result should be(expectedValue)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star") {
-      ignore("ignore test getBanksFuture -- status.hasNoError,  if it is mapped connector", kafkaTest) {}
-    } else 
-      scenario(s"test getBanksFuture -- status.hasNoError", kafkaTest) {
-        val inbound = Connector.connector.vend.messageDocs
-          .filter(_.exampleInboundMessage.isInstanceOf[InBoundGetBanks])
-          .map(_.exampleInboundMessage).head.asInstanceOf[InBoundGetBanks]
-          .copy(status = Status("", Nil)) // This will set errorCode to "", no it works
-        //This inBound.status.errorCode != "", so it will throw the error back.
-        val expectedValue = Full(inbound.data.head.bankId).toString
-        dispathResponse(inbound)
-        val future = Connector.connector.vend.getBanks(callContext)
-  
-        dispathResponse(inbound)
-        val result = future.getContent
-        result.map(_._1.head.bankId).toString should be(expectedValue)
-      }
-
-  }
-}
\ No newline at end of file
diff --git a/obp-api/src/test/scala/code/kafka/KafkaTest.scala b/obp-api/src/test/scala/code/kafka/KafkaTest.scala
deleted file mode 100644
index 5f342f1ae..000000000
--- a/obp-api/src/test/scala/code/kafka/KafkaTest.scala
+++ /dev/null
@@ -1,401 +0,0 @@
-package code.kafka
-
-import java.util.{Date, UUID}
-
-import code.api.JSONFactoryGateway.PayloadOfJwtJSON
-import code.api.util.{APIUtil, CallContext, CustomJsonFormats}
-import code.api.v2_1_0.TransactionRequestBodyCommonJSON
-import code.bankconnectors.Connector
-import code.bankconnectors.vSept2018._
-import code.setup.{KafkaSetup, ServerSetupWithTestData}
-import com.openbankproject.commons.dto.{InBoundGetKycChecks, InBoundGetKycMedias, InBoundGetKycStatuses}
-import com.openbankproject.commons.model._
-import net.liftweb.common.{Box, Full}
-import org.scalatest.Tag
-
-import scala.collection.immutable.List
-
-class KafkaTest extends KafkaSetup with ServerSetupWithTestData {
-
-  override implicit val formats = CustomJsonFormats.formats
-  
-  object kafkaTest extends Tag("kafkaTest")
-  
-  val callContext = Some(
-    CallContext(
-      gatewayLoginRequestPayload = Some(PayloadOfJwtJSON(
-        login_user_name = "",
-        is_first = false,
-        app_id = "",
-        app_name = "",
-        time_stamp = "",
-        cbs_token = Some(""),
-        cbs_id = "",
-        session_id = Some(""))),
-      user = Full(resourceUser1)
-    )
-  )
-
-  val PropsConnectorVersion = APIUtil.getPropsValue("connector").openOrThrowException("connector props filed `connector` not set")
-  
- 
-  feature("Send and retrieve message") {
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getObpConnectorLoopback, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("1st test `getObpConnectorLoopback` method, there no need Adapter message for this method!", kafkaTest) {
-        //This method is only used for `kafka` connector, should first set `connector=kafka_vSept2018` in test.default.props. 
-        //and also need to set up `api_instance_id` and `remotedata.timeout` field for it.
-        val propsApiInstanceId = code.api.Constant.ApiInstanceId
-        val propsRemotedataTimeout = APIUtil.getPropsValue("remotedata.timeout").openOrThrowException("connector props filed `remotedata.timeout` not set")
-  
-        PropsConnectorVersion contains ("kafka") should be (true)
-        propsApiInstanceId should be ("1")
-        propsRemotedataTimeout should be ("10")
-  
-        When("We call this method, and get the response. ")
-        val future = KafkaHelper.echoKafkaServer
-        val result =  future.getContent
-  
-        Then("If it return value successfully, that mean api <--> kafka is working well. We only need check one filed of response.")
-        val connectorVersion= result.connectorVersion
-        connectorVersion should be (PropsConnectorVersion)
-  
-        Then("For KafkaMappedConnector_vSept2018 connector, we need to make these two methods work `getAuthInfoFirstCbsCall` and `getAuthInfo`")
-        
-        val firstAuthInfo: Box[AuthInfo] = for{
-          firstGetAuthInfo <- KafkaMappedConnector_vSept2018.getAuthInfoFirstCbsCall("","", callContext)
-        } yield {
-          (firstGetAuthInfo)
-        }
-        firstAuthInfo.openOrThrowException("firstAuthInfo Can not be empty here. ")
-
-        val authInfo: Box[AuthInfo] =  for{
-          getAuthInfo <- KafkaMappedConnector_vSept2018.getAuthInfo(callContext)
-        } yield {
-          getAuthInfo
-        }
-        authInfo.openOrThrowException("firstAuthInfo Can not be empty here. ")
-
-    }
-
-//    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-//      ignore("ignore test processRequest, if it is mapped connector", kafkaTest) {}
-//    } else
-//      scenario("Send and retrieve message directly to and from kafka", kafkaTest) {
-//        val emptyStatusMessage = InboundStatusMessage("", "", "", "")
-//        val inBound = InboundGetBanks(InboundAuthInfo("", ""), Status("", List(emptyStatusMessage)), List(InboundBank("1", "2", "3", "4")))
-//        When("send a OutboundGetBanks message")
-//
-//        dispathResponse(inBound)
-//        val req = OutboundGetBanks(AuthInfo())
-//
-//        val future = processRequest[InboundGetBanks](req)
-//        val result: Box[InboundGetBanks] = future.getContent
-//
-//        result should be (Full(inBound))
-//    }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getKycStatuses, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("test `getKycStatuses` method",kafkaTest) {
-        When("send a OutboundGetKycStatuses api message")
-        val emptyStatusMessage = InboundStatusMessage("", "", "", "")
-        val kycStatusCommons = KycStatusCommons(bankId = "hello_bank_id", customerId = "hello_customer_id", customerNumber = "hello_customer_number", ok = true, date = new Date())
-        val singleInboundBank = List(kycStatusCommons)
-        val inboundAdapterCallContext = InboundAdapterCallContext(correlationId="some_correlationId")
-        val inBound = InBoundGetKycStatuses(inboundAdapterCallContext, Status("", List(emptyStatusMessage)), singleInboundBank)
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getKycStatuses(kycStatusCommons.customerId, Some(CallContext()))
-
-        val result: (Box[List[KycStatus]], Option[CallContext]) =  future.getContent
-        val expectResult = Full(singleInboundBank)
-        result._1.toString should be (expectResult.toString)
-     }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getKycChecks, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("test `getKycChecks` method", kafkaTest) {
-        When("send a OutboundGetKycChecks api message")
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process =="obp.getKycChecks").map(_.exampleInboundMessage).head.asInstanceOf[InBoundGetKycChecks]
-
-        dispathResponse(inBound)
-
-        val future = Connector.connector.vend.getKycChecks(inBound.data.head.customerId, Some(CallContext()))
-        val result: (Box[List[KycCheck]], Option[CallContext]) =  future.getContent
-        val expectResult = Full(inBound.data)
-        result._1.toString should be (expectResult.toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getKycMedias, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario("test `getKycMedias` method",kafkaTest) {
-        When("send a OutboundetKycMedias api message")
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process =="obp.getKycMedias").map(_.exampleInboundMessage).head.asInstanceOf[InBoundGetKycMedias]
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getKycMedias(inBound.data.head.customerId, Some(CallContext()))
-
-        val result: (Box[List[KycMedia]], Option[CallContext]) =  future.getContent
-        val expectResult = Full(inBound.data)
-        result._1.toString should be (expectResult.toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getAdapterInfo, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getAdapterInfo method",kafkaTest) {
-        When("send a getAdapterInfo api message")
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process.toString.contains("getAdapterInfo")).map(_.exampleInboundMessage).head.asInstanceOf[InboundAdapterInfo]
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getAdapterInfo(None)
-
-        val result: Box[(InboundAdapterInfoInternal, Option[CallContext])] =  future.getContent
-        result.map(_._1) should be (Full(inBound.data))
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getUser, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getUser method",kafkaTest) {
-        When("send a getUser api message")
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process.toString.contains("getUser")).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetUserByUsernamePassword]
-
-        dispathResponse(inBound)
-        val box = Connector.connector.vend.getUser("username","password")
-
-        box.map(_.displayName) should be (Full(inBound.data.displayName))
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBanksFuture, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBanksFuture method", kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process.toString.contains("getBanks")).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetBanks]
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getBanks(None)
-
-        val result =  future.getContent
-        result.map(_._1.head.bankId).toString should be (Full(inBound.data.head.bankId).toString)
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBanks, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBanks method", kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.process.toString.contains("getBanks")).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetBanks]
-
-        dispathResponse(inBound)
-        val box = Connector.connector.vend.getBanksLegacy(None)
-
-        box.map(_._1.head.bankId).toString should be (Full(inBound.data.head.bankId).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBank, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBank method", kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetBank]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetBank]
-
-        dispathResponse(inBound)
-        val box = Connector.connector.vend.getBankLegacy(BankId(""), None)
-
-        box.map(_._1.bankId).toString should be (Full(inBound.data.bankId).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBankFuture, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBankFuture method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetBank]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetBank]
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getBank(BankId(""), None)
-        val result = future.getContent
-
-        result.map(_._1.bankId).toString should be (Full(inBound.data.bankId).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBankAccountsForUserFuture, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBankAccountsForUserFuture method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetAccounts]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetAccounts]
-
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getBankAccountsForUser("", "", callContext)
-        val result = future.getContent
-
-        result.map(_._1.head).toString should be (Full(inBound.data.head).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBankAccountsForUser, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBankAccountsForUser method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetAccounts]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetAccounts]
-        dispathResponse(inBound)
-        val box = Connector.connector.vend.getBankAccountsForUserLegacy("","", callContext)
-
-        box.map(_._1.head).toString should be (Full(inBound.data.head).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBankAccount, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBankAccount method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetAccountbyAccountID]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetAccountbyAccountID]
-        dispathResponse(inBound)
-        val box = Connector.connector.vend.getBankAccountLegacy(BankId(""), AccountId(""), callContext)
-
-        box.map(_._1.bankId).toString should be (Full(inBound.data.head.bankId).toString)
-        box.map(_._1.accountId).toString should be (Full(inBound.data.head.accountId).toString)
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getBankAccountFuture, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getBankAccountFuture method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetAccountbyAccountID]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetAccountbyAccountID]
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.checkBankAccountExists(BankId(""), AccountId(""), callContext)
-
-        val result = future.getContent
-
-        result._1.map(_.accountId.value).toString should be (Full(inBound.data.head.accountId).toString)
-        result._1.map(_.bankId.value).toString should be (Full(inBound.data.head.bankId).toString)
-        
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getChallengeThreshold, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getChallengeThreshold method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetChallengeThreshold]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetChallengeThreshold]
-        dispathResponse(inBound)
-        val future = Connector.connector.vend.getChallengeThreshold("","","","","","","", callContext)
-
-        val result = future.getContent
-
-        result._1.map(_.amount).toString should be (Full(inBound.data.amount).toString)
-        result._1.map(_.currency).toString should be (Full(inBound.data.currency).toString)
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test makePaymentv210, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test makePaymentv210 method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundCreateTransactionId]).map(_.exampleInboundMessage).head.asInstanceOf[InboundCreateTransactionId]
-        dispathResponse(inBound)
-        
-        val fromAccount = BankAccountSept2018(KafkaMappedConnector_vSept2018.inboundAccountSept2018Example)
-        val toAccount = BankAccountSept2018(KafkaMappedConnector_vSept2018.inboundAccountSept2018Example)
-        val transactionRequestId = TransactionRequestId(UUID.randomUUID().toString)
-        val transactionRequestCommonBody = TransactionRequestBodyCommonJSON(AmountOfMoneyJsonV121("",""),"")
-        val future = Connector.connector.vend.makePaymentv210(
-          fromAccount,
-          toAccount,
-          transactionRequestId,
-          transactionRequestCommonBody,
-          10,
-          "",
-          TransactionRequestType("SANDBOX_TAN"),
-          "", 
-          callContext)
-
-        val result = future.getContent
-
-        result._1.map(_.value).toString should be (Full(inBound.data.id).toString)
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test createChallenge, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test createChallenge method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundCreateChallengeSept2018]).map(_.exampleInboundMessage).head.asInstanceOf[InboundCreateChallengeSept2018]
-        dispathResponse(inBound)
-
-        val account = BankAccountSept2018(KafkaMappedConnector_vSept2018.inboundAccountSept2018Example)
-        val transactionRequestCommonBody = TransactionRequestBodyCommonJSON(AmountOfMoneyJsonV121("",""),"")
-        val future = Connector.connector.vend.createChallenge(
-          account.bankId,
-          account.accountId,
-          "",
-          TransactionRequestType("SANDBOX_TAN"),
-          "",
-          None,
-          callContext)
-
-        val result = future.getContent
-
-        result._1.toString should be (Full(inBound.data.answer).toString)
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test createCounterparty, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test createCounterparty method",kafkaTest) {
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundCreateCounterparty]).map(_.exampleInboundMessage).head.asInstanceOf[InboundCreateCounterparty]
-        val outBound = Connector.connector.vend.messageDocs.filter(_.exampleOutboundMessage.isInstanceOf[OutboundCreateCounterparty]).map(_.exampleOutboundMessage).head.asInstanceOf[OutboundCreateCounterparty]
-        dispathResponse(inBound)
-
-        val account = BankAccountSept2018(KafkaMappedConnector_vSept2018.inboundAccountSept2018Example)
-        val transactionRequestCommonBody = TransactionRequestBodyCommonJSON(AmountOfMoneyJsonV121("",""),"")
-        val box = Connector.connector.vend.createCounterparty(
-          outBound.counterparty.name,
-          outBound.counterparty.description,                       
-          outBound.counterparty.currency,
-          outBound.counterparty.createdByUserId,
-          outBound.counterparty.thisBankId,                
-          outBound.counterparty.thisAccountId,             
-          outBound.counterparty.thisViewId,                
-          outBound.counterparty.otherAccountRoutingScheme, 
-          outBound.counterparty.otherAccountRoutingAddress,
-          outBound.counterparty.otherAccountSecondaryRoutingScheme,
-          outBound.counterparty.otherAccountSecondaryRoutingAddress,
-          outBound.counterparty.otherBankRoutingScheme,    
-          outBound.counterparty.otherBankRoutingAddress,   
-          outBound.counterparty.otherBranchRoutingScheme,  
-          outBound.counterparty.otherBranchRoutingAddress,  
-          outBound.counterparty.isBeneficiary,  
-          outBound.counterparty.bespoke,  
-          callContext)
-
-
-        box.map(_._1.counterpartyId) should be (Full(inBound.data.get.counterpartyId))
-        box.map(_._1.createdByUserId) should be (Full(inBound.data.get.createdByUserId))
-
-      }
-
-    if (PropsConnectorVersion =="mapped" || PropsConnectorVersion =="star"){
-      ignore("ignore test getTransactionRequests210, if it is mapped connector", kafkaTest) {}
-    } else
-      scenario(s"test getTransactionRequests210 method",kafkaTest) {
-
-        val inBound = Connector.connector.vend.messageDocs.filter(_.exampleInboundMessage.isInstanceOf[InboundGetTransactionRequests210]).map(_.exampleInboundMessage).head.asInstanceOf[InboundGetTransactionRequests210]
-        dispathResponse(inBound)
-
-        val account = BankAccountSept2018(KafkaMappedConnector_vSept2018.inboundAccountSept2018Example)
-        val transactionRequestCommonBody = TransactionRequestBodyCommonJSON(AmountOfMoneyJsonV121("",""),"")
-        val box = Connector.connector.vend.getTransactionRequests210(
-          resourceUser1,
-          account,
-          callContext)
-
-       box.map(_._1.head.body) should be (inBound.data.head.body) 
-
-
-      }
-
-  }
-}
diff --git a/obp-api/src/test/scala/code/setup/KafkaSetup.scala b/obp-api/src/test/scala/code/setup/KafkaSetup.scala
deleted file mode 100644
index 4c881e9d6..000000000
--- a/obp-api/src/test/scala/code/setup/KafkaSetup.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-package code.setup
-
-import code.actorsystem.ObpActorSystem
-import code.api.util.CustomJsonFormats
-import code.kafka._
-import code.util.Helper.MdcLoggable
-import net.liftweb.json
-import net.liftweb.json.Extraction
-import net.manub.embeddedkafka.{EmbeddedKafka, EmbeddedKafkaConfig}
-import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
-import org.scalatest.{FeatureSpec, _}
-
-import com.openbankproject.commons.ExecutionContext.Implicits.global
-import scala.concurrent.{Await, Future}
-import scala.concurrent.duration.{Duration, _}
-
-trait KafkaSetup extends FeatureSpec with EmbeddedKafka with KafkaHelper
-  with GivenWhenThen with BeforeAndAfterAll
-  with Matchers with MdcLoggable {
-
-
-
-  implicit val formats = CustomJsonFormats.formats
-  implicit val config = EmbeddedKafkaConfig(kafkaPort = 9092, zooKeeperPort = 2181) //TODO the port should read from test.default.props, but fail
-  implicit val stringSerializer = new StringSerializer
-  implicit val stringDeserializer = new StringDeserializer
-
-  val requestMapResponseTopics:Map[String, String] =  NorthSideConsumer.listOfTopics
-    .map(Topics.createTopicByClassName)
-    .map(pair => (pair.request, pair.response))
-    .toMap
-  val requestTopics = requestMapResponseTopics.keySet
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-
-    EmbeddedKafka.start()
-
-    if(!OBPKafkaConsumer.primaryConsumer.started){
-      val actorSystem = ObpActorSystem.startLocalActorSystem
-      KafkaHelperActors.startLocalKafkaHelperWorkers(actorSystem)
-      // Start North Side Consumer if it's not already started
-      OBPKafkaConsumer.primaryConsumer.start()
-    }
-  }
-
-  override def afterAll(): Unit = {
-    super.afterAll()
-    OBPKafkaConsumer.primaryConsumer.complete()
-    EmbeddedKafka.stop()
-  }
-
-  /**
-    * send an object to kafka as response
-    *
-    * @param inBound inBound object that will send to kafka as a response
-    * @tparam T Outbound type
-    */
-  def dispathResponse(inBound: AnyRef): Unit = {
-    val inBoundStr = inBound match {
-      case str: String => str
-      case _ =>json.compactRender(Extraction.decompose(inBound))
-    }
-    Future{
-      val requestKeyValue = consumeNumberKeyedMessagesFromTopics(requestTopics, 1, true)
-      val (requestTopic, keyValueList) = requestKeyValue.find(_._2.nonEmpty).get
-      val (key, _) = keyValueList.head
-      val responseTopic = requestMapResponseTopics(requestTopic)
-      publishToKafka(responseTopic, key, inBoundStr)
-    }
-  }
-
-    implicit class FutureExtract[T](future: Future[T]) {
-      def getContent: T = Await.result(future, 10 seconds)
-    }
-}

From 35b7c5755e4bc73d0cbc3930051eba335026898e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 6 Dec 2023 11:41:49 +0100
Subject: [PATCH 21/36] feature/Dynamic Registration of Consumer - WIP 2

---
 .../src/main/scala/code/api/util/X509.scala   | 36 +++++++++++++++++--
 .../code/api/util/newstyle/Consumer.scala     | 11 +++---
 .../scala/code/api/v5_1_0/APIMethods510.scala | 17 ++++-----
 .../code/api/v5_1_0/JSONFactory5.1.0.scala    |  8 ++---
 4 files changed, 53 insertions(+), 19 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/X509.scala b/obp-api/src/main/scala/code/api/util/X509.scala
index 1d2d6d71c..051353e07 100644
--- a/obp-api/src/main/scala/code/api/util/X509.scala
+++ b/obp-api/src/main/scala/code/api/util/X509.scala
@@ -10,7 +10,7 @@ import code.util.Helper.MdcLoggable
 import com.github.dwickern.macros.NameOf
 import com.nimbusds.jose.jwk.RSAKey
 import com.nimbusds.jose.util.X509CertUtils
-import net.liftweb.common.{Box, Failure, Full}
+import net.liftweb.common.{Box, Failure, Full, Empty}
 import org.bouncycastle.asn1._
 import org.bouncycastle.asn1.x509.Extension
 import org.bouncycastle.asn1.x509.qualified.QCStatement
@@ -246,5 +246,37 @@ object X509 extends MdcLoggable {
       case None => Failure(ErrorMessages.X509CannotGetCertificate)
     }
   }
-  
+
+  def getCommonName(pem: Option[String]): Box[String] = {
+    getFieldCommon(pem, "CN")
+  }
+  def getOrganization(pem: Option[String]): Box[String] = {
+    getFieldCommon(pem, "O")
+  }
+  def getOrganizationUnit(pem: Option[String]): Box[String] = {
+    getFieldCommon(pem, "OU")
+  }
+  def getEmailAddress(pem: Option[String]): Box[String] = {
+    getFieldCommon(pem, "EMAILADDRESS")
+      .or(getFieldCommon(pem, "EMAILADDRESS".toLowerCase()))
+  }
+
+  private def getFieldCommon(pem: Option[String], field: String) = {
+    pem match {
+      case Some(unboxedPem) =>
+        extractCertificateInfo(unboxedPem).map { item =>
+          val splitByComma: Array[String] = item.subject_domain_name.split(",")
+          val splitByKeyValuePair: Array[(String, String)] = splitByComma.map(i => i.split("=")(0).trim -> i.split("=")(1).trim)
+          val valuesAsMap: Map[String, List[String]] = splitByKeyValuePair.toList.groupBy(_._1).map { case (k, v) => (k, v.map(_._2)) }
+          val result: String = valuesAsMap.get(field).map(_.mkString).getOrElse("")
+          result
+        } match {
+          case Full(value) if value.isEmpty => Empty
+          case everythingElse => everythingElse
+        }
+      case _ =>
+        Empty
+    }
+  }
+
 }
diff --git a/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala b/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
index 9625229a4..251ac8051 100644
--- a/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
+++ b/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
@@ -1,7 +1,8 @@
 package code.api.util.newstyle
 
-import code.api.util.APIUtil.{OBPReturnType, unboxFull}
+import code.api.util.APIUtil.{OBPReturnType, unboxFull, unboxFullOrFail}
 import code.api.util.CallContext
+import code.api.util.ErrorMessages.CreateConsumerError
 import code.consumer.Consumers
 import code.model.{AppType, Consumer}
 
@@ -34,11 +35,11 @@ object Consumer {
         redirectURL,
         createdByUserId,
         clientCertificate
-      ) map {
-        (_, callContext)
-      }
+      )
     } map {
-      unboxFull(_)
+      (_, callContext)
+    } map {
+      x => (unboxFullOrFail(x._1, callContext, CreateConsumerError, 400), x._2)
     }
   }
 
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 0630947f8..ecfe6f6c5 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -9,6 +9,7 @@ import code.api.util.ApiTag._
 import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _}
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
+import code.api.util.X509.{getCommonName, getEmailAddress}
 import code.api.util._
 import code.api.util.newstyle.Consumer.createConsumerNewStyle
 import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle}
@@ -1784,11 +1785,11 @@ trait APIMethods510 {
          |""",
       ConsumerPostJsonV510(
         "TESOBE GmbH",
-        "Test",
-        "Web",
+        Some("Test"),
+        Some("Web"),
         "Description",
-        "some@email.com",
-        "redirecturl",
+        Some("some@email.com"),
+        Some("redirecturl"),
       ),
       consumerJsonV510,
       List(
@@ -1819,11 +1820,11 @@ trait APIMethods510 {
               key = Some(Helpers.randomString(40).toLowerCase),
               secret = Some(Helpers.randomString(40).toLowerCase),
               isActive = Some(true),
-              name = Some(postedJson.app_name),
-              appType = Some(AppType.valueOf(postedJson.app_type)),
+              name = getCommonName(pem).or(postedJson.app_name) ,
+              appType = postedJson.app_type.map(AppType.valueOf).orElse(Some(AppType.valueOf("Confidential"))),
               description = Some(postedJson.description),
-              developerEmail = Some(postedJson.developer_email),
-              redirectURL = Some(postedJson.redirect_url),
+              developerEmail = getEmailAddress(pem).or(postedJson.developer_email),
+              redirectURL = postedJson.redirect_url,
               createdByUserId = None,
               clientCertificate = pem,
               cc.callContext
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 9ec054fed..0046fda95 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -282,11 +282,11 @@ case class MetricJsonV510(
 case class MetricsJsonV510(metrics: List[MetricJsonV510])
 
 case class ConsumerPostJsonV510(entity_name: String,
-                                app_name: String,
-                                app_type: String,
+                                app_name: Option[String],
+                                app_type: Option[String],
                                 description: String,
-                                developer_email: String,
-                                redirect_url: String,
+                                developer_email: Option[String],
+                                redirect_url: Option[String],
                                )
 case class ConsumerJsonV510(consumer_id: String,
                             consumer_key: String,

From f59448effc580a2ceba3a321e9a6f58e1f6f464d Mon Sep 17 00:00:00 2001
From: Simon Redfern <simon@tesobe.com>
Date: Wed, 6 Dec 2023 12:24:38 +0100
Subject: [PATCH 22/36] Tweaking documentation for getMetrics in 5.1.0

---
 .../main/scala/code/api/v5_1_0/APIMethods510.scala    | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 41545ead0..2134c45f9 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -1422,19 +1422,22 @@ trait APIMethods510 {
       "GET",
       "/management/metrics",
       "Get Metrics",
-      s"""Get the all metrics
+      s"""Get API metrics rows. These are records of each REST API call.
          |
          |require CanReadMetrics role
          |
          |Filters Part 1.*filtering* (no wilde cards etc.) parameters to GET /management/metrics
          |
-         |Should be able to filter on the following metrics fields
+         |You can filter by the following fields by applying url parameters
          |
          |eg: /management/metrics?from_date=$DateWithMsExampleString&to_date=$DateWithMsExampleString&limit=50&offset=2
          |
-         |1 from_date (defaults to one week before current date): eg:from_date=$DateWithMsExampleString
+         |1 from_date e.g.:from_date=$DateWithMsExampleString Defaults to the Unix Epoch i.e. ${theEpochTime}
          |
-         |2 to_date (defaults to current date) eg:to_date=$DateWithMsExampleString
+         |2 to_date e.g.:to_date=$DateWithMsExampleString Defaults to a far future date i.e. ${APIUtil.ToDateInFuture}
+         |
+         |Note: it is recommended you send a valid from_date (e.g. 5 seconds ago) and to_date (now + 1 second) if you want to get the latest records
+         | Otherwise you may receive stale cached results.
          |
          |3 limit (for pagination: defaults to 50)  eg:limit=200
          |

From e12fecd8ed8ccd5a4d4a045734f98e9b2e29b155 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 6 Dec 2023 13:48:03 +0100
Subject: [PATCH 23/36] feature/Dynamic Registration of Consumer - WIP 3

---
 .../ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala   |  1 +
 .../main/scala/code/api/util/newstyle/Consumer.scala |  4 +++-
 .../main/scala/code/api/v4_0_0/APIMethods400.scala   |  1 +
 .../main/scala/code/api/v5_1_0/APIMethods510.scala   | 12 ++++++------
 .../scala/code/api/v5_1_0/JSONFactory5.1.0.scala     |  5 +++--
 5 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
index a91347362..5914c12b4 100644
--- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
+++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
@@ -2684,6 +2684,7 @@ object SwaggerDefinitionsJSON {
     app_type = "Web",
     description = "Account Management",
     developer_email = ExampleValue.emailExample.value,
+    company = ExampleValue.companyExample.value,
     redirect_url = "www.openbankproject.com",
     certificate_pem = pem,
     certificate_info = Some(certificateInfoJsonV510),
diff --git a/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala b/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
index 251ac8051..e71cab7db 100644
--- a/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
+++ b/obp-api/src/main/scala/code/api/util/newstyle/Consumer.scala
@@ -19,6 +19,7 @@ object Consumer {
                              appType: Option[AppType],
                              description: Option[String],
                              developerEmail: Option[String],
+                             company: Option[String],
                              redirectURL: Option[String],
                              createdByUserId: Option[String],
                              clientCertificate: Option[String],
@@ -34,7 +35,8 @@ object Consumer {
         developerEmail,
         redirectURL,
         createdByUserId,
-        clientCertificate
+        clientCertificate,
+        company
       )
     } map {
       (_, callContext)
diff --git a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala
index 0f812e84a..855044d64 100644
--- a/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala
+++ b/obp-api/src/main/scala/code/api/v4_0_0/APIMethods400.scala
@@ -5295,6 +5295,7 @@ trait APIMethods400 extends MdcLoggable {
               appType = None,
               description = Some(postedJson.description),
               developerEmail = Some(postedJson.developer_email),
+              company = None,
               redirectURL = Some(postedJson.redirect_url),
               createdByUserId = Some(u.userId),
               clientCertificate = Some(postedJson.clientCertificate),
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index ecfe6f6c5..b5c0c418f 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -9,7 +9,7 @@ import code.api.util.ApiTag._
 import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _}
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
 import code.api.util.NewStyle.HttpCode
-import code.api.util.X509.{getCommonName, getEmailAddress}
+import code.api.util.X509.{getCommonName, getEmailAddress, getOrganization}
 import code.api.util._
 import code.api.util.newstyle.Consumer.createConsumerNewStyle
 import code.api.util.newstyle.RegulatedEntityNewStyle.{createRegulatedEntityNewStyle, deleteRegulatedEntityNewStyle, getRegulatedEntitiesNewStyle, getRegulatedEntityByEntityIdNewStyle}
@@ -1784,12 +1784,11 @@ trait APIMethods510 {
          |
          |""",
       ConsumerPostJsonV510(
-        "TESOBE GmbH",
-        Some("Test"),
-        Some("Web"),
+        None,
+        None,
         "Description",
-        Some("some@email.com"),
-        Some("redirecturl"),
+        None,
+        None,
       ),
       consumerJsonV510,
       List(
@@ -1824,6 +1823,7 @@ trait APIMethods510 {
               appType = postedJson.app_type.map(AppType.valueOf).orElse(Some(AppType.valueOf("Confidential"))),
               description = Some(postedJson.description),
               developerEmail = getEmailAddress(pem).or(postedJson.developer_email),
+              company = getOrganization(pem),
               redirectURL = postedJson.redirect_url,
               createdByUserId = None,
               clientCertificate = pem,
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 0046fda95..cad6d562f 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -281,8 +281,7 @@ case class MetricJsonV510(
                      )
 case class MetricsJsonV510(metrics: List[MetricJsonV510])
 
-case class ConsumerPostJsonV510(entity_name: String,
-                                app_name: Option[String],
+case class ConsumerPostJsonV510(app_name: Option[String],
                                 app_type: Option[String],
                                 description: String,
                                 developer_email: Option[String],
@@ -295,6 +294,7 @@ case class ConsumerJsonV510(consumer_id: String,
                             app_type: String,
                             description: String,
                             developer_email: String,
+                            company: String,
                             redirect_url: String,
                             certificate_pem: String,
                             certificate_info: Option[CertificateInfoJsonV510],
@@ -651,6 +651,7 @@ object JSONFactory510 extends CustomJsonFormats {
       app_type = c.appType.toString(),
       description = c.description.get,
       developer_email = c.developerEmail.get,
+      company = c.company.get,
       redirect_url = c.redirectURL.get,
       certificate_pem = c.clientCertificate.get,
       certificate_info = certificateInfo,

From c0f354f10f2b9c21706a94f4e7a4b1bff0ef5fab Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Wed, 6 Dec 2023 15:37:26 +0100
Subject: [PATCH 24/36] bugfix/move the future out of the cache

---
 obp-api/src/main/scala/code/api/util/NewStyle.scala    |  4 ++--
 .../src/main/scala/code/metrics/MappedMetrics.scala    | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/NewStyle.scala b/obp-api/src/main/scala/code/api/util/NewStyle.scala
index 20325761d..5042e4402 100644
--- a/obp-api/src/main/scala/code/api/util/NewStyle.scala
+++ b/obp-api/src/main/scala/code/api/util/NewStyle.scala
@@ -3056,7 +3056,7 @@ object NewStyle extends MdcLoggable{
 
     private[this] val endpointMappingTTL = APIUtil.getPropsValue(s"endpointMapping.cache.ttl.seconds", "0").toInt
 
-    def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = {
+    def getEndpointMappings(bankId: Option[String], callContext: Option[CallContext]): OBPReturnType[List[EndpointMappingT]] = Future{
       import scala.concurrent.duration._
 
       validateBankId(bankId, callContext)
@@ -3064,7 +3064,7 @@ object NewStyle extends MdcLoggable{
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
         Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(endpointMappingTTL second) {
-          Future{(EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId), callContext)}
+          {(EndpointMappingProvider.endpointMappingProvider.vend.getAllEndpointMappings(bankId), callContext)}
         }
       }
     }
diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala
index 0a7c7c747..9f04090db 100644
--- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala
+++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala
@@ -360,7 +360,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{
   }
 
   // TODO Cache this as long as fromDate and toDate are in the past (before now)
-  override def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] = {
+  override def getTopApisFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopApi]]] = Future{
   /**                                                                                        
   * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUU
   * is just a temporary value field with UUID values in order to prevent any ambiguity.
@@ -369,7 +369,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{
   */                                                                                       
   var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)       
   CacheKeyFromArguments.buildCacheKey {Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cachedTopApis seconds){   
-    Future{
+    {
       val fromDate = queryParams.collect { case OBPFromDate(value) => value }.headOption
       val toDate = queryParams.collect { case OBPToDate(value) => value }.headOption
       val consumerId = queryParams.collect { case OBPConsumerId(value) => value }.headOption.flatMap(consumerIdToPrimaryKey)
@@ -440,7 +440,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{
   }}
 
   // TODO Cache this as long as fromDate and toDate are in the past (before now)
-  override def getTopConsumersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = {
+  override def getTopConsumersFuture(queryParams: List[OBPQueryParam]): Future[Box[List[TopConsumer]]] = Future {
   /**                                                                                        
   * Please note that "var cacheKey = (randomUUID().toString, randomUUID().toString, randomUU
   * is just a temporary value field with UUID values in order to prevent any ambiguity.
@@ -449,7 +449,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{
   */                                                                                       
   var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)       
   CacheKeyFromArguments.buildCacheKey {Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cachedTopConsumers seconds){   
-    Future {
+  
       val fromDate = queryParams.collect { case OBPFromDate(value) => value }.headOption
       val toDate = queryParams.collect { case OBPToDate(value) => value }.headOption
       val consumerId = queryParams.collect { case OBPConsumerId(value) => value }.headOption.flatMap(consumerIdToPrimaryKey)
@@ -519,7 +519,7 @@ object MappedMetrics extends APIMetrics with MdcLoggable{
       }
       tryo(result)
     }
-  }}}
+  }}
 
 }
 

From 34a8954edcd79e33e80b87f92765c9d703ed82dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 6 Dec 2023 17:56:30 +0100
Subject: [PATCH 25/36] feature/Dynamic Registration of Consumer - WIP 4

---
 .../scala/code/api/util/ErrorMessages.scala   |  1 +
 .../main/scala/code/api/util/JwtUtil.scala    |  9 +++++
 .../scala/code/api/v5_1_0/APIMethods510.scala | 36 +++++++++++++------
 .../code/api/v5_1_0/JSONFactory5.1.0.scala    |  2 ++
 4 files changed, 37 insertions(+), 11 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
index 934207029..429cddc2c 100644
--- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
+++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala
@@ -520,6 +520,7 @@ object ErrorMessages {
   val RegulatedEntityNotFound = "OBP-34100: Regulated Entity not found. Please specify a valid value for REGULATED_ENTITY_ID."
   val RegulatedEntityNotDeleted = "OBP-34101: Regulated Entity cannot be deleted. Please specify a valid value for REGULATED_ENTITY_ID."
   val RegulatedEntityNotFoundByCertificate = "OBP-34102: Regulated Entity cannot be found by provided certificate."
+  val PostJsonIsNotSigned = "OBP-34110: JWT at the post json cannot be verified."
 
   // Consents
   val ConsentNotFound = "OBP-35001: Consent not found by CONSENT_ID. "
diff --git a/obp-api/src/main/scala/code/api/util/JwtUtil.scala b/obp-api/src/main/scala/code/api/util/JwtUtil.scala
index f77b05087..96ec3c33b 100644
--- a/obp-api/src/main/scala/code/api/util/JwtUtil.scala
+++ b/obp-api/src/main/scala/code/api/util/JwtUtil.scala
@@ -269,6 +269,15 @@ object JwtUtil extends MdcLoggable {
     jwk.toPublicJWK.toRSAKey
   }
 
+  def verifyJwt(jwtString: String, pemEncodedRsaPublicKey: String): Boolean = {
+    // Parse PEM-encoded key to RSA public / private JWK
+    val jwk: JWK = JWK.parseFromPEMEncodedObjects(pemEncodedRsaPublicKey);
+    val rsaPublicKey: RSAKey = jwk.toPublicJWK.toRSAKey
+    val signedJWT = SignedJWT.parse(jwtString)
+    val verifier = new RSASSAVerifier(rsaPublicKey)
+    signedJWT.verify(verifier)
+  }
+
 
   def main(args: Array[String]): Unit = {
     val jwtToken = "eyJhbGciOiJSUzI1NiIsImtpZCI6IjhhYWQ2NmJkZWZjMWI0M2Q4ZGIyN2U2NWUyZTJlZjMwMTg3OWQzZTgiLCJ0eXAiOiJKV1QifQ.eyJpc3MiOiJodHRwczovL2FjY291bnRzLmdvb2dsZS5jb20iLCJhenAiOiI0MDc0MDg3MTgxOTIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJhdWQiOiI0MDc0MDg3MTgxOTIuYXBwcy5nb29nbGV1c2VyY29udGVudC5jb20iLCJzdWIiOiIxMTM5NjY4NTQyNDU3ODA4OTI5NTkiLCJhdF9oYXNoIjoiWGlpckZ1cnJ2X0ZxN3RHd25rLWt1QSIsIm5hbWUiOiJNYXJrbyBNaWxpxIciLCJwaWN0dXJlIjoiaHR0cHM6Ly9saDUuZ29vZ2xldXNlcmNvbnRlbnQuY29tLy1YZDQ0aG5KNlREby9BQUFBQUFBQUFBSS9BQUFBQUFBQUFBQS9BS3hyd2NhZHd6aG00TjR0V2s1RThBdnhpLVpLNmtzNHFnL3M5Ni1jL3Bob3RvLmpwZyIsImdpdmVuX25hbWUiOiJNYXJrbyIsImZhbWlseV9uYW1lIjoiTWlsacSHIiwibG9jYWxlIjoiZW4iLCJpYXQiOjE1NDczMTE3NjAsImV4cCI6MTU0NzMxNTM2MH0.UyOmM0rsO0-G_ibDH3DFogS94GcsNd9GtYVw7j3vSMjO1rZdIraV-N2HUtQN3yHopwdf35A2FEJaag6X8dbvEkJC7_GAynyLIpodoaHNtaLbww6XQSYuQYyF27aPMpROoGZUYkMpB_82LF3PbD4ecDPC2IA5oSyDF4Eya4yn-MzxYmXS7usVWvanREg8iNQSxpu7zZqj4UwhvSIv7wH0vskr_M-PnefQzNTrdUx74i-v9lVqC4E_bF5jWeDGO8k5dqWqg55QuZdyJdSh89KNiIjJXGZDWUBzGfsbetWRnObIgX264fuOW4SpRglUc8fzv41Sc7SSqjqRAFm05t60kg"
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index b5c0c418f..18b1e75a2 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -8,6 +8,7 @@ import code.api.util.ApiRole._
 import code.api.util.ApiTag._
 import code.api.util.ErrorMessages.{$UserNotLoggedIn, BankNotFound, ConsentNotFound, InvalidJsonFormat, UnknownError, UserNotFoundByUserId, UserNotLoggedIn, _}
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
+import code.api.util.JwtUtil.{getSignedPayloadAsJson, verifyJwt}
 import code.api.util.NewStyle.HttpCode
 import code.api.util.X509.{getCommonName, getEmailAddress, getOrganization}
 import code.api.util._
@@ -42,7 +43,7 @@ import com.openbankproject.commons.model._
 import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion}
 import net.liftweb.common.Full
 import net.liftweb.http.rest.RestHelper
-import net.liftweb.json.parse
+import net.liftweb.json.{compactRender, parse}
 import net.liftweb.mapper.By
 import net.liftweb.util.Helpers
 import net.liftweb.util.Helpers.tryo
@@ -1781,21 +1782,28 @@ trait APIMethods510 {
       "/dynamic-registration/consumers",
       "Create a Consumer",
       s"""Create a Consumer (mTLS access).
+         |
+         | JWT payload:
+         |  - minimal
+         |    { "description":"Description" }
+         |  - full
+         |    {
+         |     "description": "Description",
+         |     "app_name": "Tesobe GmbH",
+         |     "app_type": "Sofit",
+         |     "developer_email": "marko@tesobe.com",
+         |     "redirect_url": "http://localhost:8082"
+         |    }
+         | Please note that JWT must be signed with the counterpart private kew of the public key used to establish mTLS
          |
          |""",
-      ConsumerPostJsonV510(
-        None,
-        None,
-        "Description",
-        None,
-        None,
-      ),
+      ConsumerJwtPostJsonV510("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkZXNjcmlwdGlvbiI6IkRlc2NyaXB0aW9uIn0.qDnzk1dGK8akdLFRl8fmJV_SeoDjRTDG_eMogCIzZ7M"),
       consumerJsonV510,
       List(
         InvalidJsonFormat,
         UnknownError
       ),
-      List(apiTagConsumer),
+      List(apiTagDirectory, apiTagConsumer),
       Some(Nil))
 
 
@@ -1804,10 +1812,16 @@ trait APIMethods510 {
         cc =>
           implicit val ec = EndpointContext(Some(cc))
           for {
-            postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, cc.callContext) {
-              json.extract[ConsumerPostJsonV510]
+            postedJwt <- NewStyle.function.tryons(InvalidJsonFormat, 400, cc.callContext) {
+              json.extract[ConsumerJwtPostJsonV510]
             }
             pem = APIUtil.`getPSD2-CERT`(cc.requestHeaders)
+            _ <- Helper.booleanToFuture(PostJsonIsNotSigned, 400, cc.callContext) {
+              verifyJwt(postedJwt.jwt, pem.getOrElse(""))
+            }
+            postedJson <- NewStyle.function.tryons(InvalidJsonFormat, 400, cc.callContext) {
+              parse(getSignedPayloadAsJson(postedJwt.jwt).getOrElse("{}")).extract[ConsumerPostJsonV510]
+            }
             certificateInfo: CertificateInfoJsonV510 <- Future(X509.getCertificateInfo(pem)) map {
               unboxFullOrFail(_, cc.callContext, X509GeneralError)
             }
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index cad6d562f..6ff9d348e 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -281,6 +281,8 @@ case class MetricJsonV510(
                      )
 case class MetricsJsonV510(metrics: List[MetricJsonV510])
 
+
+case class ConsumerJwtPostJsonV510(jwt: String)
 case class ConsumerPostJsonV510(app_name: Option[String],
                                 app_type: Option[String],
                                 description: String,

From efe45e9c86a753afa3c1b3c90595ba1f7e05f89b Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Wed, 6 Dec 2023 21:52:11 +0100
Subject: [PATCH 26/36] refactor/added the pagination for getConsumers

---
 .../main/scala/bootstrap/liftweb/Boot.scala   |  2 +-
 .../scala/code/api/v3_1_0/APIMethods310.scala |  8 +++++-
 .../code/consumer/ConsumerProvider.scala      |  6 ++---
 obp-api/src/main/scala/code/model/OAuth.scala | 25 +++++++++++++++----
 .../code/remotedata/RemotedataConsumers.scala |  5 ++--
 .../remotedata/RemotedataConsumersActor.scala |  5 ++--
 6 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
index 1526b7e9a..286662e31 100644
--- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
+++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala
@@ -850,7 +850,7 @@ class Boot extends MdcLoggable {
         .map[String](_.getClientId)
         .collect(Collectors.toSet())
 
-      Consumers.consumers.vend.getConsumersFuture().foreach{ consumers =>
+      Consumers.consumers.vend.getConsumersFuture(Nil, None).foreach{ consumers =>
         consumers.filter(consumer => consumer.isActive.get && !oAuth2ClientIds.contains(consumer.key.get))
           .foreach(HydraUtil.createHydraClient(_))
       }
diff --git a/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala b/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala
index d0860b0ba..6448a4596 100644
--- a/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala
+++ b/obp-api/src/main/scala/code/api/v3_1_0/APIMethods310.scala
@@ -771,6 +771,10 @@ trait APIMethods310 {
       "Get Consumers",
       s"""Get the all Consumers.
          |
+         |${authenticationRequiredMessage(true)}
+         |
+         |${urlParametersDocument(true, true)}
+         |
         |""",
       EmptyBody,
       consumersJson310,
@@ -790,7 +794,9 @@ trait APIMethods310 {
           for {
             (Full(u), callContext) <- authenticatedAccess(cc)
             _ <- NewStyle.function.hasEntitlement("", u.userId, ApiRole.canGetConsumers, callContext)
-            consumers <- Consumers.consumers.vend.getConsumersFuture()
+            httpParams <- NewStyle.function.extractHttpParamsFromUrl(cc.url)
+            (obpQueryParams, callContext) <- createQueriesByHttpParamsFuture(httpParams, callContext)
+            consumers <- Consumers.consumers.vend.getConsumersFuture(obpQueryParams, callContext)
             users <- Users.users.vend.getUsersByUserIdsFuture(consumers.map(_.createdByUserId.get))
           } yield {
             (createConsumersJson(consumers, users), HttpCode.`200`(callContext))
diff --git a/obp-api/src/main/scala/code/consumer/ConsumerProvider.scala b/obp-api/src/main/scala/code/consumer/ConsumerProvider.scala
index 351e4e6a0..25725e19f 100644
--- a/obp-api/src/main/scala/code/consumer/ConsumerProvider.scala
+++ b/obp-api/src/main/scala/code/consumer/ConsumerProvider.scala
@@ -1,6 +1,6 @@
 package code.consumer
 
-import code.api.util.APIUtil
+import code.api.util.{APIUtil, CallContext, OBPQueryParam}
 import code.model.{AppType, Consumer, MappedConsumersProvider}
 import code.remotedata.RemotedataConsumers
 import com.openbankproject.commons.model.{BankIdAccountId, User, View}
@@ -31,7 +31,7 @@ trait ConsumersProvider {
   def getConsumerByConsumerId(consumerId: String): Box[Consumer]
   def getConsumerByConsumerIdFuture(consumerId: String): Future[Box[Consumer]]
   def getConsumersByUserIdFuture(userId: String): Future[List[Consumer]]
-  def getConsumersFuture(): Future[List[Consumer]]
+  def getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext]): Future[List[Consumer]]
   def createConsumer(key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String], clientCertificate: Option[String] = None, company: Option[String] = None): Box[Consumer]
   def deleteConsumer(consumer: Consumer): Boolean
   def updateConsumer(id: Long, key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String]): Box[Consumer]
@@ -64,7 +64,7 @@ class RemotedataConsumersCaseClasses {
   case class getConsumerByConsumerId(consumerId: String)
   case class getConsumerByConsumerIdFuture(consumerId: String)
   case class getConsumersByUserIdFuture(userId: String)
-  case class getConsumersFuture()
+  case class getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext])
   case class createConsumer(key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String], clientCertificate: Option[String], company: Option[String])
   case class updateConsumer(id: Long, key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String])
   case class deleteConsumer(consumer: Consumer)
diff --git a/obp-api/src/main/scala/code/model/OAuth.scala b/obp-api/src/main/scala/code/model/OAuth.scala
index 7368ff9e9..14da03e21 100644
--- a/obp-api/src/main/scala/code/model/OAuth.scala
+++ b/obp-api/src/main/scala/code/model/OAuth.scala
@@ -27,7 +27,7 @@ TESOBE (http://www.tesobe.com/)
 package code.model
 import java.util.{Collections, Date}
 
-import code.api.util.APIUtil
+import code.api.util.{APIUtil, CallContext, OBPAscending, OBPDescending, OBPFromDate, OBPLimit, OBPOffset, OBPOrdering, OBPQueryParam, OBPToDate}
 import code.api.util.CommonFunctions.validUri
 import code.api.util.migration.Migration.DbFunction
 import code.consumer.{Consumers, ConsumersProvider}
@@ -119,11 +119,26 @@ object MappedConsumersProvider extends ConsumersProvider with MdcLoggable {
     Future(getConsumersByUserId(userId))
   }
 
-  def getConsumers(): List[Consumer] = {
-    Consumer.findAll()
+  def getConsumers(queryParams: List[OBPQueryParam], callContext: Option[CallContext]): List[Consumer] = {
+    val limit = queryParams.collect { case OBPLimit(value) => MaxRows[Consumer](value) }.headOption
+    val offset = queryParams.collect { case OBPOffset(value) => StartAt[Consumer](value) }.headOption
+    val fromDate = queryParams.collect { case OBPFromDate(date) => By_>=(Consumer.createdAt, date) }.headOption
+    val toDate = queryParams.collect { case OBPToDate(date) => By_<=(Consumer.createdAt, date) }.headOption
+    val ordering = queryParams.collect {
+      case OBPOrdering(_, direction) =>
+        direction match {
+          case OBPAscending => OrderBy(Consumer.createdAt, Ascending)
+          case OBPDescending => OrderBy(Consumer.createdAt, Descending)
+        }
+    }
+
+    val mapperParams: Seq[QueryParam[Consumer]] = Seq(limit.toSeq, offset.toSeq, fromDate.toSeq, toDate.toSeq, ordering.toSeq).flatten
+    
+    Consumer.findAll(mapperParams: _*)
   }
-  override def getConsumersFuture(): Future[List[Consumer]] = {
-    Future(getConsumers())
+  
+  override def getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext]): Future[List[Consumer]] = {
+    Future(getConsumers(httpParams: List[OBPQueryParam], callContext: Option[CallContext]))
   }
 
   override def createConsumer(key: Option[String],
diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataConsumers.scala b/obp-api/src/main/scala/code/remotedata/RemotedataConsumers.scala
index 06f991656..714644782 100644
--- a/obp-api/src/main/scala/code/remotedata/RemotedataConsumers.scala
+++ b/obp-api/src/main/scala/code/remotedata/RemotedataConsumers.scala
@@ -2,6 +2,7 @@ package code.remotedata
 
 import akka.pattern.ask
 import code.actorsystem.ObpActorInit
+import code.api.util.{CallContext, OBPQueryParam}
 import code.consumer.{ConsumersProvider, RemotedataConsumersCaseClasses}
 import code.model._
 import net.liftweb.common._
@@ -35,8 +36,8 @@ object RemotedataConsumers extends ObpActorInit with ConsumersProvider {
   def getConsumersByUserIdFuture(id: String): Future[List[Consumer]] =
     (actor ? cc.getConsumersByUserIdFuture(id)).mapTo[List[Consumer]]
 
-  def getConsumersFuture(): Future[List[Consumer]] =
-    (actor ? cc.getConsumersFuture()).mapTo[List[Consumer]]
+  def getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext]): Future[List[Consumer]] =
+    (actor ? cc.getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext])).mapTo[List[Consumer]]
 
   def createConsumer(key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String], clientCertificate: Option[String] = None, company: Option[String] = None): Box[Consumer] = getValueFromFuture(
     (actor ? cc.createConsumer(key, secret, isActive, name, appType, description, developerEmail, redirectURL, createdByUserId, clientCertificate, company)).mapTo[Box[Consumer]]
diff --git a/obp-api/src/main/scala/code/remotedata/RemotedataConsumersActor.scala b/obp-api/src/main/scala/code/remotedata/RemotedataConsumersActor.scala
index fe6db3f0a..84a7264da 100644
--- a/obp-api/src/main/scala/code/remotedata/RemotedataConsumersActor.scala
+++ b/obp-api/src/main/scala/code/remotedata/RemotedataConsumersActor.scala
@@ -2,6 +2,7 @@ package code.remotedata
 
 import akka.actor.Actor
 import code.actorsystem.ObpActorHelper
+import code.api.util.{CallContext, OBPQueryParam}
 import code.consumer.RemotedataConsumersCaseClasses
 import code.model.{MappedConsumersProvider, _}
 import code.util.Helper.MdcLoggable
@@ -41,9 +42,9 @@ class RemotedataConsumersActor extends Actor with ObpActorHelper with MdcLoggabl
       logger.debug(s"getConsumersByUserIdFuture($id)")
       sender ! (mapper.getConsumersByUserId(id))
 
-    case cc.getConsumersFuture() =>
+    case cc.getConsumersFuture(httpParams: List[OBPQueryParam], callContext: Option[CallContext]) =>
       logger.debug(s"getConsumersFuture()")
-      sender ! (mapper.getConsumers())
+      sender ! (mapper.getConsumers(httpParams: List[OBPQueryParam], callContext: Option[CallContext]))
 
     case cc.createConsumer(key: Option[String], secret: Option[String], isActive: Option[Boolean], name: Option[String], appType: Option[AppType], description: Option[String], developerEmail: Option[String], redirectURL: Option[String], createdByUserId: Option[String], clientCertificate: Option[String], company: Option[String]) =>
       logger.debug(s"createConsumer(*****, *****, ${isActive.getOrElse("None")}, ${name.getOrElse("None")}, ${appType.getOrElse("None")}, ${description.getOrElse("None")}, ${developerEmail.getOrElse("None")}, ${redirectURL.getOrElse("None")}, ${createdByUserId.getOrElse("None")}, ${clientCertificate.getOrElse("None")}, ${company.getOrElse("None")})")

From ed3a89df77dd518cda05e2ec6edb4bfbc07d99e4 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Wed, 6 Dec 2023 21:56:55 +0100
Subject: [PATCH 27/36] docfix/the default limit is Constant.Pagination.limit

---
 obp-api/src/main/scala/code/api/util/APIUtil.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala
index 239e2fb3a..3a2d93e77 100644
--- a/obp-api/src/main/scala/code/api/util/APIUtil.scala
+++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala
@@ -2086,7 +2086,7 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
          |
          |Possible custom url parameters for pagination:
          |
-         |* limit=NUMBER ==> default value: 50
+         |* limit=NUMBER ==> default value: ${Constant.Pagination.limit}
          |* offset=NUMBER ==> default value: 0
          |
          |eg1:?limit=100&offset=0

From 4089c7eea9c71486c1812bfd53aad59bef8abf2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Thu, 7 Dec 2023 10:01:50 +0100
Subject: [PATCH 28/36] feature/Add entity_name to GET /regulated-entities

---
 .../ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala  |  2 ++
 .../code/api/util/newstyle/RegulatedEntity.scala    |  2 ++
 .../main/scala/code/api/v5_1_0/APIMethods510.scala  |  1 +
 .../scala/code/api/v5_1_0/JSONFactory5.1.0.scala    |  3 +++
 .../MappedRegulatedEntitiyProvider.scala            | 13 ++++++++++---
 .../code/regulatedentities/RegulatedEntity.scala    |  1 +
 .../commons/model/CommonModelTrait.scala            |  1 +
 7 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
index 5914c12b4..84ce088ae 100644
--- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
+++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala
@@ -57,6 +57,7 @@ object SwaggerDefinitionsJSON {
     entity_id = "0af807d7-3c39-43ef-9712-82bcfde1b9ca",
     certificate_authority_ca_owner_id = "CY_CBC",
     entity_certificate_public_key = "-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----",
+    entity_name = "EXAMPLE COMPANY LTD",
     entity_code = "PSD_PICY_CBC!12345",
     entity_type = "PSD_PI",
     entity_address = "EXAMPLE COMPANY LTD, 5 SOME STREET",
@@ -70,6 +71,7 @@ object SwaggerDefinitionsJSON {
   lazy val regulatedEntityPostJsonV510: RegulatedEntityPostJsonV510 = RegulatedEntityPostJsonV510(
     certificate_authority_ca_owner_id = "CY_CBC",
     entity_certificate_public_key = "-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----",
+    entity_name = "EXAMPLE COMPANY LTD",
     entity_code = "PSD_PICY_CBC!12345",
     entity_type = "PSD_PI",
     entity_address = "EXAMPLE COMPANY LTD, 5 SOME STREET",
diff --git a/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
index c4d79bbd2..58f5f11bd 100644
--- a/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
+++ b/obp-api/src/main/scala/code/api/util/newstyle/RegulatedEntity.scala
@@ -17,6 +17,7 @@ object RegulatedEntityNewStyle {
 
   def createRegulatedEntityNewStyle(certificateAuthorityCaOwnerId: Option[String],
                                     entityCertificatePublicKey: Option[String],
+                                    entityName: Option[String],
                                     entityCode: Option[String],
                                     entityType: Option[String],
                                     entityAddress: Option[String],
@@ -30,6 +31,7 @@ object RegulatedEntityNewStyle {
       MappedRegulatedEntityProvider.createRegulatedEntity(
         certificateAuthorityCaOwnerId: Option[String],
         entityCertificatePublicKey: Option[String],
+        entityName: Option[String],
         entityCode: Option[String],
         entityType: Option[String],
         entityAddress: Option[String],
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
index 1ef1b831c..06677998b 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/APIMethods510.scala
@@ -218,6 +218,7 @@ trait APIMethods510 {
             (entity, callContext) <- createRegulatedEntityNewStyle(
               certificateAuthorityCaOwnerId = Some(postedData.certificate_authority_ca_owner_id),
               entityCertificatePublicKey = Some(postedData.entity_certificate_public_key),
+              entityName = Some(postedData.entity_name),
               entityCode = Some(postedData.entity_code),
               entityType = Some(postedData.entity_type),
               entityAddress = Some(postedData.entity_address),
diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 6ff9d348e..735b4fe75 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -72,6 +72,7 @@ case class RegulatedEntityJsonV510(
                                     entity_id: String,
                                     certificate_authority_ca_owner_id: String,
                                     entity_certificate_public_key: String,
+                                    entity_name: String,
                                     entity_code: String,
                                     entity_type: String,
                                     entity_address: String,
@@ -84,6 +85,7 @@ case class RegulatedEntityJsonV510(
 case class RegulatedEntityPostJsonV510(
                                       certificate_authority_ca_owner_id: String,
                                       entity_certificate_public_key: String,
+                                      entity_name: String,
                                       entity_code: String,
                                       entity_type: String,
                                       entity_address: String,
@@ -594,6 +596,7 @@ object JSONFactory510 extends CustomJsonFormats {
       entity_id = entity.entityId,
       certificate_authority_ca_owner_id = entity.certificateAuthorityCaOwnerId,
       entity_certificate_public_key = entity.entityCertificatePublicKey,
+      entity_name = entity.entityName,
       entity_code = entity.entityCode,
       entity_type = entity.entityType,
       entity_address = entity.entityAddress,
diff --git a/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
index b1295e0f5..e84a6d042 100644
--- a/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
+++ b/obp-api/src/main/scala/code/regulatedentities/MappedRegulatedEntitiyProvider.scala
@@ -19,6 +19,7 @@ object MappedRegulatedEntityProvider extends RegulatedEntityProvider {
 
   override def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
                                      entityCertificatePublicKey: Option[String],
+                                     entityName: Option[String],
                                      entityCode: Option[String],
                                      entityType: Option[String],
                                      entityAddress: Option[String],
@@ -38,6 +39,10 @@ object MappedRegulatedEntityProvider extends RegulatedEntityProvider {
         case Some(v) => entity.EntityCertificatePublicKey(v)
         case None =>
       }
+      entityName match {
+        case Some(v) => entity.EntityName(v)
+        case None =>
+      }
       entityCode match {
         case Some(v) => entity.EntityCode(v)
         case None =>
@@ -90,20 +95,22 @@ object MappedRegulatedEntityProvider extends RegulatedEntityProvider {
 class MappedRegulatedEntity extends RegulatedEntityTrait with LongKeyedMapper[MappedRegulatedEntity] with IdPK {
   override def getSingleton = MappedRegulatedEntity
   object EntityId extends MappedUUID(this)
-  object CertificateAuthorityCaOwnerId extends MappedString(this, 50)
+  object CertificateAuthorityCaOwnerId extends MappedString(this, 256)
+  object EntityName extends MappedString(this, 256)
   object EntityCode extends MappedString(this, 50)
   object EntityCertificatePublicKey extends MappedText(this)
   object EntityType extends MappedString(this, 50)
-  object EntityAddress extends MappedString(this, 50)
+  object EntityAddress extends MappedString(this, 256)
   object EntityTownCity extends MappedString(this, 50)
   object EntityPostCode extends MappedString(this, 50)
   object EntityCountry extends MappedString(this, 50)
-  object EntityWebSite extends MappedString(this, 50)
+  object EntityWebSite extends MappedString(this, 256)
   object Services extends MappedText(this)
 
 
   override def entityId: String = EntityId.get
   override def certificateAuthorityCaOwnerId: String = CertificateAuthorityCaOwnerId.get
+  override def entityName: String = EntityName.get
   override def entityCode: String = EntityCode.get
   override def entityCertificatePublicKey: String = EntityCertificatePublicKey.get
   override def entityType: String = EntityType.get
diff --git a/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
index 25af53275..2a73a3e41 100644
--- a/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
+++ b/obp-api/src/main/scala/code/regulatedentities/RegulatedEntity.scala
@@ -20,6 +20,7 @@ trait RegulatedEntityProvider {
 
   def createRegulatedEntity(certificateAuthorityCaOwnerId: Option[String],
                             entityCertificatePublicKey: Option[String],
+                            entityName: Option[String],
                             entityCode: Option[String],
                             entityType: Option[String],
                             entityAddress: Option[String],
diff --git a/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala b/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
index 54c4cac53..7707a607b 100644
--- a/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
+++ b/obp-commons/src/main/scala/com/openbankproject/commons/model/CommonModelTrait.scala
@@ -100,6 +100,7 @@ trait AccountApplication {
 trait RegulatedEntityTrait {
   def entityId: String
   def certificateAuthorityCaOwnerId: String
+  def entityName: String
   def entityCode: String
   def entityCertificatePublicKey: String
   def entityType: String

From 861b65752f5c1f4c87f15b0d2aff4cdc7039f43e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 12 Dec 2023 09:37:25 +0100
Subject: [PATCH 29/36] feature/Remove log4j library

---
 obp-api/pom.xml | 10 ----------
 pom.xml         |  1 -
 2 files changed, 11 deletions(-)

diff --git a/obp-api/pom.xml b/obp-api/pom.xml
index 211c527f2..4b8194138 100644
--- a/obp-api/pom.xml
+++ b/obp-api/pom.xml
@@ -79,16 +79,6 @@
       <groupId>org.slf4j</groupId>
       <version>1.7.26</version>
     </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-api</artifactId>
-      <version>${log4j.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-core</artifactId>
-      <version>${log4j.version}</version>
-    </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
diff --git a/pom.xml b/pom.xml
index 2b8287314..b8f7e7923 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,6 @@
     <avro.version>1.8.2</avro.version>
     <lift.version>3.5.0</lift.version>
     <jetty.version>9.4.50.v20221201</jetty.version>
-    <log4j.version>2.17.1</log4j.version>
     <obp-ri.version>2016.11-RC6-SNAPSHOT</obp-ri.version>
     <!-- Common plugin settings -->
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

From a35ed553146214bb9cb7158ed213adf8afdcff7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 12 Dec 2023 13:26:59 +0100
Subject: [PATCH 30/36] bugfix/Switch source_ip/target_ip values at getMetrics
 endpoint v5.1.0

---
 obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
index 735b4fe75..73dff2c05 100644
--- a/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
+++ b/obp-api/src/main/scala/code/api/v5_1_0/JSONFactory5.1.0.scala
@@ -625,8 +625,8 @@ object JSONFactory510 extends CustomJsonFormats {
       implemented_by_partial_function = metric.getImplementedByPartialFunction(),
       correlation_id = metric.getCorrelationId(),
       duration = metric.getDuration(),
-      source_ip = metric.getTargetIp(),
-      target_ip = metric.getSourceIp(),
+      source_ip = metric.getSourceIp(),
+      target_ip = metric.getTargetIp(),
       response_body = metric.getResponseBody()
     )
   }

From d69c73e806e59a7cb0997b6396fe130576a33ea1 Mon Sep 17 00:00:00 2001
From: Simon Redfern <simon@tesobe.com>
Date: Tue, 12 Dec 2023 14:22:59 +0100
Subject: [PATCH 31/36] added token endpoint url

---
 obp-api/src/main/scala/code/api/util/Glossary.scala | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala
index b60d9ab57..cee7c95e1 100644
--- a/obp-api/src/main/scala/code/api/util/Glossary.scala
+++ b/obp-api/src/main/scala/code/api/util/Glossary.scala
@@ -2115,12 +2115,11 @@ object Glossary extends MdcLoggable  {
         |
         |Register your App key [HERE]($getServerUrl/consumer-registration)
         |
-        |Copy and paste the CONSUMER_KEY, CONSUMER_SECRET and REDIRECT_URL for the subsequent steps below.
+        |Copy and paste the CLIENT ID (AKA CONSUMER KEY), CLIENT SECRET (AKA CONSUMER SECRET) and REDIRECT_URL for the subsequent steps below.
         |
         |
         |### Step 2: Initiate the OAuth 2.0 / OpenID Connect Flow
         |
-        |
         |Once you have registered your App you should initiate the OAuth2 / OIDC flow using the following URL
         |
         |${APIUtil.getHydraPublicServerUrl}/oauth2/auth
@@ -2129,6 +2128,12 @@ object Glossary extends MdcLoggable  {
         |
         |${APIUtil.getHydraPublicServerUrl}/oauth2/auth?client_id=YOUR-CLIENT-ID&response_type=code&state=GENERATED_BY_YOUR_APP&scope=openid+offline+ReadAccountsBasic+ReadAccountsDetail+ReadBalances+ReadTransactionsBasic+ReadTransactionsDebits+ReadTransactionsDetail&redirect_uri=https%3A%2F%2FYOUR-APP.com%2Fmain.html
         |
+        |### Step 3: Exchange the authorisation code for an access token
+        |
+        |The token endpoint is:
+        |
+        |${APIUtil.getHydraPublicServerUrl}/oauth2/token
+        |
         |
         |For further information please see [here](https://www.ory.sh/hydra/docs/concepts/login#initiating-the-oauth-20--openid-connect-flow)
         |

From 3ee6fde057bcf08dde6ffa6e526a1fdc9737c422 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 12 Dec 2023 15:34:49 +0100
Subject: [PATCH 32/36] docfix/Tweak props mirror_request_headers_to_response

---
 obp-api/src/main/resources/props/sample.props.template | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index 3b2fcf9b6..91eacabcc 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -1039,6 +1039,13 @@ outboundAdapterCallContext.generalContext
 # ------------------------------ default entitlements end ------------------------------
 
 ## Mirror request headers to response
+# This feature is driven by FAPI requirements. For instance:
+# The resource server with the FAPI endpoints
+# - shall set the response header x-fapi-interaction-id to the value
+#   received from the corresponding fapi client request header or to a RFC4122 UUID value
+#   if the request header was not provided to track the interaction, e.g.,
+#   x-fapi-interaction-id: c770aef3-6784-41f7-8e0e-ff5f97bddb3a; and
+# - shall log the value of x-fapi-interaction-id in the log entry.
 # mirror_request_headers_to_response=x-fapi-interaction-id,x-jws-signature
 
 ## Echo all request headers to response

From cc825f2a453fa12e7accbaac1a9b968d047ba3b8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Tue, 12 Dec 2023 15:49:48 +0100
Subject: [PATCH 33/36] docfix/Tweak props echo_request_headers

---
 obp-api/src/main/resources/props/sample.props.template | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index 91eacabcc..517b43c26 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -1049,6 +1049,8 @@ outboundAdapterCallContext.generalContext
 # mirror_request_headers_to_response=x-fapi-interaction-id,x-jws-signature
 
 ## Echo all request headers to response
+# Rename all request headers by prepending the word "echo_" and sends them back as response headers
+# This feature helps to reveal information does every request header sent at a client side really reach a server side
 echo_request_headers=false
 
 ### enable or disable the feature of send "Force-Error" header, default value is false

From ff8cb866dc361ff43e579aeb730281f1b8387ee7 Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Tue, 12 Dec 2023 12:58:00 +0100
Subject: [PATCH 34/36] feature/show ms for getMetrics and getConnectorMetrics
 endpoints

---
 .../src/main/scala/code/api/util/APIUtil.scala    | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala
index 3a2d93e77..16105575f 100644
--- a/obp-api/src/main/scala/code/api/util/APIUtil.scala
+++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala
@@ -118,6 +118,8 @@ import java.security.AccessControlException
 import java.util.regex.Pattern
 
 import code.api.util.FutureUtil.{EndpointContext, EndpointTimeout}
+import code.api.v2_1_0.OBPAPI2_1_0.Implementations2_1_0
+import code.api.v2_2_0.OBPAPI2_2_0.Implementations2_2_0
 import code.etag.MappedETag
 import code.users.Users
 import net.liftweb.mapper.By
@@ -643,7 +645,18 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{
   val excludedFieldValues = APIUtil.getPropsValue("excluded.response.field.values").map[JArray](it => json.parse(it).asInstanceOf[JArray])
 
   def successJsonResponseNewStyle(cc: Any, callContext: Option[CallContext], httpCode : Int = 200)(implicit headers: CustomResponseHeaders = CustomResponseHeaders(Nil)) : JsonResponse = {
-    val jsonAst: JValue = ApiSession.processJson((Extraction.decompose(cc)), callContext)
+    val jsonAst: JValue = {
+      val partialFunctionName = callContext.map(_.resourceDocument.map(_.partialFunctionName)).flatten.getOrElse("")
+      if (
+        nameOf(code.api.v5_1_0.APIMethods510.Implementations5_1_0.getMetrics).equals(partialFunctionName) ||
+        nameOf(code.api.v5_0_0.APIMethods500.Implementations5_0_0.getMetricsAtBank).equals(partialFunctionName) ||
+        nameOf(Implementations2_2_0.getConnectorMetrics).equals(partialFunctionName)
+      ) {
+        ApiSession.processJson(Extraction.decompose(cc)(CustomJsonFormats.losslessFormats), callContext)
+      } else {
+        ApiSession.processJson((Extraction.decompose(cc)), callContext)
+      }
+    }
     val excludeOptionalFieldsParam = getHttpRequestUrlParam(callContext.map(_.url).getOrElse(""),"exclude-optional-fields")
     val excludedResponseBehaviour = APIUtil.getPropsAsBoolValue("excluded.response.behaviour", false)
     //excludeOptionalFieldsParamValue has top priority, then the excludedResponseBehaviour props.

From 84fa1a46f961f715c6159062353fc71aa8eb9676 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20Mili=C4=87?= <marko@tesobe.com>
Date: Wed, 13 Dec 2023 09:42:14 +0100
Subject: [PATCH 35/36] docfix/Tweak props create_just_in_time_entitlements

---
 obp-api/src/main/resources/props/sample.props.template | 4 +++-
 obp-api/src/main/scala/code/api/util/Glossary.scala    | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template
index 517b43c26..1468010a1 100644
--- a/obp-api/src/main/resources/props/sample.props.template
+++ b/obp-api/src/main/resources/props/sample.props.template
@@ -865,7 +865,9 @@ create_just_in_time_entitlements=false
 # if create_just_in_time_entitlements=true then OBP does the following:
 # If a user is trying to use a Role (via an endpoint) and the user could grant them selves the required Role(s), then OBP automatically grants the Role!
 # i.e. if the User already has canCreateEntitlementAtOneBank or canCreateEntitlementAtAnyBank and then OBP will auto grant a role that could be granted by a manual process anyway.
-# This speeds up the process of granting of roles. Certain roles are excluded from this automation.
+# This speeds up the process of granting of roles. Certain roles are excluded from this automation:
+# - CanCreateEntitlementAtOneBank
+# - CanCreateEntitlementAtAnyBank
 # If create_just_in_time_entitlements is again set to false after it was true for a while, any auto granted Entitlements to roles are kept in place.
 # Note: In the entitlements model we set createdbyprocess="create_just_in_time_entitlements". For manual operations we set createdbyprocess="manual"
 # -------------------------------------------------------------
diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala
index cee7c95e1..7a7657934 100644
--- a/obp-api/src/main/scala/code/api/util/Glossary.scala
+++ b/obp-api/src/main/scala/code/api/util/Glossary.scala
@@ -643,7 +643,9 @@ object Glossary extends MdcLoggable  {
 				 |If Just in Time Entitlements are enabled then OBP does the following:
 				 |If a user is trying to use a Role (via an endpoint) and the user could grant them selves the required Role(s), then OBP automatically grants the Role.
 				 |i.e. if the User already has canCreateEntitlementAtOneBank or canCreateEntitlementAtAnyBank then OBP will automatically grant a role that would be granted by a manual process anyway.
-				 |This speeds up the process of granting of roles. Certain roles are excluded from this automation.
+				 |This speeds up the process of granting of roles. Certain roles are excluded from this automation:
+				 |  - CanCreateEntitlementAtOneBank
+				 |  - CanCreateEntitlementAtAnyBank
 				 |If create_just_in_time_entitlements is again set to false after it was true for a while, any auto granted Entitlements to roles are kept in place.
 				 |Note: In the entitlements model we set createdbyprocess=create_just_in_time_entitlements. For manual operations we set createdbyprocess=manual
 				 |

From 0a245bce197ea900577388debb77fbeb15eeaffd Mon Sep 17 00:00:00 2001
From: hongwei <hongwei@tesobe.com>
Date: Wed, 13 Dec 2023 15:11:23 +0100
Subject: [PATCH 36/36] docfix/added the comments for redirect ulrs

---
 obp-api/src/main/webapp/WEB-INF/web.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/obp-api/src/main/webapp/WEB-INF/web.xml b/obp-api/src/main/webapp/WEB-INF/web.xml
index 16bd7588d..f421ec503 100644
--- a/obp-api/src/main/webapp/WEB-INF/web.xml
+++ b/obp-api/src/main/webapp/WEB-INF/web.xml
@@ -15,6 +15,13 @@
 
     <filter-mapping>
         <filter-name>LiftFilter</filter-name>
+<!--TODO can prepare the white list later    -->
+<!--        We can specify folders or files that we want Liftweb to process.-->
+<!--        For example, we generally want Liftweb to process all HTML files because they are dynamic, and all the RESTful endpoints.-->
+<!--        On the other hand, we generally do not want Liftweb to process JS, JPG, and CSS files because they are static.-->
+<!--        Any processing we can offload from Liftweb is good for performance.-->
+<!--        Note: Jetty will serve the static files, so they will still appear in the Jetty logs.-->
+<!--        Ideally, NGINX is used to serve the static files, preventing them from cluttering the Jetty logs.-->
         <url-pattern>/*</url-pattern>
     </filter-mapping>