From c6f4df7a031d0413cfe6f8859b9476ca1905f3b8 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 2 Feb 2026 23:07:18 +0100 Subject: [PATCH] DBUtil for MS SQL Server handling of NVARCHAR (JDBC type -9) --- .../src/main/scala/code/api/util/DBUtil.scala | 68 +++++++++++++++++++ .../scala/code/metrics/MappedMetrics.scala | 35 ++++++---- .../code/model/dataAccess/ResourceUser.scala | 5 +- .../scala/code/util/AttributeQueryTrait.scala | 9 ++- .../code/util/NewAttributeQueryTrait.scala | 9 ++- 5 files changed, 104 insertions(+), 22 deletions(-) diff --git a/obp-api/src/main/scala/code/api/util/DBUtil.scala b/obp-api/src/main/scala/code/api/util/DBUtil.scala index f3f4901da..d9c6d9b50 100644 --- a/obp-api/src/main/scala/code/api/util/DBUtil.scala +++ b/obp-api/src/main/scala/code/api/util/DBUtil.scala @@ -1,9 +1,77 @@ package code.api.util import code.api.Constant +import net.liftweb.db.{DB, DefaultConnectionIdentifier} +import net.liftweb.util.Helpers.tryo + +import java.sql.{ResultSet, Types} object DBUtil { def dbUrl: String = APIUtil.getPropsValue("db.url") openOr Constant.h2DatabaseDefaultUrlValue + + def isSqlServer: Boolean = dbUrl.contains("sqlserver") + + /** + * SQL Server-safe alternative to Lift's DB.runQuery. + * + * Lift's DB.runQuery uses DB.asString which doesn't handle SQL Server's NVARCHAR type + * (JDBC type -9), causing MatchError. This function handles all JDBC types properly. + * + * @param query SQL query string + * @param params Query parameters (for prepared statement) + * @return Tuple of (column names, rows as List[List[String]]) + */ + def runQuery(query: String, params: List[String] = Nil): (List[String], List[List[String]]) = { + DB.use(DefaultConnectionIdentifier) { conn => + val stmt = conn.prepareStatement(query) + try { + // Set parameters + params.zipWithIndex.foreach { case (param, idx) => + stmt.setString(idx + 1, param) + } + + val rs = stmt.executeQuery() + val meta = rs.getMetaData + val colCount = meta.getColumnCount + + // Get column names + val colNames = (1 to colCount).map(i => meta.getColumnName(i)).toList + + // Get rows - convert all types to String safely + var rows = List[List[String]]() + while (rs.next()) { + val row = (1 to colCount).map { i => + safeGetString(rs, i, meta.getColumnType(i)) + }.toList + rows = rows :+ row + } + + (colNames, rows) + } finally { + stmt.close() + } + } + } + + /** + * Safely convert any JDBC type to String, including SQL Server's NVARCHAR (-9). + */ + private def safeGetString(rs: ResultSet, columnIndex: Int, jdbcType: Int): String = { + val value = jdbcType match { + case Types.NVARCHAR | Types.NCHAR | Types.LONGNVARCHAR | Types.NCLOB => + // SQL Server NVARCHAR types that Lift doesn't handle + rs.getNString(columnIndex) + case Types.CLOB => + val clob = rs.getClob(columnIndex) + if (clob != null) clob.getSubString(1, clob.length().toInt) else null + case Types.BLOB => + val blob = rs.getBlob(columnIndex) + if (blob != null) new String(blob.getBytes(1, blob.length().toInt)) else null + case _ => + rs.getString(columnIndex) + } + if (rs.wasNull()) null else value + } def getDbConnectionParameters: (String, String, String) = { dbUrl.contains("jdbc:h2") match { diff --git a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala index c9ba35b15..350a5c4a2 100644 --- a/obp-api/src/main/scala/code/metrics/MappedMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/MappedMetrics.scala @@ -436,7 +436,8 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) """.stripMargin - val (_, rows) = DB.runQuery(sqlQuery, List()) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, rows) = DBUtil.runQuery(sqlQuery) logger.debug("code.metrics.MappedMetrics.getAllAggregateMetricsBox.sqlQuery --: " + sqlQuery) logger.info(s"getAllAggregateMetricsBox - Query executed, returned ${rows.length} rows") val sqlResult = rows.map( @@ -504,13 +505,13 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val (dbUrl, _, _) = DBUtil.getDbConnectionParameters - val result: List[TopApi] = { + val result: Box[List[TopApi]] = tryo { // MS SQL server has the specific syntax for limiting number of rows val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" // TODO Make it work in case of Oracle database val otherDbLimit = if (dbUrl.contains("sqlserver")) s"" else s"LIMIT $limit" val sqlQuery: String = - s"""SELECT ${msSqlLimit} count(*), metric.implementedbypartialfunction, metric.implementedinversion + s"""SELECT ${msSqlLimit} count(*), metric.implementedbypartialfunction, metric.implementedinversion FROM metric WHERE date_c >= '${sqlTimestamp(fromDate.get)}' AND @@ -522,29 +523,35 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ AND (${trueOrFalse(url.isEmpty)} or url = ${url.getOrElse("null")}) AND (${trueOrFalse(appName.isEmpty)} or appname = ${appName.getOrElse("null")}) AND (${trueOrFalse(verb.isEmpty)} or verb = ${verb.getOrElse("null")}) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = null) - AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != null) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(true)))} or userid = null) + AND (${falseOrTrue(anon.isDefined && anon.equals(Some(false)))} or userid != null) AND (${trueOrFalse(httpStatusCode.isEmpty)} or httpcode = ${sqlFriendlyInt(httpStatusCode)}) AND (${trueOrFalse(excludeUrlPatterns.isEmpty)} or (url NOT LIKE ($excludeUrlPatternsQueries))) AND (${trueOrFalse(excludeAppNames.isEmpty)} or appname not in ($excludeAppNamesNumberList)) AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty)} or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsNumberList)) - GROUP BY metric.implementedbypartialfunction, metric.implementedinversion + GROUP BY metric.implementedbypartialfunction, metric.implementedinversion ORDER BY count(*) DESC ${otherDbLimit} """.stripMargin - val (_, rows) = DB.runQuery(sqlQuery, List()) + logger.debug(s"getTopApisFuture SQL query: $sqlQuery") + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, rows) = DBUtil.runQuery(sqlQuery) + logger.debug(s"getTopApisFuture returned ${rows.length} rows") + if (rows.nonEmpty) { + logger.debug(s"getTopApisFuture first row sample: ${rows.head}") + } val sqlResult = rows.map { rs => // Map result to case class TopApi( - rs(0).toInt, + tryo(rs(0).toInt).getOrElse(0), // Safe conversion with fallback rs(1), rs(2) ) } sqlResult } - tryo(result) + result }} }} @@ -591,11 +598,10 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ val msSqlLimit = if (dbUrl.contains("sqlserver")) s"TOP ($limit)" else s"" // TODO Make it work in case of Oracle database val otherDbLimit: String = if (dbUrl.contains("sqlserver")) s"" else s"LIMIT $limit" - val result: List[TopConsumer] = { val sqlQuery = - s"""SELECT ${msSqlLimit} count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, - consumer.developeremail as email, consumer.consumerid as consumerid + s"""SELECT ${msSqlLimit} count(*) as count, consumer.id as consumerprimaryid, metric.appname as appname, + consumer.developeremail as email, consumer.consumerid as consumerid FROM metric, consumer WHERE metric.appname = consumer.name AND date_c >= '${sqlTimestamp(fromDate.get)}' @@ -613,11 +619,12 @@ object MappedMetrics extends APIMetrics with MdcLoggable{ AND (${trueOrFalse(excludeUrlPatterns.isEmpty) } or (url NOT LIKE ($excludeUrlPatternsQueries))) AND (${trueOrFalse(excludeAppNames.isEmpty) } or appname not in ($excludeAppNamesList)) AND (${trueOrFalse(excludeImplementedByPartialFunctions.isEmpty) } or implementedbypartialfunction not in ($excludeImplementedByPartialFunctionsList)) - GROUP BY appname, consumer.developeremail, consumer.id, consumer.consumerid + GROUP BY appname, consumer.developeremail, consumer.id, consumer.consumerid ORDER BY count DESC ${otherDbLimit} """.stripMargin - val (_, rows) = DB.runQuery(sqlQuery, List()) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, rows) = DBUtil.runQuery(sqlQuery) val sqlResult = rows.map { rs => // Map result to case class TopConsumer( diff --git a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala index 810d1bc6c..896afc789 100644 --- a/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala +++ b/obp-api/src/main/scala/code/model/dataAccess/ResourceUser.scala @@ -31,7 +31,7 @@ import java.util.UUID.randomUUID import code.api.Constant import code.api.cache.Caching -import code.api.util.APIUtil +import code.api.util.{APIUtil, DBUtil} import code.util.MappedUUID import com.openbankproject.commons.model.{User, UserPrimaryKey} import com.tesobe.CacheKeyFromArguments @@ -141,7 +141,8 @@ object ResourceUser extends ResourceUser with LongKeyedMetaMapper[ResourceUser]{ CacheKeyFromArguments.buildCacheKey { Caching.memoizeSyncWithProvider(Some(cacheKey.toString()))(cacheTTL.seconds) { val sql = "SELECT DISTINCT provider_ FROM resourceuser ORDER BY provider_" - val (_, rows) = DB.runQuery(sql, List()) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, rows) = DBUtil.runQuery(sql) rows.flatten } } diff --git a/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala b/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala index 37ea818ff..3714f3a03 100644 --- a/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala +++ b/obp-api/src/main/scala/code/util/AttributeQueryTrait.scala @@ -1,7 +1,8 @@ package code.util +import code.api.util.DBUtil import com.openbankproject.commons.model.BankId -import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper, DB} +import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper} import scala.collection.immutable.List @@ -39,7 +40,8 @@ trait AttributeQueryTrait { self: BaseMetaMapper => def getParentIdByParams(bankId: BankId, params: Map[String, List[String]]): List[String] = { if (params.isEmpty) { val sql = s"SELECT DISTINCT attr.$parentIdColumn FROM $tableName attr where attr.$bankIdColumn = ? " - val (_, list) = DB.runQuery(sql, List(bankId.value)) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, list) = DBUtil.runQuery(sql, List(bankId.value)) list.flatten } else { val paramList = params.toList @@ -67,7 +69,8 @@ trait AttributeQueryTrait { self: BaseMetaMapper => | AND ($sqlParametersFilter) |""".stripMargin - val (columnNames: List[String], list: List[List[String]]) = DB.runQuery(sql, bankId.value :: parameters) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (columnNames: List[String], list: List[List[String]]) = DBUtil.runQuery(sql, bankId.value :: parameters) val columnNamesLowerCase = columnNames.map(_.toLowerCase) val parentIdIndex = columnNamesLowerCase.indexOf(parentIdColumn.toLowerCase) val nameIndex = columnNamesLowerCase.indexOf(nameColumn.toLowerCase) diff --git a/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala b/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala index 4d5f14769..a0a579574 100644 --- a/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala +++ b/obp-api/src/main/scala/code/util/NewAttributeQueryTrait.scala @@ -1,7 +1,8 @@ package code.util +import code.api.util.DBUtil import com.openbankproject.commons.model.BankId -import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper, DB} +import net.liftweb.mapper.{BaseMappedField, BaseMetaMapper} import scala.collection.immutable.List @@ -36,7 +37,8 @@ trait NewAttributeQueryTrait { def getParentIdByParams(bankId: BankId, params: Map[String, List[String]]): List[String] = { if (params.isEmpty) { val sql = s"SELECT DISTINCT attr.$parentIdColumn FROM $tableName attr where attr.$bankIdColumn = ? " - val (_, list) = DB.runQuery(sql, List(bankId.value)) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (_, list) = DBUtil.runQuery(sql, List(bankId.value)) list.flatten } else { val paramList = params.toList @@ -64,7 +66,8 @@ trait NewAttributeQueryTrait { | AND ($sqlParametersFilter) |""".stripMargin - val (columnNames: List[String], list: List[List[String]]) = DB.runQuery(sql, bankId.value :: parameters) + // Use DBUtil.runQuery which handles SQL Server NVARCHAR properly + val (columnNames: List[String], list: List[List[String]]) = DBUtil.runQuery(sql, bankId.value :: parameters) val columnNamesLowerCase = columnNames.map(_.toLowerCase) val parentIdIndex = columnNamesLowerCase.indexOf(parentIdColumn.toLowerCase) val nameIndex = columnNamesLowerCase.indexOf(nameColumn.toLowerCase)