artin/OBP-API
1
0
Fork 0
mirror of https://github.com/OpenBankProject/OBP-API.git synced 2026-02-06 11:06:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "refactor/Implement a streaming YAML generation to avoid large memory spikes"

Browse Source 
This reverts commit e6009facca.
This commit is contained in:
Marko Milić 2026-02-04 09:45:55 +01:00
parent 8c377f3b52
commit 1cfcbf3442
2 changed files with 22 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocs140.scala
View File

@ -11,7 +11,7 @@ import code.apicollectionendpoint.MappedApiCollectionEndpointsProvider
import code.util.Helper.{MdcLoggable, SILENCE_IS_GOLDEN}
import com.openbankproject.commons.model.enums.ContentParam.{DYNAMIC, STATIC}
import com.openbankproject.commons.util.{ApiVersion, ApiVersionStatus}
import net.liftweb.http.{GetRequest, InMemoryResponse, PlainTextResponse, Req, S, StreamingResponse}
import net.liftweb.http.{GetRequest, InMemoryResponse, PlainTextResponse, Req, S}
object ResourceDocs140 extends OBPRestHelper with ResourceDocsAPIMethods with MdcLoggable {
@ -198,15 +198,10 @@ object ResourceDocs300 extends OBPRestHelper with ResourceDocsAPIMethods with Md
)
val cacheValueFromRedis = Caching.getStaticSwaggerDocCache(cacheKey)
if (cacheValueFromRedis.isDefined) {
// If we already have a cached YAML string, serve it as before.
val yamlString = cacheValueFromRedis.get
val headers = List("Content-Type" -> YAMLUtils.getYAMLContentType, (ResponseHeader.`Correlation-Id` -> getCorrelationId()))
val bytes = yamlString.getBytes("UTF-8")
InMemoryResponse(bytes, headers, Nil, 200)
val yamlString = if (cacheValueFromRedis.isDefined) {
cacheValueFromRedis.get
} else {
// Generate OpenAPI JSON JValue (this may be large) but stream YAML generation to avoid
// building a huge YAML string in memory.
// Generate OpenAPI JSON and convert to YAML
val openApiJValue = try {
val resourceDocsJsonFiltered = locale match {
case _ if (apiCollectionIdParam.isDefined) =>
@ -236,30 +231,14 @@ object ResourceDocs300 extends OBPRestHelper with ResourceDocsAPIMethods with Md
throw e
}
// Attempt to obtain an InputStream that streams YAML
YAMLUtils.jValueToYAMLInputStream(openApiJValue) match {
case scala.util.Success(in) =>
val headers = List("Content-Type" -> YAMLUtils.getYAMLContentType, (ResponseHeader.`Correlation-Id` -> getCorrelationId()))
// StreamingResponse takes a function that returns an InputStream when called by Lift
// StreamingResponse constructor expects: data, onEnd, size, headers, cookies, code
// Provide the InputStream directly as `data`, an onEnd that closes the stream,
// -1L for unknown size, the headers, empty cookies list, and HTTP 200 code.
StreamingResponse(in, () => { try { in.close() } catch { case _: Throwable => () } }, -1L, headers, Nil, 200)
case scala.util.Failure(e) =>
logger.error(s"Error streaming OpenAPI YAML: ${e.getMessage}", e)
// Fallback: try a safe conversion to a string (may be memory heavy) and return that, or an error.
// We attempt a safe string conversion as a last resort.
val yamlResult = YAMLUtils.jValueToYAMLSafe(openApiJValue, s"# Error converting OpenAPI to YAML: ${openApiJValue.toString}")
if (yamlResult.nonEmpty) {
Caching.setStaticSwaggerDocCache(cacheKey, yamlResult)
val headers = List("Content-Type" -> YAMLUtils.getYAMLContentType, (ResponseHeader.`Correlation-Id` -> getCorrelationId()))
val bytes = yamlResult.getBytes("UTF-8")
InMemoryResponse(bytes, headers, Nil, 200)
} else {
PlainTextResponse(s"Error generating OpenAPI YAML: ${e.getMessage}", 500)
}
}
val yamlResult = YAMLUtils.jValueToYAMLSafe(openApiJValue, s"# Error converting OpenAPI to YAML: ${openApiJValue.toString}")
Caching.setStaticSwaggerDocCache(cacheKey, yamlResult)
yamlResult
}
val headers = List("Content-Type" -> YAMLUtils.getYAMLContentType, (ResponseHeader.`Correlation-Id` -> getCorrelationId()))
val bytes = yamlString.getBytes("UTF-8")
InMemoryResponse(bytes, headers, Nil, 200)
}
} catch {
case _: Exception =>

127
obp-api/src/main/scala/code/api/util/YAMLUtils.scala
View File

@ -28,22 +28,17 @@ package code.api.util
import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.core.{JsonGenerator, JsonFactory}
import net.liftweb.json.JsonAST.{JObject, JArray, JBool, JNull, JNothing, JDouble, JInt, JString, JField, JValue}
import net.liftweb.json.JsonAST.JValue
import net.liftweb.json._
import net.liftweb.json.compactRender
import code.util.Helper.MdcLoggable
import scala.util.{Try, Success, Failure}
import java.io.{OutputStream, InputStream, PipedInputStream, PipedOutputStream}
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
/**
* Utility object for YAML conversion operations
*
* This utility provides methods to convert Lift's JValue objects to YAML format
* using Jackson's YAML support. It provides both simple string-based conversion
* and streaming conversion APIs to avoid building huge intermediate strings.
* using Jackson's YAML support.
*/
object YAMLUtils extends MdcLoggable {
@ -51,121 +46,21 @@ object YAMLUtils extends MdcLoggable {
private val yamlMapper = new ObjectMapper(new YAMLFactory())
/**
* Convert a Lift JValue by writing it token-by-token to a Jackson JsonGenerator.
* This avoids creating a very large intermediate JSON string.
*/
private def writeJValueToGenerator(gen: JsonGenerator, j: JValue): Unit = {
j match {
case JObject(fields) =>
gen.writeStartObject()
fields.foreach {
case JField(name, value) =>
gen.writeFieldName(name)
writeJValueToGenerator(gen, value)
}
gen.writeEndObject()
case JArray(items) =>
gen.writeStartArray()
items.foreach(item => writeJValueToGenerator(gen, item))
gen.writeEndArray()
case JString(s) =>
gen.writeString(s)
case JInt(num) =>
// Jackson supports BigInteger via writeNumber(String)
gen.writeNumber(num.toString)
case JDouble(d) =>
gen.writeNumber(d)
// JDecimal is not available in this Lift version; high-precision decimals will
// fall through to the fallback case (written via compactRender) or be represented
// as JDouble/JInt depending on creation site.
case JBool(b) =>
gen.writeBoolean(b)
case JNull | JNothing =>
gen.writeNull()
case other =>
// fallback: write compact rendering as string
gen.writeString(compactRender(other))
}
}
/**
* Stream a JValue as YAML into a supplied OutputStream.
* The caller is responsible for closing the OutputStream when appropriate.
*
* @param jValue the JValue to serialize
* @param out the OutputStream to write YAML bytes to
* @return Try[Unit] indicating success or failure
*/
def jValueToYAMLStream(jValue: JValue, out: OutputStream): Try[Unit] = {
Try {
val gen = yamlMapper.getFactory.createGenerator(out)
try {
writeJValueToGenerator(gen, jValue)
gen.flush()
} finally {
// Do not close the provided OutputStream here; just close the generator
try { gen.close() } catch { case _: Throwable => }
}
}.recoverWith {
case ex: Exception =>
logger.error(s"Failed to stream JValue to YAML: ${ex.getMessage}", ex)
Failure(new RuntimeException(s"YAML streaming failed: ${ex.getMessage}", ex))
}
}
/**
* Provide an InputStream that streams YAML representation of the provided JValue.
* Writing is performed on a background thread into a PipedOutputStream connected to
* the returned PipedInputStream. Caller must close the InputStream when done.
*
* @param jValue the JValue to serialize
* @return Try[InputStream] that will yield the YAML bytes
*/
def jValueToYAMLInputStream(jValue: JValue): Try[InputStream] = {
Try {
val in = new PipedInputStream(64 * 1024)
val out = new PipedOutputStream(in)
// Write in a background thread so the caller can read as we generate
val writerThread = new Thread(new Runnable {
override def run(): Unit = {
try {
jValueToYAMLStream(jValue, out) match {
case Success(_) => // done
case Failure(e) =>
// attempt to write an error message into the stream so the reader sees something useful
try {
val msg = s"# Error generating YAML: ${e.getMessage}\n"
out.write(msg.getBytes("UTF-8"))
} catch { case _: Throwable => }
}
} finally {
try { out.close() } catch { case _: Throwable => }
}
}
}, "yaml-stream-writer")
writerThread.setDaemon(true)
writerThread.start()
in
}.recoverWith {
case ex: Exception =>
logger.error(s"Failed to create YAML InputStream: ${ex.getMessage}", ex)
Failure(new RuntimeException(s"Failed to create YAML InputStream: ${ex.getMessage}", ex))
}
}
/**
* Converts a JValue to YAML string (keeps compatibility). This method uses the streaming
* generator internally but still accumulates into a String (for callers that need a String).
* Prefer streaming APIs for large documents.
* Converts a JValue to YAML string
*
* @param jValue The Lift JValue to convert
* @return Try containing the YAML string or error
*/
def jValueToYAML(jValue: JValue): Try[String] = {
Try {
val baos = new java.io.ByteArrayOutputStream()
jValueToYAMLStream(jValue, baos).get
baos.toString("UTF-8")
// First convert JValue to JSON string
val jsonString = compactRender(jValue)
// Parse JSON string to Jackson JsonNode
val jsonNode: JsonNode = jsonMapper.readTree(jsonString)
// Convert JsonNode to YAML string
yamlMapper.writeValueAsString(jsonNode)
}.recoverWith {
case ex: Exception =>
logger.error(s"Failed to convert JValue to YAML: ${ex.getMessage}", ex)