artin/gameyfin
1
0
Fork 0
mirror of https://github.com/grimsi/gameyfin.git synced 2026-02-06 11:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement file size calculation

Browse Source 
Implement compression level config for DirectDownloadPlugin
Fix download of games hogging RAM
This commit is contained in:
grimsi 2025-05-15 22:57:25 +02:00
parent fc3a6fd52f
commit f91b289cee
13 changed files with 157 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
gameyfin/src/main/frontend/endpoints/DownloadEndpoint.ts
View File

@ -1,30 +1,3 @@
export async function downloadGame(gameId: number, provider: string) {
try {
const response = await fetch(`/download/${gameId}?provider=${provider}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
});
if (!response.ok) {
throw new Error('Network response was not ok');
}
const blob = await response.blob();
const url = window.URL.createObjectURL(blob);
const contentDisposition = response.headers.get('Content-Disposition');
const filename = contentDisposition
? contentDisposition.split('filename=')[1].replace(/"/g, '')
: 'downloaded_file';
const link = document.createElement('a');
link.href = url;
link.setAttribute('download', filename);
document.body.appendChild(link);
link.click();
link.remove();
} catch (error) {
console.error('Error downloading the file', error);
}
export function downloadGame(gameId: number, provider: string) {
window.open(`/download/${gameId}?provider=${provider}`, '_top');
}

32
gameyfin/src/main/frontend/util/utils.ts
View File

@ -87,6 +87,38 @@ export function timeUntil(instantString: string, timeZone: string = moment.tz.gu
return "just now";
}
/**
* Format bytes as human-readable text.
*
* @param bytes Number of bytes.
* @param si True to use metric (SI) units, aka powers of 1000. False to use
* binary (IEC), aka powers of 1024.
* @param dp Number of decimal places to display.
*
* @return Formatted string.
*/
export function humanFileSize(bytes: number, si: boolean = false, dp: number = 1) {
const thresh = si ? 1000 : 1024;
if (Math.abs(bytes) < thresh) {
return bytes + ' B';
}
const units = si
? ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'];
let u = -1;
const r = 10 ** dp;
do {
bytes /= thresh;
++u;
} while (Math.round(Math.abs(bytes) * r) / r >= thresh && u < units.length - 1);
return bytes.toFixed(dp) + ' ' + units[u];
}
/**
* Select a random number of games from the library based on the library ID.
* @param library

4
gameyfin/src/main/frontend/views/GameView.tsx
View File

@ -6,7 +6,7 @@ import {GameCover} from "Frontend/components/general/covers/GameCover";
import ComboButton, {ComboButtonOption} from "Frontend/components/general/input/ComboButton";
import ImageCarousel from "Frontend/components/general/covers/ImageCarousel";
import {Chip} from "@heroui/react";
import {toTitleCase} from "Frontend/util/utils";
import {humanFileSize, toTitleCase} from "Frontend/util/utils";
import {DownloadEndpoint} from "Frontend/endpoints/endpoints";
export default function GameView() {
@ -61,7 +61,7 @@ export default function GameView() {
<p className="text-foreground/60">{game.release !== undefined ? new Date(game.release).getFullYear() : "unknown"}</p>
</div>
</div>
<ComboButton description="64 GiB"
<ComboButton description={humanFileSize(game.fileSize)}
options={downloadOptions}
preferredOptionKey="preferred-download-method"
/>

22
gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadEndpoint.kt
View File

@ -4,10 +4,9 @@ import de.grimsi.gameyfin.core.annotations.DynamicPublicAccess
import de.grimsi.gameyfin.games.GameService
import de.grimsi.gameyfin.pluginapi.download.FileDownload
import de.grimsi.gameyfin.pluginapi.download.LinkDownload
import org.springframework.core.io.InputStreamResource
import org.springframework.core.io.Resource
import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.*
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody
@RestController
@RequestMapping("/download")
@ -21,19 +20,20 @@ class DownloadEndpoint(
}
@GetMapping("/{gameId}")
fun downloadGame(@PathVariable gameId: Long, @RequestParam provider: String): ResponseEntity<Resource> {
fun downloadGame(
@PathVariable gameId: Long,
@RequestParam provider: String
): ResponseEntity<StreamingResponseBody> {
val game = gameService.getGame(gameId)
val downloadElement = downloadService.getDownloadElement(game.path, provider)
val download = downloadService.getDownload(game.path, provider)
return when (downloadElement) {
return when (download) {
is FileDownload -> {
val resource = InputStreamResource(downloadElement.data)
ResponseEntity.ok()
.header(
"Content-Disposition",
"attachment; filename=\"${game.title}.${downloadElement.fileExtension}\""
)
.body(resource)
.header("Content-Disposition", "attachment; filename=\"${game.title}.zip\"")
.body(StreamingResponseBody { outputStream ->
download.data.copyTo(outputStream)
})
}
is LinkDownload -> {

4
gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/download/DownloadService.kt
View File

@ -17,10 +17,10 @@ class DownloadService(
return downloadPlugins.map { it.javaClass.name }
}
fun getDownloadElement(path: String, provider: String): Download {
fun getDownload(path: String, provider: String): Download {
val provider = downloadPlugins.firstOrNull { it.javaClass.name == provider }
?: throw IllegalArgumentException("Download provider $provider not found")
return provider.getDownloadSources(Path(path))
return provider.download(Path(path))
}
}

17
gameyfin/src/main/kotlin/de/grimsi/gameyfin/core/filesystem/FilesystemService.kt
View File

@ -6,6 +6,7 @@ import de.grimsi.gameyfin.libraries.Library
import io.github.oshai.kotlinlogging.KotlinLogging
import org.apache.commons.io.FilenameUtils
import org.springframework.stereotype.Service
import java.io.File
import java.nio.file.FileSystems
import java.nio.file.Path
import kotlin.io.path.*
@ -125,6 +126,22 @@ class FilesystemService(
)
}
fun calculateFileSize(path: String): Long {
return try {
val file = File(path)
if (file.isFile) {
file.length()
} else if (file.isDirectory) {
File(path).walkTopDown().filter { it.isFile }.map { it.length() }.sum()
} else {
0L
}
} catch (e: Exception) {
log.warn { "Error calculating file size for $path: ${e.message}" }
0L
}
}
private fun safeReadDirectoryContents(path: String): List<FileDto> {
return safeReadDirectoryContents(Path(path))
.map { FileDto(it.name, if (it.isDirectory()) FileType.DIRECTORY else FileType.FILE, it.hashCode()) }

1
gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/GameService.kt
View File

@ -345,6 +345,7 @@ fun Game.toDto(): GameDto {
imageIds = this.images.mapNotNull { it.id },
videoUrls = this.videoUrls.map { it.toString() },
path = this.path,
fileSize = this.fileSize ?: 0L,
metadata = toDto(this.metadata),
originalIds = this.originalIds.mapKeys { it.key.pluginId }
)

1
gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/dto/GameDto.kt
View File

@ -24,6 +24,7 @@ class GameDto(
val imageIds: List<Long>?,
val videoUrls: List<String>?,
val path: String,
val fileSize: Long,
val metadata: Map<String, GameMetadataDto>,
val originalIds: Map<String, String>
)

2
gameyfin/src/main/kotlin/de/grimsi/gameyfin/games/entities/Game.kt
View File

@ -78,6 +78,8 @@ class Game(
@Column(unique = true)
val path: String,
var fileSize: Long? = null,
@OneToMany(cascade = [CascadeType.ALL], orphanRemoval = true)
var metadata: Map<String, FieldMetadata> = emptyMap(),

23
gameyfin/src/main/kotlin/de/grimsi/gameyfin/libraries/LibraryService.kt
View File

@ -170,6 +170,7 @@ class LibraryService(
val totalPaths = gamePaths.size
val completedMetadata = AtomicInteger(0)
val completedImageDownload = AtomicInteger(0)
val calculatedFileSize = AtomicInteger(0)
log.info { "Scanning library '${library.name}' with $totalPaths paths..." }
@ -238,14 +239,30 @@ class LibraryService(
val gamesWithImages = executor.invokeAll(imageDownloadTasks).mapNotNull { it.get() }
// 3. Persist new games
// 3. Calculate game file sizes
val calculateFileSizeTask = matchedGames.map { game ->
Callable {
game.path.let { path ->
val fileSize = filesystemService.calculateFileSize(path)
game.fileSize = fileSize
val progress = calculatedFileSize.incrementAndGet()
log.debug { "${progress}/${totalPaths} file sizes calculated" }
game
}
}
}
val gamesWithFileSizes = executor.invokeAll(calculateFileSizeTask).map { it.get() }
// 4. Persist new games
val persistedGames = gameService.create(gamesWithImages)
log.debug { "${persistedGames.size}/${totalPaths} saved to database" }
// 4. Add new games to library
// 5. Add new games to library
addGamesToLibrary(persistedGames, library)
// 5. Persist library
// 6. Persist library
libraryRepository.save(library)
return LibraryScanResult(

4
gameyfin/src/main/resources/application.yml
View File

@ -39,6 +39,10 @@ spring:
fs.filesystem-root: ./data/
application:
name: Gameyfin
threads:
virtual.enabled: true
mvc:
async.request-timeout: 0
vaadin:
# To improve the performance during development.

2
plugin-api/src/main/kotlin/de/grimsi/gameyfin/pluginapi/download/DownloadProvider.kt
View File

@ -5,5 +5,5 @@ import java.nio.file.Path
interface DownloadProvider : ExtensionPoint {
fun getDownloadSources(path: Path): Download
fun download(path: Path): Download
}

103
plugins/directdownload/src/main/kotlin/de/grimsi/gameyfin/plugins/directdownload/DirectDownloadPlugin.kt
View File

@ -10,13 +10,12 @@ import org.pf4j.Extension
import org.pf4j.PluginWrapper
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.io.IOException
import java.io.InputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.nio.file.*
import java.nio.file.attribute.BasicFileAttributes
import java.util.concurrent.BlockingQueue
import java.util.concurrent.LinkedBlockingQueue
import java.util.zip.Deflater
import java.util.zip.ZipEntry
import java.util.zip.ZipOutputStream
@ -26,6 +25,15 @@ import kotlin.io.path.fileSize
import kotlin.io.path.isDirectory
class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Configurable {
companion object {
lateinit var plugin: DirectDownloadPlugin
private set
}
init {
plugin = this
}
val log: Logger = LoggerFactory.getLogger(javaClass)
enum class CompressionMode {
@ -68,64 +76,77 @@ class DirectDownloadPlugin(wrapper: PluginWrapper) : GameyfinPlugin(wrapper), Co
@Extension
class DirectDownloadProvider : DownloadProvider {
companion object {
private val END_OF_QUEUE = Pair<ZipEntry, Path>(ZipEntry("__END__"), Paths.get(""))
}
override fun getDownloadSources(path: Path): Download {
override fun download(path: Path): Download {
if (!path.exists()) throw IllegalArgumentException("Path $path does not exist")
return FileDownload(
data = readContentAsSingleFile(path),
data = streamContentAsSingleFile(path),
fileExtension = if (path.isDirectory()) "zip" else path.extension,
size = path.fileSize()
)
}
fun readContentAsSingleFile(path: Path): InputStream {
if (path.isDirectory()) return zipFilesInPath(path)
return Files.newInputStream(path, StandardOpenOption.READ)
fun streamContentAsSingleFile(path: Path): InputStream {
if (path.isDirectory()) return streamFolderAsZip(path)
return streamFile(path)
}
private fun zipFilesInPath(path: Path): InputStream {
val pipedIn = PipedInputStream(64 * 1024)
val pipedOut = PipedOutputStream(pipedIn)
val queue: BlockingQueue<Pair<ZipEntry, Path>?> = LinkedBlockingQueue()
fun streamFile(path: Path): InputStream {
val pipeIn = PipedInputStream(512 * 1024)
val pipeOut = PipedOutputStream(pipeIn)
// Producer: walks the file tree and enqueues files
Thread.startVirtualThread {
Thread.ofVirtual().start {
try {
Files.walkFileTree(path, object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
val entry = ZipEntry(path.relativize(file).toString())
queue.put(entry to file)
return FileVisitResult.CONTINUE
}
})
Files.newInputStream(path, StandardOpenOption.READ).use { input ->
input.copyTo(pipeOut, 512 * 1024)
}
} catch (_: IOException) {
} finally {
queue.put(END_OF_QUEUE) // signal end
try {
pipeOut.close()
} catch (_: IOException) {
}
}
}
// Consumer: zips files in parallel, but writes entries in order
Thread {
ZipOutputStream(pipedOut).use { zos ->
zos.setLevel(Deflater.NO_COMPRESSION)
while (true) {
val item = queue.take()
if (item === END_OF_QUEUE || item == null) break
val (entry, file) = item
zos.putNextEntry(entry)
Files.newInputStream(file, StandardOpenOption.READ).use { input ->
input.copyTo(zos, 128 * 1024)
}
zos.closeEntry()
return pipeIn
}
fun streamFolderAsZip(path: Path): InputStream {
val pipeIn = PipedInputStream(512 * 1024) // 512 KB buffer
val pipeOut = PipedOutputStream(pipeIn)
Thread.ofVirtual().start {
try {
ZipOutputStream(pipeOut).use { zos ->
zos.setLevel(CompressionMode.toDeflaterLevel(plugin.config["compressionMode"]?.let {
CompressionMode.valueOf(it.uppercase())
} ?: CompressionMode.NONE))
Files.walkFileTree(path, object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
val entry = ZipEntry(path.relativize(file).toString())
zos.putNextEntry(entry)
Files.newInputStream(file, StandardOpenOption.READ).use { input ->
input.copyTo(zos, 512 * 1024)
}
zos.closeEntry()
return FileVisitResult.CONTINUE
}
})
}
pipeOut.close()
} catch (_: IOException) {
} finally {
try {
pipeOut.close()
} catch (_: IOException) {
}
}
pipedOut.close()
}.start()
}
return pipedIn
return pipeIn
}
}
}