From 37fd3ae76d00f7506a8a19eb746e6ef336d224a2 Mon Sep 17 00:00:00 2001 From: TheOtherP Date: Fri, 18 Jul 2025 07:13:28 +0200 Subject: [PATCH] Fuck this more --- .../downloading/FileDownloadEntity.java | 168 +- .../java/org/nzbhydra/historystats/Stats.java | 1424 ++++++++--------- .../indexers/IndexerApiAccessEntity.java | 170 +- .../org/nzbhydra/indexers/IndexerEntity.java | 120 +- .../indexers/IndexerSearchEntity.java | 136 +- .../searching/db/IdentifierKeyValuePair.java | 5 +- .../resources/config/application.properties | 22 +- 7 files changed, 1022 insertions(+), 1023 deletions(-) diff --git a/core/src/main/java/org/nzbhydra/downloading/FileDownloadEntity.java b/core/src/main/java/org/nzbhydra/downloading/FileDownloadEntity.java index c9d8387af..d4ee3c489 100644 --- a/core/src/main/java/org/nzbhydra/downloading/FileDownloadEntity.java +++ b/core/src/main/java/org/nzbhydra/downloading/FileDownloadEntity.java @@ -1,84 +1,84 @@ -package org.nzbhydra.downloading; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import jakarta.persistence.Column; -import jakarta.persistence.Convert; -import jakarta.persistence.Entity; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Index; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.Table; -import lombok.Data; -import org.hibernate.annotations.OnDelete; -import org.hibernate.annotations.OnDeleteAction; -import org.nzbhydra.config.SearchSource; -import org.nzbhydra.config.downloading.FileDownloadAccessType; -import org.nzbhydra.searching.db.SearchResultEntity; -import org.nzbhydra.springnative.ReflectionMarker; -import org.nzbhydra.web.SessionStorage; - -import java.time.Duration; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -@Data -@ReflectionMarker -@Entity -@Table(name = "indexernzbdownload", indexes = {@Index(name = "NZB_DOWNLOAD_EXT_ID", columnList = "EXTERNAL_ID")}) -public final class FileDownloadEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private int id; - @ManyToOne - @JsonIgnoreProperties(value = {"handler", "hibernateLazyInitializer"}) - @OnDelete(action = OnDeleteAction.CASCADE) - private SearchResultEntity searchResult; - @Enumerated(EnumType.STRING) - private FileDownloadAccessType nzbAccessType; - @Enumerated(EnumType.STRING) - private SearchSource accessSource; - @Convert(converter = org.springframework.data.jpa.convert.threeten.Jsr310JpaConverters.InstantConverter.class) - private Instant time = Instant.now(); - @Enumerated(EnumType.STRING) - private FileDownloadStatus status; - private String error; - private String username; - private String ip; - private String userAgent; - /** - * The age of the NZB at the time of downloading. - */ - private Integer age; - @Column(name = "EXTERNAL_ID") - private String externalId; - - public FileDownloadEntity(SearchResultEntity searchResult, FileDownloadAccessType nzbAccessType, SearchSource accessSource, FileDownloadStatus status, String error) { - this.searchResult = searchResult; - this.nzbAccessType = nzbAccessType; - this.accessSource = accessSource; - this.status = status; - this.time = Instant.now(); - this.username = SessionStorage.username.get(); - this.userAgent = SessionStorage.userAgent.get(); - this.ip = SessionStorage.IP.get(); - this.age = (int) (Duration.between(searchResult.getPubDate(), searchResult.getFirstFound()).get(ChronoUnit.SECONDS) / (24 * 60 * 60)); - setError(error); - } - - public void setError(String error) { - if (error != null && error.length() > 4000) { - this.error = error.substring(0,4000); - } else { - this.error = error; - } - } - - public FileDownloadEntity() { - this.time = Instant.now(); - } -} +package org.nzbhydra.downloading; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.Data; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; +import org.nzbhydra.config.SearchSource; +import org.nzbhydra.config.downloading.FileDownloadAccessType; +import org.nzbhydra.searching.db.SearchResultEntity; +import org.nzbhydra.springnative.ReflectionMarker; +import org.nzbhydra.web.SessionStorage; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +@Data +@ReflectionMarker +@Entity +@Table(name = "indexernzbdownload", indexes = {@Index(name = "NZB_DOWNLOAD_EXT_ID", columnList = "EXTERNAL_ID")}) +public final class FileDownloadEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private int id; + @ManyToOne + @JsonIgnoreProperties(value = {"handler", "hibernateLazyInitializer"}) + @OnDelete(action = OnDeleteAction.CASCADE) + private SearchResultEntity searchResult; + @Enumerated(EnumType.STRING) + private FileDownloadAccessType nzbAccessType; + @Enumerated(EnumType.STRING) + private SearchSource accessSource; + @Convert(converter = org.springframework.data.jpa.convert.threeten.Jsr310JpaConverters.InstantConverter.class) + private Instant time = Instant.now(); + @Enumerated(EnumType.STRING) + private FileDownloadStatus status; + private String error; + private String username; + private String ip; + private String userAgent; + /** + * The age of the NZB at the time of downloading. + */ + private Integer age; + @Column(name = "EXTERNAL_ID") + private String externalId; + + public FileDownloadEntity(SearchResultEntity searchResult, FileDownloadAccessType nzbAccessType, SearchSource accessSource, FileDownloadStatus status, String error) { + this.searchResult = searchResult; + this.nzbAccessType = nzbAccessType; + this.accessSource = accessSource; + this.status = status; + this.time = Instant.now(); + this.username = SessionStorage.username.get(); + this.userAgent = SessionStorage.userAgent.get(); + this.ip = SessionStorage.IP.get(); + this.age = (int) (Duration.between(searchResult.getPubDate(), searchResult.getFirstFound()).get(ChronoUnit.SECONDS) / (24 * 60 * 60)); + setError(error); + } + + public void setError(String error) { + if (error != null && error.length() > 4000) { + this.error = error.substring(0,4000); + } else { + this.error = error; + } + } + + public FileDownloadEntity() { + this.time = Instant.now(); + } +} diff --git a/core/src/main/java/org/nzbhydra/historystats/Stats.java b/core/src/main/java/org/nzbhydra/historystats/Stats.java index 40c9b8593..65fbe1eec 100644 --- a/core/src/main/java/org/nzbhydra/historystats/Stats.java +++ b/core/src/main/java/org/nzbhydra/historystats/Stats.java @@ -1,712 +1,712 @@ -package org.nzbhydra.historystats; - -import com.google.common.base.Stopwatch; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; -import jakarta.persistence.Query; -import org.nzbhydra.config.indexer.IndexerConfig; -import org.nzbhydra.config.indexer.SearchModuleType; -import org.nzbhydra.historystats.stats.AverageResponseTime; -import org.nzbhydra.historystats.stats.CountPerDayOfWeek; -import org.nzbhydra.historystats.stats.CountPerHourOfDay; -import org.nzbhydra.historystats.stats.DownloadOrSearchSharePerUserOrIp; -import org.nzbhydra.historystats.stats.DownloadPerAge; -import org.nzbhydra.historystats.stats.DownloadPerAgeStats; -import org.nzbhydra.historystats.stats.IndexerApiAccessStatsEntry; -import org.nzbhydra.historystats.stats.IndexerDownloadShare; -import org.nzbhydra.historystats.stats.IndexerScore; -import org.nzbhydra.historystats.stats.StatsRequest; -import org.nzbhydra.historystats.stats.SuccessfulDownloadsPerIndexer; -import org.nzbhydra.historystats.stats.UserAgentShare; -import org.nzbhydra.indexers.Indexer; -import org.nzbhydra.indexers.IndexerAccessResult; -import org.nzbhydra.indexers.IndexerApiAccessEntityShortRepository; -import org.nzbhydra.indexers.IndexerEntity; -import org.nzbhydra.indexers.IndexerRepository; -import org.nzbhydra.logging.LoggingMarkers; -import org.nzbhydra.searching.SearchModuleProvider; -import org.nzbhydra.searching.db.SearchResultRepository; -import org.nzbhydra.searching.uniqueness.IndexerUniquenessScoreEntity; -import org.nzbhydra.searching.uniqueness.IndexerUniquenessScoreEntityRepository; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.bind.annotation.RestController; - -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.OptionalDouble; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -@SuppressWarnings("OptionalGetWithoutIsPresent") -@RestController -public class Stats { - - private static final Logger logger = LoggerFactory.getLogger(Stats.class); - private static final int TIMEOUT = 120; - - @Autowired - private SearchModuleProvider searchModuleProvider; - @Autowired - private IndexerRepository indexerRepository; - @PersistenceContext - private EntityManager entityManager; - @Autowired - private IndexerApiAccessEntityShortRepository shortRepository; - @Autowired - private SearchResultRepository searchResultRepository; - @Autowired - private IndexerUniquenessScoreEntityRepository uniquenessScoreEntityRepository; - - @Transactional(readOnly = true) - public StatsResponse getAllStats(StatsRequest statsRequest) throws InterruptedException { - logger.debug("Request for stats between {} and {}", statsRequest.getAfter(), statsRequest.getBefore()); - Stopwatch stopwatch = Stopwatch.createStarted(); - - StatsResponse statsResponse = new StatsResponse(); - statsResponse.setAfter(statsRequest.getAfter()); - statsResponse.setBefore(statsRequest.getBefore()); - - ExecutorService executor = Executors.newFixedThreadPool(1); //Multithreading doesn't improve performance but it allows us to stop calculation when the time is over - - List futures = new ArrayList<>(); - - - if (statsRequest.isAvgResponseTimes()) { - futures.add(executor.submit(() -> statsResponse.setAvgResponseTimes(averageResponseTimes(statsRequest)))); - } - if (statsRequest.isIndexerApiAccessStats()) { - futures.add(executor.submit(() -> statsResponse.setIndexerApiAccessStats(indexerApiAccesses(statsRequest)))); - } - if (statsRequest.isAvgIndexerUniquenessScore()) { - futures.add(executor.submit(() -> statsResponse.setIndexerScores(indexerScores(statsRequest)))); - } - - if (statsRequest.isSearchesPerDayOfWeek()) { - futures.add(executor.submit(() -> statsResponse.setSearchesPerDayOfWeek(countPerDayOfWeek("SEARCH", statsRequest)))); - } - if (statsRequest.isDownloadsPerDayOfWeek()) { - futures.add(executor.submit(() -> statsResponse.setDownloadsPerDayOfWeek(countPerDayOfWeek("INDEXERNZBDOWNLOAD", statsRequest)))); - } - - if (statsRequest.isSearchesPerHourOfDay()) { - futures.add(executor.submit(() -> statsResponse.setSearchesPerHourOfDay(countPerHourOfDay("SEARCH", statsRequest)))); - } - if (statsRequest.isDownloadsPerHourOfDay()) { - futures.add(executor.submit(() -> statsResponse.setDownloadsPerHourOfDay(countPerHourOfDay("INDEXERNZBDOWNLOAD", statsRequest)))); - } - - if (statsRequest.isIndexerDownloadShares()) { - futures.add(executor.submit(() -> statsResponse.setIndexerDownloadShares(indexerDownloadShares(statsRequest)))); - } - - - if (statsRequest.isDownloadsPerAgeStats()) { - futures.add(executor.submit(() -> statsResponse.setDownloadsPerAgeStats(downloadsPerAgeStats()))); - } - - if (statsRequest.isSuccessfulDownloadsPerIndexer()) { - futures.add(executor.submit(() -> statsResponse.setSuccessfulDownloadsPerIndexer(successfulDownloadsPerIndexer(statsRequest)))); - } - - if (statsRequest.isUserAgentSearchShares()) { - futures.add(executor.submit(() -> statsResponse.setUserAgentSearchShares(userAgentSearchShares(statsRequest)))); - } - - if (statsRequest.isUserAgentDownloadShares()) { - futures.add(executor.submit(() -> statsResponse.setUserAgentDownloadShares(userAgentDownloadShares(statsRequest)))); - } - - - if (statsRequest.isSearchSharesPerUser()) { - Long countSearchesWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM SEARCH t WHERE t.USERNAME IS NOT NULL").getSingleResult(); - if (countSearchesWithData.intValue() > 0) { - futures.add(executor.submit(() -> statsResponse.setSearchSharesPerUser(downloadsOrSearchesPerUserOrIp(statsRequest, "SEARCH", "USERNAME")))); - } - } - if (statsRequest.isDownloadSharesPerUser()) { - Long countDownloadsWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM INDEXERNZBDOWNLOAD t WHERE t.USERNAME IS NOT NULL").getSingleResult(); - if (countDownloadsWithData > 0) { - futures.add(executor.submit(() -> statsResponse.setDownloadSharesPerUser(downloadsOrSearchesPerUserOrIp(statsRequest, "INDEXERNZBDOWNLOAD", "USERNAME")))); - } - } - if (statsRequest.isSearchSharesPerIp()) { - Long countSearchesWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM SEARCH t WHERE t.IP IS NOT NULL").getSingleResult(); - if (countSearchesWithData > 0) { - futures.add(executor.submit(() -> statsResponse.setSearchSharesPerIp(downloadsOrSearchesPerUserOrIp(statsRequest, "SEARCH", "IP")))); - } - } - if (statsRequest.isDownloadSharesPerIp()) { - Long countDownloadsWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM INDEXERNZBDOWNLOAD t WHERE t.IP IS NOT NULL").getSingleResult(); - if (countDownloadsWithData > 0) { - futures.add(executor.submit(() -> statsResponse.setDownloadSharesPerIp(downloadsOrSearchesPerUserOrIp(statsRequest, "INDEXERNZBDOWNLOAD", "IP")))); - } - } - - - executor.shutdown(); - boolean wasCompleted = executor.awaitTermination(TIMEOUT, TimeUnit.SECONDS); - if (!wasCompleted) { - executor.shutdownNow(); - logger.error("Aborted stats generation because it took longer than {} seconds. Please restart", TIMEOUT); - } else { - for (Future future : futures) { - try { - future.get(); - } catch (ExecutionException e) { - logger.error("Error during calculation of stats", e.getCause()); - } - } - - } - - statsResponse.setNumberOfConfiguredIndexers(searchModuleProvider.getIndexers().size()); - statsResponse.setNumberOfEnabledIndexers(searchModuleProvider.getEnabledIndexers().size()); - - logger.info("Stats calculation took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return statsResponse; - } - - - List indexerDownloadShares(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating indexer download shares"); - if (searchModuleProvider.getEnabledIndexers().size() == 0 && !statsRequest.isIncludeDisabled()) { - logger.warn("Unable to generate any stats without any enabled indexers"); - return Collections.emptyList(); - } - - List indexerDownloadShares = new ArrayList<>(); - - String sqlQueryByIndexer = - "SELECT\n" + - " indexer.name,\n" + - " count(*) AS total,\n" + - " countall.countall\n" + - "FROM\n" + - " indexernzbdownload dl LEFT JOIN SEARCHRESULT ON dl.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + - " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + - " ,\n" + - " (SELECT count(*) AS countall\n" + - " FROM\n" + - " indexernzbdownload dl LEFT JOIN SEARCHRESULT ON dl.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + - buildWhereFromStatsRequest(false, statsRequest) + - ")\n" + - " countall\n" + - buildWhereFromStatsRequest(false, statsRequest) + - "GROUP BY\n" + - " INDEXER.NAME"; - - Query query = entityManager.createNativeQuery(sqlQueryByIndexer); - Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); - List resultList = query.getResultList(); - for (Object result : resultList) { - Object[] resultSet = (Object[]) result; - String indexerName = (String) resultSet[0]; - if (!indexerNamesToInclude.contains(indexerName)) { - continue; - } - long total = ((Long) resultSet[1]).longValue(); - long countAll = ((Long) resultSet[2]).longValue(); - float share = total > 0 ? (100F / ((float) countAll / total)) : 0F; - indexerDownloadShares.add(new IndexerDownloadShare(indexerName, total, share)); - } - indexerDownloadShares.sort((IndexerDownloadShare a, IndexerDownloadShare b) -> Float.compare(b.getShare(), a.getShare())); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated indexer download shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return indexerDownloadShares; - } - - List averageResponseTimes(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating average response times for indexers"); - List averageResponseTimes = new ArrayList<>(); - String sql = "SELECT\n" + - " NAME,\n" + - " avg(RESPONSE_TIME) AS avg\n" + - "FROM INDEXERAPIACCESS\n" + - " LEFT JOIN indexer i ON INDEXERAPIACCESS.INDEXER_ID = i.ID\n" + - buildWhereFromStatsRequest(false, statsRequest) + - "GROUP BY INDEXER_ID\n" + - "ORDER BY avg ASC"; - - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); - OptionalDouble overallAverage = resultList.stream().filter(x -> ((Object[]) x)[1] != null).mapToLong(x -> ((BigDecimal) ((Object[]) x)[1]).longValue()).average(); - - for (Object result : resultList) { - Object[] resultSet = (Object[]) result; - String indexerName = (String) resultSet[0]; - - if (resultSet[0] == null || resultSet[1] == null || !indexerNamesToInclude.contains(indexerName)) { - continue; - } - long averageResponseTime = ((BigDecimal) resultSet[1]).longValue(); - averageResponseTimes.add(new AverageResponseTime(indexerName, averageResponseTime, averageResponseTime - overallAverage.orElse(0D))); - } - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated average response times for indexers. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return averageResponseTimes; - } - - /** - * Calculates how unique a downloaded result was, i.e. how many other indexers could've (or not could've) provided the same result. - */ - @Transactional(readOnly = true) - public List indexerScores(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating indexer result uniqueness scores"); - - List typesToUse = Arrays.asList(SearchModuleType.NEWZNAB, SearchModuleType.TORZNAB, SearchModuleType.ANIZB); - final Set indexersToInclude = (statsRequest.isIncludeDisabled() ? searchModuleProvider.getIndexers() : searchModuleProvider.getEnabledIndexers().stream().filter(x -> typesToUse.contains(x.getConfig().getSearchModuleType())).toList()).stream().map(Indexer::getName).collect(Collectors.toSet()); - - List indexerUniquenessScores = calculateIndexerScores(indexersToInclude, uniquenessScoreEntityRepository.findAll()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated indexer result uniqueness scores. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return indexerUniquenessScores; - } - - List calculateIndexerScores(Set indexersToInclude, List scoreEntities) { - List scores = new ArrayList<>(); - Map> entities = scoreEntities.stream() - .filter(x -> indexersToInclude.contains(x.getIndexer().getName())) - .filter(IndexerUniquenessScoreEntity::isHasResult) - .collect(Collectors.groupingBy(IndexerUniquenessScoreEntity::getIndexer)); - for (Entry> indexerEntityListEntry : entities.entrySet()) { - Integer averageScore; - if (!indexerEntityListEntry.getValue().isEmpty()) { - OptionalDouble average = indexerEntityListEntry.getValue().stream().mapToDouble(x -> (100D * (double) x.getInvolved() / (double) x.getHave())).average(); - averageScore = (int) average.getAsDouble(); - } else { - averageScore = null; - } - IndexerScore indexerScore = new IndexerScore(); - indexerScore.setIndexerName(indexerEntityListEntry.getKey().getName()); - indexerScore.setAverageUniquenessScore(averageScore); - indexerScore.setInvolvedSearches(indexerEntityListEntry.getValue().size()); - long uniqueDownloads = indexerEntityListEntry.getValue().stream().filter(x -> x.getHave() == 1 && x.getInvolved() > 1).count(); - indexerScore.setUniqueDownloads(uniqueDownloads); - scores.add(indexerScore); - } - scores.sort(Comparator.comparing(IndexerScore::getAverageUniquenessScore).reversed()); - return scores; - } - - - List indexerApiAccesses(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating indexer API stats"); - Set indexerIdsToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(x -> x.getIndexerEntity().getId()).filter(id -> indexerRepository.findById(id) != null).collect(Collectors.toSet()); - - String averageIndexerAccessesPerDay = "SELECT\n" + - " indexer_id,\n" + - " avg(count)\n" + - "FROM (\n" + - " (SELECT\n" + - " INDEXER_ID,\n" + - " cast(count(INDEXER_ID) AS FLOAT) AS count" + - " FROM INDEXERAPIACCESS\n" + - buildWhereFromStatsRequest(false, statsRequest) + - " GROUP BY INDEXER_ID,\n" + - " date(time)))\n" + - "GROUP BY INDEXER_ID"; - - Map accessesPerDayCountMap = new HashMap<>(); - Query query = entityManager.createNativeQuery(averageIndexerAccessesPerDay); - //query = query.setParameter("indexerIds", indexerIdsToInclude); - List results = query.getResultList(); - for (Object resultObject : results) { - Object[] array = (Object[]) resultObject; - Integer indexerId = (Integer) array[0]; - if (!indexerIdsToInclude.contains(indexerId)) { - continue; - } - Double avg = ((BigDecimal) array[1]).doubleValue(); - accessesPerDayCountMap.put(indexerId, avg); - } - logger.debug(LoggingMarkers.PERFORMANCE, "Calculating accesses per day took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - stopwatch.reset(); - stopwatch.start(); - - String countByResultSql = "SELECT\n" + - " INDEXER_ID,\n" + - " RESULT,\n" + - " count(result) AS count\n" + - " FROM INDEXERAPIACCESS\n" + - buildWhereFromStatsRequest(false, statsRequest) + - " GROUP BY INDEXER_ID, RESULT\n" + - " ORDER BY INDEXER_ID, RESULT"; - - Map successCountMap = new HashMap<>(); - Map connectionErrorCountMap = new HashMap<>(); - Map allAccessesCountMap = new HashMap<>(); - query = entityManager.createNativeQuery(countByResultSql); - //query = query.setParameter("indexerIds", indexerIdsToInclude); - results = query.getResultList(); - for (Object resultObject : results) { - Object[] array = (Object[]) resultObject; - Integer indexerId = (Integer) array[0]; - if (!indexerIdsToInclude.contains(indexerId)) { - continue; - } - String result = (String) array[1]; - int count = ((Long) array[2]).intValue(); - if (result.equals(IndexerAccessResult.SUCCESSFUL.name())) { - successCountMap.put(indexerId, count); - } else if (result.equals(IndexerAccessResult.CONNECTION_ERROR.name())) { - connectionErrorCountMap.put(indexerId, count); - } - if (allAccessesCountMap.containsKey(indexerId)) { - allAccessesCountMap.put(indexerId, allAccessesCountMap.get(indexerId) + count); - } else { - allAccessesCountMap.put(indexerId, count); - } - } - - List indexerApiAccessStatsEntries = new ArrayList<>(); - for (Integer id : indexerIdsToInclude) { - IndexerApiAccessStatsEntry entry = new IndexerApiAccessStatsEntry(); - IndexerEntity indexerEntity = indexerRepository.findById(id).get(); - entry.setIndexerName(indexerEntity.getName()); - - if (allAccessesCountMap.containsKey(id) && allAccessesCountMap.get(id) != null) { - if (successCountMap.get(id) != null) { - Double percentSuccessFul = 100D / (allAccessesCountMap.get(id).doubleValue() / successCountMap.get(id).doubleValue()); - entry.setPercentSuccessful(percentSuccessFul); - } - - if (connectionErrorCountMap.get(id) != null) { - Double percentConnectionError = 100D / (allAccessesCountMap.get(id).doubleValue() / connectionErrorCountMap.get(id).doubleValue()); - entry.setPercentConnectionError(percentConnectionError); - } - } - - if (accessesPerDayCountMap.containsKey(id) && accessesPerDayCountMap.get(id) != null) { - entry.setAverageAccessesPerDay(accessesPerDayCountMap.get(id)); - } - - indexerApiAccessStatsEntries.add(entry); - } - logger.debug(LoggingMarkers.PERFORMANCE, "Calculating success/failure stats took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return indexerApiAccessStatsEntries; - } - - List countPerDayOfWeek(final String table, final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating count for day of week for table {}", table); - String sql = "SELECT \n" + - " strftime('%w', time) AS dayofweek, \n" + - " count(*) AS counter \n" + - "FROM " + table + " \n" + - buildWhereFromStatsRequest(false, statsRequest) + - "GROUP BY strftime('%w', time)"; - - List dayOfWeekCounts = new ArrayList<>(); - for (int i = 0; i < 7; i++) { - dayOfWeekCounts.add(new CountPerDayOfWeek(i + 1, 0)); - } - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - for (Object o : resultList) { - Object[] resultSet = (Object[]) o; - Integer index = (Integer) resultSet[0]; - //SQLite strftime('%w') returns 0 for sunday, 1 for monday, etc. - //We want sunday in index 6, monday in index 0 - //SQLite S M T W T F S - //index 0 1 2 3 4 5 6 - - //want 6 0 1 2 3 4 5 - // S M T W T F S - Long counter = (Long) resultSet[1]; - int indexInList = (index + 6) % 7; - dayOfWeekCounts.get(indexInList).setCount(counter.intValue()); - } - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated count for day of week for table {}. Took {}ms", table, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return dayOfWeekCounts; - } - - - List countPerHourOfDay(final String table, final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating count for hour of day for table {}", table); - String sql = "SELECT \n" + - " strftime('%H', time) AS hourofday, \n" + - " count(*) AS counter \n" + - "FROM " + table + " \n" + - buildWhereFromStatsRequest(false, statsRequest) + - "GROUP BY strftime('%H', time)"; - - List hourOfDayCounts = new ArrayList<>(); - for (int i = 0; i < 24; i++) { - hourOfDayCounts.add(new CountPerHourOfDay(i, 0)); - } - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - Integer index = (Integer) o2[0]; - Long counter = (Long) o2[1]; - hourOfDayCounts.get(index).setCount(counter.intValue()); - } - - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated count for hour of day for table {}. Took {}ms", table, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return hourOfDayCounts; - } - - List successfulDownloadsPerIndexer(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - String sql = "SELECT\n" + - " name1,\n" + - " count_all,\n" + - " count_success,\n" + - " count_error\n" + - "FROM\n" + - " (SELECT\n" + - " indexer.NAME AS name1,\n" + - " count(*) AS count_success\n" + - " FROM INDEXERNZBDOWNLOAD\n" + - " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + - " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + - " WHERE\n" + - " status = 'CONTENT_DOWNLOAD_SUCCESSFUL'\n" + - buildWhereFromStatsRequest(true, statsRequest) + - " GROUP BY name1)\n" + - " LEFT JOIN\n" + - " (SELECT\n" + - " indexer.NAME AS name2,\n" + - " count(*) AS count_error\n" + - " FROM INDEXERNZBDOWNLOAD\n" + - " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + - " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + - " WHERE\n" + - " status IN ('CONTENT_DOWNLOAD_ERROR', 'CONTENT_DOWNLOAD_WARNING')\n" + - buildWhereFromStatsRequest(true, statsRequest) + - " GROUP BY name2) ON name1 = name2\n" + - " LEFT JOIN\n" + - " (SELECT\n" + - " indexer.NAME AS name3,\n" + - " count(*) AS count_all\n" + - " FROM INDEXERNZBDOWNLOAD\n" + - " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + - " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + - buildWhereFromStatsRequest(false, statsRequest) + - " GROUP BY name3) ON name1 = name3;"; - Query query = entityManager.createNativeQuery(sql); - Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); - List resultList = query.getResultList(); - List result = new ArrayList<>(); - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - String indexerName = (String) o2[0]; - if (!indexerNamesToInclude.contains(indexerName)) { - continue; - } - Long countAll = (Long) o2[1]; - Long countSuccess = (Long) o2[2]; - Long countError = (Long) o2[3]; - if (countAll == null) { - countAll = 0L; - } - if (countSuccess == null) { - countSuccess = 0L; - } - if (countError == null) { - countError = 0L; - } - - Float percentSuccessful; - if (countSuccess.intValue() > 0) { - percentSuccessful = 100F / ((countSuccess.floatValue() + countError.floatValue()) / countSuccess.floatValue()); - } else if (countAll.intValue() > 0) { - percentSuccessful = 0F; - } else { - percentSuccessful = null; - } - result.add(new SuccessfulDownloadsPerIndexer(indexerName, countAll.intValue(), countSuccess.intValue(), countError.intValue(), percentSuccessful)); - } - result.sort(Comparator.comparingDouble(SuccessfulDownloadsPerIndexer::getPercentSuccessful).reversed()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated successful download percentages for indexers. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return result; - } - - List downloadsOrSearchesPerUserOrIp(final StatsRequest statsRequest, String tablename, final String column) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating download or search shares for table {} and column {}", tablename, column); - String sql = "" + - "SELECT\n" + - " " + column + ",\n" + - " count(*) AS peruser,\n" + - " (SELECT count(*)\n" + - " FROM " + tablename + "\n" + - " WHERE " + column + " IS NOT NULL AND " + column + " != ''" + - buildWhereFromStatsRequest(true, statsRequest) + - ") AS countall\n" + - "FROM " + tablename + "\n" + - " WHERE " + column + " IS NOT NULL AND " + column + " != ''\n" + - buildWhereFromStatsRequest(true, statsRequest) + - "GROUP BY " + column; - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - List result = new ArrayList<>(); - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - String usernameOrIp = (String) o2[0]; - int countForUser = ((Long) o2[1]).intValue(); - float percentSuccessful = 100F / (((Long) o2[2]).floatValue() / ((Long) o2[1]).floatValue()); - result.add(new DownloadOrSearchSharePerUserOrIp(usernameOrIp, countForUser, percentSuccessful)); - } - result.sort(Comparator.comparingDouble(DownloadOrSearchSharePerUserOrIp::getPercentage).reversed()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated download or search shares for table {} and column {}. Took {}ms", tablename, column, stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return result; - } - - List userAgentSearchShares(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating user agent search shares"); - String sql = "SELECT\n" + - " user_agent,\n" + - " count(*)\n" + - "FROM SEARCH\n" + - "WHERE user_agent IS NOT NULL\n" + - "AND SOURCE = 'API'" + - buildWhereFromStatsRequest(true, statsRequest) + - "GROUP BY user_agent"; - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - List result = new ArrayList<>(); - int countAll = 0; - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - String userAgent = (String) o2[0]; - int countForUserAgent = ((Long) o2[1]).intValue(); - countAll += countForUserAgent; - result.add(new UserAgentShare(userAgent, countForUserAgent)); - } - for (UserAgentShare userAgentShare : result) { - userAgentShare.setPercentage(100F / ((float) countAll / userAgentShare.getCount())); - } - - result.sort(Comparator.comparingDouble(UserAgentShare::getPercentage).reversed()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated user agent search shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return result; - } - - List userAgentDownloadShares(final StatsRequest statsRequest) { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating user agent download shares"); - String sql = "SELECT\n" + - " user_agent,\n" + - " count(*)\n" + - "FROM INDEXERNZBDOWNLOAD\n" + - "WHERE user_agent IS NOT NULL\n" + - "and ACCESS_SOURCE = 'API' \n" + - buildWhereFromStatsRequest(true, statsRequest) + - "GROUP BY user_agent"; - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - List result = new ArrayList<>(); - int countAll = 0; - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - String userAgent = (String) o2[0]; - int countForUserAgent = ((Long) o2[1]).intValue(); - countAll += countForUserAgent; - result.add(new UserAgentShare(userAgent, countForUserAgent)); - } - for (UserAgentShare userAgentShare : result) { - userAgentShare.setPercentage(100F / ((float) countAll / userAgentShare.getCount())); - } - - result.sort(Comparator.comparingDouble(UserAgentShare::getPercentage).reversed()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated user agent download shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return result; - } - - List downloadsPerAge() { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating downloads per age"); - String sql = """ - SELECT - steps, - count(*) - FROM - (SELECT age / 100 AS steps - FROM INDEXERNZBDOWNLOAD - WHERE age IS NOT NULL) - GROUP BY steps - ORDER BY steps ASC"""; - Query query = entityManager.createNativeQuery(sql); - List resultList = query.getResultList(); - List results = new ArrayList<>(); - Map agesAndCountsMap = new HashMap<>(); - for (Object o : resultList) { - Object[] o2 = (Object[]) o; - int ageStep = (Integer) o2[0]; - int count = ((Long) o2[1]).intValue(); - agesAndCountsMap.put(ageStep, count); - } - for (int i = 0; i <= 34; i += 1) { - if (!agesAndCountsMap.containsKey(i)) { - agesAndCountsMap.put(i, 0); - } - } - for (Entry entry : agesAndCountsMap.entrySet()) { - results.add(new DownloadPerAge(entry.getKey() * 100, entry.getValue())); - } - results.sort(Comparator.comparingInt(DownloadPerAge::getAge)); - - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated downloads per age. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - return results; - } - - DownloadPerAgeStats downloadsPerAgeStats() { - Stopwatch stopwatch = Stopwatch.createStarted(); - logger.debug("Calculating downloads per age percentages"); - DownloadPerAgeStats result = new DownloadPerAgeStats(); - String percentage = """ - SELECT CASE - WHEN (SELECT CAST(COUNT(*) AS FLOAT) AS COUNT - FROM INDEXERNZBDOWNLOAD - WHERE AGE > %d) > 0 - THEN SELECT CAST(100 AS FLOAT) / (CAST(COUNT(i.*) AS FLOAT)/ x.COUNT) - FROM INDEXERNZBDOWNLOAD i, - ( SELECT COUNT(*) AS COUNT - FROM INDEXERNZBDOWNLOAD - WHERE AGE > %d) AS x - ELSE 0 END"""; - result.setPercentOlder1000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 1000, 1000)).getResultList().get(0)).intValue()); - result.setPercentOlder2000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 2000, 2000)).getResultList().get(0)).intValue()); - result.setPercentOlder3000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 3000, 3000)).getResultList().get(0)).intValue()); - final Double averageAge = (Double) entityManager.createNativeQuery("SELECT AVG(AGE) FROM INDEXERNZBDOWNLOAD").getResultList().get(0); - result.setAverageAge(averageAge == null ? 0 : averageAge.intValue()); - logger.debug(LoggingMarkers.PERFORMANCE, "Calculated downloads per age percentages . Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); - - result.setDownloadsPerAge(downloadsPerAge()); - return result; - } - - - private String buildWhereFromStatsRequest(boolean useAnd, StatsRequest statsRequest) { - if (statsRequest.getAfter() == null && statsRequest.getBefore() == null) { - return " "; - } - return (useAnd ? " AND " : " WHERE ") + - (statsRequest.getAfter() != null ? " TIME > datetime(" + statsRequest.getAfter().getEpochSecond() + ", 'unixepoch') " : "") + - ((statsRequest.getBefore() != null && statsRequest.getAfter() != null) ? " AND " : " ") + - (statsRequest.getBefore() != null ? " TIME < datetime(" + statsRequest.getBefore().getEpochSecond() + ", 'unixepoch') " : ""); - } - - -} +package org.nzbhydra.historystats; + +import com.google.common.base.Stopwatch; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; +import org.nzbhydra.config.indexer.IndexerConfig; +import org.nzbhydra.config.indexer.SearchModuleType; +import org.nzbhydra.historystats.stats.AverageResponseTime; +import org.nzbhydra.historystats.stats.CountPerDayOfWeek; +import org.nzbhydra.historystats.stats.CountPerHourOfDay; +import org.nzbhydra.historystats.stats.DownloadOrSearchSharePerUserOrIp; +import org.nzbhydra.historystats.stats.DownloadPerAge; +import org.nzbhydra.historystats.stats.DownloadPerAgeStats; +import org.nzbhydra.historystats.stats.IndexerApiAccessStatsEntry; +import org.nzbhydra.historystats.stats.IndexerDownloadShare; +import org.nzbhydra.historystats.stats.IndexerScore; +import org.nzbhydra.historystats.stats.StatsRequest; +import org.nzbhydra.historystats.stats.SuccessfulDownloadsPerIndexer; +import org.nzbhydra.historystats.stats.UserAgentShare; +import org.nzbhydra.indexers.Indexer; +import org.nzbhydra.indexers.IndexerAccessResult; +import org.nzbhydra.indexers.IndexerApiAccessEntityShortRepository; +import org.nzbhydra.indexers.IndexerEntity; +import org.nzbhydra.indexers.IndexerRepository; +import org.nzbhydra.logging.LoggingMarkers; +import org.nzbhydra.searching.SearchModuleProvider; +import org.nzbhydra.searching.db.SearchResultRepository; +import org.nzbhydra.searching.uniqueness.IndexerUniquenessScoreEntity; +import org.nzbhydra.searching.uniqueness.IndexerUniquenessScoreEntityRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.bind.annotation.RestController; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.OptionalDouble; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +@RestController +public class Stats { + + private static final Logger logger = LoggerFactory.getLogger(Stats.class); + private static final int TIMEOUT = 120; + + @Autowired + private SearchModuleProvider searchModuleProvider; + @Autowired + private IndexerRepository indexerRepository; + @PersistenceContext + private EntityManager entityManager; + @Autowired + private IndexerApiAccessEntityShortRepository shortRepository; + @Autowired + private SearchResultRepository searchResultRepository; + @Autowired + private IndexerUniquenessScoreEntityRepository uniquenessScoreEntityRepository; + + @Transactional(readOnly = true) + public StatsResponse getAllStats(StatsRequest statsRequest) throws InterruptedException { + logger.debug("Request for stats between {} and {}", statsRequest.getAfter(), statsRequest.getBefore()); + Stopwatch stopwatch = Stopwatch.createStarted(); + + StatsResponse statsResponse = new StatsResponse(); + statsResponse.setAfter(statsRequest.getAfter()); + statsResponse.setBefore(statsRequest.getBefore()); + + ExecutorService executor = Executors.newFixedThreadPool(1); //Multithreading doesn't improve performance but it allows us to stop calculation when the time is over + + List futures = new ArrayList<>(); + + + if (statsRequest.isAvgResponseTimes()) { + futures.add(executor.submit(() -> statsResponse.setAvgResponseTimes(averageResponseTimes(statsRequest)))); + } + if (statsRequest.isIndexerApiAccessStats()) { + futures.add(executor.submit(() -> statsResponse.setIndexerApiAccessStats(indexerApiAccesses(statsRequest)))); + } + if (statsRequest.isAvgIndexerUniquenessScore()) { + futures.add(executor.submit(() -> statsResponse.setIndexerScores(indexerScores(statsRequest)))); + } + + if (statsRequest.isSearchesPerDayOfWeek()) { + futures.add(executor.submit(() -> statsResponse.setSearchesPerDayOfWeek(countPerDayOfWeek("SEARCH", statsRequest)))); + } + if (statsRequest.isDownloadsPerDayOfWeek()) { + futures.add(executor.submit(() -> statsResponse.setDownloadsPerDayOfWeek(countPerDayOfWeek("INDEXERNZBDOWNLOAD", statsRequest)))); + } + + if (statsRequest.isSearchesPerHourOfDay()) { + futures.add(executor.submit(() -> statsResponse.setSearchesPerHourOfDay(countPerHourOfDay("SEARCH", statsRequest)))); + } + if (statsRequest.isDownloadsPerHourOfDay()) { + futures.add(executor.submit(() -> statsResponse.setDownloadsPerHourOfDay(countPerHourOfDay("INDEXERNZBDOWNLOAD", statsRequest)))); + } + + if (statsRequest.isIndexerDownloadShares()) { + futures.add(executor.submit(() -> statsResponse.setIndexerDownloadShares(indexerDownloadShares(statsRequest)))); + } + + + if (statsRequest.isDownloadsPerAgeStats()) { + futures.add(executor.submit(() -> statsResponse.setDownloadsPerAgeStats(downloadsPerAgeStats()))); + } + + if (statsRequest.isSuccessfulDownloadsPerIndexer()) { + futures.add(executor.submit(() -> statsResponse.setSuccessfulDownloadsPerIndexer(successfulDownloadsPerIndexer(statsRequest)))); + } + + if (statsRequest.isUserAgentSearchShares()) { + futures.add(executor.submit(() -> statsResponse.setUserAgentSearchShares(userAgentSearchShares(statsRequest)))); + } + + if (statsRequest.isUserAgentDownloadShares()) { + futures.add(executor.submit(() -> statsResponse.setUserAgentDownloadShares(userAgentDownloadShares(statsRequest)))); + } + + + if (statsRequest.isSearchSharesPerUser()) { + Long countSearchesWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM SEARCH t WHERE t.USERNAME IS NOT NULL").getSingleResult(); + if (countSearchesWithData.intValue() > 0) { + futures.add(executor.submit(() -> statsResponse.setSearchSharesPerUser(downloadsOrSearchesPerUserOrIp(statsRequest, "SEARCH", "USERNAME")))); + } + } + if (statsRequest.isDownloadSharesPerUser()) { + Long countDownloadsWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM INDEXERNZBDOWNLOAD t WHERE t.USERNAME IS NOT NULL").getSingleResult(); + if (countDownloadsWithData > 0) { + futures.add(executor.submit(() -> statsResponse.setDownloadSharesPerUser(downloadsOrSearchesPerUserOrIp(statsRequest, "INDEXERNZBDOWNLOAD", "USERNAME")))); + } + } + if (statsRequest.isSearchSharesPerIp()) { + Long countSearchesWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM SEARCH t WHERE t.IP IS NOT NULL").getSingleResult(); + if (countSearchesWithData > 0) { + futures.add(executor.submit(() -> statsResponse.setSearchSharesPerIp(downloadsOrSearchesPerUserOrIp(statsRequest, "SEARCH", "IP")))); + } + } + if (statsRequest.isDownloadSharesPerIp()) { + Long countDownloadsWithData = (Long) entityManager.createNativeQuery("SELECT count(*) FROM INDEXERNZBDOWNLOAD t WHERE t.IP IS NOT NULL").getSingleResult(); + if (countDownloadsWithData > 0) { + futures.add(executor.submit(() -> statsResponse.setDownloadSharesPerIp(downloadsOrSearchesPerUserOrIp(statsRequest, "INDEXERNZBDOWNLOAD", "IP")))); + } + } + + + executor.shutdown(); + boolean wasCompleted = executor.awaitTermination(TIMEOUT, TimeUnit.SECONDS); + if (!wasCompleted) { + executor.shutdownNow(); + logger.error("Aborted stats generation because it took longer than {} seconds. Please restart", TIMEOUT); + } else { + for (Future future : futures) { + try { + future.get(); + } catch (ExecutionException e) { + logger.error("Error during calculation of stats", e.getCause()); + } + } + + } + + statsResponse.setNumberOfConfiguredIndexers(searchModuleProvider.getIndexers().size()); + statsResponse.setNumberOfEnabledIndexers(searchModuleProvider.getEnabledIndexers().size()); + + logger.info("Stats calculation took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return statsResponse; + } + + + List indexerDownloadShares(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating indexer download shares"); + if (searchModuleProvider.getEnabledIndexers().size() == 0 && !statsRequest.isIncludeDisabled()) { + logger.warn("Unable to generate any stats without any enabled indexers"); + return Collections.emptyList(); + } + + List indexerDownloadShares = new ArrayList<>(); + + String sqlQueryByIndexer = + "SELECT\n" + + " indexer.name,\n" + + " count(*) AS total,\n" + + " countall.countall\n" + + "FROM\n" + + " indexernzbdownload dl LEFT JOIN SEARCHRESULT ON dl.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + + " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + + " ,\n" + + " (SELECT count(*) AS countall\n" + + " FROM\n" + + " indexernzbdownload dl LEFT JOIN SEARCHRESULT ON dl.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + + buildWhereFromStatsRequest(false, statsRequest) + + ")\n" + + " countall\n" + + buildWhereFromStatsRequest(false, statsRequest) + + "GROUP BY\n" + + " INDEXER.NAME"; + + Query query = entityManager.createNativeQuery(sqlQueryByIndexer); + Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); + List resultList = query.getResultList(); + for (Object result : resultList) { + Object[] resultSet = (Object[]) result; + String indexerName = (String) resultSet[0]; + if (!indexerNamesToInclude.contains(indexerName)) { + continue; + } + long total = ((Long) resultSet[1]).longValue(); + long countAll = ((Long) resultSet[2]).longValue(); + float share = total > 0 ? (100F / ((float) countAll / total)) : 0F; + indexerDownloadShares.add(new IndexerDownloadShare(indexerName, total, share)); + } + indexerDownloadShares.sort((IndexerDownloadShare a, IndexerDownloadShare b) -> Float.compare(b.getShare(), a.getShare())); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated indexer download shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return indexerDownloadShares; + } + + List averageResponseTimes(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating average response times for indexers"); + List averageResponseTimes = new ArrayList<>(); + String sql = "SELECT\n" + + " NAME,\n" + + " avg(RESPONSE_TIME) AS avg\n" + + "FROM INDEXERAPIACCESS\n" + + " LEFT JOIN indexer i ON INDEXERAPIACCESS.INDEXER_ID = i.ID\n" + + buildWhereFromStatsRequest(false, statsRequest) + + "GROUP BY INDEXER_ID\n" + + "ORDER BY avg ASC"; + + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); + OptionalDouble overallAverage = resultList.stream().filter(x -> ((Object[]) x)[1] != null).mapToLong(x -> ((BigDecimal) ((Object[]) x)[1]).longValue()).average(); + + for (Object result : resultList) { + Object[] resultSet = (Object[]) result; + String indexerName = (String) resultSet[0]; + + if (resultSet[0] == null || resultSet[1] == null || !indexerNamesToInclude.contains(indexerName)) { + continue; + } + long averageResponseTime = ((BigDecimal) resultSet[1]).longValue(); + averageResponseTimes.add(new AverageResponseTime(indexerName, averageResponseTime, averageResponseTime - overallAverage.orElse(0D))); + } + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated average response times for indexers. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return averageResponseTimes; + } + + /** + * Calculates how unique a downloaded result was, i.e. how many other indexers could've (or not could've) provided the same result. + */ + @Transactional(readOnly = true) + public List indexerScores(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating indexer result uniqueness scores"); + + List typesToUse = Arrays.asList(SearchModuleType.NEWZNAB, SearchModuleType.TORZNAB, SearchModuleType.ANIZB); + final Set indexersToInclude = (statsRequest.isIncludeDisabled() ? searchModuleProvider.getIndexers() : searchModuleProvider.getEnabledIndexers().stream().filter(x -> typesToUse.contains(x.getConfig().getSearchModuleType())).toList()).stream().map(Indexer::getName).collect(Collectors.toSet()); + + List indexerUniquenessScores = calculateIndexerScores(indexersToInclude, uniquenessScoreEntityRepository.findAll()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated indexer result uniqueness scores. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return indexerUniquenessScores; + } + + List calculateIndexerScores(Set indexersToInclude, List scoreEntities) { + List scores = new ArrayList<>(); + Map> entities = scoreEntities.stream() + .filter(x -> indexersToInclude.contains(x.getIndexer().getName())) + .filter(IndexerUniquenessScoreEntity::isHasResult) + .collect(Collectors.groupingBy(IndexerUniquenessScoreEntity::getIndexer)); + for (Entry> indexerEntityListEntry : entities.entrySet()) { + Integer averageScore; + if (!indexerEntityListEntry.getValue().isEmpty()) { + OptionalDouble average = indexerEntityListEntry.getValue().stream().mapToDouble(x -> (100D * (double) x.getInvolved() / (double) x.getHave())).average(); + averageScore = (int) average.getAsDouble(); + } else { + averageScore = null; + } + IndexerScore indexerScore = new IndexerScore(); + indexerScore.setIndexerName(indexerEntityListEntry.getKey().getName()); + indexerScore.setAverageUniquenessScore(averageScore); + indexerScore.setInvolvedSearches(indexerEntityListEntry.getValue().size()); + long uniqueDownloads = indexerEntityListEntry.getValue().stream().filter(x -> x.getHave() == 1 && x.getInvolved() > 1).count(); + indexerScore.setUniqueDownloads(uniqueDownloads); + scores.add(indexerScore); + } + scores.sort(Comparator.comparing(IndexerScore::getAverageUniquenessScore).reversed()); + return scores; + } + + + List indexerApiAccesses(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating indexer API stats"); + Set indexerIdsToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(x -> x.getIndexerEntity().getId()).filter(id -> indexerRepository.findById(id) != null).collect(Collectors.toSet()); + + String averageIndexerAccessesPerDay = "SELECT\n" + + " indexer_id,\n" + + " avg(count)\n" + + "FROM (\n" + + " (SELECT\n" + + " INDEXER_ID,\n" + + " cast(count(INDEXER_ID) AS FLOAT) AS count" + + " FROM INDEXERAPIACCESS\n" + + buildWhereFromStatsRequest(false, statsRequest) + + " GROUP BY INDEXER_ID,\n" + + " date(time)))\n" + + "GROUP BY INDEXER_ID"; + + Map accessesPerDayCountMap = new HashMap<>(); + Query query = entityManager.createNativeQuery(averageIndexerAccessesPerDay); + //query = query.setParameter("indexerIds", indexerIdsToInclude); + List results = query.getResultList(); + for (Object resultObject : results) { + Object[] array = (Object[]) resultObject; + Integer indexerId = (Integer) array[0]; + if (!indexerIdsToInclude.contains(indexerId)) { + continue; + } + Double avg = ((BigDecimal) array[1]).doubleValue(); + accessesPerDayCountMap.put(indexerId, avg); + } + logger.debug(LoggingMarkers.PERFORMANCE, "Calculating accesses per day took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + stopwatch.reset(); + stopwatch.start(); + + String countByResultSql = "SELECT\n" + + " INDEXER_ID,\n" + + " RESULT,\n" + + " count(result) AS count\n" + + " FROM INDEXERAPIACCESS\n" + + buildWhereFromStatsRequest(false, statsRequest) + + " GROUP BY INDEXER_ID, RESULT\n" + + " ORDER BY INDEXER_ID, RESULT"; + + Map successCountMap = new HashMap<>(); + Map connectionErrorCountMap = new HashMap<>(); + Map allAccessesCountMap = new HashMap<>(); + query = entityManager.createNativeQuery(countByResultSql); + //query = query.setParameter("indexerIds", indexerIdsToInclude); + results = query.getResultList(); + for (Object resultObject : results) { + Object[] array = (Object[]) resultObject; + Integer indexerId = (Integer) array[0]; + if (!indexerIdsToInclude.contains(indexerId)) { + continue; + } + String result = (String) array[1]; + int count = ((Long) array[2]).intValue(); + if (result.equals(IndexerAccessResult.SUCCESSFUL.name())) { + successCountMap.put(indexerId, count); + } else if (result.equals(IndexerAccessResult.CONNECTION_ERROR.name())) { + connectionErrorCountMap.put(indexerId, count); + } + if (allAccessesCountMap.containsKey(indexerId)) { + allAccessesCountMap.put(indexerId, allAccessesCountMap.get(indexerId) + count); + } else { + allAccessesCountMap.put(indexerId, count); + } + } + + List indexerApiAccessStatsEntries = new ArrayList<>(); + for (Integer id : indexerIdsToInclude) { + IndexerApiAccessStatsEntry entry = new IndexerApiAccessStatsEntry(); + IndexerEntity indexerEntity = indexerRepository.findById(id).get(); + entry.setIndexerName(indexerEntity.getName()); + + if (allAccessesCountMap.containsKey(id) && allAccessesCountMap.get(id) != null) { + if (successCountMap.get(id) != null) { + Double percentSuccessFul = 100D / (allAccessesCountMap.get(id).doubleValue() / successCountMap.get(id).doubleValue()); + entry.setPercentSuccessful(percentSuccessFul); + } + + if (connectionErrorCountMap.get(id) != null) { + Double percentConnectionError = 100D / (allAccessesCountMap.get(id).doubleValue() / connectionErrorCountMap.get(id).doubleValue()); + entry.setPercentConnectionError(percentConnectionError); + } + } + + if (accessesPerDayCountMap.containsKey(id) && accessesPerDayCountMap.get(id) != null) { + entry.setAverageAccessesPerDay(accessesPerDayCountMap.get(id)); + } + + indexerApiAccessStatsEntries.add(entry); + } + logger.debug(LoggingMarkers.PERFORMANCE, "Calculating success/failure stats took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return indexerApiAccessStatsEntries; + } + + List countPerDayOfWeek(final String table, final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating count for day of week for table {}", table); + String sql = "SELECT \n" + + " strftime('%w', time) AS dayofweek, \n" + + " count(*) AS counter \n" + + "FROM " + table + " \n" + + buildWhereFromStatsRequest(false, statsRequest) + + "GROUP BY strftime('%w', time)"; + + List dayOfWeekCounts = new ArrayList<>(); + for (int i = 0; i < 7; i++) { + dayOfWeekCounts.add(new CountPerDayOfWeek(i + 1, 0)); + } + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + for (Object o : resultList) { + Object[] resultSet = (Object[]) o; + Integer index = (Integer) resultSet[0]; + //SQLite strftime('%w') returns 0 for sunday, 1 for monday, etc. + //We want sunday in index 6, monday in index 0 + //SQLite S M T W T F S + //index 0 1 2 3 4 5 6 + + //want 6 0 1 2 3 4 5 + // S M T W T F S + Long counter = (Long) resultSet[1]; + int indexInList = (index + 6) % 7; + dayOfWeekCounts.get(indexInList).setCount(counter.intValue()); + } + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated count for day of week for table {}. Took {}ms", table, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return dayOfWeekCounts; + } + + + List countPerHourOfDay(final String table, final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating count for hour of day for table {}", table); + String sql = "SELECT \n" + + " strftime('%H', time) AS hourofday, \n" + + " count(*) AS counter \n" + + "FROM " + table + " \n" + + buildWhereFromStatsRequest(false, statsRequest) + + "GROUP BY strftime('%H', time)"; + + List hourOfDayCounts = new ArrayList<>(); + for (int i = 0; i < 24; i++) { + hourOfDayCounts.add(new CountPerHourOfDay(i, 0)); + } + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + Integer index = (Integer) o2[0]; + Long counter = (Long) o2[1]; + hourOfDayCounts.get(index).setCount(counter.intValue()); + } + + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated count for hour of day for table {}. Took {}ms", table, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return hourOfDayCounts; + } + + List successfulDownloadsPerIndexer(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + String sql = "SELECT\n" + + " name1,\n" + + " count_all,\n" + + " count_success,\n" + + " count_error\n" + + "FROM\n" + + " (SELECT\n" + + " indexer.NAME AS name1,\n" + + " count(*) AS count_success\n" + + " FROM INDEXERNZBDOWNLOAD\n" + + " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + + " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + + " WHERE\n" + + " status = 'CONTENT_DOWNLOAD_SUCCESSFUL'\n" + + buildWhereFromStatsRequest(true, statsRequest) + + " GROUP BY name1)\n" + + " LEFT JOIN\n" + + " (SELECT\n" + + " indexer.NAME AS name2,\n" + + " count(*) AS count_error\n" + + " FROM INDEXERNZBDOWNLOAD\n" + + " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + + " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + + " WHERE\n" + + " status IN ('CONTENT_DOWNLOAD_ERROR', 'CONTENT_DOWNLOAD_WARNING')\n" + + buildWhereFromStatsRequest(true, statsRequest) + + " GROUP BY name2) ON name1 = name2\n" + + " LEFT JOIN\n" + + " (SELECT\n" + + " indexer.NAME AS name3,\n" + + " count(*) AS count_all\n" + + " FROM INDEXERNZBDOWNLOAD\n" + + " LEFT JOIN SEARCHRESULT ON INDEXERNZBDOWNLOAD.SEARCH_RESULT_ID = SEARCHRESULT.ID\n" + + " LEFT JOIN indexer ON SEARCHRESULT.INDEXER_ID = INDEXER.ID\n" + + buildWhereFromStatsRequest(false, statsRequest) + + " GROUP BY name3) ON name1 = name3;"; + Query query = entityManager.createNativeQuery(sql); + Set indexerNamesToInclude = searchModuleProvider.getIndexers().stream().filter(x -> x.getConfig().getState() == IndexerConfig.State.ENABLED || statsRequest.isIncludeDisabled()).map(Indexer::getName).collect(Collectors.toSet()); + List resultList = query.getResultList(); + List result = new ArrayList<>(); + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + String indexerName = (String) o2[0]; + if (!indexerNamesToInclude.contains(indexerName)) { + continue; + } + Long countAll = (Long) o2[1]; + Long countSuccess = (Long) o2[2]; + Long countError = (Long) o2[3]; + if (countAll == null) { + countAll = 0L; + } + if (countSuccess == null) { + countSuccess = 0L; + } + if (countError == null) { + countError = 0L; + } + + Float percentSuccessful; + if (countSuccess.intValue() > 0) { + percentSuccessful = 100F / ((countSuccess.floatValue() + countError.floatValue()) / countSuccess.floatValue()); + } else if (countAll.intValue() > 0) { + percentSuccessful = 0F; + } else { + percentSuccessful = null; + } + result.add(new SuccessfulDownloadsPerIndexer(indexerName, countAll.intValue(), countSuccess.intValue(), countError.intValue(), percentSuccessful)); + } + result.sort(Comparator.comparingDouble(SuccessfulDownloadsPerIndexer::getPercentSuccessful).reversed()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated successful download percentages for indexers. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return result; + } + + List downloadsOrSearchesPerUserOrIp(final StatsRequest statsRequest, String tablename, final String column) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating download or search shares for table {} and column {}", tablename, column); + String sql = "" + + "SELECT\n" + + " " + column + ",\n" + + " count(*) AS peruser,\n" + + " (SELECT count(*)\n" + + " FROM " + tablename + "\n" + + " WHERE " + column + " IS NOT NULL AND " + column + " != ''" + + buildWhereFromStatsRequest(true, statsRequest) + + ") AS countall\n" + + "FROM " + tablename + "\n" + + " WHERE " + column + " IS NOT NULL AND " + column + " != ''\n" + + buildWhereFromStatsRequest(true, statsRequest) + + "GROUP BY " + column; + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + List result = new ArrayList<>(); + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + String usernameOrIp = (String) o2[0]; + int countForUser = ((Long) o2[1]).intValue(); + float percentSuccessful = 100F / (((Long) o2[2]).floatValue() / ((Long) o2[1]).floatValue()); + result.add(new DownloadOrSearchSharePerUserOrIp(usernameOrIp, countForUser, percentSuccessful)); + } + result.sort(Comparator.comparingDouble(DownloadOrSearchSharePerUserOrIp::getPercentage).reversed()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated download or search shares for table {} and column {}. Took {}ms", tablename, column, stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return result; + } + + List userAgentSearchShares(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating user agent search shares"); + String sql = "SELECT\n" + + " user_agent,\n" + + " count(*)\n" + + "FROM SEARCH\n" + + "WHERE user_agent IS NOT NULL\n" + + "AND SOURCE = 'API'" + + buildWhereFromStatsRequest(true, statsRequest) + + "GROUP BY user_agent"; + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + List result = new ArrayList<>(); + int countAll = 0; + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + String userAgent = (String) o2[0]; + int countForUserAgent = ((Long) o2[1]).intValue(); + countAll += countForUserAgent; + result.add(new UserAgentShare(userAgent, countForUserAgent)); + } + for (UserAgentShare userAgentShare : result) { + userAgentShare.setPercentage(100F / ((float) countAll / userAgentShare.getCount())); + } + + result.sort(Comparator.comparingDouble(UserAgentShare::getPercentage).reversed()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated user agent search shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return result; + } + + List userAgentDownloadShares(final StatsRequest statsRequest) { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating user agent download shares"); + String sql = "SELECT\n" + + " user_agent,\n" + + " count(*)\n" + + "FROM INDEXERNZBDOWNLOAD\n" + + "WHERE user_agent IS NOT NULL\n" + + "and ACCESS_SOURCE = 'API' \n" + + buildWhereFromStatsRequest(true, statsRequest) + + "GROUP BY user_agent"; + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + List result = new ArrayList<>(); + int countAll = 0; + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + String userAgent = (String) o2[0]; + int countForUserAgent = ((Long) o2[1]).intValue(); + countAll += countForUserAgent; + result.add(new UserAgentShare(userAgent, countForUserAgent)); + } + for (UserAgentShare userAgentShare : result) { + userAgentShare.setPercentage(100F / ((float) countAll / userAgentShare.getCount())); + } + + result.sort(Comparator.comparingDouble(UserAgentShare::getPercentage).reversed()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated user agent download shares. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return result; + } + + List downloadsPerAge() { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating downloads per age"); + String sql = """ + SELECT + steps, + count(*) + FROM + (SELECT age / 100 AS steps + FROM INDEXERNZBDOWNLOAD + WHERE age IS NOT NULL) + GROUP BY steps + ORDER BY steps ASC"""; + Query query = entityManager.createNativeQuery(sql); + List resultList = query.getResultList(); + List results = new ArrayList<>(); + Map agesAndCountsMap = new HashMap<>(); + for (Object o : resultList) { + Object[] o2 = (Object[]) o; + int ageStep = (Integer) o2[0]; + int count = ((Long) o2[1]).intValue(); + agesAndCountsMap.put(ageStep, count); + } + for (int i = 0; i <= 34; i += 1) { + if (!agesAndCountsMap.containsKey(i)) { + agesAndCountsMap.put(i, 0); + } + } + for (Entry entry : agesAndCountsMap.entrySet()) { + results.add(new DownloadPerAge(entry.getKey() * 100, entry.getValue())); + } + results.sort(Comparator.comparingInt(DownloadPerAge::getAge)); + + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated downloads per age. Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + return results; + } + + DownloadPerAgeStats downloadsPerAgeStats() { + Stopwatch stopwatch = Stopwatch.createStarted(); + logger.debug("Calculating downloads per age percentages"); + DownloadPerAgeStats result = new DownloadPerAgeStats(); + String percentage = """ + SELECT CASE + WHEN (SELECT CAST(COUNT(*) AS FLOAT) AS COUNT + FROM INDEXERNZBDOWNLOAD + WHERE AGE > %d) > 0 + THEN SELECT CAST(100 AS FLOAT) / (CAST(COUNT(i.*) AS FLOAT)/ x.COUNT) + FROM INDEXERNZBDOWNLOAD i, + ( SELECT COUNT(*) AS COUNT + FROM INDEXERNZBDOWNLOAD + WHERE AGE > %d) AS x + ELSE 0 END"""; + result.setPercentOlder1000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 1000, 1000)).getResultList().get(0)).intValue()); + result.setPercentOlder2000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 2000, 2000)).getResultList().get(0)).intValue()); + result.setPercentOlder3000(((BigDecimal) entityManager.createNativeQuery(String.format(percentage, 3000, 3000)).getResultList().get(0)).intValue()); + final Double averageAge = (Double) entityManager.createNativeQuery("SELECT AVG(AGE) FROM INDEXERNZBDOWNLOAD").getResultList().get(0); + result.setAverageAge(averageAge == null ? 0 : averageAge.intValue()); + logger.debug(LoggingMarkers.PERFORMANCE, "Calculated downloads per age percentages . Took {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); + + result.setDownloadsPerAge(downloadsPerAge()); + return result; + } + + + private String buildWhereFromStatsRequest(boolean useAnd, StatsRequest statsRequest) { + if (statsRequest.getAfter() == null && statsRequest.getBefore() == null) { + return " "; + } + return (useAnd ? " AND " : " WHERE ") + + (statsRequest.getAfter() != null ? " TIME > datetime(" + statsRequest.getAfter().getEpochSecond() + ", 'unixepoch') " : "") + + ((statsRequest.getBefore() != null && statsRequest.getAfter() != null) ? " AND " : " ") + + (statsRequest.getBefore() != null ? " TIME < datetime(" + statsRequest.getBefore().getEpochSecond() + ", 'unixepoch') " : ""); + } + + +} diff --git a/core/src/main/java/org/nzbhydra/indexers/IndexerApiAccessEntity.java b/core/src/main/java/org/nzbhydra/indexers/IndexerApiAccessEntity.java index b865c71c7..1266a97fa 100644 --- a/core/src/main/java/org/nzbhydra/indexers/IndexerApiAccessEntity.java +++ b/core/src/main/java/org/nzbhydra/indexers/IndexerApiAccessEntity.java @@ -1,85 +1,85 @@ -package org.nzbhydra.indexers; - -import com.google.common.base.MoreObjects; -import jakarta.persistence.Column; -import jakarta.persistence.Convert; -import jakarta.persistence.Entity; -import jakarta.persistence.EnumType; -import jakarta.persistence.Enumerated; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.Table; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.hibernate.annotations.OnDelete; -import org.hibernate.annotations.OnDeleteAction; -import org.nzbhydra.springnative.ReflectionMarker; - -import java.time.Instant; -import java.util.Objects; - - -@Data -@ReflectionMarker -@Entity -@NoArgsConstructor -@Table(name = "indexerapiaccess") -public final class IndexerApiAccessEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - protected int id; - - @ManyToOne - @OnDelete(action = OnDeleteAction.CASCADE) - private IndexerEntity indexer; - - @Convert(converter = org.springframework.data.jpa.convert.threeten.Jsr310JpaConverters.InstantConverter.class) - private Instant time; - - @Enumerated(EnumType.STRING) - private IndexerAccessResult result; - - @Enumerated(EnumType.STRING) - private IndexerApiAccessType accessType; - private Long responseTime; - @Column(length = 4000) - private String error; - //later username / user ? - - - public IndexerApiAccessEntity(IndexerEntity indexerEntity) { - this.indexer = indexerEntity; - time = Instant.now(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - IndexerApiAccessEntity that = (IndexerApiAccessEntity) o; - return id == that.id; - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("time", time) - .add("result", result) - .add("accessType", accessType) - .add("responseTime", responseTime) - .add("error", error) - .toString(); - } -} +package org.nzbhydra.indexers; + +import com.google.common.base.MoreObjects; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; +import org.nzbhydra.springnative.ReflectionMarker; + +import java.time.Instant; +import java.util.Objects; + + +@Data +@ReflectionMarker +@Entity +@NoArgsConstructor +@Table(name = "indexerapiaccess") +public final class IndexerApiAccessEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + protected int id; + + @ManyToOne + @OnDelete(action = OnDeleteAction.CASCADE) + private IndexerEntity indexer; + + @Convert(converter = org.springframework.data.jpa.convert.threeten.Jsr310JpaConverters.InstantConverter.class) + private Instant time; + + @Enumerated(EnumType.STRING) + private IndexerAccessResult result; + + @Enumerated(EnumType.STRING) + private IndexerApiAccessType accessType; + private Long responseTime; + @Column(length = 4000) + private String error; + //later username / user ? + + + public IndexerApiAccessEntity(IndexerEntity indexerEntity) { + this.indexer = indexerEntity; + time = Instant.now(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IndexerApiAccessEntity that = (IndexerApiAccessEntity) o; + return id == that.id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("time", time) + .add("result", result) + .add("accessType", accessType) + .add("responseTime", responseTime) + .add("error", error) + .toString(); + } +} diff --git a/core/src/main/java/org/nzbhydra/indexers/IndexerEntity.java b/core/src/main/java/org/nzbhydra/indexers/IndexerEntity.java index ede012e83..00e46c2cc 100644 --- a/core/src/main/java/org/nzbhydra/indexers/IndexerEntity.java +++ b/core/src/main/java/org/nzbhydra/indexers/IndexerEntity.java @@ -1,60 +1,60 @@ -package org.nzbhydra.indexers; - -import com.google.common.base.MoreObjects; -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import lombok.Data; -import org.nzbhydra.springnative.ReflectionMarker; - -import java.util.Objects; - - -@Data -@ReflectionMarker -@Entity -@Table(name = "indexer") -public class IndexerEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private int id; - - @Column(unique = true) - private String name; - - public IndexerEntity() { - } - - public IndexerEntity(String name) { - this.name = name; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - IndexerEntity that = (IndexerEntity) o; - return id == that.id; - } - - @Override - public int hashCode() { - return Objects.hash(id, name); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("id", id) - .add("name", name) - .toString(); - } -} +package org.nzbhydra.indexers; + +import com.google.common.base.MoreObjects; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Data; +import org.nzbhydra.springnative.ReflectionMarker; + +import java.util.Objects; + + +@Data +@ReflectionMarker +@Entity +@Table(name = "indexer") +public class IndexerEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private int id; + + @Column(unique = true) + private String name; + + public IndexerEntity() { + } + + public IndexerEntity(String name) { + this.name = name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IndexerEntity that = (IndexerEntity) o; + return id == that.id; + } + + @Override + public int hashCode() { + return Objects.hash(id, name); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", id) + .add("name", name) + .toString(); + } +} diff --git a/core/src/main/java/org/nzbhydra/indexers/IndexerSearchEntity.java b/core/src/main/java/org/nzbhydra/indexers/IndexerSearchEntity.java index 4507199f7..25b1c0973 100644 --- a/core/src/main/java/org/nzbhydra/indexers/IndexerSearchEntity.java +++ b/core/src/main/java/org/nzbhydra/indexers/IndexerSearchEntity.java @@ -1,68 +1,68 @@ -package org.nzbhydra.indexers; - -import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; -import jakarta.persistence.Id; -import jakarta.persistence.Index; -import jakarta.persistence.ManyToOne; -import jakarta.persistence.Table; -import lombok.Data; -import org.hibernate.annotations.OnDelete; -import org.hibernate.annotations.OnDeleteAction; -import org.nzbhydra.searching.db.SearchEntity; -import org.nzbhydra.springnative.ReflectionMarker; - -import java.util.Objects; - - -@Data -@ReflectionMarker -@Entity -@Table(name = "indexersearch", indexes = {@Index(name = "ISINDEX1", columnList = "INDEXER_ENTITY_ID"), @Index(name = "ISINDEX2", columnList = "SEARCH_ENTITY_ID")}) -public final class IndexerSearchEntity { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - private int id; - - @ManyToOne - @OnDelete(action = OnDeleteAction.CASCADE) - private IndexerEntity indexerEntity; - @ManyToOne - @OnDelete(action = OnDeleteAction.CASCADE) - private SearchEntity searchEntity; - - private Boolean successful; - - /** - * Number of total results reported by the indexer - */ - private Integer resultsCount; - - public IndexerSearchEntity() { - } - - public IndexerSearchEntity(IndexerEntity indexerEntity, SearchEntity searchEntity, int id) { - this.indexerEntity = indexerEntity; - this.searchEntity = searchEntity; - this.id = id; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - IndexerSearchEntity entity = (IndexerSearchEntity) o; - return id == entity.id; - } - - @Override - public int hashCode() { - return Objects.hash(id); - } -} +package org.nzbhydra.indexers; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; +import lombok.Data; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; +import org.nzbhydra.searching.db.SearchEntity; +import org.nzbhydra.springnative.ReflectionMarker; + +import java.util.Objects; + + +@Data +@ReflectionMarker +@Entity +@Table(name = "indexersearch", indexes = {@Index(name = "ISINDEX1", columnList = "INDEXER_ENTITY_ID"), @Index(name = "ISINDEX2", columnList = "SEARCH_ENTITY_ID")}) +public final class IndexerSearchEntity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private int id; + + @ManyToOne + @OnDelete(action = OnDeleteAction.CASCADE) + private IndexerEntity indexerEntity; + @ManyToOne + @OnDelete(action = OnDeleteAction.CASCADE) + private SearchEntity searchEntity; + + private Boolean successful; + + /** + * Number of total results reported by the indexer + */ + private Integer resultsCount; + + public IndexerSearchEntity() { + } + + public IndexerSearchEntity(IndexerEntity indexerEntity, SearchEntity searchEntity, int id) { + this.indexerEntity = indexerEntity; + this.searchEntity = searchEntity; + this.id = id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IndexerSearchEntity entity = (IndexerSearchEntity) o; + return id == entity.id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/core/src/main/java/org/nzbhydra/searching/db/IdentifierKeyValuePair.java b/core/src/main/java/org/nzbhydra/searching/db/IdentifierKeyValuePair.java index 3f354ae4d..6e1994711 100644 --- a/core/src/main/java/org/nzbhydra/searching/db/IdentifierKeyValuePair.java +++ b/core/src/main/java/org/nzbhydra/searching/db/IdentifierKeyValuePair.java @@ -19,8 +19,8 @@ package org.nzbhydra.searching.db; import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.persistence.Entity; import jakarta.persistence.GeneratedValue; -import jakarta.persistence.GenerationType; import jakarta.persistence.Id; +import jakarta.persistence.SequenceGenerator; import lombok.Data; import lombok.NoArgsConstructor; import org.nzbhydra.springnative.ReflectionMarker; @@ -35,8 +35,9 @@ import java.util.Objects; public final class IdentifierKeyValuePair { @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) + @GeneratedValue @JsonIgnore + @SequenceGenerator(allocationSize = 1, name = "IDENTIFIER_KEY_VALUE_PAIR_SEQ") private Integer id; public IdentifierKeyValuePair(String identifierKey, String identifierValue) { diff --git a/core/src/main/resources/config/application.properties b/core/src/main/resources/config/application.properties index 9e5effef0..807a9241b 100644 --- a/core/src/main/resources/config/application.properties +++ b/core/src/main/resources/config/application.properties @@ -23,16 +23,13 @@ spring.profiles.active=default #Database connection, hibernate config -spring.datasource.url=jdbc:sqlite:${nzbhydra.dataFolder:.}/database/nzbhydra.db -spring.datasource.jdbc-url=jdbc:sqlite:${nzbhydra.dataFolder:.}/database/nzbhydra.db -spring.datasource.username= -spring.datasource.password= -spring.datasource.driver-class-name=org.sqlite.JDBC - -# SQLite connection pooling settings are configured in SQLiteConfiguration.java - -# SQLite-specific JPA settings for multithreading -# Let Hibernate auto-detect the dialect for SQLite +spring.datasource.url=jdbc:h2:file:${nzbhydra.dataFolder:.}/database/nzbhydra;MAX_COMPACT_TIME=${main.databaseCompactTime:15000};WRITE_DELAY=${main.databaseWriteDelay:5000};TRACE_MAX_FILE_SIZE=16;RETENTION_TIME=${main.databaseRetentionTime:1000};NON_KEYWORDS=YEAR,DATA,KEY +spring.datasource.jdbc-url=jdbc:h2:file:${nzbhydra.dataFolder:.}/database/nzbhydra;MAX_COMPACT_TIME=${main.databaseCompactTime:15000};WRITE_DELAY=${main.databaseWriteDelay:5000};TRACE_MAX_FILE_SIZE=16;RETENTION_TIME=${main.databaseRetentionTime:1000};NON_KEYWORDS=YEAR,DATA,KEY +spring.datasource.username=sa +spring.datasource.password=sa +spring.datasource.driver-class-name=org.h2.Driver +spring.jpa.database-platform=org.nzbhydra.database.H2DialectExtended +spring.jpa.properties.hibernate.dialect=org.nzbhydra.database.H2DialectExtended spring.jpa.properties.hibernate.current_session_context_class=org.springframework.orm.hibernate5.SpringSessionContext spring.jpa.properties.hibernate.show_sql=false spring.jpa.properties.hibernate.use_sql_comments=false @@ -48,8 +45,9 @@ spring.h2.console.enabled=false #Migration spring.flyway.enabled=true spring.flyway.locations=classpath:/migration,classpath:/org/nzbhydra/database/migration -# For fresh databases, let Hibernate create the schema -spring.jpa.hibernate.ddl-auto=validate +spring.flyway.schemas=PUBLIC +#spring.flyway.baseline-on-migrate=true +#spring.flyway.ignore-future-migrations=false #Jackson stuff spring.jackson.deserialization.unwrap-single-value-arrays=true