Add post-search attribute filter

Closes #983
2026-02-06 11:17:18 +00:00 · 2026-01-03 15:19:17 +01:00 · 2026-01-03 15:19:17 +01:00 · 0fa5b1ddc6
commit 0fa5b1ddc6
parent a32dc300f6
10 changed files with 404 additions and 5 deletions
--- a/core/.claude/settings.local.json
+++ b/core/.claude/settings.local.json
@ -34,7 +34,10 @@
      "Bash(python3:*)",
      "Bash(/c/Programme/python312/python:*)",
      "mcp__jetbrains__get_run_configurations",
-      "mcp__jetbrains__execute_run_configuration"
+      "mcp__jetbrains__execute_run_configuration",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:raw.githubusercontent.com)",
+      "WebFetch(domain:api.github.com)"
    ],
    "deny": [],
    "ask": []
--- a/core/src/main/java/org/nzbhydra/config/validation/IndexerConfigValidator.java
+++ b/core/src/main/java/org/nzbhydra/config/validation/IndexerConfigValidator.java
@ -64,6 +64,12 @@ public class IndexerConfigValidator implements ConfigValidator<IndexerConfig> {
            }
        });

+        newConfig.getAttributeWhitelist().forEach(x -> {
+            if (Strings.isNullOrEmpty(x) || StringUtils.countMatches(x, '=') != 1) {
+                validationResult.getErrorMessages().add("The attribute whitelist entry '" + x + "' is invalid. You must use the format name=value or name=value1,value2.");
+            }
+        });
+
        return validationResult;
    }

--- a/core/src/main/java/org/nzbhydra/searching/SearchResultAcceptor.java
+++ b/core/src/main/java/org/nzbhydra/searching/SearchResultAcceptor.java
@ -100,6 +100,9 @@ public class SearchResultAcceptor {
            if (!checkMinSeeders(indexerConfig, reasonsForRejection, item)) {
                continue;
            }
+            if (!checkAttributeWhitelist(indexerConfig, reasonsForRejection, item)) {
+                continue;
+            }

            //Forbidden words from query
            if (!checkForForbiddenWords(indexerConfig, reasonsForRejection, searchRequest.getInternalData().getForbiddenWords(), item, "internal data")) {
@ -396,6 +399,98 @@ public class SearchResultAcceptor {
        return true;
    }

+    /**
+     * Checks if an item matches the configured attribute whitelist for the indexer.
+     * <p>
+     * The whitelist uses OR logic between different entries (e.g., ["subs=English", "subs=French"] accepts if EITHER matches).
+     * Comma-separated values within one entry use AND logic (e.g., "subs=English,French" requires BOTH to be present).
+     * Items without any attributes are rejected when a whitelist is configured.
+     * <p>
+     * The whitelist can be limited to specific categories. If no categories are configured, the whitelist applies to all results.
+     *
+     * @param indexerConfig       the indexer configuration containing the whitelist
+     * @param reasonsForRejection multiset to track rejection reasons
+     * @param item                the search result item to check
+     * @return true if the item should be accepted, false if it should be rejected
+     */
+    protected boolean checkAttributeWhitelist(IndexerConfig indexerConfig, Multiset<String> reasonsForRejection, SearchResultItem item) {
+        List<String> whitelist = indexerConfig.getAttributeWhitelist();
+        if (whitelist == null || whitelist.isEmpty()) {
+            return true;
+        }
+
+        // Check if the whitelist should apply to this item's category
+        List<String> whitelistCategories = indexerConfig.getAttributeWhitelistCategories();
+        if (whitelistCategories != null && !whitelistCategories.isEmpty()) {
+            String itemCategoryName = item.getCategory() != null ? item.getCategory().getName() : null;
+            if (itemCategoryName == null || !whitelistCategories.contains(itemCategoryName)) {
+                // Category not in the whitelist categories - skip attribute filtering for this item
+                return true;
+            }
+        }
+
+        Map<String, String> itemAttributes = item.getAttributes();
+        if (itemAttributes == null || itemAttributes.isEmpty()) {
+            logger.debug(LoggingMarkers.RESULT_ACCEPTOR, "Rejecting '{}' because it has no attributes but attribute whitelist is configured", item.getTitle());
+            reasonsForRejection.add("No attributes but whitelist configured");
+            return false;
+        }
+
+        // Check each whitelist entry - OR logic between entries
+        for (String whitelistEntry : whitelist) {
+            if (matchesWhitelistEntry(whitelistEntry, itemAttributes)) {
+                return true;
+            }
+        }
+
+        logger.debug(LoggingMarkers.RESULT_ACCEPTOR, "Rejecting '{}' because it doesn't match any attribute whitelist entry", item.getTitle());
+        reasonsForRejection.add("Attribute whitelist not matched");
+        return false;
+    }
+
+    /**
+     * Checks if item attributes match a single whitelist entry.
+     * For comma-separated values, ALL values must be present (AND logic).
+     */
+    private boolean matchesWhitelistEntry(String whitelistEntry, Map<String, String> itemAttributes) {
+        int equalsIndex = whitelistEntry.indexOf('=');
+        if (equalsIndex <= 0 || equalsIndex >= whitelistEntry.length() - 1) {
+            logger.error("Invalid whitelist entry format: '{}'", whitelistEntry);
+            return false;
+        }
+
+        String attributeName = whitelistEntry.substring(0, equalsIndex).trim().toLowerCase();
+        String requiredValuesStr = whitelistEntry.substring(equalsIndex + 1).trim();
+
+        // Get the item's value for this attribute (case-insensitive key lookup)
+        String itemValue = null;
+        for (Map.Entry<String, String> entry : itemAttributes.entrySet()) {
+            if (entry.getKey().equalsIgnoreCase(attributeName)) {
+                itemValue = entry.getValue();
+                break;
+            }
+        }
+
+        if (itemValue == null) {
+            return false;
+        }
+
+        // Check if the required values contain commas (AND logic for multiple values)
+        if (requiredValuesStr.contains(",")) {
+            String[] requiredValues = requiredValuesStr.split(",");
+            // ALL values must be present in the item's attribute value
+            for (String requiredValue : requiredValues) {
+                if (!itemValue.toLowerCase().contains(requiredValue.trim().toLowerCase())) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            // Single value - simple case-insensitive comparison
+            return itemValue.equalsIgnoreCase(requiredValuesStr);
+        }
+    }
+
    @Data
@ReflectionMarker
    @AllArgsConstructor
--- a/core/src/main/resources/META-INF/native-image/reflect-config.json
+++ b/core/src/main/resources/META-INF/native-image/reflect-config.json
@ -13938,6 +13938,14 @@
          "name": "getCustomParameters",
          "parameterTypes": []
      },
+      {
+          "name": "getAttributeWhitelist",
+          "parameterTypes": []
+      },
+      {
+          "name": "getAttributeWhitelistCategories",
+          "parameterTypes": []
+      },
      {
          "name": "getDisabledAt",
          "parameterTypes": []
@ -14102,6 +14110,18 @@
              "java.util.List"
          ]
      },
+      {
+          "name": "setAttributeWhitelist",
+          "parameterTypes": [
+              "java.util.List"
+          ]
+      },
+      {
+          "name": "setAttributeWhitelistCategories",
+          "parameterTypes": [
+              "java.util.List"
+          ]
+      },
      {
          "name": "setDisabledAt",
          "parameterTypes": [
@ -47757,6 +47777,26 @@
        "name": "getCustomParameters",
        "parameterTypes": []
      },
+      {
+        "name": "getAttributeWhitelist",
+        "parameterTypes": []
+      },
+      {
+        "name": "getAttributeWhitelistCategories",
+        "parameterTypes": []
+      },
+      {
+        "name": "setAttributeWhitelist",
+        "parameterTypes": [
+          "java.util.List"
+        ]
+      },
+      {
+        "name": "setAttributeWhitelistCategories",
+        "parameterTypes": [
+          "java.util.List"
+        ]
+      },
      {
        "name": "setName",
        "parameterTypes": [
--- a/core/src/main/resources/changelog.yaml
+++ b/core/src/main/resources/changelog.yaml
@ -1,6 +1,8 @@
 #@formatter:off
-   version: "v8.2.0"
+-   version: "v8.3.0"
    changes:
+      -   type: "feature"
+          text: "Add attribute whitelist filtering for indexers. Allows filtering search results by newznab attributes (e.g., subtitles). Configure per-indexer with optional category restrictions. See #983"
      -   type: "feature"
          text: "Option to import prowlarr indexers. They will be added as separate entries with a (Prowlarr) suffix. This different than using the external tool configuration as the prowlarr indexers will be searched via prowlarr, i.e. when you search them in hydra it will call prowlarr which then calls the indexers. See #922"
      -   type: "feature"
--- a/core/src/main/resources/static/js/nzbhydra.js
+++ b/core/src/main/resources/static/js/nzbhydra.js
@ -4215,13 +4215,51 @@ function getIndexerBoxFields(indexerModel, parentModel, isInitial, CategoriesSer
                    type: 'text',
                    required: false,
                    label: 'Custom parameters',
-                    help: 'Define custom parameters to be sent to the indexer when searching. Use the format "name=value"Apply values with return key.',
+                    help: 'Define custom parameters to be sent to the indexer when searching. Use the format "name=value". Apply values with return key.',
                    advanced: 'true'
                }
            }
        )
    }

+    if (['NEWZNAB', 'TORZNAB'].includes(indexerModel.searchModuleType)) {
+        fieldset.push(
+            {
+                key: 'attributeWhitelist',
+                type: 'horizontalChips',
+                templateOptions: {
+                    type: 'text',
+                    required: false,
+                    label: 'Attribute whitelist',
+                    help: 'Only accept results with matching attributes. Use format "name=value" (e.g., "subs=English"). Multiple entries use OR logic. Comma-separated values use AND logic (e.g., "subs=English,French" requires both). Apply values with return key. Results with no attributes will be rejected.',
+                    advanced: 'true'
+                }
+            }
+        );
+
+        var cats = CategoriesService.getWithoutAll();
+        var categoryOptions = _.map(cats, function (x) {
+            return {id: x.name, label: x.name}
+        });
+        fieldset.push(
+            {
+                key: 'attributeWhitelistCategories',
+                type: 'horizontalMultiselect',
+                hideExpression: '!model.attributeWhitelist || model.attributeWhitelist.length === 0',
+                templateOptions: {
+                    label: 'Whitelist categories',
+                    help: 'Apply attribute whitelist only to results in these categories. Selecting none applies the whitelist to all categories.',
+                    options: categoryOptions,
+                    settings: {
+                        showSelectedValues: false,
+                        noSelectedText: "All"
+                    },
+                    advanced: true
+                }
+            }
+        );
+    }
+
    fieldset.push(
        {
            key: 'preselect',
--- a/core/src/main/resources/static/js/nzbhydra.js.map
+++ b/core/src/main/resources/static/js/nzbhydra.js.map
--- a/core/src/test/java/org/nzbhydra/searching/SearchResultAcceptorTest.java
+++ b/core/src/test/java/org/nzbhydra/searching/SearchResultAcceptorTest.java
@ -313,5 +313,180 @@ public class SearchResultAcceptorTest {

    }

+    @Test
+    void shouldAcceptWhenNoWhitelistConfigured() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Collections.emptyList());
+        item.setTitle("Some result");
+
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+
+    @Test
+    void shouldRejectItemWithNoAttributesWhenWhitelistConfigured() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        item.setTitle("Some result");
+        // item has no attributes set (empty map by default)
+
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldAcceptItemMatchingSingleWhitelistEntry() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        item.setTitle("Some result");
+        item.getAttributes().put("subs", "English");
+
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+
+    @Test
+    void shouldAcceptItemMatchingWhitelistEntryCaseInsensitive() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=english"));
+        item.setTitle("Some result");
+        item.getAttributes().put("SUBS", "ENGLISH");
+
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+
+    @Test
+    void shouldRejectItemNotMatchingWhitelistEntry() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        item.setTitle("Some result");
+        item.getAttributes().put("subs", "French");
+
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldAcceptItemMatchingAnyWhitelistEntryOrLogic() {
+        // OR logic: accept if subs=English OR subs=French
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English", "subs=French"));
+        item.setTitle("Some result");
+
+        // English matches first entry
+        item.getAttributes().put("subs", "English");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // French matches second entry
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "French");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // German doesn't match any
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "German");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldRequireAllValuesForCommaSeparatedAndLogic() {
+        // AND logic for comma-separated: subs=English,French requires BOTH to be in the value
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English,French"));
+        item.setTitle("Some result");
+
+        // Value contains both English and French
+        item.getAttributes().put("subs", "English French German");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // Value contains only English
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "English");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+
+        // Value contains only French
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "French");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldCombineOrAndAndLogicCorrectly() {
+        // Two entries: "subs=English,French" (requires both) OR "subs=Japanese"
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English,French", "subs=Japanese"));
+        item.setTitle("Some result");
+
+        // Match first entry (both English and French)
+        item.getAttributes().put("subs", "English French");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // Match second entry (Japanese)
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "Japanese");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // Only English doesn't match either entry
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "English");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldRejectItemWithWrongAttribute() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        item.setTitle("Some result");
+        // Item has a different attribute, not 'subs'
+        item.getAttributes().put("language", "English");
+
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+    }
+
+    @Test
+    void shouldApplyWhitelistOnlyToConfiguredCategories() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        when(indexerConfig.getAttributeWhitelistCategories()).thenReturn(Arrays.asList("Movies"));
+        item.setTitle("Some result");
+
+        // Item in Movies category without matching attributes - should be rejected
+        Category moviesCategory = new Category();
+        moviesCategory.setName("Movies");
+        item.setCategory(moviesCategory);
+        item.getAttributes().put("subs", "French");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+
+        // Item in Movies category with matching attributes - should be accepted
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "English");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+
+        // Item in TV category without matching attributes - should be accepted (category not in whitelist categories)
+        Category tvCategory = new Category();
+        tvCategory.setName("TV");
+        item.setCategory(tvCategory);
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "French");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+
+    @Test
+    void shouldApplyWhitelistToAllCategoriesWhenNoCategoriesConfigured() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        when(indexerConfig.getAttributeWhitelistCategories()).thenReturn(Collections.emptyList());
+        item.setTitle("Some result");
+
+        // Item in any category without matching attributes - should be rejected
+        Category moviesCategory = new Category();
+        moviesCategory.setName("Movies");
+        item.setCategory(moviesCategory);
+        item.getAttributes().put("subs", "French");
+        assertThat(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item)).isFalse();
+
+        // Item with matching attributes - should be accepted
+        item.getAttributes().clear();
+        item.getAttributes().put("subs", "English");
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+
+    @Test
+    void shouldAcceptItemWithNullCategoryWhenCategoriesConfigured() {
+        when(indexerConfig.getAttributeWhitelist()).thenReturn(Arrays.asList("subs=English"));
+        when(indexerConfig.getAttributeWhitelistCategories()).thenReturn(Arrays.asList("Movies"));
+        item.setTitle("Some result");
+        item.setCategory(null);
+        item.getAttributes().put("subs", "French");
+
+        // Item with null category should be accepted (skip filtering) when categories are configured
+        assertTrue(testee.checkAttributeWhitelist(indexerConfig, HashMultiset.create(), item));
+    }
+

 }
--- a/core/ui-src/js/config/formly-indexers.js
+++ b/core/ui-src/js/config/formly-indexers.js
@ -414,13 +414,51 @@ function getIndexerBoxFields(indexerModel, parentModel, isInitial, CategoriesSer
                    type: 'text',
                    required: false,
                    label: 'Custom parameters',
-                    help: 'Define custom parameters to be sent to the indexer when searching. Use the format "name=value"Apply values with return key.',
+                    help: 'Define custom parameters to be sent to the indexer when searching. Use the format "name=value". Apply values with return key.',
                    advanced: 'true'
                }
            }
        )
    }

+    if (['NEWZNAB', 'TORZNAB'].includes(indexerModel.searchModuleType)) {
+        fieldset.push(
+            {
+                key: 'attributeWhitelist',
+                type: 'horizontalChips',
+                templateOptions: {
+                    type: 'text',
+                    required: false,
+                    label: 'Attribute whitelist',
+                    help: 'Only accept results with matching attributes. Use format "name=value" (e.g., "subs=English"). Multiple entries use OR logic. Comma-separated values use AND logic (e.g., "subs=English,French" requires both). Apply values with return key. Results with no attributes will be rejected.',
+                    advanced: 'true'
+                }
+            }
+        );
+
+        var cats = CategoriesService.getWithoutAll();
+        var categoryOptions = _.map(cats, function (x) {
+            return {id: x.name, label: x.name}
+        });
+        fieldset.push(
+            {
+                key: 'attributeWhitelistCategories',
+                type: 'horizontalMultiselect',
+                hideExpression: '!model.attributeWhitelist || model.attributeWhitelist.length === 0',
+                templateOptions: {
+                    label: 'Whitelist categories',
+                    help: 'Apply attribute whitelist only to results in these categories. Selecting none applies the whitelist to all categories.',
+                    options: categoryOptions,
+                    settings: {
+                        showSelectedValues: false,
+                        noSelectedText: "All"
+                    },
+                    advanced: true
+                }
+            }
+        );
+    }
+
    fieldset.push(
        {
            key: 'preselect',
--- a/shared/mapping/src/main/java/org/nzbhydra/config/indexer/IndexerConfig.java
+++ b/shared/mapping/src/main/java/org/nzbhydra/config/indexer/IndexerConfig.java
@ -113,6 +113,8 @@ public class IndexerConfig {
    private String username = null;
    private String userAgent = null;
    private String vipExpirationDate;
+    private List<String> attributeWhitelist = new ArrayList<>();
+    private List<String> attributeWhitelistCategories = new ArrayList<>();

    public Optional<String> getApiPath() {
        return Optional.ofNullable(apiPath);