Skip to content

Commit

Permalink
Automatically determine forbidden word prefix to use
Browse files Browse the repository at this point in the history
  • Loading branch information
@theotherp
theotherp committed Nov 12, 2024
1 parent 214aabe commit 711d0e8
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 12 deletions.
4 changes: 4 additions & 0 deletions core/src/main/java/org/nzbhydra/indexers/Indexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.slf4j.LoggerFactory;
import org.slf4j.Marker;
import org.springframework.aot.hint.annotation.Reflective;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
Expand Down Expand Up @@ -105,6 +106,8 @@ public abstract class Indexer<T> {

private BaseConfigHandler baseConfigHandler;

protected AutowireCapableBeanFactory beanFactory;


protected Indexer() {
}
Expand All @@ -126,6 +129,7 @@ public Indexer(ConfigProvider configProvider, IndexerRepository indexerRepositor
this.baseConfigHandler = baseConfigHandler;
}


public void initialize(IndexerConfig config, IndexerEntity indexer) {
this.indexer = indexer;
this.config = config;
Expand Down
21 changes: 18 additions & 3 deletions core/src/main/java/org/nzbhydra/indexers/Newznab.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.google.common.base.Strings;
import lombok.Getter;
import lombok.Setter;
import org.nzbhydra.NzbHydra;
import org.nzbhydra.NzbHydraException;
import org.nzbhydra.config.BaseConfigHandler;
import org.nzbhydra.config.ConfigProvider;
Expand All @@ -17,6 +18,7 @@
import org.nzbhydra.config.indexer.SearchModuleType;
import org.nzbhydra.config.mediainfo.MediaIdType;
import org.nzbhydra.config.searching.SearchType;
import org.nzbhydra.indexers.capscheck.IndexerChecker;
import org.nzbhydra.indexers.exceptions.IndexerAccessException;
import org.nzbhydra.indexers.exceptions.IndexerAuthException;
import org.nzbhydra.indexers.exceptions.IndexerErrorCodeException;
Expand Down Expand Up @@ -111,12 +113,15 @@ public class Newznab extends Indexer<Xml> {
@Autowired
private IndexerLimitRepository indexerStatusRepository;

private BaseConfigHandler baseConfigHandler;

private final ConcurrentHashMap<Integer, Category> idToCategory = new ConcurrentHashMap<>();

public Newznab(ConfigProvider configProvider, IndexerRepository indexerRepository, SearchResultRepository searchResultRepository, IndexerApiAccessRepository indexerApiAccessRepository, IndexerApiAccessEntityShortRepository indexerApiAccessShortRepository, IndexerLimitRepository indexerStatusRepository, IndexerWebAccess indexerWebAccess, SearchResultAcceptor resultAcceptor, CategoryProvider categoryProvider, InfoProvider infoProvider, ApplicationEventPublisher eventPublisher, QueryGenerator queryGenerator, CustomQueryAndTitleMappingHandler titleMapping, Unmarshaller unmarshaller, BaseConfigHandler baseConfigHandler) {
super(configProvider, indexerRepository, searchResultRepository, indexerApiAccessRepository, indexerApiAccessShortRepository, indexerStatusRepository, indexerWebAccess, resultAcceptor, categoryProvider, infoProvider, eventPublisher, queryGenerator, titleMapping, baseConfigHandler);
this.unmarshaller = unmarshaller;
this.indexerStatusRepository = indexerStatusRepository;
this.baseConfigHandler = baseConfigHandler;
}

protected UriComponentsBuilder getBaseUri() {
Expand Down Expand Up @@ -275,17 +280,27 @@ protected String addRequiredAndforbiddenWordsToQuery(SearchRequest searchRequest
}

protected String addForbiddenWords(SearchRequest searchRequest, String query) {
if (config.getForbiddenWordPrefix() == IndexerConfig.ForbiddenWordPrefix.UNKNOWN) {
info("Forbidden word prefix unknown - running check to determine it");
//Horrible but easier...
config.setForbiddenWordPrefix(NzbHydra.getApplicationContext().getAutowireCapableBeanFactory().getBean(IndexerChecker.class).determineForbiddenWordPrefix(config));
baseConfigHandler.save(false);
}
if (config.getForbiddenWordPrefix() == IndexerConfig.ForbiddenWordPrefix.UNSUPPORTED) {
debug("Not adding forbidden words as this indexer doesn't support them.");
return query;
}
List<String> allForbiddenWords = new ArrayList<>(searchRequest.getInternalData().getForbiddenWords());
allForbiddenWords.addAll(configProvider.getBaseConfig().getSearching().getForbiddenWords());
allForbiddenWords.addAll(searchRequest.getCategory().getForbiddenWords());
List<String> allPossibleForbiddenWords = allForbiddenWords.stream().filter(x -> !(x.contains(" ") || x.contains("-") || x.contains("."))).collect(Collectors.toList());
if (allForbiddenWords.size() > allPossibleForbiddenWords.size()) {
debug("Not using some forbidden words in query because characters forbidden by newznab are contained");
debug("Not using some forbidden words in query because characters forbidden by newznab are contained.");
}
if (!allPossibleForbiddenWords.isEmpty()) {
if (config.getBackend().equals(BackendType.NZEDB) || config.getBackend().equals(BackendType.NNTMUX) || config.getHost().toLowerCase().contains("omgwtf") || config.getHost().toLowerCase().contains("nzbfinder")) {
if (config.getForbiddenWordPrefix() == IndexerConfig.ForbiddenWordPrefix.EXCLAMATION_MARK) {
query += (query.isEmpty() ? "" : " ") + "!" + Joiner.on(",!").join(allPossibleForbiddenWords);
} else {
} else if (config.getForbiddenWordPrefix() == IndexerConfig.ForbiddenWordPrefix.DOUBLE_DASH) {
query += (query.isEmpty() ? "" : " ") + "--" + Joiner.on(" --").join(allPossibleForbiddenWords);
}
}
Expand Down
55 changes: 46 additions & 9 deletions core/src/main/java/org/nzbhydra/indexers/capscheck/IndexerChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ public CheckCapsResponse checkCaps(IndexerConfig indexerConfig) {
try {
logger.info("Will check capabilities of indexer {} using {} concurrent connections and a delay of {}ms", indexerConfig.getName(), capsCheckLimit.maxConnections, capsCheckLimit.delayInMiliseconds);
List<Future<SingleCheckCapsResponse>> futures = executor.invokeAll(callables);
Future<IndexerConfig.ForbiddenWordPrefix> forbiddenWordPrefixFuture = executor.submit(() -> determineForbiddenWordPrefix(indexerConfig));
for (Future<SingleCheckCapsResponse> future : futures) {
try {
SingleCheckCapsResponse response = future.get(timeout, TimeUnit.SECONDS);
Expand All @@ -209,7 +210,7 @@ public CheckCapsResponse checkCaps(IndexerConfig indexerConfig) {
responses.add(response);
} catch (ExecutionException e) {
if (e.getCause() instanceof IndexerAccessException) {
logger.error("Error while communicating with indexer: " + e.getMessage());
logger.error("Error while communicating with indexer: {}", e.getMessage());
} else {
logger.error("Unexpected error while checking caps", e);
}
Expand All @@ -219,6 +220,14 @@ public CheckCapsResponse checkCaps(IndexerConfig indexerConfig) {
allChecked = false;
}
}
try {
IndexerConfig.ForbiddenWordPrefix prefix = forbiddenWordPrefixFuture.get();
logger.info("Prefix for forbidden words: {}", prefix);
indexerConfig.setForbiddenWordPrefix(prefix);
} catch (ExecutionException e) {
allChecked = false;
logger.error("Error checking forbidden word prefix", e);
}
supportedIds = responses.stream().filter(SingleCheckCapsResponse::isSupported).map(SingleCheckCapsResponse::getIdType).collect(Collectors.toSet());
Optional<SingleCheckCapsResponse> responseWithLimits = responses.stream().filter(x -> x.getApiMax() != null).findFirst();
if (responseWithLimits.isPresent()) {
Expand Down Expand Up @@ -252,7 +261,7 @@ public CheckCapsResponse checkCaps(IndexerConfig indexerConfig) {
logger.info("Indexer {} supports the following search types: {}", indexerConfig.getName(), indexerConfig.getSupportedSearchTypes().stream().map(Enum::name).collect(Collectors.joining(", ")));
}
} catch (IndexerAccessException e) {
logger.error("Error while accessing indexer: " + e.getMessage());
logger.error("Error while accessing indexer: {}", e.getMessage());
configComplete = false;
}

Expand All @@ -276,15 +285,43 @@ public CheckCapsResponse checkCaps(IndexerConfig indexerConfig) {
return new CheckCapsResponse(indexerConfig, allChecked, configComplete);
}

public IndexerConfig.ForbiddenWordPrefix determineForbiddenWordPrefix(IndexerConfig indexerConfig) {
URI uri = getBaseUri(indexerConfig).queryParam("t", "search").queryParam("q", "Avengers --1080p").build().toUri();
try {
NewznabXmlRoot root = indexerWebAccess.get(uri, indexerConfig, NewznabXmlRoot.class);
if (!root.getRssChannel().getItems().isEmpty() && root.getRssChannel().getItems().stream().noneMatch(x -> x.getTitle().contains("1080p"))) {
logger.info("Determined forbidden word prefix \"--\" for indexer {}", indexerConfig.getName());
return IndexerConfig.ForbiddenWordPrefix.DOUBLE_DASH;
}
} catch (IndexerAccessException e) {
logger.error("Error while determining forbidden word prefix", e);
return IndexerConfig.ForbiddenWordPrefix.UNKNOWN;
}

uri = getBaseUri(indexerConfig).queryParam("t", "search").queryParam("q", "Avengers !1080p").build().toUri();
try {
NewznabXmlRoot root = indexerWebAccess.get(uri, indexerConfig, NewznabXmlRoot.class);
if (!root.getRssChannel().getItems().isEmpty() && root.getRssChannel().getItems().stream().noneMatch(x -> x.getTitle().contains("1080p"))) {
logger.info("Determined forbidden word prefix \"!\" for indexer {}", indexerConfig.getName());
return IndexerConfig.ForbiddenWordPrefix.EXCLAMATION_MARK;
}
} catch (IndexerAccessException e) {
logger.error("Error while determining forbidden word prefix", e);
return IndexerConfig.ForbiddenWordPrefix.UNKNOWN;
}

return IndexerConfig.ForbiddenWordPrefix.UNSUPPORTED;
}

private List<CheckCapsResponse> checkCaps(CapsCheckRequest.CheckType checkType) {
Predicate<IndexerConfig> isToBeCheckedPredicate = x ->
x.getState() == IndexerConfig.State.ENABLED
x.getState() == IndexerConfig.State.ENABLED
&& (x.getSearchModuleType() == SearchModuleType.NEWZNAB || x.getSearchModuleType() == SearchModuleType.TORZNAB)
&& x.isConfigComplete()
&& (checkType == CheckType.ALL || !x.isAllCapsChecked());
List<IndexerConfig> configsToCheck = configProvider.getBaseConfig().getIndexers().stream()
.filter(isToBeCheckedPredicate)
.toList();
.filter(isToBeCheckedPredicate)
.toList();
if (configsToCheck.isEmpty()) {
logger.info("No indexers to check");
return Collections.emptyList();
Expand Down Expand Up @@ -452,7 +489,7 @@ private SingleCheckCapsResponse singleCheckCaps(CheckCapsRequest request, Indexe
}

@Data
@ReflectionMarker
@ReflectionMarker
@AllArgsConstructor
@NoArgsConstructor
public static class CheckerEvent {
Expand All @@ -461,7 +498,7 @@ public static class CheckerEvent {
}

@Data
@ReflectionMarker
@ReflectionMarker
@AllArgsConstructor
@NoArgsConstructor
public static class ConnectionCheckResponse {
Expand All @@ -471,7 +508,7 @@ public static class ConnectionCheckResponse {


@Data
@ReflectionMarker
@ReflectionMarker
@AllArgsConstructor
@NoArgsConstructor
private static class CheckCapsRequest {
Expand All @@ -484,7 +521,7 @@ private static class CheckCapsRequest {
}

@Data
@ReflectionMarker
@ReflectionMarker
@AllArgsConstructor
@NoArgsConstructor
private static class SingleCheckCapsResponse {
Expand Down
6 changes: 6 additions & 0 deletions core/src/main/resources/changelog.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
#@formatter:off
- version: "v7.10.0"
date: "2024-11-12"
changes:
- type: "feature"
text: "Previously the way forbidden words would be excluded from queries depended on the backend and, to a certain degree, on the indexer but in some cases -- would be used as a prefix which is only supported by many indexers. NZBHydra will now attempt to automatically detect if and how certain words may be excluded from a query. This will either happen during a caps check or on the first search request where needed."
final: true
- version: "v7.9.0"
date: "2024-11-10"
changes:
Expand Down
8 changes: 8 additions & 0 deletions shared/mapping/src/main/java/org/nzbhydra/config/indexer/IndexerConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ public String humanize() {
}
}

public enum ForbiddenWordPrefix {
UNKNOWN,
EXCLAMATION_MARK,
DOUBLE_DASH,
UNSUPPORTED
}

private boolean allCapsChecked;
@SensitiveData
private String apiKey;
Expand All @@ -71,6 +78,7 @@ public String humanize() {
private boolean configComplete = true;
private List<String> enabledCategories = new ArrayList<>();
private Integer downloadLimit = null;
private ForbiddenWordPrefix forbiddenWordPrefix = ForbiddenWordPrefix.UNKNOWN;
@JsonFormat(shape = Shape.STRING)
private State state = State.ENABLED;
@JsonFormat(shape = Shape.STRING)
Expand Down

0 comments on commit 711d0e8

Please sign in to comment.