56 Commits

Author SHA1 Message Date
pcjones
dbac09bf36 Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr 2024-02-23 12:25:51 +01:00
pcjones
0bde5d5d24 Fix sync process not finishing if there was an error 2024-02-23 12:25:42 +01:00
Jonas F
99af842fc6 Update README.md 2024-02-20 15:16:19 +01:00
pcjones
fbfbeadb3e set FORCE_TEXT_SEARCH_ORIGINAL_TITLE to true by default 2024-02-19 21:35:16 +01:00
pcjones
7cfae00511 Fix SearchItem lookup not working for newly added items in Readarr and Lidarr 2024-02-19 21:04:32 +01:00
pcjones
cac920ae88 Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr 2024-02-19 14:20:27 +01:00
pcjones
bab60771a4 Merge branch 'develop' 2024-02-19 14:20:10 +01:00
pcjones
828faae486 Lower minimum delay between two requests to 1.5 seconds; no longer use delay if cache can be used 2024-02-19 14:19:57 +01:00
Jonas F
333a18ecd5 Update README.md 2024-02-19 14:07:47 +01:00
pcjones
a4a57d899a Merge branch 'develop' 2024-02-19 14:05:16 +01:00
pcjones
f804dd796f Add title without umlauts as base variation 2024-02-19 06:20:47 +01:00
pcjones
b2e4dbbda6 Made apiKey parameter lowercase 2024-02-19 05:08:24 +01:00
Jonas F
cd70997507 Update README.md 2024-02-18 21:20:39 +01:00
pcjones
b741239194 Lidarr optimizations 2024-02-14 23:59:53 +01:00
pcjones
96f8ff9332 Refactor TItleMatchingService 2024-02-14 21:00:24 +01:00
pcjones
e739affb39 Use TitleMatchVariations instead of TitleSearchVariations in SearchItemByTitle 2024-02-14 20:55:13 +01:00
pcjones
92bdf14618 Remove test variable 2024-02-14 20:40:13 +01:00
pcjones
4260b07bc4 Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr 2024-02-14 11:15:51 +01:00
pcjones
4d2ac194aa Ignore case when filtering distinct title match variations 2024-02-14 11:15:44 +01:00
pcjones
a6f332fd99 Fix hyphen in indexer url not being accepted 2024-02-14 11:14:26 +01:00
Jonas F
9c364cb652 Update README.md 2024-02-13 22:34:49 +01:00
pcjones
7e7ff15f75 Workaround for weird lidarr album title parsing 2024-02-13 01:47:08 +01:00
pcjones
4ee55fc14a Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr 2024-02-13 01:38:11 +01:00
pcjones
2ae236b68c Add Lidarr album matching workaround 2024-02-13 01:38:06 +01:00
Jonas F
5fe257f5d6 Update README.md 2024-02-13 01:26:58 +01:00
pcjones
525036e08f Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr 2024-02-13 01:22:03 +01:00
pcjones
687ba9b924 Add workaround for (DE) titles 2024-02-13 01:21:59 +01:00
Jonas F
0a048c92b8 Update README.md 2024-02-13 00:14:45 +01:00
Jonas F
eef0822ce7 Update README.md 2024-02-13 00:13:35 +01:00
pcjones
a25c950a81 Add RSS sync for Lidarr 2024-02-13 00:04:50 +01:00
Jonas F
14b7bc8e60 Update README.md 2024-02-12 21:37:02 +01:00
Jonas F
9cf590b7e5 Update README.md 2024-02-12 21:34:19 +01:00
Jonas F
7c5ff4c77b Update README.md 2024-02-12 21:30:01 +01:00
pcjones
4df02f48ff Update docker compose 2024-02-12 21:28:51 +01:00
pcjones
8c07d48038 Update docker compose 2024-02-12 21:26:31 +01:00
pcjones
4ca89f8bdd Intermediate commit 2024-02-12 21:04:18 +01:00
Jonas F
0071b0c080 Update README.md 2024-02-12 14:14:34 +01:00
Jonas F
7e7100d715 Update README.md 2024-02-12 13:54:16 +01:00
Jonas F
4f5f369339 Update README.md 2024-02-12 13:29:20 +01:00
Jonas F
fc32682493 Update README.md 2024-02-12 05:33:34 +01:00
Jonas F
e78573fd03 Update docker-compose.yml 2024-02-12 05:30:47 +01:00
Jonas F
a59e00288f Update README.md 2024-02-12 05:30:34 +01:00
Jonas F
baeb9bb771 Update README.md 2024-02-12 05:29:04 +01:00
Jonas F
2d44de0212 Update docker-compose.yml 2024-02-12 05:28:44 +01:00
Jonas F
1434f3d52f Update README.md 2024-02-12 05:27:52 +01:00
Jonas F
87e34f156e Update README.md 2024-02-12 05:10:10 +01:00
Jonas F
a4dfa62c4f Update Dockerfile 2024-02-12 05:06:20 +01:00
Jonas F
196c948e8b Update README.md 2024-02-12 04:37:56 +01:00
Jonas F
8c3878a935 Update README.md 2024-02-12 04:18:04 +01:00
Jonas F
02664231e1 Update README.md 2024-02-12 04:17:00 +01:00
pcjones
d159850f67 Fix wrong title renaming when expectedTitle startsWith variation 2024-02-12 04:06:39 +01:00
Jonas F
5c87f3a0af Update README.md 2024-02-12 03:46:59 +01:00
pcjones
614906287a Intermediate commit 2024-02-12 03:46:06 +01:00
pcjones
4784867277 Remove networks from docker compose 2024-02-12 02:38:02 +01:00
pcjones
458d23e4a2 Update docker-compose 2024-02-12 02:36:12 +01:00
pcjones
8df44d514f Don't log api keys 2024-02-12 02:31:36 +01:00
21 changed files with 1175 additions and 177 deletions

View File

@@ -1,13 +1,10 @@
# Use the official Microsoft .NET Core SDK image as the build environment
FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build-env
WORKDIR /app
# Copy everything and build the project
COPY . ./
RUN dotnet restore
RUN dotnet publish -c Release -o out
# Generate the runtime image
FROM mcr.microsoft.com/dotnet/aspnet:8.0
WORKDIR /app
COPY --from=build-env /app/out .

106
README.md
View File

@@ -1,25 +1,99 @@
# UmlautAdaptarr
## Testversion kommt in wenigen Tagen
## English description coming soon
### Kleine Preview
## Erste Testversion
Wer möchte kann den UmlautAdaptarr jetzt gerne testen! Über Feedback würde ich mich sehr freuen!
Es sollte mit allen *arrs funktionieren, hat aber nur bei Sonarr und Lidarr schon Auswirkungen (abgesehen vom Caching).
Momentan ist docker dafür nötig, wer kein Docker nutzt muss sich noch etwas gedulden.
[Link zum Docker Image](https://hub.docker.com/r/pcjones/umlautadaptarr)
Zusätzlich müsst ihr in Sonarr oder Prowlarr einen neuen Indexer hinzufügen (für jeden Indexer, bei dem UmlautAdapdarr greifen soll).
Am Beispiel von sceneNZBs:
![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/07c7ca45-e0e5-4a82-af63-365bb23c55c9)
Also alles wie immer, nur dass ihr als API-URL nicht direkt z.B. `https://scenenzbs.com` eingebt, sondern
`http://localhost:5005/_/scenenzbs.com`
Den API-Key müsst ihr natürlich auch ganz normal setzen.
## Was macht UmlautAdaptarr überhaupt?
UmlautAdaptarr löst mehrere Probleme:
- Releases mit Umlauten werden grundsätzlich nicht korrekt von den *Arrs importiert
- Releases mit Umlauten werden oft nicht korrekt gefunden (*Arrs suchen nach "o" statt "ö" & es fehlt häufig die korrekte Zuordnung zur Serie/zum Film beim Indexer)
- Sonarr & Radarr erwarten immer den englischen Titel von https://thetvdb.com/ bzw. https://www.themoviedb.org/. Das führt bei deutschen Produktionen oder deutschen Übersetzungen oft zu Problemen - falls die *arrs schon mal etwas mit der Meldung `Found matching series/movie via grab history, but release was matched to series by ID. Automatic import is not possible/` nicht importiert haben, dann war das der Grund.
# Wie macht UmlautAdaptarr das?
UmlautAdaptarr tut so, als wäre es ein Indexer. In Wahrheit schaltet sich UmlautAdaptarr aber nur zwischen die *arrs und den echten Indexer und kann somit die Suchen sowie die Ergebnisse abfangen und bearbeiten.
Am Ende werden die gefundenen Releases immer so umbenannt, das die Arrs sie einwandfrei erkennen.
Einige Beispiele findet ihr unter Features.
## Features
| Feature | Status |
|-------------------------------------------------------------------|---------------|
| Prowlarr & NZB Hydra Support | ✓|
| Sonarr Support | ✓ |
| Lidarr Support | ✓|
| Readarr Support | ✓ |
| Releases mit deutschem Titel werden erkannt | ✓ |
| Releases mit TVDB-Alias Titel werden erkannt | ✓ |
| Korrekte Suche und Erkennung von Titel mit Umlauten | ✓ |
| Anfragen-Caching für 5 Minuten zur Reduzierung der API-Zugriffe | ✓ |
| Radarr Support | Geplant |
| Prowlarr Unterstützung für "DE" SceneNZBs Kategorien | Geplant |
| Unterstützung weiterer Sprachen neben Deutsch | Geplant |
| Wünsche? | Vorschläge? |
## Beispiel-Funktionalität
In den Klammern am Ende des Releasenamens (Bild 2 & 4) steht zu Anschauungszwecken der deutsche Titel der vorher nicht gefunden bzw. akzeptiert wurde. Das bleibt natürlich nicht so ;)
**Vorher:**
![Vorherige Suche ohne deutsche Titel](https://i.imgur.com/7pfRzgH.png)
Release wird zwar gefunden, kann aber kann nicht zu geordnet werden.
**Vorher:** Release wird zwar gefunden, kann aber kann nicht zu geordnet werden.
![Vorherige Suche ohne deutsche Titel](https://github.com/PCJones/UmlautAdaptarr/assets/377223/1fce2909-a36c-4f1b-8497-85903357fee3)
**Jetzt:**
![Jetzige Suche mit deutschen Titeln](https://i.imgur.com/k55YIN9.png)
2-3 weitere Releases werden gefunden, außerdem meckert Sonarr nicht mehr über den Namen und würde es bei einer automatischen Suche ohne Probleme importieren.
**Jetzt:** 2-3 weitere Releases werden gefunden, außerdem meckert Sonarr nicht mehr über den Namen und würde es bei einer automatischen Suche ohne Probleme importieren.
![Jetzige Suche mit deutschen Titeln](https://github.com/PCJones/UmlautAdaptarr/assets/377223/0edf43ba-2beb-4f22-aaf4-30f9a619dbd6)
**Vorher:**
![Vorherige Suche, englische Titel](https://i.imgur.com/pbRlOeX.png)
Es werden nur Releases mit dem englischen Titel der Serie gefunden
**Jetzt:**
![Jetzige Suche, deutsche und englische Titel](https://i.imgur.com/eeq0Voj.png)
Es werden auch Titel mit dem deutschen Namen gefunden :D (haben nicht alle Suchergebnisse auf den Screenshot gepasst)
**Vorher:** Es werden nur Releases mit dem englischen Titel der Serie gefunden
![Vorherige Suche, englische Titel](https://github.com/PCJones/UmlautAdaptarr/assets/377223/ed7ca0fa-ac36-4584-87ac-b29f32dd9ace)
PS:
Das Problem, dass Prowlarr mit SceneNZBs nicht richtig funktioniert ist damit auch behoben :D
**Jetzt:** Es werden auch Titel mit dem deutschen Namen gefunden :D (haben nicht alle Suchergebnisse auf den Screenshot gepasst)
![Jetzige Suche, deutsche und englische Titel](https://github.com/PCJones/UmlautAdaptarr/assets/377223/1c2dbe1a-5943-4fc4-91ef-29708082900e)
**Vorher:** Die deutsche Produktion `Alone - Überlebe die Wildnis` hat auf [TheTVDB](https://thetvdb.com/series/alone-uberlebe-die-wildnis) den Englischen Namen `Alone Germany`.
Sonarr erwartet immer den Englischen Namen, der hier natürlich nicht gegeben ist.
![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/62158f77-ecc2-4747-af85-4b8f94f51ab4)
**Jetzt:** UmlautAdaptarr hat die Releases in `Alone Germany` umbenannt und Sonarr hat keine Probleme mehr
![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/57539ffc-b8a6-4255-a7f8-03079c10b1e8)
**Vorher:** Hier wird der komplette deutsche Titel im Release angegeben (also mit `- Das Lied von Eis und Feuer`) - glücklicherweise stellt uns [TheTVDB](https://thetvdb.com/series/game-of-thrones) aber diesen längeren Titel als Alias zur Verfügung - nur nutzt Sonarr diese Informationen (bisher) einfach nicht.
![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/8f3297bd-ebe4-42de-b4e6-952882c8b902)
**Jetzt:** UmlautAdapatarr erkennt alle auf TheTVDB angegebenen Aliase und benennt das Release in den Englischen Titel um
![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/52f0caf5-6e9d-442e-9018-ba29f954a890)
## Kontakt & Support
- Öffne gerne ein Issue auf GitHub falls du Unterstützung benötigst.
- [Telegram](https://t.me/pc_jones)
- Discord: pcjones1 - oder komm in den UsenetDE Discord Server: [https://discord.gg/pZrrMcJMQM](https://discord.gg/pZrrMcJMQM)
## Spenden
Über eine Spende freue ich mich natürlich immer :D
PayPal: https://paypal.me/pcjones1
Für andere Spendenmöglichkeiten gerne auf Discord oder Telegram melden - danke!
### Licenses & Metadata source
- TV Metadata source: https://thetvdb.com
- Movie Metadata source: https://themoviedb.org
- Licenses: TODO

View File

@@ -1,7 +1,5 @@
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
using System.Text;
using System.Xml.Linq;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Services;
using UmlautAdaptarr.Utilities;
@@ -10,7 +8,8 @@ namespace UmlautAdaptarr.Controllers
{
public abstract class SearchControllerBase(ProxyService proxyService, TitleMatchingService titleMatchingService) : ControllerBase
{
private readonly bool TODO_FORCE_TEXT_SEARCH_ORIGINAL_TITLE = false;
// TODO evaluate if this should be set to true by default
private readonly bool TODO_FORCE_TEXT_SEARCH_ORIGINAL_TITLE = true;
private readonly bool TODO_FORCE_TEXT_SEARCH_GERMAN_TITLE = false;
protected async Task<IActionResult> BaseSearch(string options,
string domain,
@@ -34,7 +33,7 @@ namespace UmlautAdaptarr.Controllers
// Rename titles in the single search content
if (!string.IsNullOrEmpty(initialSearchResult?.Content))
{
inititalProcessedContent = ProcessContent(initialSearchResult.Content, searchItem?.TitleMatchVariations, searchItem?.ExpectedTitle);
inititalProcessedContent = ProcessContent(initialSearchResult.Content, searchItem);
}
var additionalTextSearch = searchItem != null
@@ -54,7 +53,7 @@ namespace UmlautAdaptarr.Controllers
var titleSearchVariations = new List<string>(searchItem?.TitleSearchVariations);
string searchQuery = string.Empty;
var searchQuery = string.Empty;
if (queryParameters.TryGetValue("q", out string? q))
{
searchQuery = q ?? string.Empty;
@@ -76,7 +75,7 @@ namespace UmlautAdaptarr.Controllers
}
// Handle multiple search requests based on German title variations
var aggregatedResult = await AggregateSearchResults(domain, queryParameters, titleSearchVariations, searchItem.TitleMatchVariations, expectedTitle);
var aggregatedResult = await AggregateSearchResults(domain, queryParameters, titleSearchVariations, searchItem);
aggregatedResult.AggregateItems(inititalProcessedContent);
return Content(aggregatedResult.Content, aggregatedResult.ContentType, aggregatedResult.ContentEncoding);
@@ -109,17 +108,17 @@ namespace UmlautAdaptarr.Controllers
}
private string ProcessContent(string content, string[]? titleMatchVariations = null, string? expectedTitle = null)
private string ProcessContent(string content, SearchItem? searchItem)
{
return titleMatchingService.RenameTitlesInContent(content, titleMatchVariations, expectedTitle);
return titleMatchingService.RenameTitlesInContent(content, searchItem);
}
public async Task<AggregatedSearchResult> AggregateSearchResults(
string domain,
IDictionary<string, string> queryParameters,
IEnumerable<string> titleSearchVariations,
string[] titleMatchVariations,
string expectedTitle)
SearchItem? searchItem
)
{
string defaultContentType = "application/xml";
Encoding defaultEncoding = Encoding.UTF8;
@@ -143,7 +142,7 @@ namespace UmlautAdaptarr.Controllers
}
// Process and rename titles in the content
content = ProcessContent(content, titleMatchVariations, expectedTitle);
content = ProcessContent(content, searchItem);
// Aggregate the items into a single document
aggregatedResult.AggregateItems(content);
@@ -157,6 +156,9 @@ namespace UmlautAdaptarr.Controllers
TitleMatchingService titleMatchingService,
SearchItemLookupService searchItemLookupService) : SearchControllerBase(proxyService, titleMatchingService)
{
public readonly string[] LIDARR_CATEGORY_IDS = ["3000", "3010", "3020", "3040", "3050"];
public readonly string[] READARR_CATEGORY_IDS = ["3030", "3130", "7000", "7010", "7020", "7030", "7100", "7110", "7120", "7130"];
[HttpGet]
public async Task<IActionResult> MovieSearch([FromRoute] string options, [FromRoute] string domain)
{
@@ -169,10 +171,34 @@ namespace UmlautAdaptarr.Controllers
[HttpGet]
public async Task<IActionResult> GenericSearch([FromRoute] string options, [FromRoute] string domain)
{
var queryParameters = HttpContext.Request.Query.ToDictionary(
q => q.Key,
q => string.Join(",", q.Value));
return await BaseSearch(options, domain, queryParameters);
SearchItem? searchItem = null;
if (queryParameters.TryGetValue("q", out string? title) && !string.IsNullOrEmpty(title))
{
if (queryParameters.TryGetValue("cat", out string? categories) && !string.IsNullOrEmpty(categories))
{
// look for (audio-)book
if (categories.Split(',').Any(category => READARR_CATEGORY_IDS.Contains(category)))
{
var mediaType = "book";
searchItem = await searchItemLookupService.GetOrFetchSearchItemByExternalId(mediaType, title.GetReadarrTitleForExternalId());
}
// look for audio (lidarr)
if (searchItem == null && categories.Split(',').Any(category => LIDARR_CATEGORY_IDS.Contains(category)))
{
var mediaType = "audio";
searchItem = await searchItemLookupService.GetOrFetchSearchItemByExternalId(mediaType, title.GetLidarrTitleForExternalId());
}
}
}
return await BaseSearch(options, domain, queryParameters, searchItem);
}
[HttpGet]
@@ -192,7 +218,7 @@ namespace UmlautAdaptarr.Controllers
q => string.Join(",", q.Value));
SearchItem? searchItem = null;
string mediaType = "tv";
var mediaType = "tv";
if (queryParameters.TryGetValue("tvdbid", out string? tvdbId) && !string.IsNullOrEmpty(tvdbId))
{

View File

@@ -1,4 +1,5 @@
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging.Abstractions;
using System.Text.RegularExpressions;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Models
@@ -8,23 +9,47 @@ namespace UmlautAdaptarr.Models
public int ArrId { get; set; }
public string ExternalId { get; set; }
public string Title { get; set; }
public bool HasGermanUmlaut => Title?.HasGermanUmlauts() ?? false;
public bool HasUmlaut => Title?.HasUmlauts() ?? false;
public string ExpectedTitle { get; set; }
public string? ExpectedAuthor { get; set; }
// TODO rename GermanTitle into Foreign or LocalTitle?
public string? GermanTitle { get; set; }
public string[] TitleSearchVariations { get; set; }
public string[] TitleMatchVariations { get; set; }
public string[] AuthorMatchVariations { get; set; }
public string MediaType { get; set; }
// TODO public MediaType instead of string
public SearchItem(int arrId, string externalId, string title, string expectedTitle, string? germanTitle, string mediaType, string[]? aliases)
public SearchItem(
int arrId,
string externalId,
string title,
string expectedTitle,
string? germanTitle,
string mediaType,
string[]? aliases,
string? expectedAuthor = null)
{
ArrId = arrId;
ExternalId = externalId;
Title = title;
ExpectedTitle = expectedTitle;
ExpectedAuthor = expectedAuthor;
GermanTitle = germanTitle;
TitleSearchVariations = GenerateTitleVariations(germanTitle).ToArray();
MediaType = mediaType;
if ((mediaType == "audio" || mediaType == "book") && expectedAuthor != null)
{
GenerateVariationsForBooksAndAudio(expectedTitle, mediaType, expectedAuthor);
}
else
{
GenerateVariationsForTV(germanTitle, mediaType, aliases);
}
}
private void GenerateVariationsForTV(string? germanTitle, string mediaType, string[]? aliases)
{
TitleSearchVariations = GenerateVariations(germanTitle, mediaType).ToArray();
var allTitleVariations = new List<string>(TitleSearchVariations);
@@ -34,20 +59,76 @@ namespace UmlautAdaptarr.Models
{
foreach (var alias in aliases)
{
allTitleVariations.AddRange(GenerateTitleVariations(alias));
allTitleVariations.AddRange(GenerateVariations(alias, mediaType));
}
}
TitleMatchVariations = allTitleVariations.Distinct().ToArray();
}
AuthorMatchVariations = [];
private IEnumerable<string> GenerateTitleVariations(string? germanTitle)
// if a german title ends with (DE) also add a search string that replaces (DE) with GERMAN
// also add a matching title without (DE)
if (germanTitle?.EndsWith("(DE)") ?? false)
{
if (germanTitle == null)
TitleSearchVariations = [.. TitleSearchVariations,
..
GenerateVariations(
germanTitle.Replace("(DE)", " GERMAN").RemoveExtraWhitespaces(),
mediaType)];
allTitleVariations.AddRange(GenerateVariations(germanTitle.Replace("(DE)", "").Trim(), mediaType));
}
TitleMatchVariations = allTitleVariations.Distinct(StringComparer.InvariantCultureIgnoreCase).ToArray();
}
private void GenerateVariationsForBooksAndAudio(string expectedTitle, string mediaType, string? expectedAuthor)
{
// e.g. Die Ärzte - best of die Ärzte
if (expectedTitle.Contains(expectedAuthor))
{
var titleWithoutAuthorName = expectedTitle.Replace(expectedAuthor, string.Empty).RemoveExtraWhitespaces().Trim();
if (titleWithoutAuthorName.Length < 2)
{
// TODO log warning that this album can't be searched for automatically
}
TitleMatchVariations = GenerateVariations(titleWithoutAuthorName, mediaType).ToArray();
}
else
{
TitleMatchVariations = GenerateVariations(expectedTitle, mediaType).ToArray();
}
TitleSearchVariations = GenerateVariations($"{expectedAuthor} {expectedTitle}", mediaType).ToArray();
AuthorMatchVariations = GenerateVariations(expectedAuthor, mediaType).ToArray();
if (mediaType == "book")
{
if (expectedAuthor?.Contains(' ') ?? false)
{
var nameParts = expectedAuthor.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var lastName = nameParts.Last();
var firstNames = nameParts.Take(nameParts.Length - 1);
var alternativeExpectedAuthor = $"{lastName}, {string.Join(" ", firstNames)}";
AuthorMatchVariations = [.. AuthorMatchVariations, .. GenerateVariations(alternativeExpectedAuthor, mediaType)];
}
}
}
private IEnumerable<string> GenerateVariations(string? title, string mediaType)
{
if (title == null)
{
return [];
}
var cleanTitle = title.GetCleanTitle();
if (cleanTitle?.Length == 0)
{
return [];
}
var cleanTitle = germanTitle.RemoveAccentButKeepGermanUmlauts();
// Start with base variations including handling umlauts
var baseVariations = new List<string>
@@ -57,6 +138,11 @@ namespace UmlautAdaptarr.Models
cleanTitle.RemoveGermanUmlautDots()
};
if (mediaType == "book" || mediaType == "audio")
{
baseVariations.Add(cleanTitle.RemoveGermanUmlauts());
}
// TODO: determine if this is really needed
// Additional variations to accommodate titles with "-"
if (cleanTitle.Contains('-'))
@@ -76,13 +162,23 @@ namespace UmlautAdaptarr.Models
});
}
// If a title starts with der/die/das also accept variations without it
// Same for english the, an, a
if (cleanTitle.StartsWith("Der ") || cleanTitle.StartsWith("Die ") || cleanTitle.StartsWith("Das ")
|| cleanTitle.StartsWith("The ") || cleanTitle.StartsWith("An "))
{
var cleanTitleWithoutArticle = title[3..].Trim();
baseVariations.AddRange(GenerateVariations(cleanTitleWithoutArticle, mediaType));
} else if (cleanTitle.StartsWith("A "))
{
var cleanTitleWithoutArticle = title[2..].Trim();
baseVariations.AddRange(GenerateVariations(cleanTitleWithoutArticle, mediaType));
}
// Remove multiple spaces
var cleanedVariations = baseVariations.Select(variation => MultipleWhitespaceRegex().Replace(variation, " "));
var cleanedVariations = baseVariations.Select(variation => variation.RemoveExtraWhitespaces());
return cleanedVariations.Distinct();
}
[GeneratedRegex(@"\s+")]
private static partial Regex MultipleWhitespaceRegex();
}
}

View File

@@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using System.Net;
using UmlautAdaptarr.Providers;
using UmlautAdaptarr.Routing;
@@ -10,8 +11,6 @@ internal class Program
// TODO:
// add option to sort by nzb age
// TODO
// add delay between requests
var builder = WebApplication.CreateBuilder(args);
@@ -33,12 +32,26 @@ internal class Program
//options.SizeLimit = 20000;
});
// TODO workaround to not log api keys
builder.Logging.AddFilter((category, level) =>
{
// Prevent logging of HTTP request and response if the category is HttpClient
if (category.Contains("System.Net.Http.HttpClient") || category.Contains("Microsoft.Extensions.Http.DefaultHttpClientFactory"))
{
return false;
}
return true;
});
builder.Services.AddControllers();
builder.Services.AddHostedService<ArrSyncBackgroundService>();
builder.Services.AddSingleton<TitleApiService>(); // TODO rename
builder.Services.AddSingleton<TitleApiService>();
builder.Services.AddSingleton<SearchItemLookupService>();
builder.Services.AddSingleton<TitleMatchingService>();
builder.Services.AddSingleton<SonarrClient>();
builder.Services.AddSingleton<LidarrClient>();
builder.Services.AddSingleton<ReadarrClient>();
builder.Services.AddSingleton<CacheService>();
builder.Services.AddSingleton<ProxyService>();

View File

@@ -0,0 +1,149 @@
using Microsoft.Extensions.Caching.Memory;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Services;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Providers
{
public class LidarrClient(
IHttpClientFactory clientFactory,
IConfiguration configuration,
CacheService cacheService,
IMemoryCache cache,
ILogger<LidarrClient> logger) : ArrClientBase()
{
private readonly string _lidarrHost = configuration.GetValue<string>("LIDARR_HOST") ?? throw new ArgumentException("LIDARR_HOST environment variable must be set");
private readonly string _lidarrApiKey = configuration.GetValue<string>("LIDARR_API_KEY") ?? throw new ArgumentException("LIDARR_API_KEY environment variable must be set");
private readonly string _mediaType = "audio";
public override async Task<IEnumerable<SearchItem>> FetchAllItemsAsync()
{
var httpClient = clientFactory.CreateClient();
var items = new List<SearchItem>();
try
{
var lidarrArtistsUrl = $"{_lidarrHost}/api/v1/artist?apikey={_lidarrApiKey}";
logger.LogInformation($"Fetching all artists from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}");
var artistsApiResponse = await httpClient.GetStringAsync(lidarrArtistsUrl);
var artists = JsonConvert.DeserializeObject<List<dynamic>>(artistsApiResponse);
if (artists == null)
{
logger.LogError($"Lidarr artists API request resulted in null");
return items;
}
logger.LogInformation($"Successfully fetched {artists.Count} artists from Lidarr.");
foreach (var artist in artists)
{
var artistId = (int)artist.id;
var lidarrAlbumUrl = $"{_lidarrHost}/api/v1/album?artistId={artistId}&apikey={_lidarrApiKey}";
// TODO add caching here
// Disable cache for now as it can result in problems when adding new albums that aren't displayed on the artists page initially
//if (cache.TryGetValue(lidarrAlbumUrl, out List<dynamic>? albums))
//{
// logger.LogInformation($"Using cached albums for {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}");
//}
//else
//{
logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}");
var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl);
var albums = JsonConvert.DeserializeObject<List<dynamic>>(albumApiResponse);
//}
if (albums == null)
{
logger.LogWarning($"Lidarr album API request for artistId {artistId} resulted in null");
continue;
}
logger.LogInformation($"Successfully fetched {albums.Count} albums for artistId {artistId} from Lidarr.");
// Cache albums for 3 minutes
cache.Set(lidarrAlbumUrl, albums, TimeSpan.FromMinutes(3));
foreach (var album in albums)
{
var artistName = (string)album.artist.artistName;
var albumTitle = (string)album.title;
var expectedTitle = $"{artistName} {albumTitle}";
string[]? aliases = null;
// Abuse externalId to set the search string Lidarr uses
var externalId = expectedTitle.GetLidarrTitleForExternalId();
var searchItem = new SearchItem
(
arrId: artistId,
externalId: externalId,
title: albumTitle,
expectedTitle: albumTitle,
germanTitle: null,
aliases: aliases,
mediaType: _mediaType,
expectedAuthor: artistName
);
items.Add(searchItem);
}
}
logger.LogInformation($"Finished fetching all items from Lidarr");
}
catch (Exception ex)
{
logger.LogError($"Error fetching all artists from Lidarr: {ex.Message}");
}
return items;
}
public override async Task<SearchItem?> FetchItemByExternalIdAsync(string externalId)
{
try
{
// For now we have to fetch all items every time
// TODO if possible look at the author in search query and only update for author
var searchItems = await FetchAllItemsAsync();
foreach (var searchItem in searchItems ?? [])
{
try
{
cacheService.CacheSearchItem(searchItem);
}
catch (Exception ex)
{
logger.LogError(ex, $"An error occurred while caching search item with ID {searchItem.ArrId}.");
}
}
}
catch (Exception ex)
{
logger.LogError($"Error fetching single artist from Lidarr: {ex.Message}");
}
return null;
}
public override async Task<SearchItem?> FetchItemByTitleAsync(string title)
{
try
{
// this should never be called at the moment
throw new NotImplementedException();
}
catch (Exception ex)
{
logger.LogError($"Error fetching single artist from Lidarr: {ex.Message}");
}
return null;
}
}
}

View File

@@ -0,0 +1,172 @@
using Microsoft.Extensions.Caching.Memory;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Services;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Providers
{
public class ReadarrClient(
IHttpClientFactory clientFactory,
IConfiguration configuration,
CacheService cacheService,
IMemoryCache cache,
ILogger<ReadarrClient> logger) : ArrClientBase()
{
private readonly string _readarrHost = configuration.GetValue<string>("READARR_HOST") ?? throw new ArgumentException("READARR_HOST environment variable must be set");
private readonly string _readarrApiKey = configuration.GetValue<string>("READARR_API_KEY") ?? throw new ArgumentException("READARR_API_KEY environment variable must be set");
private readonly string _mediaType = "book";
public override async Task<IEnumerable<SearchItem>> FetchAllItemsAsync()
{
var httpClient = clientFactory.CreateClient();
var items = new List<SearchItem>();
try
{
var readarrAuthorUrl = $"{_readarrHost}/api/v1/author?apikey={_readarrApiKey}";
logger.LogInformation($"Fetching all authors from Readarr: {UrlUtilities.RedactApiKey(readarrAuthorUrl)}");
var authorApiResponse = await httpClient.GetStringAsync(readarrAuthorUrl);
var authors = JsonConvert.DeserializeObject<List<dynamic>>(authorApiResponse);
if (authors == null)
{
logger.LogError($"Readarr authors API request resulted in null");
return items;
}
logger.LogInformation($"Successfully fetched {authors.Count} authors from Readarr.");
foreach (var author in authors)
{
var authorId = (int)author.id;
var readarrBookUrl = $"{_readarrHost}/api/v1/book?authorId={authorId}&apikey={_readarrApiKey}";
// TODO add caching here
logger.LogInformation($"Fetching all books from authorId {authorId} from Readarr: {UrlUtilities.RedactApiKey(readarrBookUrl)}");
var bookApiResponse = await httpClient.GetStringAsync(readarrBookUrl);
var books = JsonConvert.DeserializeObject<List<dynamic>>(bookApiResponse);
if (books == null)
{
logger.LogWarning($"Readarr book API request for authorId {authorId} resulted in null");
continue;
}
logger.LogInformation($"Successfully fetched {books.Count} books for authorId {authorId} from Readarr.");
// Cache books for 3 minutes
cache.Set(readarrBookUrl, books, TimeSpan.FromMinutes(3));
foreach (var book in books)
{
var authorName = (string)author.authorName;
var bookTitle = GetSearchBookTitle((string)book.title, authorName);
var expectedTitle = $"{bookTitle} {authorName}";
string[]? aliases = null;
// Abuse externalId to set the search string Readarr uses
// TODO use own method or rename
var externalId = expectedTitle.GetReadarrTitleForExternalId();
var searchItem = new SearchItem
(
arrId: authorId,
externalId: externalId,
title: bookTitle,
expectedTitle: bookTitle,
germanTitle: null,
aliases: aliases,
mediaType: _mediaType,
expectedAuthor: authorName
);
items.Add(searchItem);
}
}
logger.LogInformation($"Finished fetching all items from Readarr");
}
catch (Exception ex)
{
logger.LogError($"Error fetching all authors from Readarr: {ex.Message}");
}
return items;
}
// Logic based on https://github.com/Readarr/Readarr/blob/develop/src/NzbDrone.Core/Parser/Parser.cs#L541
public static string GetSearchBookTitle(string bookTitle, string authorName)
{
// Remove author prefix from title if present, e.g., "Tom Clancy: Ghost Protocol"
if (!string.IsNullOrEmpty(authorName) && bookTitle.StartsWith($"{authorName}:"))
{
bookTitle = bookTitle[(authorName.Length + 1)..].Trim();
}
// Remove subtitles or additional info enclosed in parentheses or following a colon, if any
int firstParenthesisIndex = bookTitle.IndexOf('(');
int firstColonIndex = bookTitle.IndexOf(':');
if (firstParenthesisIndex > -1)
{
int endParenthesisIndex = bookTitle.IndexOf(')', firstParenthesisIndex);
if (endParenthesisIndex > -1 && bookTitle.Substring(firstParenthesisIndex + 1, endParenthesisIndex - firstParenthesisIndex - 1).Contains(' '))
{
bookTitle = bookTitle[..firstParenthesisIndex].Trim();
}
}
else if (firstColonIndex > -1)
{
bookTitle = bookTitle[..firstColonIndex].Trim();
}
return bookTitle;
}
public override async Task<SearchItem?> FetchItemByExternalIdAsync(string externalId)
{
try
{
// For now we have to fetch all items every time
// TODO if possible look at the author in search query and only update for author
var searchItems = await FetchAllItemsAsync();
foreach (var searchItem in searchItems ?? [])
{
try
{
cacheService.CacheSearchItem(searchItem);
}
catch (Exception ex)
{
logger.LogError(ex, $"An error occurred while caching search item with ID {searchItem.ArrId}.");
}
}
}
catch (Exception ex)
{
logger.LogError($"Error fetching single author from Readarr: {ex.Message}");
}
return null;
}
public override async Task<SearchItem?> FetchItemByTitleAsync(string title)
{
try
{
// this should never be called at the moment
throw new NotImplementedException();
}
catch (Exception ex)
{
logger.LogError($"Error fetching single author from Readarr: {ex.Message}");
}
return null;
}
}
}

View File

@@ -1,6 +1,4 @@
using Microsoft.Extensions.Caching.Memory;
using Newtonsoft.Json;
using System.Net.Http;
using Newtonsoft.Json;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Services;
using UmlautAdaptarr.Utilities;
@@ -24,7 +22,6 @@ namespace UmlautAdaptarr.Providers
try
{
var sonarrUrl = $"{_sonarrHost}/api/v3/series?includeSeasonImages=false&apikey={_sonarrApiKey}";
logger.LogInformation($"Fetching all items from Sonarr: {UrlUtilities.RedactApiKey(sonarrUrl)}");
var response = await httpClient.GetStringAsync(sonarrUrl);
@@ -41,6 +38,7 @@ namespace UmlautAdaptarr.Providers
logger.LogWarning($"Sonarr Show {show.id} doesn't have a tvdbId.");
continue;
}
(var germanTitle, var aliases) = await titleService.FetchGermanTitleAndAliasesByExternalIdAsync(_mediaType, tvdbId);
var searchItem = new SearchItem
(

View File

@@ -13,53 +13,127 @@ namespace UmlautAdaptarr.Services
{
public class ArrSyncBackgroundService(
SonarrClient sonarrClient,
LidarrClient lidarrClient,
ReadarrClient readarrClient,
CacheService cacheService,
IConfiguration configuration,
ILogger<ArrSyncBackgroundService> logger) : BackgroundService
{
private readonly bool _sonarrEnabled = configuration.GetValue<bool>("SONARR_ENABLED");
private readonly bool _lidarrEnabled = configuration.GetValue<bool>("LIDARR_ENABLED");
private readonly bool _readarrEnabled = configuration.GetValue<bool>("READARR_ENABLED");
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
logger.LogInformation("ArrSyncBackgroundService is starting.");
bool lastRunSuccess = true;
while (!stoppingToken.IsCancellationRequested)
{
logger.LogInformation("ArrSyncBackgroundService is running.");
await FetchAndUpdateDataAsync();
var syncSuccess = await FetchAndUpdateDataAsync();
logger.LogInformation("ArrSyncBackgroundService has completed an iteration.");
if (syncSuccess)
{
lastRunSuccess = true;
await Task.Delay(TimeSpan.FromHours(12), stoppingToken);
}
else
{
if (lastRunSuccess)
{
lastRunSuccess = false;
logger.LogInformation("ArrSyncBackgroundService is trying again in 2 minutes because not all syncs were successful.");
await Task.Delay(TimeSpan.FromMinutes(2), stoppingToken);
}
else
{
logger.LogInformation("ArrSyncBackgroundService is trying again in one hour only because not all syncs were successful twice in a row.");
await Task.Delay(TimeSpan.FromHours(1), stoppingToken);
}
}
}
logger.LogInformation("ArrSyncBackgroundService is stopping.");
}
private async Task FetchAndUpdateDataAsync()
private async Task<bool> FetchAndUpdateDataAsync()
{
try
{
await FetchItemsFromSonarrAsync();
var success = true;
if (_readarrEnabled)
{
var syncSuccess = await FetchItemsFromReadarrAsync();
success = success && syncSuccess;
}
if (_sonarrEnabled)
{
var syncSuccess = await FetchItemsFromSonarrAsync();
success = success && syncSuccess;
}
if (_lidarrEnabled)
{
var syncSuccess = await FetchItemsFromLidarrAsync();
success = success && syncSuccess;
}
return success;
}
catch (Exception ex)
{
logger.LogError(ex, "An error occurred while fetching items from the Arrs.");
}
return false;
}
private async Task FetchItemsFromSonarrAsync()
private async Task<bool> FetchItemsFromSonarrAsync()
{
try
{
var items = await sonarrClient.FetchAllItemsAsync();
UpdateSearchItems(items);
return items?.Any()?? false;
}
catch (Exception ex)
{
logger.LogError(ex, "An error occurred while updating search item from Sonarr.");
}
return false;
}
private void UpdateSearchItems(IEnumerable<SearchItem> searchItems)
private async Task<bool> FetchItemsFromLidarrAsync()
{
foreach (var searchItem in searchItems)
try
{
var items = await lidarrClient.FetchAllItemsAsync();
UpdateSearchItems(items);
return items?.Any() ?? false;
}
catch (Exception ex)
{
logger.LogError(ex, "An error occurred while updating search item from Lidarr.");
}
return false;
}
private async Task<bool> FetchItemsFromReadarrAsync()
{
try
{
var items = await readarrClient.FetchAllItemsAsync();
UpdateSearchItems(items);
return items?.Any() ?? false;
}
catch (Exception ex)
{
logger.LogError(ex, "An error occurred while updating search item from Lidarr.");
}
return false;
}
private void UpdateSearchItems(IEnumerable<SearchItem>? searchItems)
{
foreach (var searchItem in searchItems ?? [])
{
try
{

View File

@@ -1,43 +1,98 @@
using Microsoft.Extensions.Caching.Memory;
using Microsoft.AspNetCore.Mvc.Formatters;
using Microsoft.Extensions.Caching.Memory;
using System.Reflection.Metadata.Ecma335;
using System.Text.RegularExpressions;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Services
{
public class CacheService(IMemoryCache cache)
public partial class CacheService(IMemoryCache cache)
{
private readonly Dictionary<string, HashSet<string>> VariationIndex = [];
private readonly Dictionary<string, List<(HashSet<string> TitleVariations, string CacheKey)>> BookVariationIndex = [];
private readonly Dictionary<string, List<(HashSet<string> TitleVariations, string CacheKey)>> AudioVariationIndex = [];
private const int VARIATION_LOOKUP_CACHE_LENGTH = 5;
public void CacheSearchItem(SearchItem item)
{
var prefix = item.MediaType;
var normalizedTitle = item.Title.RemoveAccentButKeepGermanUmlauts().ToLower();
// TODO maybe we need to also add the media type (movie/book/show etc)
var cacheKey = $"{prefix}_extid_{item.ExternalId}";
cache.Set(cacheKey, item);
if (item.MediaType == "audio")
{
CacheAudioSearchItem(item, cacheKey);
return;
}
else if (item.MediaType == "book")
{
CacheBookSearchItem(item, cacheKey);
return;
}
var normalizedTitle = item.Title.RemoveAccentButKeepGermanUmlauts().ToLower();
cache.Set($"{prefix}_extid_{item.ExternalId}", item);
cache.Set($"{prefix}_title_{normalizedTitle}", item);
foreach (var variation in item.TitleSearchVariations)
foreach (var variation in item.TitleMatchVariations)
{
var normalizedVariation = item.Title.RemoveAccentButKeepGermanUmlauts().ToLower();
var cacheKey = $"{prefix}_var_{normalizedVariation}";
var normalizedVariation = variation.RemoveAccentButKeepGermanUmlauts().ToLower();
cacheKey = $"{prefix}_var_{normalizedVariation}";
cache.Set(cacheKey, item);
// Indexing by prefix
var indexPrefix = normalizedVariation[..Math.Min(VARIATION_LOOKUP_CACHE_LENGTH, variation.Length)].ToLower();
if (!VariationIndex.ContainsKey(indexPrefix))
{
VariationIndex[indexPrefix] = new HashSet<string>();
VariationIndex[indexPrefix] = [];
}
VariationIndex[indexPrefix].Add(cacheKey);
}
}
public void CacheAudioSearchItem(SearchItem item, string cacheKey)
{
// Index author and title variations
foreach (var authorVariation in item.AuthorMatchVariations)
{
var normalizedAuthor = authorVariation.NormalizeForComparison();
if (!AudioVariationIndex.ContainsKey(normalizedAuthor))
{
AudioVariationIndex[normalizedAuthor] = [];
}
var titleVariations = item.TitleMatchVariations.Select(titleMatchVariation => titleMatchVariation.NormalizeForComparison()).ToHashSet();
AudioVariationIndex[normalizedAuthor].Add((titleVariations, cacheKey));
}
}
public void CacheBookSearchItem(SearchItem item, string cacheKey)
{
// Index author and title variations
foreach (var authorVariation in item.AuthorMatchVariations)
{
var normalizedAuthor = authorVariation.NormalizeForComparison();
if (!BookVariationIndex.ContainsKey(normalizedAuthor))
{
BookVariationIndex[normalizedAuthor] = [];
}
var titleVariations = item.TitleMatchVariations.Select(titleMatchVariation => titleMatchVariation.NormalizeForComparison()).ToHashSet();
BookVariationIndex[normalizedAuthor].Add((titleVariations, cacheKey));
}
}
public SearchItem? SearchItemByTitle(string mediaType, string title)
{
var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower();
if (mediaType == "audio" || mediaType == "book")
{
return FindBestMatchForBooksAndAudio(normalizedTitle.NormalizeForComparison(), mediaType);
}
// Use the first few characters of the normalized title for cache prefix search
var cacheSearchPrefix = normalizedTitle[..Math.Min(VARIATION_LOOKUP_CACHE_LENGTH, normalizedTitle.Length)];
@@ -52,7 +107,7 @@ namespace UmlautAdaptarr.Services
continue;
}
// After finding a potential item, compare normalizedTitle with each German title variation
foreach (var variation in item?.TitleSearchVariations ?? [])
foreach (var variation in item?.TitleMatchVariations ?? [])
{
var normalizedVariation = variation.RemoveAccentButKeepGermanUmlauts().ToLower();
if (normalizedTitle.StartsWith(variation, StringComparison.OrdinalIgnoreCase))
@@ -80,10 +135,12 @@ namespace UmlautAdaptarr.Services
{
var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower();
if (mediaType == "generic")
{
// TODO
}
cache.TryGetValue($"{mediaType}_var_{normalizedTitle}", out SearchItem? item);
if (item == null)
{
@@ -91,5 +148,35 @@ namespace UmlautAdaptarr.Services
}
return item;
}
private SearchItem? FindBestMatchForBooksAndAudio(string normalizedOriginalTitle, string mediaType)
{
var index = mediaType == "audio" ? AudioVariationIndex : BookVariationIndex;
foreach (var authorEntry in index)
{
if (normalizedOriginalTitle.Contains(authorEntry.Key))
{
var sortedEntries = authorEntry.Value.OrderByDescending(entry => entry.TitleVariations.FirstOrDefault()?.Length).ToList();
foreach (var (titleVariations, cacheKey) in sortedEntries)
{
if (titleVariations.Any(normalizedOriginalTitle.Contains))
{
if (cache.TryGetValue(cacheKey, out SearchItem? item))
{
return item;
}
}
}
}
}
return null;
}
[GeneratedRegex("\\s")]
private static partial Regex WhiteSpaceRegex();
}
}

View File

@@ -20,6 +20,20 @@ namespace UmlautAdaptarr.Services
_cache = cache;
}
private static async Task EnsureMinimumDelayAsync(string targetUri)
{
var host = new Uri(targetUri).Host;
if (_lastRequestTimes.TryGetValue(host, out var lastRequestTime))
{
var timeSinceLastRequest = DateTimeOffset.Now - lastRequestTime;
if (timeSinceLastRequest < TimeSpan.FromMilliseconds(1500))
{
await Task.Delay(TimeSpan.FromMilliseconds(1500) - timeSinceLastRequest);
}
}
_lastRequestTimes[host] = DateTimeOffset.Now;
}
public async Task<HttpResponseMessage> ProxyRequestAsync(HttpContext context, string targetUri)
{
if (!HttpMethods.IsGet(context.Request.Method))
@@ -27,18 +41,6 @@ namespace UmlautAdaptarr.Services
throw new ArgumentException("Only GET requests are supported", context.Request.Method);
}
// Throttling mechanism
var host = new Uri(targetUri).Host;
if (_lastRequestTimes.TryGetValue(host, out var lastRequestTime))
{
var timeSinceLastRequest = DateTimeOffset.Now - lastRequestTime;
if (timeSinceLastRequest < TimeSpan.FromSeconds(3))
{
await Task.Delay(TimeSpan.FromSeconds(3) - timeSinceLastRequest);
}
}
_lastRequestTimes[host] = DateTimeOffset.Now;
// Check cache
if (_cache.TryGetValue(targetUri, out HttpResponseMessage cachedResponse))
{
@@ -46,6 +48,8 @@ namespace UmlautAdaptarr.Services
return cachedResponse!;
}
await EnsureMinimumDelayAsync(targetUri);
var requestMessage = new HttpRequestMessage
{
RequestUri = new Uri(targetUri),

View File

@@ -3,8 +3,15 @@ using UmlautAdaptarr.Providers;
namespace UmlautAdaptarr.Services
{
public class SearchItemLookupService(CacheService cacheService, SonarrClient sonarrClient)
public class SearchItemLookupService(CacheService cacheService,
SonarrClient sonarrClient,
ReadarrClient readarrClient,
LidarrClient lidarrClient,
IConfiguration configuration)
{
private readonly bool _sonarrEnabled = configuration.GetValue<bool>("SONARR_ENABLED");
private readonly bool _lidarrEnabled = configuration.GetValue<bool>("LIDARR_ENABLED");
private readonly bool _readarrEnabled = configuration.GetValue<bool>("READARR_ENABLED");
public async Task<SearchItem?> GetOrFetchSearchItemByExternalId(string mediaType, string externalId)
{
// Attempt to get the item from the cache first
@@ -19,9 +26,25 @@ namespace UmlautAdaptarr.Services
switch (mediaType)
{
case "tv":
if (_sonarrEnabled)
{
fetchedItem = await sonarrClient.FetchItemByExternalIdAsync(externalId);
}
break;
case "audio":
if (_lidarrEnabled)
{
fetchedItem = await lidarrClient.FetchItemByExternalIdAsync(externalId);
fetchedItem = cacheService.GetSearchItemByExternalId(mediaType, externalId);
}
break;
case "book":
if (_readarrEnabled)
{
await readarrClient.FetchItemByExternalIdAsync(externalId);
fetchedItem = cacheService.GetSearchItemByExternalId(mediaType, externalId);
}
break;
// TODO Add cases for other sources like Radarr, Lidarr, etc.
}
// If an item is fetched, cache it
@@ -47,7 +70,14 @@ namespace UmlautAdaptarr.Services
switch (mediaType)
{
case "tv":
if (_sonarrEnabled)
{
fetchedItem = await sonarrClient.FetchItemByTitleAsync(title);
}
break;
case "audio":
break;
case "book":
break;
// TODO add cases for other sources as needed, such as Radarr, Lidarr, etc.
}
@@ -61,5 +91,4 @@ namespace UmlautAdaptarr.Services
return fetchedItem;
}
}
}

View File

@@ -1,5 +1,6 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Services
{
@@ -12,13 +13,14 @@ namespace UmlautAdaptarr.Services
private async Task EnsureMinimumDelayAsync()
{
var sinceLastRequest = DateTime.Now - lastRequestTime;
if (sinceLastRequest < TimeSpan.FromSeconds(2))
if (sinceLastRequest < TimeSpan.FromSeconds(1))
{
await Task.Delay(TimeSpan.FromSeconds(2) - sinceLastRequest);
await Task.Delay(TimeSpan.FromSeconds(1) - sinceLastRequest);
}
lastRequestTime = DateTime.Now;
}
// TODO add cache, TODO add bulk request
public async Task<(string? germanTitle, string[]? aliases)> FetchGermanTitleAndAliasesByExternalIdAsync(string mediaType, string externalId)
{
try
@@ -27,6 +29,7 @@ namespace UmlautAdaptarr.Services
var httpClient = clientFactory.CreateClient();
var titleApiUrl = $"{_umlautAdaptarrApiHost}/tvshow_german.php?tvdbid={externalId}";
logger.LogInformation($"TitleApiService GET {UrlUtilities.RedactApiKey(titleApiUrl)}");
var response = await httpClient.GetStringAsync(titleApiUrl);
var titleApiResponseData = JsonConvert.DeserializeObject<dynamic>(response);
@@ -72,6 +75,7 @@ namespace UmlautAdaptarr.Services
var httpClient = clientFactory.CreateClient();
var tvdbCleanTitle = title.Replace("ß", "ss");
var titleApiUrl = $"{_umlautAdaptarrApiHost}/tvshow_german.php?title={tvdbCleanTitle}";
logger.LogInformation($"TitleApiService GET {UrlUtilities.RedactApiKey(titleApiUrl)}");
var titleApiResponse = await httpClient.GetStringAsync(titleApiUrl);
var titleApiResponseData = JsonConvert.DeserializeObject<dynamic>(titleApiResponse);

View File

@@ -1,17 +1,18 @@
using System.Text.RegularExpressions;
using Microsoft.Extensions.FileSystemGlobbing.Internal;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using UmlautAdaptarr.Models;
using UmlautAdaptarr.Utilities;
namespace UmlautAdaptarr.Services
{
public partial class TitleMatchingService(CacheService cacheService, ILogger<TitleMatchingService> logger)
{
public string RenameTitlesInContent(string content, string[]? titleMatchVariations, string? expectedTitle)
public string RenameTitlesInContent(string content, SearchItem? searchItem)
{
var xDoc = XDocument.Parse(content);
// If expectedTitle and titleMatchVariations are provided use them, if not use the CacheService to find matches.
bool useCacheService = string.IsNullOrEmpty(expectedTitle) || titleMatchVariations?.Length == 0;
bool useCacheService = searchItem == null;
foreach (var item in xDoc.Descendants("item"))
{
@@ -19,35 +20,155 @@ namespace UmlautAdaptarr.Services
if (titleElement != null)
{
var originalTitle = titleElement.Value;
var normalizedOriginalTitle = NormalizeTitle(originalTitle);
var cleanTitleSeperatedBySpace = ReplaceSeperatorsWithSpace(originalTitle.RemoveAccentButKeepGermanUmlauts());
if (useCacheService)
{
var categoryElement = item.Element("category");
var category = categoryElement?.Value;
var mediaType = GetMediaTypeFromCategory(category);
if (mediaType == null)
{
continue;
}
// Use CacheService to find a matching SearchItem by title
var searchItem = cacheService.SearchItemByTitle(mediaType, originalTitle);
if (searchItem != null)
if (useCacheService)
{
// If a SearchItem is found, use its ExpectedTitle and titleMatchVariations for renaming
expectedTitle = searchItem.ExpectedTitle;
titleMatchVariations = searchItem.TitleMatchVariations;
// Use CacheService to find a matching SearchItem by title
searchItem = cacheService.SearchItemByTitle(mediaType, cleanTitleSeperatedBySpace);
}
else
if (searchItem == null)
{
// Skip processing this item if no matching SearchItem is found
continue;
}
switch (mediaType)
{
case "tv":
FindAndReplaceForMoviesAndTV(logger, searchItem, titleElement, originalTitle, cleanTitleSeperatedBySpace!);
break;
case "movie":
FindAndReplaceForMoviesAndTV(logger, searchItem, titleElement, originalTitle, cleanTitleSeperatedBySpace!);
break;
case "audio":
FindAndReplaceForBooksAndAudio(searchItem, titleElement, originalTitle!);
break;
case "book":
FindAndReplaceForBooksAndAudio(searchItem, titleElement, originalTitle!);
break;
default:
throw new NotImplementedException();
}
}
}
return xDoc.ToString();
}
public void FindAndReplaceForBooksAndAudio(SearchItem searchItem, XElement? titleElement, string originalTitle)
{
var authorMatch = FindBestMatch(searchItem.AuthorMatchVariations, originalTitle.NormalizeForComparison(), originalTitle);
var titleMatch = FindBestMatch(searchItem.TitleMatchVariations, originalTitle.NormalizeForComparison(), originalTitle);
if (authorMatch.foundMatch && titleMatch.foundMatch)
{
int matchEndPositionInOriginal = Math.Max(authorMatch.bestEndInOriginal, titleMatch.bestEndInOriginal);
// Check and adjust for immediate following delimiter
char[] delimiters = [' ', '-', '_', '.'];
if (matchEndPositionInOriginal < originalTitle.Length && delimiters.Contains(originalTitle[matchEndPositionInOriginal]))
{
matchEndPositionInOriginal++; // Skip the delimiter if it's immediately after the match
}
// Ensure we trim any leading delimiters from the suffix
string suffix = originalTitle[matchEndPositionInOriginal..].TrimStart([' ', '-', '_', '.']).Trim();
// Concatenate the expected title with the remaining suffix
var updatedTitle = $"{searchItem.ExpectedAuthor} - {searchItem.ExpectedTitle}";
if (suffix.Length >= 3)
{
updatedTitle += $"-[{suffix}]";
}
// Update the title element
titleElement.Value = updatedTitle;
logger.LogInformation($"TitleMatchingService - Title changed: '{originalTitle}' to '{updatedTitle}'");
}
else
{
logger.LogDebug($"TitleMatchingService - No satisfactory fuzzy match found for both author and title for {originalTitle}.");
}
}
private (bool foundMatch, int bestStart, int bestEndInOriginal) FindBestMatch(string[] variations, string normalizedOriginal, string originalTitle)
{
bool found = false;
int bestStart = int.MaxValue;
int bestEndInOriginal = -1;
foreach (var variation in variations)
{
var normalizedVariation = variation.NormalizeForComparison();
int startNormalized = normalizedOriginal.IndexOf(normalizedVariation);
if (startNormalized >= 0)
{
found = true;
// Map the start position from the normalized string back to the original string
int startOriginal = MapNormalizedIndexToOriginal(normalizedOriginal, originalTitle, startNormalized);
int endOriginal = MapNormalizedIndexToOriginal(normalizedOriginal, originalTitle, startNormalized + normalizedVariation.Length);
bestStart = Math.Min(bestStart, startOriginal);
bestEndInOriginal = Math.Max(bestEndInOriginal, endOriginal);
}
}
if (!found) return (false, 0, 0);
return (found, bestStart, bestEndInOriginal);
}
// Maps an index from the normalized string back to a corresponding index in the original string
private int MapNormalizedIndexToOriginal(string normalizedOriginal, string originalTitle, int normalizedIndex)
{
// Count non-special characters up to the given index in the normalized string
int nonSpecialCharCount = 0;
for (int i = 0; i < normalizedIndex && i < normalizedOriginal.Length; i++)
{
if (char.IsLetterOrDigit(normalizedOriginal[i]))
{
nonSpecialCharCount++;
}
}
// Count non-special characters in the original title to find the corresponding index
int originalIndex = 0;
for (int i = 0; i < originalTitle.Length; i++)
{
if (char.IsLetterOrDigit(originalTitle[i]))
{
if (--nonSpecialCharCount < 0)
{
break;
}
}
originalIndex = i;
}
return originalIndex;
}
// This method replaces the first variation that starts at the beginning of the release title
private static void FindAndReplaceForMoviesAndTV(ILogger<TitleMatchingService> logger, SearchItem searchItem, XElement? titleElement, string originalTitle, string normalizedOriginalTitle)
{
var titleMatchVariations = searchItem.TitleMatchVariations;
var expectedTitle = searchItem.ExpectedTitle;
var variationsOrderedByLength = titleMatchVariations!.OrderByDescending(variation => variation.Length);
// Attempt to find a variation that matches the start of the original title
foreach (var variation in titleMatchVariations!)
foreach (var variation in variationsOrderedByLength)
{
// Skip variations that are already the expectedTitle
if (variation == expectedTitle)
@@ -56,11 +177,13 @@ namespace UmlautAdaptarr.Services
}
// Variation is already normalized at creation
var pattern = "^" + Regex.Escape(variation).Replace("\\ ", "[._ ]");
var variationMatchPattern = "^" + Regex.Escape(variation).Replace("\\ ", "[._ ]");
// Check if the originalTitle starts with the variation (ignoring case and separators)
if (Regex.IsMatch(normalizedOriginalTitle, pattern, RegexOptions.IgnoreCase))
if (Regex.IsMatch(normalizedOriginalTitle, variationMatchPattern, RegexOptions.IgnoreCase))
{
var originalTitleMatchPattern = "^" + Regex.Escape(variation).Replace("\\ ", "[._ ]");
// Find the first separator used in the original title for consistent replacement
var separator = FindFirstSeparator(originalTitle);
// Reconstruct the expected title using the original separator
@@ -70,8 +193,21 @@ namespace UmlautAdaptarr.Services
var variationLength = variation.Length;
var suffix = originalTitle[Math.Min(variationLength, originalTitle.Length)..];
// Clean up any leading separators from the suffix
suffix = Regex.Replace(suffix, "^[._ ]+", "");
// Workaround for the rare case of e.g. "Frieren: Beyond Journey's End" that also has the alias "Frieren"
if (expectedTitle!.StartsWith(variation, StringComparison.OrdinalIgnoreCase))
{
// See if we already matched the whole title by checking if S01E01 pattern is coming next to avoid false positives
// - that won't help with movies but with tv shows
var seasonMatchingPattern = $"^{separator}S\\d{{1,2}}E\\d{{1,2}}";
if (!Regex.IsMatch(suffix, seasonMatchingPattern))
{
logger.LogWarning($"TitleMatchingService - Didn't rename: '{originalTitle}' because the expected title '{expectedTitle}' starts with the variation '{variation}'");
continue;
}
}
// Clean up any leading separator from the suffix
suffix = Regex.Replace(suffix, "^ +", "");
// TODO EVALUTE! definitely make this optional - this adds GERMAN to the title is the title is german to make sure it's recognized as german
// can lead to problems with shows such as "dark" that have international dubs
@@ -96,15 +232,9 @@ namespace UmlautAdaptarr.Services
}
}
}
}
return xDoc.ToString();
}
private static string NormalizeTitle(string title)
private static string ReplaceSeperatorsWithSpace(string title)
{
title = title.RemoveAccentButKeepGermanUmlauts();
// Replace all known separators with space for normalization
return WordSeperationCharRegex().Replace(title, " ".ToString());
}
@@ -117,7 +247,11 @@ namespace UmlautAdaptarr.Services
private static string ReconstructTitleWithSeparator(string title, char separator)
{
// Replace spaces with the original separator found in the title
if (separator != ' ')
{
return title;
}
return title.Replace(' ', separator);
}
@@ -144,6 +278,10 @@ namespace UmlautAdaptarr.Services
{
return "book";
}
else if (category.StartsWith("Audio"))
{
return "audio";
}
return null;
}
@@ -151,5 +289,6 @@ namespace UmlautAdaptarr.Services
[GeneratedRegex("[._ ]")]
private static partial Regex WordSeperationCharRegex();
}
}

View File

@@ -141,7 +141,7 @@ namespace UmlautAdaptarr.Services
germanTitle = titleApiResponseData.germanTitle;
hasGermanTitle = true;
var hasGermanUmlaut = germanTitle?.HasGermanUmlauts() ?? false;
var hasGermanUmlaut = germanTitle?.HasUmlauts() ?? false;
var result = (hasGermanUmlaut, germanTitle, expectedTitle);
memoryCache.Set(cacheKey, result, new MemoryCacheEntryOptions

View File

@@ -1,9 +1,10 @@
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
namespace UmlautAdaptarr.Utilities
{
public static class Extensions
public static partial class Extensions
{
public static string GetQuery(this HttpContext context, string key)
{
@@ -46,6 +47,67 @@ namespace UmlautAdaptarr.Utilities
return stringBuilder.ToString().Normalize(NormalizationForm.FormC);
}
public static string GetLidarrTitleForExternalId(this string text)
{
text = text.RemoveGermanUmlautDots()
.Replace("-", "")
.GetCleanTitle()
.ToLower();
// Lidarr removes the, an and a at the beginning
return TitlePrefixRegex()
.Replace(text, "")
.RemoveExtraWhitespaces()
.Trim();
}
public static string GetReadarrTitleForExternalId(this string text)
{
text = text.ToLower();
// Readarr removes "the" at the beginning
if (text.StartsWith("the "))
{
text = text[4..];
}
return text.RemoveGermanUmlautDots()
.Replace(".", " ")
.Replace("-", " ")
.Replace(":", " ")
.GetCleanTitle();
}
public static string GetCleanTitle(this string text)
{
return text
.Replace(".", " ")
.Replace(":", " ")
.RemoveAccentButKeepGermanUmlauts()
.RemoveSpecialCharacters(removeUmlauts: false)
.RemoveExtraWhitespaces()
.Trim();
}
public static string NormalizeForComparison(this string text)
{
// TODO see if we can replace RemoveGermanUmlautDots() with RemoveSpecialCharacters(removeUmlauts: false);
return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower();
}
public static string RemoveSpecialCharacters(this string text, bool removeUmlauts = true)
{
if (removeUmlauts)
{
return NoSpecialCharactersExceptHypenRegex().Replace(text, "");
}
else
{
return NoSpecialCharactersExceptHyphenAndUmlautsRegex().Replace(text, "");
}
}
public static string ReplaceGermanUmlautsWithLatinEquivalents(this string text)
{
return text
@@ -70,11 +132,40 @@ namespace UmlautAdaptarr.Utilities
.Replace("ß", "ss");
}
public static bool HasGermanUmlauts(this string text)
public static string RemoveGermanUmlauts(this string text)
{
return text
.Replace("ö", "")
.Replace("ü", "")
.Replace("ä", "")
.Replace("Ö", "")
.Replace("Ü", "")
.Replace("Ä", "")
.Replace("ß", "");
}
public static string RemoveExtraWhitespaces(this string text)
{
return MultipleWhitespaceRegex().Replace(text, " ");
}
public static bool HasUmlauts(this string text)
{
if (text == null) return false;
var umlauts = new[] { 'ö', 'ä', 'ü', 'Ä', 'Ü', 'Ö', 'ß' };
return umlauts.Any(text.Contains);
}
[GeneratedRegex("[^a-zA-Z0-9 -]+", RegexOptions.Compiled)]
private static partial Regex NoSpecialCharactersExceptHypenRegex();
[GeneratedRegex("[^a-zA-Z0-9 -öäüßÖÄÜß]+", RegexOptions.Compiled)]
private static partial Regex NoSpecialCharactersExceptHyphenAndUmlautsRegex();
[GeneratedRegex(@"\s+")]
private static partial Regex MultipleWhitespaceRegex();
[GeneratedRegex(@"\b(the|an|a)\b", RegexOptions.IgnoreCase, "de-DE")]
private static partial Regex TitlePrefixRegex();
}
}

View File

@@ -5,7 +5,7 @@ namespace UmlautAdaptarr.Utilities
{
public partial class UrlUtilities
{
[GeneratedRegex(@"^(?!http:\/\/)([a-zA-Z0-9]+(\.[a-zA-Z0-9]+)+.*)$")]
[GeneratedRegex(@"^(?!http:\/\/)([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+.*)$")]
private static partial Regex UrlMatchingRegex();
public static bool IsValidDomain(string domain)
{
@@ -35,7 +35,7 @@ namespace UmlautAdaptarr.Utilities
if (!string.IsNullOrEmpty(apiKey))
{
queryParameters["apiKey"] = apiKey;
queryParameters["apikey"] = apiKey;
}
return BuildUrl(domain, queryParameters);

View File

@@ -0,0 +1,5 @@
{
"version": "1.0",
"defaultProvider": "cdnjs",
"libraries": []
}

View File

@@ -1,4 +1,11 @@
{
"SONARR_ENABLED": false,
"SONARR_HOST": "http://localhost:8989",
"SONARR_API_KEY": ""
"SONARR_API_KEY": "",
"LIDARR_ENABLED": false,
"LIDARR_HOST": "http://localhost:8686",
"LIDARR_API_KEY": "",
"READARR_ENABLED": false,
"READARR_HOST": "http://localhost:8787",
"READARR_API_KEY": ""
}

View File

@@ -0,0 +1,18 @@
@echo off
SET IMAGE_NAME=pcjones/umlautadaptarr
echo Enter the version number for the Docker image:
set /p VERSION="Version: "
echo Building Docker image with version %VERSION%...
docker build -t %IMAGE_NAME%:%VERSION% .
docker tag %IMAGE_NAME%:%VERSION% %IMAGE_NAME%:latest
echo Pushing Docker image with version %VERSION%...
docker push %IMAGE_NAME%:%VERSION%
echo Pushing Docker image with tag latest...
docker push %IMAGE_NAME%:latest
echo Done.
pause

View File

@@ -1,9 +1,24 @@
version: '3.8'
services:
umlautadaptarr:
build: .
#uncomment this to get the development branch
#build: https://github.com/PCJones/UmlautAdaptarr.git#develop
build: https://github.com/PCJones/UmlautAdaptarr.git#master
image: umlautadaptarr
restart: unless-stopped
environment:
SONARR_HOST: "http://localhost:8989"
SONARR_API_KEY: ""
- TZ=Europe/Berlin
- SONARR_ENABLED=false
- SONARR_HOST=http://localhost:8989
- SONARR_API_KEY=API_KEY
- RADARR_ENABLED=false
- RADARR_HOST=http://localhost:7878
- RADARR_API_KEY=API_KEY
- READARR_ENABLED=false
- READARR_HOST=http://localhost:8787
- READARR_API_KEY=API_KEY
- LIDARR_ENABLED=false
- LIDARR_HOST=http://localhost:8686
- LIDARR_API_KEY=API_KEY
ports:
- "5005:5005"
- "5005":"5005"