Merge branch 'master' of https://github.com/PCJones/UmlautAdaptarr

v0.4

README.md

@@ -5,7 +5,7 @@
 ## Erste Testversion
 Wer möchte kann den UmlautAdaptarr jetzt gerne testen! Über Feedback würde ich mich sehr freuen!
 
-Es sollte mit allen *arrs funktionieren, hat aber nur bei Sonarr und Lidarr schon Auswirkungen (abgesehen vom Caching).
+Es sollte mit allen *arrs funktionieren, hat aber nur bei Sonarr, Readarr und Lidarr schon Auswirkungen (abgesehen vom Caching).
 
 Momentan ist docker dafür nötig, wer kein Docker nutzt muss sich noch etwas gedulden. 
 
@@ -30,7 +30,7 @@ UmlautAdaptarr löst mehrere Probleme:
 
 # Wie macht UmlautAdaptarr das?
 UmlautAdaptarr tut so, als wäre es ein Indexer. In Wahrheit schaltet sich UmlautAdaptarr aber nur zwischen die *arrs und den echten Indexer und kann somit die Suchen sowie die Ergebnisse abfangen und bearbeiten.
-Am Ende werden die gefundenen Releases immer so umbenannt, das die Arrs sie einwandfrei erkennen.
+Am Ende werden die gefundenen Releases immer so umbenannt, dass die Arrs sie einwandfrei erkennen.
 Einige Beispiele findet ihr unter Features.
 
 
@@ -38,15 +38,17 @@ Einige Beispiele findet ihr unter Features.
 
 | Feature                                                           | Status        |
 |-------------------------------------------------------------------|---------------|
-| Prowlarr Support                                                    | ✓|
+| Prowlarr & NZB Hydra Support                                                    | ✓|
 | Sonarr Support                                         | ✓             |
 | Lidarr Support                                                    | ✓|
+| Readarr Support                                                   | ✓       |
 | Releases mit deutschem Titel werden erkannt   | ✓             |
 | Releases mit TVDB-Alias Titel werden erkannt  | ✓             |
 | Korrekte Suche und Erkennung von Titel mit Umlauten                            | ✓             |
 | Anfragen-Caching für 5 Minuten zur Reduzierung der API-Zugriffe   | ✓             |
+| Usenet (newznab) Support                                          |✓|
+| Torrent (torznab) Support                                         |✓|
 | Radarr Support                                                    | Geplant       |
-| Readarr Support                                                   | Geplant       |
 | Prowlarr Unterstützung für "DE" SceneNZBs Kategorien              | Geplant       |
 | Unterstützung weiterer Sprachen neben Deutsch                     | Geplant       |
 | Wünsche?                                                          | Vorschläge?   |
@@ -87,6 +89,12 @@ Sonarr erwartet immer den Englischen Namen, der hier natürlich nicht gegeben is
 - [Telegram](https://t.me/pc_jones)
 - Discord: pcjones1 - oder komm in den UsenetDE Discord Server: [https://discord.gg/pZrrMcJMQM](https://discord.gg/pZrrMcJMQM)
 
+## Spenden
+Über eine Spende freue ich mich natürlich immer :D
+PayPal: https://paypal.me/pcjones1
+
+Für andere Spendenmöglichkeiten gerne auf Discord oder Telegram melden - danke!
+
 ### Licenses & Metadata source
 - TV Metadata source: https://thetvdb.com
 - Movie Metadata source: https://themoviedb.org

UmlautAdaptarr/Controllers/SearchController.cs

@@ -8,7 +8,8 @@ namespace UmlautAdaptarr.Controllers
 {
     public abstract class SearchControllerBase(ProxyService proxyService, TitleMatchingService titleMatchingService) : ControllerBase
     {
-        private readonly bool TODO_FORCE_TEXT_SEARCH_ORIGINAL_TITLE = false;
+        // TODO evaluate if this should be set to true by default
+        private readonly bool TODO_FORCE_TEXT_SEARCH_ORIGINAL_TITLE = true;
         private readonly bool TODO_FORCE_TEXT_SEARCH_GERMAN_TITLE = false;
         protected async Task<IActionResult> BaseSearch(string options,
                                                        string domain,
@@ -52,7 +53,7 @@ namespace UmlautAdaptarr.Controllers
 
                     var titleSearchVariations = new List<string>(searchItem?.TitleSearchVariations);
 
-                    string searchQuery = string.Empty;
+                    var searchQuery = string.Empty;
                     if (queryParameters.TryGetValue("q", out string? q))
                     {
                         searchQuery = q ?? string.Empty;
@@ -185,7 +186,6 @@ namespace UmlautAdaptarr.Controllers
                     if (categories.Split(',').Any(category => READARR_CATEGORY_IDS.Contains(category)))
                     {
                         var mediaType = "book";
-                        // TODO rename function or use own
                         searchItem = await searchItemLookupService.GetOrFetchSearchItemByExternalId(mediaType, title.GetReadarrTitleForExternalId());
                     }
 

UmlautAdaptarr/Models/SearchItem.cs

@@ -43,7 +43,30 @@ namespace UmlautAdaptarr.Models
             }
             else
             {
-                GenerateVariationsForTV(germanTitle, mediaType, aliases);
+                // if mediatype is movie/tv and the Expected Title ends with a year but the german title doesn't then append the year to the german title and to aliases
+                // example: https://thetvdb.com/series/385925-avatar-the-last-airbender -> german Title is without 2024
+                var yearAtEndOfTitleMatch = YearAtEndOfTitleRegex().Match(expectedTitle);
+                if (yearAtEndOfTitleMatch.Success)
+                {
+                    string year = yearAtEndOfTitleMatch.Value[1..^1];
+                    if (GermanTitle != null && !GermanTitle.Contains(year))
+                    {
+                        GermanTitle = $"{germanTitle} {year}";
+                    }
+
+                    if (aliases != null)
+                    {
+                        for (int i = 0; i < aliases.Length; i++)
+                        {
+                            if (!aliases[i].Contains(year))
+                            {
+                                aliases[i] = $"{aliases[i]} {year}";
+                            }
+                        }
+                    }
+                }
+
+                GenerateVariationsForTV(GermanTitle, mediaType, aliases);
             }
         }
 
@@ -60,6 +83,12 @@ namespace UmlautAdaptarr.Models
                 foreach (var alias in aliases)
                 {
                     allTitleVariations.AddRange(GenerateVariations(alias, mediaType));
+
+                    // If title contains ":" also match for "-"
+                    if (alias.Contains(':'))
+                    {
+                        allTitleVariations.Add(alias.Replace(":", " -"));
+                    }
                 }
             }
 
@@ -79,6 +108,12 @@ namespace UmlautAdaptarr.Models
 
             }
 
+            // If title contains ":" also match for "-"
+            if (germanTitle?.Contains(':') ?? false)
+            {
+                allTitleVariations.Add(germanTitle.Replace(":", " -"));
+            }
+
             TitleMatchVariations = allTitleVariations.Distinct(StringComparer.InvariantCultureIgnoreCase).ToArray();
         }
 
@@ -123,6 +158,7 @@ namespace UmlautAdaptarr.Models
             {
                 return [];
             }
+
             var cleanTitle = title.GetCleanTitle();
 
             if (cleanTitle?.Length == 0)
@@ -180,5 +216,8 @@ namespace UmlautAdaptarr.Models
 
             return cleanedVariations.Distinct();
         }
+
+        [GeneratedRegex(@"\(\d{4}\)$")]
+        private static partial Regex YearAtEndOfTitleRegex();
     }
 }

UmlautAdaptarr/Providers/SonarrClient.cs

@@ -22,7 +22,6 @@ namespace UmlautAdaptarr.Providers
 
             try
             {
-
                 var sonarrUrl = $"{_sonarrHost}/api/v3/series?includeSeasonImages=false&apikey={_sonarrApiKey}";
                 logger.LogInformation($"Fetching all items from Sonarr: {UrlUtilities.RedactApiKey(sonarrUrl)}");
                 var response = await httpClient.GetStringAsync(sonarrUrl);
@@ -99,7 +98,7 @@ namespace UmlautAdaptarr.Providers
                         mediaType: _mediaType
                     );
 
-                    logger.LogInformation($"Successfully fetched show {searchItem} from Sonarr.");
+                    logger.LogInformation($"Successfully fetched show {searchItem.Title} from Sonarr.");
                     return searchItem;
                 }
             }
@@ -157,7 +156,7 @@ namespace UmlautAdaptarr.Providers
                     mediaType: _mediaType
                 );
 
-                logger.LogInformation($"Successfully fetched show {searchItem} from Sonarr.");
+                logger.LogInformation($"Successfully fetched show {searchItem.Title} from Sonarr.");
                 return searchItem;
             }
             catch (Exception ex)

UmlautAdaptarr/Services/ArrSyncBackgroundService.cs

@@ -64,15 +64,18 @@ namespace UmlautAdaptarr.Services
                 var success = true;
                 if (_readarrEnabled)
                 {
-                    success = success && await FetchItemsFromReadarrAsync();
+                    var syncSuccess = await FetchItemsFromReadarrAsync();
+                    success = success && syncSuccess;
                 }
                 if (_sonarrEnabled)
                 {
-                    success = success && await FetchItemsFromSonarrAsync();
+                    var syncSuccess = await FetchItemsFromSonarrAsync();
+                    success = success && syncSuccess;
                 }
                 if (_lidarrEnabled)
                 {
-                    success = success && await FetchItemsFromLidarrAsync();
+                    var syncSuccess = await FetchItemsFromLidarrAsync();
+                    success = success && syncSuccess;
                 }
                 return success;
             }

UmlautAdaptarr/Services/CacheService.cs

@@ -96,30 +96,51 @@ namespace UmlautAdaptarr.Services
             // Use the first few characters of the normalized title for cache prefix search
             var cacheSearchPrefix = normalizedTitle[..Math.Min(VARIATION_LOOKUP_CACHE_LENGTH, normalizedTitle.Length)];
 
+            SearchItem? bestSearchItemMatch = null;
+            var bestVariationMatchLength = 0;
+            HashSet<string> checkedSearchItems = [];
+
             if (VariationIndex.TryGetValue(cacheSearchPrefix, out var cacheKeys))
             {
                 foreach (var cacheKey in cacheKeys)
                 {
                     if (cache.TryGetValue(cacheKey, out SearchItem? item))
                     {
-                        if (item?.MediaType != mediaType)
+                        if (item == null || item.MediaType != mediaType)
                         {
                             continue;
                         }
+
+                        var searchItemIdentifier = $"{item.MediaType}_{item.ExternalId}";
+
+                        if (checkedSearchItems.Contains(searchItemIdentifier))
+                        {
+                            continue;
+                        }
+                        else
+                        {
+                            checkedSearchItems.Add(searchItemIdentifier);
+                        }
+
                         // After finding a potential item, compare normalizedTitle with each German title variation
-                        foreach (var variation in item?.TitleMatchVariations ?? [])
+                        foreach (var variation in item.TitleMatchVariations ?? [])
                         {
                             var normalizedVariation = variation.RemoveAccentButKeepGermanUmlauts().ToLower();
                             if (normalizedTitle.StartsWith(variation, StringComparison.OrdinalIgnoreCase))
                             {
-                                return item;
+                                // If we find a variation match that is "longer" then most likely that one is correct and the earlier match was wrong (if it was from another searchItem)
+                                if (variation.Length > bestVariationMatchLength)
+                                {
+                                    bestSearchItemMatch = item;
+                                    bestVariationMatchLength = variation.Length;
+                                }
                             }
                         }
                     }
                 }
             }
 
-            return null;
+            return bestSearchItemMatch;
         }
 
         public SearchItem? GetSearchItemByExternalId(string mediaType, string externalId)

UmlautAdaptarr/Services/ProxyService.cs

@@ -11,6 +11,7 @@ namespace UmlautAdaptarr.Services
         private readonly ILogger<ProxyService> _logger;
         private readonly IMemoryCache _cache;
         private static readonly ConcurrentDictionary<string, DateTimeOffset> _lastRequestTimes = new();
+        private static readonly TimeSpan MINIMUM_DELAY_FOR_SAME_HOST = new(0, 0, 0, 1);
 
         public ProxyService(IHttpClientFactory clientFactory, IConfiguration configuration, ILogger<ProxyService> logger, IMemoryCache cache)
         {
@@ -20,6 +21,20 @@ namespace UmlautAdaptarr.Services
             _cache = cache;
         }
 
+        private static async Task EnsureMinimumDelayAsync(string targetUri)
+        {
+            var host = new Uri(targetUri).Host;
+            if (_lastRequestTimes.TryGetValue(host, out var lastRequestTime))
+            {
+                var timeSinceLastRequest = DateTimeOffset.Now - lastRequestTime;
+                if (timeSinceLastRequest < MINIMUM_DELAY_FOR_SAME_HOST)
+                {
+                    await Task.Delay(MINIMUM_DELAY_FOR_SAME_HOST - timeSinceLastRequest);
+                }
+            }
+            _lastRequestTimes[host] = DateTimeOffset.Now;
+        }
+
         public async Task<HttpResponseMessage> ProxyRequestAsync(HttpContext context, string targetUri)
         {
             if (!HttpMethods.IsGet(context.Request.Method))
@@ -27,18 +42,6 @@ namespace UmlautAdaptarr.Services
                 throw new ArgumentException("Only GET requests are supported", context.Request.Method);
             }
 
-            // Throttling mechanism
-            var host = new Uri(targetUri).Host;
-            if (_lastRequestTimes.TryGetValue(host, out var lastRequestTime))
-            {
-                var timeSinceLastRequest = DateTimeOffset.Now - lastRequestTime;
-                if (timeSinceLastRequest < TimeSpan.FromSeconds(3))
-                {
-                    await Task.Delay(TimeSpan.FromSeconds(3) - timeSinceLastRequest);
-                }
-            }
-            _lastRequestTimes[host] = DateTimeOffset.Now;
-
             // Check cache
             if (_cache.TryGetValue(targetUri, out HttpResponseMessage cachedResponse))
             {
@@ -46,6 +49,8 @@ namespace UmlautAdaptarr.Services
                 return cachedResponse!;
             }
 
+            await EnsureMinimumDelayAsync(targetUri);
+
             var requestMessage = new HttpRequestMessage
             {
                 RequestUri = new Uri(targetUri),

UmlautAdaptarr/Services/SearchItemLookupService.cs

@@ -35,12 +35,14 @@ namespace UmlautAdaptarr.Services
                     if (_lidarrEnabled)
                     {
                         fetchedItem = await lidarrClient.FetchItemByExternalIdAsync(externalId);
+                        fetchedItem = cacheService.GetSearchItemByExternalId(mediaType, externalId);
                     }
                     break;
                 case "book":
                     if (_readarrEnabled)
                     {
-                        fetchedItem = await readarrClient.FetchItemByExternalIdAsync(externalId);
+                        await readarrClient.FetchItemByExternalIdAsync(externalId);
+                        fetchedItem = cacheService.GetSearchItemByExternalId(mediaType, externalId);
                     }
                     break;
             }

UmlautAdaptarr/Services/TitleApiService.cs

@@ -13,13 +13,14 @@ namespace UmlautAdaptarr.Services
         private async Task EnsureMinimumDelayAsync()
         {
             var sinceLastRequest = DateTime.Now - lastRequestTime;
-            if (sinceLastRequest < TimeSpan.FromSeconds(2))
+            if (sinceLastRequest < TimeSpan.FromSeconds(1))
             {
-                await Task.Delay(TimeSpan.FromSeconds(2) - sinceLastRequest);
+                await Task.Delay(TimeSpan.FromSeconds(1) - sinceLastRequest);
             }
             lastRequestTime = DateTime.Now;
         }
 
+        // TODO add cache, TODO add bulk request
         public async Task<(string? germanTitle, string[]? aliases)> FetchGermanTitleAndAliasesByExternalIdAsync(string mediaType, string externalId)
         {
             try

UmlautAdaptarr/Services/TitleMatchingService.cs

@@ -221,7 +221,7 @@ namespace UmlautAdaptarr.Services
                     */
 
                     // Construct the new title with the original suffix
-                    var newTitle = newTitlePrefix + (string.IsNullOrEmpty(suffix) ? "" : separator + suffix);
+                    var newTitle = newTitlePrefix + (string.IsNullOrEmpty(suffix) ? "" : suffix.StartsWith(separator) ? suffix : $"{separator}{suffix}");
 
                     // Update the title element's value with the new title
                     //titleElement.Value = newTitle + $"({originalTitle.Substring(0, variationLength)})";
@@ -262,23 +262,23 @@ namespace UmlautAdaptarr.Services
                 return null;
             }
 
-            if (category.StartsWith("EBook", StringComparison.OrdinalIgnoreCase) || category.StartsWith("Book", StringComparison.OrdinalIgnoreCase))
+            if (category == "7000" || category.StartsWith("EBook", StringComparison.OrdinalIgnoreCase) || category.StartsWith("Book", StringComparison.OrdinalIgnoreCase))
             {
                 return "book";
             }
-            else if (category.StartsWith("Movies", StringComparison.OrdinalIgnoreCase))
+            else if (category == "2000" || category.StartsWith("Movies", StringComparison.OrdinalIgnoreCase))
             {
                 return "movies";
             }
-            else if (category.StartsWith("TV", StringComparison.OrdinalIgnoreCase))
+            else if (category == "5000" || category.StartsWith("TV", StringComparison.OrdinalIgnoreCase))
             {
                 return "tv";
             }
-            else if (category.Contains("Audiobook", StringComparison.OrdinalIgnoreCase))
+            else if (category == "3030" || category.Contains("Audiobook", StringComparison.OrdinalIgnoreCase))
             {
                 return "book";
             }
-            else if (category.StartsWith("Audio"))
+            else if (category == "3000" || category.StartsWith("Audio"))
             {
                 return "audio";
             }

UmlautAdaptarr/Utilities/UrlUtilities.cs

@@ -12,7 +12,7 @@ namespace UmlautAdaptarr.Utilities
             // RegEx für eine einfache URL-Validierung ohne http:// und ohne abschließenden Schrägstrich
             // Erlaubt optionale Subdomains, Domainnamen und TLDs, aber keine Pfade oder Protokolle
             var regex = UrlMatchingRegex();
-            return regex.IsMatch(domain) && !domain.EndsWith("/");
+            return regex.IsMatch(domain);
         }
 
         public static string BuildUrl(string domain, IDictionary<string, string> queryParameters)