Workaround for weird lidarr album title parsing

README.md

@@ -2,10 +2,10 @@
 
 ## English description coming soon
 
-## 12.02.2024: Erste Testversion
+## Erste Testversion
 Wer möchte kann den UmlautAdaptarr jetzt gerne testen! Über Feedback würde ich mich sehr freuen!
 
-Es sollte mit allen *arrs funktionieren, hat aber nur bei Sonarr schon Auswirkungen (abgesehen vom Caching).
+Es sollte mit allen *arrs funktionieren, hat aber nur bei Sonarr und Lidarr schon Auswirkungen (abgesehen vom Caching).
 
 Momentan ist docker dafür nötig, wer kein Docker nutzt muss sich noch etwas gedulden. 
 
@@ -15,7 +15,7 @@ Zusätzlich müsst ihr in Sonarr oder Prowlarr einen neuen Indexer hinzufügen (
 
 Am Beispiel von sceneNZBs:
 
-![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/97ca0aef-1a9e-4560-9374-c3a8215dafd2)
+![grafik](https://github.com/PCJones/UmlautAdaptarr/assets/377223/07c7ca45-e0e5-4a82-af63-365bb23c55c9)
 
 Also alles wie immer, nur dass ihr als API-URL nicht direkt z.B. `https://scenenzbs.com` eingebt, sondern 
 `http://localhost:5005/_/scenenzbs.com`
@@ -38,14 +38,15 @@ Einige Beispiele findet ihr unter Features.
 
 | Feature                                                           | Status        |
 |-------------------------------------------------------------------|---------------|
-| Sonarr & Prowlarr Support                                         | ✓             |
+| Prowlarr Support                                                    | ✓|
+| Sonarr Support                                         | ✓             |
+| Lidarr Support                                                    | ✓|
 | Releases mit deutschem Titel werden erkannt   | ✓             |
 | Releases mit TVDB-Alias Titel werden erkannt  | ✓             |
 | Korrekte Suche und Erkennung von Titel mit Umlauten                            | ✓             |
 | Anfragen-Caching für 5 Minuten zur Reduzierung der API-Zugriffe   | ✓             |
 | Radarr Support                                                    | Geplant       |
 | Readarr Support                                                   | Geplant       |
-| Lidarr Support                                                    | Geplant       |
 | Prowlarr Unterstützung für "DE" SceneNZBs Kategorien              | Geplant       |
 | Unterstützung weiterer Sprachen neben Deutsch                     | Geplant       |
 | Wünsche?                                                          | Vorschläge?   |

UmlautAdaptarr/Controllers/SearchController.cs

@@ -1,9 +1,5 @@
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.Formatters;
-using Newtonsoft.Json.Linq;
-using System.Linq;
 using System.Text;
-using System.Xml.Linq;
 using UmlautAdaptarr.Models;
 using UmlautAdaptarr.Services;
 using UmlautAdaptarr.Utilities;

UmlautAdaptarr/Models/SearchItem.cs

@@ -54,6 +54,7 @@ namespace UmlautAdaptarr.Models
             else
             {
                 TitleSearchVariations = GenerateVariations(germanTitle, mediaType).ToArray();
+
                 var allTitleVariations = new List<string>(TitleSearchVariations);
 
                 // If aliases are not null, generate variations for each and add them to the list
@@ -66,6 +67,21 @@ namespace UmlautAdaptarr.Models
                     }
                 }
 
+                AuthorMatchVariations = [];
+
+                // if a german title ends with (DE) also add a search string that replaces (DE) with GERMAN
+                // also add a matching title without (DE)
+                if (germanTitle?.EndsWith("(DE)") ?? false)
+                {
+                    TitleSearchVariations = [.. TitleSearchVariations, .. 
+                        GenerateVariations(
+                            germanTitle.Replace("(DE)", " GERMAN").RemoveExtraWhitespaces(),
+                        mediaType)];
+
+                    allTitleVariations.AddRange(GenerateVariations(germanTitle.Replace("(DE)", "").Trim(), mediaType));
+
+                }
+
                 TitleMatchVariations = allTitleVariations.Distinct().ToArray();
             }
         }

UmlautAdaptarr/Providers/LidarrClient.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using Microsoft.Extensions.Caching.Memory;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using UmlautAdaptarr.Models;
 using UmlautAdaptarr.Services;
@@ -9,7 +10,8 @@ namespace UmlautAdaptarr.Providers
     public class LidarrClient(
         IHttpClientFactory clientFactory,
         IConfiguration configuration,
-        TitleApiService titleService,
+        CacheService cacheService,
+        IMemoryCache cache,
         ILogger<LidarrClient> logger) : ArrClientBase()
     {
         private readonly string _lidarrHost = configuration.GetValue<string>("LIDARR_HOST") ?? throw new ArgumentException("LIDARR_HOST environment variable must be set");
@@ -23,7 +25,6 @@ namespace UmlautAdaptarr.Providers
 
             try
             {
-
                 var lidarrArtistsUrl = $"{_lidarrHost}/api/v1/artist?apikey={_lidarrApiKey}";
                 logger.LogInformation($"Fetching all artists from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}");
                 var artistsApiResponse = await httpClient.GetStringAsync(lidarrArtistsUrl);
@@ -40,9 +41,17 @@ namespace UmlautAdaptarr.Providers
                     var artistId = (int)artist.id;
 
                     var lidarrAlbumUrl = $"{_lidarrHost}/api/v1/album?artistId={artistId}&apikey={_lidarrApiKey}";
-                    logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}");
-                    var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl);
-                    var albums = JsonConvert.DeserializeObject<List<dynamic>>(albumApiResponse);
+
+                    if (cache.TryGetValue(lidarrAlbumUrl, out List<dynamic>? albums))
+                    {
+                        logger.LogInformation($"Using cached albums for {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}");
+                    }
+                    else
+                    {
+                        logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}");
+                        var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl);
+                        albums = JsonConvert.DeserializeObject<List<dynamic>>(albumApiResponse);
+                    }
 
                     if (albums == null)
                     {
@@ -52,6 +61,9 @@ namespace UmlautAdaptarr.Providers
 
                     logger.LogInformation($"Successfully fetched {albums.Count} albums for artistId {artistId} from Lidarr.");
 
+                    // Cache albums for 3 minutes
+                    cache.Set(lidarrAlbumUrl, albums, TimeSpan.FromMinutes(3));
+
                     foreach (var album in albums)
                     {
                         var artistName = (string)album.artist.artistName;
@@ -92,42 +104,20 @@ namespace UmlautAdaptarr.Providers
 
         public override async Task<SearchItem?> FetchItemByExternalIdAsync(string externalId)
         {
-            var httpClient = clientFactory.CreateClient();
-
             try
             {
-                var lidarrUrl = $"{_lidarrHost}/api/v1/series?mbId={externalId}&includeSeasonImages=false&apikey={_lidarrApiKey}";
-                logger.LogInformation($"Fetching item by external ID from Lidarr: {UrlUtilities.RedactApiKey(lidarrUrl)}");
-                var response = await httpClient.GetStringAsync(lidarrUrl);
-                var artists = JsonConvert.DeserializeObject<dynamic>(response);
-                var artist = artists?[0];
-
-                if (artist != null)
+                // For now we have to fetch all items every time
+                var searchItems = await FetchAllItemsAsync();
+                foreach (var searchItem in searchItems ?? [])
                 {
-                    var mbId = (string)artist.mbId;
-                    if (mbId == null)
+                    try
                     {
-                        logger.LogWarning($"Lidarr Artist {artist.id} doesn't have a mbId.");
-                        return null;
+                        cacheService.CacheSearchItem(searchItem);
+                    }
+                    catch (Exception ex)
+                    {
+                        logger.LogError(ex, $"An error occurred while caching search item with ID {searchItem.ArrId}.");
                     }
-                    (var germanTitle, var aliases) = await titleService.FetchGermanTitleAndAliasesByExternalIdAsync(_mediaType, mbId);
-
-                    throw new NotImplementedException();
-
-                    var searchItem = new SearchItem
-                    (
-                        arrId: (int)artist.id,
-                        externalId: mbId,
-                        title: (string)artist.title,
-                        expectedTitle: (string)artist.title,
-                        germanTitle: germanTitle,
-                        aliases: aliases,
-                        mediaType: _mediaType,
-                        expectedAuthor: "TODO"
-                    ); ;
-
-                    logger.LogInformation($"Successfully fetched artist {searchItem} from Lidarr.");
-                    return searchItem;
                 }
             }
             catch (Exception ex)
@@ -140,54 +130,10 @@ namespace UmlautAdaptarr.Providers
 
         public override async Task<SearchItem?> FetchItemByTitleAsync(string title)
         {
-            var httpClient = clientFactory.CreateClient();
-
             try
             {
-                (string? germanTitle, string? mbId, string[]? aliases) = await titleService.FetchGermanTitleAndExternalIdAndAliasesByTitle(_mediaType, title);
-
-                if (mbId == null)
-                {
-                    return null;
-                }
-
-                var lidarrUrl = $"{_lidarrHost}/api/v1/series?mbId={mbId}&includeSeasonImages=false&apikey={_lidarrApiKey}";
-                var lidarrApiResponse = await httpClient.GetStringAsync(lidarrUrl);
-                var artists = JsonConvert.DeserializeObject<dynamic>(lidarrApiResponse);
-
-                if (artists == null)
-                {
-                    logger.LogError($"Parsing Lidarr API response for MB ID {mbId} resulted in null");
-                    return null;
-                }
-                else if (artists.Count == 0)
-                {
-                    logger.LogWarning($"No results found for MB ID {mbId}");
-                    return null;
-                }
-
-                var expectedTitle = (string)artists[0].title;
-                if (expectedTitle == null)
-                {
-                    logger.LogError($"Lidarr Title for MB ID {mbId} is null");
-                    return null;
-                }
-
+               // this should never be called at the moment
                 throw new NotImplementedException();
-                var searchItem = new SearchItem
-                (
-                    arrId: (int)artists[0].id,
-                    externalId: mbId,
-                    title: (string)artists[0].title,
-                    expectedTitle: (string)artists[0].title,
-                    germanTitle: germanTitle,
-                    aliases: aliases,
-                    mediaType: _mediaType,
-                    expectedAuthor: "TODO"
-                );
-
-                logger.LogInformation($"Successfully fetched artist {searchItem} from Lidarr.");
-                return searchItem;
             }
             catch (Exception ex)
             {

UmlautAdaptarr/Services/CacheService.cs

@@ -1,4 +1,6 @@
-using Microsoft.Extensions.Caching.Memory;
+using Microsoft.AspNetCore.Mvc.Formatters;
+using Microsoft.Extensions.Caching.Memory;
+using System.Reflection.Metadata.Ecma335;
 using System.Text.RegularExpressions;
 using UmlautAdaptarr.Models;
 using UmlautAdaptarr.Utilities;
@@ -8,16 +10,17 @@ namespace UmlautAdaptarr.Services
     public partial class CacheService(IMemoryCache cache)
     {
         private readonly Dictionary<string, HashSet<string>> VariationIndex = [];
-        private readonly Dictionary<string, List<SearchItem>> AudioFuzzyIndex = [];
+        private readonly Dictionary<string, List<(HashSet<string> TitleVariations, string CacheKey)>> AudioVariationIndex = [];
         private const int VARIATION_LOOKUP_CACHE_LENGTH = 5;
 
         public void CacheSearchItem(SearchItem item)
         {
             var prefix = item.MediaType;
-            cache.Set($"{prefix}_extid_{item.ExternalId}", item);
+            var cacheKey = $"{prefix}_extid_{item.ExternalId}";
+            cache.Set(cacheKey, item);
             if (item.MediaType == "audio")
             {
-                CacheAudioSearchItem(item);
+                CacheAudioSearchItem(item, cacheKey);
                 return;
             }
 
@@ -28,7 +31,7 @@ namespace UmlautAdaptarr.Services
             foreach (var variation in item.TitleMatchVariations)
             {
                 var normalizedVariation = variation.RemoveAccentButKeepGermanUmlauts().ToLower();
-                var cacheKey = $"{prefix}_var_{normalizedVariation}";
+                cacheKey = $"{prefix}_var_{normalizedVariation}";
                 cache.Set(cacheKey, item);
 
                 // Indexing by prefix
@@ -41,30 +44,32 @@ namespace UmlautAdaptarr.Services
             }
         }
 
-        private void CacheAudioSearchItem(SearchItem item)
+        public void CacheAudioSearchItem(SearchItem item, string cacheKey)
         {
-            // Normalize and simplify the title and author for fuzzy matching
-            var key = NormalizeForFuzzyMatching(item.ExternalId);
-
-            if (!AudioFuzzyIndex.ContainsKey(key))
+            // Index author and title variations
+            foreach (var authorVariation in item.AuthorMatchVariations)
             {
-                AudioFuzzyIndex[key] = new List<SearchItem>();
-            }
-            AudioFuzzyIndex[key].Add(item);
-        }
+                var normalizedAuthor = authorVariation.NormalizeForComparison();
 
-        private string NormalizeForFuzzyMatching(string input)
-        {
-            // Normalize the input string by removing accents, converting to lower case, and removing non-alphanumeric characters
-            var normalized = input.RemoveAccentButKeepGermanUmlauts().RemoveSpecialCharacters().ToLower();
-            normalized = WhiteSpaceRegex().Replace(normalized, "");
-            return normalized;
+                if (!AudioVariationIndex.ContainsKey(normalizedAuthor))
+                {
+                    AudioVariationIndex[normalizedAuthor] = [];
+                }
+
+                var titleVariations = item.TitleMatchVariations.Select(titleMatchVariation => titleMatchVariation.NormalizeForComparison()).ToHashSet();
+                AudioVariationIndex[normalizedAuthor].Add((titleVariations, cacheKey));
+            }
         }
 
         public SearchItem? SearchItemByTitle(string mediaType, string title)
         {
             var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower();
 
+            if (mediaType == "audio")
+            {
+                return FindBestMatchForAudio(normalizedTitle.NormalizeForComparison());
+            }
+
             // Use the first few characters of the normalized title for cache prefix search
             var cacheSearchPrefix = normalizedTitle[..Math.Min(VARIATION_LOOKUP_CACHE_LENGTH, normalizedTitle.Length)];
 
@@ -107,10 +112,12 @@ namespace UmlautAdaptarr.Services
         {
             var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower();
 
+
             if (mediaType == "generic")
             {
                 // TODO
             }
+
             cache.TryGetValue($"{mediaType}_var_{normalizedTitle}", out SearchItem? item);
             if (item == null)
             {
@@ -119,6 +126,31 @@ namespace UmlautAdaptarr.Services
             return item;
         }
 
+        private SearchItem? FindBestMatchForAudio(string normalizedOriginalTitle)
+        {
+            foreach (var authorEntry in AudioVariationIndex)
+            {
+                if (normalizedOriginalTitle.Contains(authorEntry.Key))
+                {
+                    var sortedEntries = authorEntry.Value.OrderByDescending(entry => entry.TitleVariations.FirstOrDefault()?.Length).ToList();
+
+                    foreach (var (titleVariations, cacheKey) in sortedEntries)
+                    {
+                        if (titleVariations.Any(normalizedOriginalTitle.Contains))
+                        {
+                            if (cache.TryGetValue(cacheKey, out SearchItem? item))
+                            {
+                                return item;
+                            }
+                        }
+                    }
+                }
+            }
+
+            return null;
+        }
+
+
         [GeneratedRegex("\\s")]
         private static partial Regex WhiteSpaceRegex();
     }

UmlautAdaptarr/Services/SearchItemLookupService.cs

@@ -62,6 +62,8 @@ namespace UmlautAdaptarr.Services
                         fetchedItem = await sonarrClient.FetchItemByTitleAsync(title);
                     }
                     break;
+                case "audio":
+                    break;
                     // TODO add cases for other sources as needed, such as Radarr, Lidarr, etc.
             }
 
@@ -74,5 +76,4 @@ namespace UmlautAdaptarr.Services
             return fetchedItem;
         }
     }
-
 }

UmlautAdaptarr/Services/TitleMatchingService.cs

@@ -1,4 +1,5 @@
-using System.Text.RegularExpressions;
+using Microsoft.Extensions.FileSystemGlobbing.Internal;
+using System.Text.RegularExpressions;
 using System.Xml.Linq;
 using UmlautAdaptarr.Models;
 using UmlautAdaptarr.Utilities;
@@ -51,7 +52,7 @@ namespace UmlautAdaptarr.Services
                             FindAndReplaceForMoviesAndTV(logger, searchItem, titleElement, originalTitle, normalizedOriginalTitle!);
                             break;
                         case "audio":
-                            ReplaceForAudio(searchItem, titleElement, originalTitle, normalizedOriginalTitle!);
+                            FindAndReplaceForAudio(searchItem, titleElement, originalTitle!);
                             break;
                         default:
                             throw new NotImplementedException();
@@ -62,26 +63,28 @@ namespace UmlautAdaptarr.Services
             return xDoc.ToString();
         }
 
-        private string NormalizeString(string text)
+        public void FindAndReplaceForAudio(SearchItem searchItem, XElement? titleElement, string originalTitle)
         {
-            return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower();
-        }
-
-
-        public void ReplaceForAudio(SearchItem searchItem, XElement? titleElement, string originalTitle, string normalizedOriginalTitle)
-        {
-            var authorMatch = FindBestMatch(searchItem.AuthorMatchVariations, NormalizeString(normalizedOriginalTitle), originalTitle);
-            var titleMatch = FindBestMatch(searchItem.TitleMatchVariations, NormalizeString(normalizedOriginalTitle), originalTitle);
+            var authorMatch = FindBestMatch(searchItem.AuthorMatchVariations, originalTitle.NormalizeForComparison(), originalTitle);
+            var titleMatch = FindBestMatch(searchItem.TitleMatchVariations, originalTitle.NormalizeForComparison(), originalTitle);
 
             if (authorMatch.Item1 && titleMatch.Item1)
             {
                 int matchEndPositionInOriginal = Math.Max(authorMatch.Item3, titleMatch.Item3);
 
+                var test = originalTitle[matchEndPositionInOriginal];
+                // Check and adjust for immediate following delimiter
+                char[] delimiters = new char[] { ' ', '-', '_', '.' };
+                if (matchEndPositionInOriginal < originalTitle.Length && delimiters.Contains(originalTitle[matchEndPositionInOriginal]))
+                {
+                    matchEndPositionInOriginal++; // Skip the delimiter if it's immediately after the match
+                }
+
                 // Ensure we trim any leading delimiters from the suffix
-                string suffix = originalTitle.Substring(matchEndPositionInOriginal).TrimStart([' ', '-', '_']);
+                string suffix = originalTitle[matchEndPositionInOriginal..].TrimStart([' ', '-', '_', '.']).Trim();
 
                 // Concatenate the expected title with the remaining suffix
-                var updatedTitle = $"{searchItem.ExpectedAuthor} - {searchItem.ExpectedTitle}-{suffix}";
+                var updatedTitle = $"{searchItem.ExpectedAuthor} - {searchItem.ExpectedTitle}-[{suffix}]";
 
                 // Update the title element
                 titleElement.Value = updatedTitle;
@@ -102,7 +105,7 @@ namespace UmlautAdaptarr.Services
 
             foreach (var variation in variations)
             {
-                var normalizedVariation = NormalizeString(variation);
+                var normalizedVariation = variation.NormalizeForComparison();
                 int startNormalized = normalizedOriginal.IndexOf(normalizedVariation);
 
                 if (startNormalized >= 0)
@@ -148,11 +151,9 @@ namespace UmlautAdaptarr.Services
                 originalIndex = i;
             }
 
-            return originalIndex + 1; // +1 to move past the matched character or to the next character in the original title
+            return originalIndex;
         }
 
-
-
         // This method replaces the first variation that starts at the beginning of the release title
         private static void FindAndReplaceForMoviesAndTV(ILogger<TitleMatchingService> logger, SearchItem searchItem, XElement? titleElement, string originalTitle, string normalizedOriginalTitle)
         {
@@ -174,12 +175,6 @@ namespace UmlautAdaptarr.Services
                 // Check if the originalTitle starts with the variation (ignoring case and separators)
                 if (Regex.IsMatch(normalizedOriginalTitle, variationMatchPattern, RegexOptions.IgnoreCase))
                 {
-                    // Workaround for the rare case of e.g. "Frieren: Beyond Journey's End" that also has the alias "Frieren"
-                    if (expectedTitle!.StartsWith(variation, StringComparison.OrdinalIgnoreCase))
-                    {
-                        logger.LogWarning($"TitleMatchingService - Didn't rename: '{originalTitle}' because the expected title '{expectedTitle}' starts with the variation '{variation}'");
-                        continue;
-                    }
                     var originalTitleMatchPattern = "^" + Regex.Escape(variation).Replace("\\ ", "[._ ]");
 
                     // Find the first separator used in the original title for consistent replacement
@@ -191,6 +186,19 @@ namespace UmlautAdaptarr.Services
                     var variationLength = variation.Length;
                     var suffix = originalTitle[Math.Min(variationLength, originalTitle.Length)..];
 
+                    // Workaround for the rare case of e.g. "Frieren: Beyond Journey's End" that also has the alias "Frieren"
+                    if (expectedTitle!.StartsWith(variation, StringComparison.OrdinalIgnoreCase))
+                    {
+                        // See if we already matched the whole title by checking if S01E01 pattern is coming next to avoid false positives
+                        // - that won't help with movies but with tv shows
+                        var seasonMatchingPattern = $"^{separator}S\\d{{1,2}}E\\d{{1,2}}";
+                        if (!Regex.IsMatch(suffix, seasonMatchingPattern))
+                        {
+                            logger.LogWarning($"TitleMatchingService - Didn't rename: '{originalTitle}' because the expected title '{expectedTitle}' starts with the variation '{variation}'");
+                            continue;
+                        }
+                    }
+
                     // Clean up any leading separators from the suffix
                     suffix = Regex.Replace(suffix, "^[._ ]+", "");
 

UmlautAdaptarr/Utilities/Extensions.cs

@@ -53,6 +53,11 @@ namespace UmlautAdaptarr.Utilities
             return text.Replace("(", "").Replace(")", "").Replace("?","").Replace(":", "").Replace("'", "");
         }
 
+        public static string NormalizeForComparison(this string text)
+        {
+            return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower();
+        }
+
         public static string RemoveSpecialCharacters(this string text)
         {
             return SpecialCharactersRegex().Replace(text, "");

build_and_push_docker_image.bat

@@ -0,0 +1,18 @@
+@echo off
+SET IMAGE_NAME=pcjones/umlautadaptarr
+
+echo Enter the version number for the Docker image:
+set /p VERSION="Version: "
+
+echo Building Docker image with version %VERSION%...
+docker build -t %IMAGE_NAME%:%VERSION% .
+docker tag %IMAGE_NAME%:%VERSION% %IMAGE_NAME%:latest
+
+echo Pushing Docker image with version %VERSION%...
+docker push %IMAGE_NAME%:%VERSION%
+
+echo Pushing Docker image with tag latest...
+docker push %IMAGE_NAME%:latest
+
+echo Done.
+pause