diff --git a/UmlautAdaptarr/Controllers/SearchController.cs b/UmlautAdaptarr/Controllers/SearchController.cs index b22dec2..a12e175 100644 --- a/UmlautAdaptarr/Controllers/SearchController.cs +++ b/UmlautAdaptarr/Controllers/SearchController.cs @@ -1,9 +1,5 @@ using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.Mvc.Formatters; -using Newtonsoft.Json.Linq; -using System.Linq; using System.Text; -using System.Xml.Linq; using UmlautAdaptarr.Models; using UmlautAdaptarr.Services; using UmlautAdaptarr.Utilities; diff --git a/UmlautAdaptarr/Models/SearchItem.cs b/UmlautAdaptarr/Models/SearchItem.cs index 91ca98a..eea582c 100644 --- a/UmlautAdaptarr/Models/SearchItem.cs +++ b/UmlautAdaptarr/Models/SearchItem.cs @@ -67,6 +67,7 @@ namespace UmlautAdaptarr.Models } TitleMatchVariations = allTitleVariations.Distinct().ToArray(); + AuthorMatchVariations = []; } } diff --git a/UmlautAdaptarr/Providers/LidarrClient.cs b/UmlautAdaptarr/Providers/LidarrClient.cs index e846483..0e88778 100644 --- a/UmlautAdaptarr/Providers/LidarrClient.cs +++ b/UmlautAdaptarr/Providers/LidarrClient.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json; +using Microsoft.Extensions.Caching.Memory; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using UmlautAdaptarr.Models; using UmlautAdaptarr.Services; @@ -9,7 +10,8 @@ namespace UmlautAdaptarr.Providers public class LidarrClient( IHttpClientFactory clientFactory, IConfiguration configuration, - TitleApiService titleService, + CacheService cacheService, + IMemoryCache cache, ILogger logger) : ArrClientBase() { private readonly string _lidarrHost = configuration.GetValue("LIDARR_HOST") ?? throw new ArgumentException("LIDARR_HOST environment variable must be set"); @@ -23,7 +25,6 @@ namespace UmlautAdaptarr.Providers try { - var lidarrArtistsUrl = $"{_lidarrHost}/api/v1/artist?apikey={_lidarrApiKey}"; logger.LogInformation($"Fetching all artists from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}"); var artistsApiResponse = await httpClient.GetStringAsync(lidarrArtistsUrl); @@ -40,9 +41,17 @@ namespace UmlautAdaptarr.Providers var artistId = (int)artist.id; var lidarrAlbumUrl = $"{_lidarrHost}/api/v1/album?artistId={artistId}&apikey={_lidarrApiKey}"; - logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}"); - var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl); - var albums = JsonConvert.DeserializeObject>(albumApiResponse); + + if (cache.TryGetValue(lidarrAlbumUrl, out List? albums)) + { + logger.LogInformation($"Using cached albums for {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}"); + } + else + { + logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}"); + var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl); + albums = JsonConvert.DeserializeObject>(albumApiResponse); + } if (albums == null) { @@ -52,6 +61,9 @@ namespace UmlautAdaptarr.Providers logger.LogInformation($"Successfully fetched {albums.Count} albums for artistId {artistId} from Lidarr."); + // Cache albums for 3 minutes + cache.Set(lidarrAlbumUrl, albums, TimeSpan.FromMinutes(3)); + foreach (var album in albums) { var artistName = (string)album.artist.artistName; @@ -92,42 +104,20 @@ namespace UmlautAdaptarr.Providers public override async Task FetchItemByExternalIdAsync(string externalId) { - var httpClient = clientFactory.CreateClient(); - try { - var lidarrUrl = $"{_lidarrHost}/api/v1/series?mbId={externalId}&includeSeasonImages=false&apikey={_lidarrApiKey}"; - logger.LogInformation($"Fetching item by external ID from Lidarr: {UrlUtilities.RedactApiKey(lidarrUrl)}"); - var response = await httpClient.GetStringAsync(lidarrUrl); - var artists = JsonConvert.DeserializeObject(response); - var artist = artists?[0]; - - if (artist != null) + // For now we have to fetch all items every time + var searchItems = await FetchAllItemsAsync(); + foreach (var searchItem in searchItems ?? []) { - var mbId = (string)artist.mbId; - if (mbId == null) + try { - logger.LogWarning($"Lidarr Artist {artist.id} doesn't have a mbId."); - return null; + cacheService.CacheSearchItem(searchItem); + } + catch (Exception ex) + { + logger.LogError(ex, $"An error occurred while caching search item with ID {searchItem.ArrId}."); } - (var germanTitle, var aliases) = await titleService.FetchGermanTitleAndAliasesByExternalIdAsync(_mediaType, mbId); - - throw new NotImplementedException(); - - var searchItem = new SearchItem - ( - arrId: (int)artist.id, - externalId: mbId, - title: (string)artist.title, - expectedTitle: (string)artist.title, - germanTitle: germanTitle, - aliases: aliases, - mediaType: _mediaType, - expectedAuthor: "TODO" - ); ; - - logger.LogInformation($"Successfully fetched artist {searchItem} from Lidarr."); - return searchItem; } } catch (Exception ex) @@ -140,54 +130,10 @@ namespace UmlautAdaptarr.Providers public override async Task FetchItemByTitleAsync(string title) { - var httpClient = clientFactory.CreateClient(); - try { - (string? germanTitle, string? mbId, string[]? aliases) = await titleService.FetchGermanTitleAndExternalIdAndAliasesByTitle(_mediaType, title); - - if (mbId == null) - { - return null; - } - - var lidarrUrl = $"{_lidarrHost}/api/v1/series?mbId={mbId}&includeSeasonImages=false&apikey={_lidarrApiKey}"; - var lidarrApiResponse = await httpClient.GetStringAsync(lidarrUrl); - var artists = JsonConvert.DeserializeObject(lidarrApiResponse); - - if (artists == null) - { - logger.LogError($"Parsing Lidarr API response for MB ID {mbId} resulted in null"); - return null; - } - else if (artists.Count == 0) - { - logger.LogWarning($"No results found for MB ID {mbId}"); - return null; - } - - var expectedTitle = (string)artists[0].title; - if (expectedTitle == null) - { - logger.LogError($"Lidarr Title for MB ID {mbId} is null"); - return null; - } - + // this should never be called at the moment throw new NotImplementedException(); - var searchItem = new SearchItem - ( - arrId: (int)artists[0].id, - externalId: mbId, - title: (string)artists[0].title, - expectedTitle: (string)artists[0].title, - germanTitle: germanTitle, - aliases: aliases, - mediaType: _mediaType, - expectedAuthor: "TODO" - ); - - logger.LogInformation($"Successfully fetched artist {searchItem} from Lidarr."); - return searchItem; } catch (Exception ex) { diff --git a/UmlautAdaptarr/Services/CacheService.cs b/UmlautAdaptarr/Services/CacheService.cs index 882f2e7..db22a0d 100644 --- a/UmlautAdaptarr/Services/CacheService.cs +++ b/UmlautAdaptarr/Services/CacheService.cs @@ -1,4 +1,6 @@ -using Microsoft.Extensions.Caching.Memory; +using Microsoft.AspNetCore.Mvc.Formatters; +using Microsoft.Extensions.Caching.Memory; +using System.Reflection.Metadata.Ecma335; using System.Text.RegularExpressions; using UmlautAdaptarr.Models; using UmlautAdaptarr.Utilities; @@ -8,16 +10,17 @@ namespace UmlautAdaptarr.Services public partial class CacheService(IMemoryCache cache) { private readonly Dictionary> VariationIndex = []; - private readonly Dictionary> AudioFuzzyIndex = []; + private readonly Dictionary TitleVariations, string CacheKey)>> AudioVariationIndex = []; private const int VARIATION_LOOKUP_CACHE_LENGTH = 5; public void CacheSearchItem(SearchItem item) { var prefix = item.MediaType; - cache.Set($"{prefix}_extid_{item.ExternalId}", item); + var cacheKey = $"{prefix}_extid_{item.ExternalId}"; + cache.Set(cacheKey, item); if (item.MediaType == "audio") { - CacheAudioSearchItem(item); + CacheAudioSearchItem(item, cacheKey); return; } @@ -28,7 +31,7 @@ namespace UmlautAdaptarr.Services foreach (var variation in item.TitleMatchVariations) { var normalizedVariation = variation.RemoveAccentButKeepGermanUmlauts().ToLower(); - var cacheKey = $"{prefix}_var_{normalizedVariation}"; + cacheKey = $"{prefix}_var_{normalizedVariation}"; cache.Set(cacheKey, item); // Indexing by prefix @@ -41,30 +44,32 @@ namespace UmlautAdaptarr.Services } } - private void CacheAudioSearchItem(SearchItem item) + public void CacheAudioSearchItem(SearchItem item, string cacheKey) { - // Normalize and simplify the title and author for fuzzy matching - var key = NormalizeForFuzzyMatching(item.ExternalId); - - if (!AudioFuzzyIndex.ContainsKey(key)) + // Index author and title variations + foreach (var authorVariation in item.AuthorMatchVariations) { - AudioFuzzyIndex[key] = new List(); - } - AudioFuzzyIndex[key].Add(item); - } + var normalizedAuthor = authorVariation.NormalizeForComparison(); - private string NormalizeForFuzzyMatching(string input) - { - // Normalize the input string by removing accents, converting to lower case, and removing non-alphanumeric characters - var normalized = input.RemoveAccentButKeepGermanUmlauts().RemoveSpecialCharacters().ToLower(); - normalized = WhiteSpaceRegex().Replace(normalized, ""); - return normalized; + if (!AudioVariationIndex.ContainsKey(normalizedAuthor)) + { + AudioVariationIndex[normalizedAuthor] = []; + } + + var titleVariations = item.TitleMatchVariations.Select(titleMatchVariation => titleMatchVariation.NormalizeForComparison()).ToHashSet(); + AudioVariationIndex[normalizedAuthor].Add((titleVariations, cacheKey)); + } } public SearchItem? SearchItemByTitle(string mediaType, string title) { var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower(); + if (mediaType == "audio") + { + return FindBestMatchForAudio(normalizedTitle.NormalizeForComparison()); + } + // Use the first few characters of the normalized title for cache prefix search var cacheSearchPrefix = normalizedTitle[..Math.Min(VARIATION_LOOKUP_CACHE_LENGTH, normalizedTitle.Length)]; @@ -107,10 +112,12 @@ namespace UmlautAdaptarr.Services { var normalizedTitle = title.RemoveAccentButKeepGermanUmlauts().ToLower(); + if (mediaType == "generic") { // TODO } + cache.TryGetValue($"{mediaType}_var_{normalizedTitle}", out SearchItem? item); if (item == null) { @@ -119,6 +126,31 @@ namespace UmlautAdaptarr.Services return item; } + private SearchItem? FindBestMatchForAudio(string normalizedOriginalTitle) + { + foreach (var authorEntry in AudioVariationIndex) + { + if (normalizedOriginalTitle.Contains(authorEntry.Key)) + { + var sortedEntries = authorEntry.Value.OrderByDescending(entry => entry.TitleVariations.FirstOrDefault()?.Length).ToList(); + + foreach (var (titleVariations, cacheKey) in sortedEntries) + { + if (titleVariations.Any(normalizedOriginalTitle.Contains)) + { + if (cache.TryGetValue(cacheKey, out SearchItem? item)) + { + return item; + } + } + } + } + } + + return null; + } + + [GeneratedRegex("\\s")] private static partial Regex WhiteSpaceRegex(); } diff --git a/UmlautAdaptarr/Services/SearchItemLookupService.cs b/UmlautAdaptarr/Services/SearchItemLookupService.cs index 955b043..4b0eff5 100644 --- a/UmlautAdaptarr/Services/SearchItemLookupService.cs +++ b/UmlautAdaptarr/Services/SearchItemLookupService.cs @@ -62,6 +62,8 @@ namespace UmlautAdaptarr.Services fetchedItem = await sonarrClient.FetchItemByTitleAsync(title); } break; + case "audio": + break; // TODO add cases for other sources as needed, such as Radarr, Lidarr, etc. } @@ -74,5 +76,4 @@ namespace UmlautAdaptarr.Services return fetchedItem; } } - } diff --git a/UmlautAdaptarr/Services/TitleMatchingService.cs b/UmlautAdaptarr/Services/TitleMatchingService.cs index c185d54..3dfadd4 100644 --- a/UmlautAdaptarr/Services/TitleMatchingService.cs +++ b/UmlautAdaptarr/Services/TitleMatchingService.cs @@ -51,7 +51,7 @@ namespace UmlautAdaptarr.Services FindAndReplaceForMoviesAndTV(logger, searchItem, titleElement, originalTitle, normalizedOriginalTitle!); break; case "audio": - ReplaceForAudio(searchItem, titleElement, originalTitle, normalizedOriginalTitle!); + FindAndReplaceForAudio(searchItem, titleElement, originalTitle!); break; default: throw new NotImplementedException(); @@ -62,23 +62,25 @@ namespace UmlautAdaptarr.Services return xDoc.ToString(); } - private string NormalizeString(string text) + public void FindAndReplaceForAudio(SearchItem searchItem, XElement? titleElement, string originalTitle) { - return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower(); - } - - - public void ReplaceForAudio(SearchItem searchItem, XElement? titleElement, string originalTitle, string normalizedOriginalTitle) - { - var authorMatch = FindBestMatch(searchItem.AuthorMatchVariations, NormalizeString(normalizedOriginalTitle), originalTitle); - var titleMatch = FindBestMatch(searchItem.TitleMatchVariations, NormalizeString(normalizedOriginalTitle), originalTitle); + var authorMatch = FindBestMatch(searchItem.AuthorMatchVariations, originalTitle.NormalizeForComparison(), originalTitle); + var titleMatch = FindBestMatch(searchItem.TitleMatchVariations, originalTitle.NormalizeForComparison(), originalTitle); if (authorMatch.Item1 && titleMatch.Item1) { int matchEndPositionInOriginal = Math.Max(authorMatch.Item3, titleMatch.Item3); + var test = originalTitle[matchEndPositionInOriginal]; + // Check and adjust for immediate following delimiter + if (matchEndPositionInOriginal < originalTitle.Length && new char[] { ' ', '-', '_', '.' }.Contains(originalTitle[matchEndPositionInOriginal])) + { + matchEndPositionInOriginal++; // Skip the delimiter if it's immediately after the match + } + // Ensure we trim any leading delimiters from the suffix - string suffix = originalTitle.Substring(matchEndPositionInOriginal).TrimStart([' ', '-', '_']); + string suffix = originalTitle[matchEndPositionInOriginal..].TrimStart([' ', '-', '_', '.']).Trim(); + suffix = suffix.Replace("-", "."); // Concatenate the expected title with the remaining suffix var updatedTitle = $"{searchItem.ExpectedAuthor} - {searchItem.ExpectedTitle}-{suffix}"; @@ -102,7 +104,7 @@ namespace UmlautAdaptarr.Services foreach (var variation in variations) { - var normalizedVariation = NormalizeString(variation); + var normalizedVariation = variation.NormalizeForComparison(); int startNormalized = normalizedOriginal.IndexOf(normalizedVariation); if (startNormalized >= 0) @@ -148,11 +150,9 @@ namespace UmlautAdaptarr.Services originalIndex = i; } - return originalIndex + 1; // +1 to move past the matched character or to the next character in the original title + return originalIndex; } - - // This method replaces the first variation that starts at the beginning of the release title private static void FindAndReplaceForMoviesAndTV(ILogger logger, SearchItem searchItem, XElement? titleElement, string originalTitle, string normalizedOriginalTitle) { diff --git a/UmlautAdaptarr/Utilities/Extensions.cs b/UmlautAdaptarr/Utilities/Extensions.cs index 89e309b..0716456 100644 --- a/UmlautAdaptarr/Utilities/Extensions.cs +++ b/UmlautAdaptarr/Utilities/Extensions.cs @@ -53,6 +53,11 @@ namespace UmlautAdaptarr.Utilities return text.Replace("(", "").Replace(")", "").Replace("?","").Replace(":", "").Replace("'", ""); } + public static string NormalizeForComparison(this string text) + { + return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower(); + } + public static string RemoveSpecialCharacters(this string text) { return SpecialCharactersRegex().Replace(text, "");