From 232f86ba85aa20d280aafe7f73e682fff4fa0da3 Mon Sep 17 00:00:00 2001 From: Pat Hartl Date: Mon, 27 Apr 2026 22:42:44 -0500 Subject: [PATCH] Refactor launcher game importing - Game importing happens on a separate thread to avoid UI blocking - Server requests per-game are done in parallel to cut down on import times - Media downloading is done in parallel - Library import progress shown in title bar - Enabled WAL mode for SQLite for better performance - Fix potential issues with deferred imports causing infinite loops or lockups --- LANCommander.Launcher.Avalonia/App.axaml.cs | 1 + LANCommander.Launcher.Avalonia/Program.cs | 2 + .../ViewModels/ShellViewModel.cs | 143 ++++++++++++++--- .../Views/MainWindow.axaml | 24 ++- LANCommander.Launcher.Data/DatabaseContext.cs | 21 ++- LANCommander.Launcher.Services/GameService.cs | 11 ++ .../Import/IImportItemInfo.cs | 2 + .../Import/ImportContext.cs | 146 ++++++++++++++---- .../Import/ImportItemInfo.cs | 7 + .../Import/Importers/GameImporter.cs | 24 ++- .../Import/Importers/MediaImporter.cs | 40 +++-- .../Import/PendingMediaDownload.cs | 8 + .../ImportService.cs | 49 ++++-- 13 files changed, 385 insertions(+), 93 deletions(-) create mode 100644 LANCommander.Launcher.Services/Import/PendingMediaDownload.cs diff --git a/LANCommander.Launcher.Avalonia/App.axaml.cs b/LANCommander.Launcher.Avalonia/App.axaml.cs index 5c335d76..a4ea8f5a 100644 --- a/LANCommander.Launcher.Avalonia/App.axaml.cs +++ b/LANCommander.Launcher.Avalonia/App.axaml.cs @@ -160,6 +160,7 @@ public partial class App : Application // Run database migrations _logger?.LogInformation("Running database migrations..."); await databaseContext.Database.MigrateAsync().ConfigureAwait(false); + await databaseContext.EnableWalModeAsync().ConfigureAwait(false); _logger?.LogInformation("Database migrations complete"); } diff --git a/LANCommander.Launcher.Avalonia/Program.cs b/LANCommander.Launcher.Avalonia/Program.cs index 47dc145b..db68fe2d 100644 --- a/LANCommander.Launcher.Avalonia/Program.cs +++ b/LANCommander.Launcher.Avalonia/Program.cs @@ -119,6 +119,7 @@ class Program } await databaseContext.Database.MigrateAsync().ConfigureAwait(false); + await databaseContext.EnableWalModeAsync().ConfigureAwait(false); await commandLineService.ParseCommandLineAsync(args); } @@ -187,6 +188,7 @@ internal class ScriptDebugApp : Application } await databaseContext.Database.MigrateAsync().ConfigureAwait(false); + await databaseContext.EnableWalModeAsync().ConfigureAwait(false); await commandLineService.ParseCommandLineAsync(args).ConfigureAwait(false); } diff --git a/LANCommander.Launcher.Avalonia/ViewModels/ShellViewModel.cs b/LANCommander.Launcher.Avalonia/ViewModels/ShellViewModel.cs index 19fb4568..b6938a04 100644 --- a/LANCommander.Launcher.Avalonia/ViewModels/ShellViewModel.cs +++ b/LANCommander.Launcher.Avalonia/ViewModels/ShellViewModel.cs @@ -2,12 +2,14 @@ using System; using System.ComponentModel; using System.Linq; using System.Threading.Tasks; +using Avalonia.Threading; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using LANCommander.Launcher.Avalonia.Services; using LANCommander.Launcher.Avalonia.ViewModels.Components; using LANCommander.Launcher.Data.Models; using LANCommander.Launcher.Services; +using LANCommander.Launcher.Services.Import; using LANCommander.Launcher.Settings.Enums; using LANCommander.SDK.Services; using Microsoft.Extensions.DependencyInjection; @@ -49,7 +51,7 @@ public partial class ShellViewModel : ViewModelBase }; public bool IsRefreshVisible => - (ContentView is GamesCollectionViewModel || ContentView is DepotViewModel) && !IsOfflineMode; + (ContentView is GamesCollectionViewModel || ContentView is DepotViewModel) && !IsOfflineMode && !IsSyncing; public bool IsTitlebarTinted => true; partial void OnContentViewChanged(ViewModelBase? oldValue, ViewModelBase? newValue) @@ -77,6 +79,29 @@ public partial class ShellViewModel : ViewModelBase OnPropertyChanged(nameof(ContentViewTitle)); } + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(IsSyncing))] + [NotifyPropertyChangedFor(nameof(IsImportIndeterminate))] + [NotifyPropertyChangedFor(nameof(ImportProgress))] + [NotifyPropertyChangedFor(nameof(IsRefreshVisible))] + private bool _isImportRunning; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(IsImportIndeterminate))] + [NotifyPropertyChangedFor(nameof(ImportProgress))] + private int _importIndex; + + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(IsImportIndeterminate))] + [NotifyPropertyChangedFor(nameof(ImportProgress))] + private int _importTotal; + + public bool IsSyncing => IsImportRunning; + + public bool IsImportIndeterminate => IsImportRunning && ImportTotal == 0; + + public double ImportProgress => ImportTotal > 0 ? (double)ImportIndex / ImportTotal : 0; + [ObservableProperty] private bool _isCheckingConnection; @@ -207,41 +232,117 @@ public partial class ShellViewModel : ViewModelBase { IsLoading = true; - if (!IsOfflineMode) - { - try - { - using var scope = _serviceProvider.CreateScope(); - var importService = scope.ServiceProvider.GetRequiredService(); - await importService.ImportLibraryAsync(); - } - catch (Exception ex) - { - _logger.LogError(ex, "Import failed"); - } - } - + // Load cached data from local DB first so the user sees games immediately try { - await GamesListViewModel.LoadGamesAsync(); - await LibraryViewModel.LoadGamesAsync(); - await DepotViewModel.LoadAsync(); + await LoadViewModelsAsync(); } catch (Exception ex) { - _logger.LogError(ex, "Failed to load library data"); + _logger.LogError(ex, "Failed to load cached library data"); } finally { IsLoading = false; } + + // Import in the background, then refresh + if (!IsOfflineMode) + { + _ = ImportInBackgroundAsync(); + } + } + + private async Task ImportInBackgroundAsync() + { + IsImportRunning = true; + + try + { + await Task.Run(async () => + { + using var scope = _serviceProvider.CreateScope(); + var importService = scope.ServiceProvider.GetRequiredService(); + + // Track progress on the UI thread + importService.OnImportStarted.EventRaised += OnImportStarted; + importService.OnImportStatusUpdate.EventRaised += OnImportStatusUpdated; + importService.OnImportComplete.EventRaised += OnImportCompleted; + + await importService.ImportLibraryAsync(); + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Background import failed"); + } + finally + { + await Dispatcher.UIThread.InvokeAsync(() => + { + ImportIndex = 0; + ImportTotal = 0; + IsImportRunning = false; + }); + } + } + + private async Task OnImportStarted(ImportStatusUpdate update) + { + await Dispatcher.UIThread.InvokeAsync(() => + { + ImportIndex = 0; + ImportTotal = update.Total; + }); + } + + private async Task OnImportStatusUpdated(ImportStatusUpdate update) + { + await Dispatcher.UIThread.InvokeAsync(() => + { + ImportIndex = update.Index; + }); + + // Refresh the library after each game's items finish importing + if (update.CurrentItem?.Type == nameof(SDK.Models.Manifest.Game)) + { + await Dispatcher.UIThread.InvokeAsync(async () => + { + try + { + await LibraryViewModel.LoadGamesAsync(); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to refresh library during import"); + } + }); + } + } + + private async Task OnImportCompleted(ImportStatusUpdate update) + { + await Dispatcher.UIThread.InvokeAsync(async () => + { + ImportIndex = update.Total; + await LoadViewModelsAsync(); + ImportIndex = 0; + ImportTotal = 0; + }); + } + + private async Task LoadViewModelsAsync() + { + await GamesListViewModel.LoadGamesAsync(); + await LibraryViewModel.LoadGamesAsync(); + await DepotViewModel.LoadAsync(); } [RelayCommand] private async Task RefreshAsync() { - if (IsOfflineMode) return; - await ImportAndLoadAsync(); + if (IsOfflineMode || IsSyncing) return; + _ = ImportInBackgroundAsync(); } [RelayCommand] diff --git a/LANCommander.Launcher.Avalonia/Views/MainWindow.axaml b/LANCommander.Launcher.Avalonia/Views/MainWindow.axaml index 74791e6d..748d55ee 100644 --- a/LANCommander.Launcher.Avalonia/Views/MainWindow.axaml +++ b/LANCommander.Launcher.Avalonia/Views/MainWindow.axaml @@ -93,16 +93,28 @@ PointerPressed="TitleBarDragRegion_PointerPressed" DoubleTapped="TitleBarDragRegion_DoubleTapped" /> - + - + + + + + + diff --git a/LANCommander.Launcher.Data/DatabaseContext.cs b/LANCommander.Launcher.Data/DatabaseContext.cs index e039544a..c49c20cc 100644 --- a/LANCommander.Launcher.Data/DatabaseContext.cs +++ b/LANCommander.Launcher.Data/DatabaseContext.cs @@ -1,6 +1,7 @@ using LANCommander.Launcher.Data.Interceptors; using LANCommander.Launcher.Data.Models; using LANCommander.SDK; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -22,9 +23,27 @@ namespace LANCommander.Launcher.Data { var dbPath = AppPaths.GetConfigPath("LANCommander.db"); + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = dbPath, + Cache = SqliteCacheMode.Shared, + }.ToString(); + optionsBuilder.AddInterceptors(new AuditingInterceptor()); optionsBuilder.UseLoggerFactory(LoggerFactory); - optionsBuilder.UseSqlite($"Data Source={dbPath};Cache=Shared"); + optionsBuilder.UseSqlite(connectionString, options => + { + options.CommandTimeout(30); + }); + } + + /// + /// Enables WAL mode for concurrent reads during background writes. + /// Should be called once after the database is created/migrated. + /// + public async Task EnableWalModeAsync() + { + await Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;"); } protected override void OnModelCreating(ModelBuilder builder) diff --git a/LANCommander.Launcher.Services/GameService.cs b/LANCommander.Launcher.Services/GameService.cs index bcc5f6b3..72e43e8d 100644 --- a/LANCommander.Launcher.Services/GameService.cs +++ b/LANCommander.Launcher.Services/GameService.cs @@ -4,6 +4,7 @@ using LANCommander.Launcher.Models; using LANCommander.SDK; using LANCommander.SDK.Extensions; using LANCommander.SDK.Helpers; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.DependencyInjection; using System.Diagnostics; @@ -23,6 +24,16 @@ namespace LANCommander.Launcher.Services { public Dictionary RunningProcesses = new Dictionary(); + public async Task> GetImportedOnMapAsync(IEnumerable ids) + { + var idSet = ids.ToHashSet(); + + return await Context.Set() + .Where(g => idSet.Contains(g.Id)) + .Select(g => new { g.Id, g.ImportedOn }) + .ToDictionaryAsync(g => g.Id, g => g.ImportedOn); + } + public delegate Task OnUninstallCompleteHandler(Game game); public event OnUninstallCompleteHandler OnUninstallComplete; diff --git a/LANCommander.Launcher.Services/Import/IImportItemInfo.cs b/LANCommander.Launcher.Services/Import/IImportItemInfo.cs index b3d0e724..b73723ce 100644 --- a/LANCommander.Launcher.Services/Import/IImportItemInfo.cs +++ b/LANCommander.Launcher.Services/Import/IImportItemInfo.cs @@ -9,4 +9,6 @@ public interface IImportItemInfo string Name { get; set; } bool Processed { get; set; } BaseManifest Manifest { get; set; } + int DeferCount { get; set; } + Guid? GameId { get; set; } } \ No newline at end of file diff --git a/LANCommander.Launcher.Services/Import/ImportContext.cs b/LANCommander.Launcher.Services/Import/ImportContext.cs index fe0389a0..ffa0d7d5 100644 --- a/LANCommander.Launcher.Services/Import/ImportContext.cs +++ b/LANCommander.Launcher.Services/Import/ImportContext.cs @@ -8,6 +8,8 @@ namespace LANCommander.Launcher.Services.Import; public class ImportContext { + private const int MaxDeferCount = 3; + private int Processed; private int Total; @@ -15,8 +17,10 @@ public class ImportContext public AsyncEventHandler OnImportComplete { get; set; } = new(); public AsyncEventHandler OnImportError { get; set; } = new(); public AsyncEventHandler OnImportStatusUpdate { get; set; } = new(); - + private Queue Queue { get; } = new(); + public List FailedItems { get; } = new(); + internal List PendingMediaDownloads { get; } = new(); private readonly CollectionImporter _collections; private readonly DeveloperImporter _developers; @@ -31,6 +35,7 @@ public class ImportContext private readonly ToolImporter _tools; private readonly ILogger _logger; + private readonly MediaService _mediaService; public ImportContext(IServiceProvider serviceProvider) { @@ -45,37 +50,40 @@ public class ImportContext _publishers = serviceProvider.GetRequiredService(); _tags = serviceProvider.GetRequiredService(); _logger = serviceProvider.GetRequiredService>(); + _mediaService = serviceProvider.GetRequiredService(); SetupContextOnImporters(); } public async Task AddAsync(Game game) { - await AddAsync(game, game.Collections, _collections); - await AddAsync(game, game.Developers, _developers); - await AddAsync(game, game.Engine, _engines); - await AddAsync(game, game.Genres, _genres); - await AddAsync(game, game.Media, _media); - await AddAsync(game, game.MultiplayerModes, _multiplayerModes); - await AddAsync(game, game.Platforms, _platforms); - await AddAsync(game, game.Publishers, _publishers); - await AddAsync(game, game.Tags, _tags); - await AddAsync(game, game, _games); + var gameId = game.Id; + + await AddAsync(game, game.Collections, _collections, gameId); + await AddAsync(game, game.Developers, _developers, gameId); + await AddAsync(game, game.Engine, _engines, gameId); + await AddAsync(game, game.Genres, _genres, gameId); + await AddAsync(game, game.Media, _media, gameId); + await AddAsync(game, game.MultiplayerModes, _multiplayerModes, gameId); + await AddAsync(game, game.Platforms, _platforms, gameId); + await AddAsync(game, game.Publishers, _publishers, gameId); + await AddAsync(game, game.Tags, _tags, gameId); + await AddAsync(game, game, _games, gameId); } public async Task AddAsync(Tool tool) { - await AddAsync(tool, tool, _tools); + await AddAsync(tool, tool, _tools, null); } - private async Task AddAsync(BaseManifest manifest, IEnumerable records, BaseImporter importer) + private async Task AddAsync(BaseManifest manifest, IEnumerable records, BaseImporter importer, Guid? gameId) where TRecord : class { foreach (var record in records) - await AddAsync(manifest, record, importer); + await AddAsync(manifest, record, importer, gameId); } - private async Task AddAsync(BaseManifest manifest, TRecord? record, BaseImporter importer) + private async Task AddAsync(BaseManifest manifest, TRecord? record, BaseImporter importer, Guid? gameId) where TRecord : class { if (record != null && !InQueue(record, importer) && await importer.CanImportAsync(record)) @@ -83,6 +91,7 @@ public class ImportContext var importInfo = await importer.GetImportInfoAsync(record, manifest); importInfo.Key = importer.GetKey(record); + importInfo.GameId = gameId; _logger.LogInformation("Queuing item {ItemName} for import with key {Key}", importInfo.Name, importInfo.Key); @@ -105,7 +114,7 @@ public class ImportContext Total = Total, })!; - int deferred = 0; + int consecutiveDefers = 0; while (Queue.Count > 0) { @@ -119,24 +128,54 @@ public class ImportContext Total = Total, })!; - var success = await TryImportAsync(queueItem); + var result = await TryImportAsync(queueItem); - if (success) + switch (result) { - _logger.LogInformation("Successfully imported item {ItemName}", queueItem.Name); - Processed++; - deferred = 0; - continue; + case ImportResult.Success: + _logger.LogInformation("Successfully imported item {ItemName}", queueItem.Name); + Processed++; + consecutiveDefers = 0; + break; + + case ImportResult.Deferred: + queueItem.DeferCount++; + + if (queueItem.DeferCount >= MaxDeferCount) + { + _logger.LogWarning("Item {ItemName} exceeded max defer count, marking as failed", queueItem.Name); + FailedItems.Add(queueItem); + Processed++; + consecutiveDefers = 0; + } + else + { + _logger.LogInformation("Deferring item {ItemName} for later import (attempt {Count})", queueItem.Name, queueItem.DeferCount); + Queue.Enqueue(queueItem); + consecutiveDefers++; + + if (consecutiveDefers >= Queue.Count) + { + _logger.LogWarning("Import deadlocked: moving all remaining {Count} items to failed", Queue.Count); + while (Queue.Count > 0) + FailedItems.Add(Queue.Dequeue()); + } + } + break; + + case ImportResult.Failed: + _logger.LogWarning("Item {ItemName} failed permanently, skipping", queueItem.Name); + FailedItems.Add(queueItem); + Processed++; + consecutiveDefers = 0; + break; } - - _logger.LogInformation("Deferring item {ItemName} for later import", queueItem.Name); - Queue.Enqueue(queueItem); - deferred++; - - if (deferred >= Queue.Count) - throw new InvalidOperationException("Import deadlocked: remaining jobs cannot be satisfied."); } + if (FailedItems.Count > 0) + _logger.LogWarning("Import completed with {FailedCount} failed items: {Items}", + FailedItems.Count, string.Join(", ", FailedItems.Select(f => f.Name))); + await OnImportComplete?.InvokeAsync(new ImportStatusUpdate { Index = Total, @@ -144,6 +183,38 @@ public class ImportContext })!; } + public async Task DownloadPendingMediaAsync(int maxConcurrency = 4) + { + if (PendingMediaDownloads.Count == 0) + return; + + _logger.LogInformation("Downloading {Count} media files with concurrency {Concurrency}", + PendingMediaDownloads.Count, maxConcurrency); + + var semaphore = new SemaphoreSlim(maxConcurrency); + + var tasks = PendingMediaDownloads.Select(async pending => + { + await semaphore.WaitAsync(); + try + { + await _mediaService.DownloadAsync(pending.Media); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download media {MediaId}", pending.Media.Id); + } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); + + _logger.LogInformation("Media download complete"); + } + private void SetupContextOnImporters() { _collections.UseContext(this); @@ -158,11 +229,11 @@ public class ImportContext _tags.UseContext(this); } - private async Task TryImportAsync(IImportItemInfo queueItem) + private async Task TryImportAsync(IImportItemInfo queueItem) { try { - return queueItem.Type switch + var success = queueItem.Type switch { nameof(Collection) => await _collections.ImportAsync(queueItem), "Developer" => await _developers.ImportAsync(queueItem), @@ -176,6 +247,8 @@ public class ImportContext nameof(Tag) => await _tags.ImportAsync(queueItem), _ => throw new InvalidOperationException($"No importer found for type {queueItem.Type}"), }; + + return success ? ImportResult.Success : ImportResult.Deferred; } catch (Exception ex) { @@ -188,8 +261,15 @@ public class ImportContext Total = Total, Error = ex.Message, })!; - } - return false; + return ImportResult.Failed; + } + } + + private enum ImportResult + { + Success, + Deferred, + Failed, } } \ No newline at end of file diff --git a/LANCommander.Launcher.Services/Import/ImportItemInfo.cs b/LANCommander.Launcher.Services/Import/ImportItemInfo.cs index 878d58f2..3f5e180d 100644 --- a/LANCommander.Launcher.Services/Import/ImportItemInfo.cs +++ b/LANCommander.Launcher.Services/Import/ImportItemInfo.cs @@ -10,4 +10,11 @@ public class ImportItemInfo : IImportItemInfo where T : class public bool Processed { get; set; } public BaseManifest Manifest { get; set; } public T Record { get; set; } + public int DeferCount { get; set; } + public Guid? GameId { get; set; } + + /// + /// Cached local entity loaded during ExistsAsync to avoid redundant DB lookups. + /// + public object? ExistingEntity { get; set; } } \ No newline at end of file diff --git a/LANCommander.Launcher.Services/Import/Importers/GameImporter.cs b/LANCommander.Launcher.Services/Import/Importers/GameImporter.cs index 983c5a96..cafa25b7 100644 --- a/LANCommander.Launcher.Services/Import/Importers/GameImporter.cs +++ b/LANCommander.Launcher.Services/Import/Importers/GameImporter.cs @@ -72,7 +72,8 @@ public class GameImporter( public override async Task UpdateAsync(ImportItemInfo importItemInfo) { - var existing = await gameService.GetAsync(importItemInfo.Record.Id); + var existing = importItemInfo.ExistingEntity as Data.Models.Game + ?? await gameService.GetAsync(importItemInfo.Record.Id); try { @@ -87,10 +88,10 @@ public class GameImporter( existing.CreatedOn = importItemInfo.Record.CreatedOn; existing.ImportedOn = DateTime.UtcNow; existing.LatestVersion = importItemInfo.Record.Version; - + await gameService.UpdateAsync(existing); - await UpdateRelationships(importItemInfo.Record); - + await UpdateRelationships(importItemInfo.Record, existing); + if (await libraryService.IsInstalledAsync(existing.Id) && existing.LatestVersion == existing.InstalledVersion) await ManifestHelper.WriteAsync(importItemInfo.Record, existing.InstallDirectory); @@ -102,9 +103,9 @@ public class GameImporter( } } - private async Task UpdateRelationships(Game manifest) + private async Task UpdateRelationships(Game manifest, Data.Models.Game? cachedGame = null) { - var game = await gameService.GetAsync(manifest.Id); + var game = cachedGame ?? await gameService.GetAsync(manifest.Id); await gameService.SyncRelatedCollectionAsync( game, @@ -143,5 +144,14 @@ public class GameImporter( r => t => t.Name == r.Name); } - public override async Task ExistsAsync(ImportItemInfo importItemInfo) => await gameService.ExistsAsync(importItemInfo.Record.Id); + public override async Task ExistsAsync(ImportItemInfo importItemInfo) + { + var existing = await gameService.GetAsync(importItemInfo.Record.Id); + if (existing != null) + { + importItemInfo.ExistingEntity = existing; + return true; + } + return false; + } } \ No newline at end of file diff --git a/LANCommander.Launcher.Services/Import/Importers/MediaImporter.cs b/LANCommander.Launcher.Services/Import/Importers/MediaImporter.cs index 16fa4a9c..531882eb 100644 --- a/LANCommander.Launcher.Services/Import/Importers/MediaImporter.cs +++ b/LANCommander.Launcher.Services/Import/Importers/MediaImporter.cs @@ -38,9 +38,9 @@ public class MediaImporter( if (importItemInfo.Manifest is not Game game) return false; - if (ImportContext is null) - { - throw new InvalidOperationException("The ImportContext has not been set. Ensure that the UseContext method is called before importing."); + if (ImportContext is null) + { + throw new InvalidOperationException("The ImportContext has not been set. Ensure that the UseContext method is called before importing."); } if (ImportContext.InQueue(game, gameImporter)) @@ -61,14 +61,15 @@ public class MediaImporter( media = await mediaService.AddAsync(media); - await mediaService.DownloadAsync(media); + // Defer download to parallel batch after queue processing + ImportContext.PendingMediaDownloads.Add(new PendingMediaDownload { Media = media }); return true; } - catch(InvalidOperationException ex) - { - logger.LogError(ex, "Failed to add media due to invalid operation | {Key}", GetKey(importItemInfo.Record)); - return false; + catch(InvalidOperationException ex) + { + logger.LogError(ex, "Failed to add media due to invalid operation | {Key}", GetKey(importItemInfo.Record)); + return false; } catch (Exception ex) { @@ -86,9 +87,10 @@ public class MediaImporter( if (ImportContext.InQueue(game, gameImporter)) return false; - - var existing = await mediaService.GetAsync(importItemInfo.Record.Id); - + + var existing = importItemInfo.ExistingEntity as Data.Models.Media + ?? await mediaService.GetAsync(importItemInfo.Record.Id); + existing.FileId = importItemInfo.Record.FileId; existing.Game = await gameService.GetAsync(game.Id); existing.CreatedOn = importItemInfo.Record.CreatedOn; @@ -99,11 +101,21 @@ public class MediaImporter( existing.Crc32 = importItemInfo.Record.Crc32 ?? string.Empty; await mediaService.UpdateAsync(existing); - await mediaService.DownloadAsync(existing); + + // Defer download to parallel batch after queue processing + ImportContext.PendingMediaDownloads.Add(new PendingMediaDownload { Media = existing }); return true; } - public override async Task ExistsAsync(ImportItemInfo importItemInfo) => - await mediaService.GetAsync(importItemInfo.Record.Id) != null; + public override async Task ExistsAsync(ImportItemInfo importItemInfo) + { + var existing = await mediaService.GetAsync(importItemInfo.Record.Id); + if (existing != null) + { + importItemInfo.ExistingEntity = existing; + return true; + } + return false; + } } \ No newline at end of file diff --git a/LANCommander.Launcher.Services/Import/PendingMediaDownload.cs b/LANCommander.Launcher.Services/Import/PendingMediaDownload.cs new file mode 100644 index 00000000..e4671d2b --- /dev/null +++ b/LANCommander.Launcher.Services/Import/PendingMediaDownload.cs @@ -0,0 +1,8 @@ +using LANCommander.Launcher.Data.Models; + +namespace LANCommander.Launcher.Services.Import; + +public class PendingMediaDownload +{ + public Media Media { get; set; } +} diff --git a/LANCommander.Launcher.Services/ImportService.cs b/LANCommander.Launcher.Services/ImportService.cs index 8dae4ecb..d3861535 100644 --- a/LANCommander.Launcher.Services/ImportService.cs +++ b/LANCommander.Launcher.Services/ImportService.cs @@ -15,6 +15,8 @@ namespace LANCommander.Launcher.Services LibraryClient libraryClient, GameService gameService) : BaseService(logger) { + private const int MaxConcurrentManifestFetches = 8; + private ImportProgress _importProgress = new(); public ImportProgress Progress => _importProgress; @@ -41,29 +43,54 @@ namespace LANCommander.Launcher.Services importContext.OnImportError = OnImportError; importContext.OnImportStatusUpdate = OnImportStatusUpdate; - foreach (var game in remoteLibrary) + // Pre-fetch all local import timestamps in a single query + var gameIds = remoteLibrary.Select(g => g.Id); + var importedOnMap = await gameService.GetImportedOnMapAsync(gameIds); + + // Filter to only games that need importing + var gamesToImport = remoteLibrary.Where(game => { + if (importedOnMap.TryGetValue(game.Id, out var importedOn) && game.UpdatedOn <= importedOn) + { + Logger?.LogDebug("Skipping unchanged game {GameId}", game.Id); + return false; + } + return true; + }).ToList(); + + Logger?.LogInformation("Importing {Count} games ({Skipped} skipped as unchanged)", + gamesToImport.Count, remoteLibrary.Count() - gamesToImport.Count); + + // Fetch manifests concurrently + var semaphore = new SemaphoreSlim(MaxConcurrentManifestFetches); + var addLock = new object(); + + var tasks = gamesToImport.Select(async game => + { + await semaphore.WaitAsync(); try { - var existing = await gameService.GetAsync(game.Id); - - if (existing != null && game.UpdatedOn <= existing.ImportedOn) - { - Logger?.LogDebug("Skipping unchanged game {GameId}", game.Id); - continue; - } - var manifest = await gameClient.GetManifestAsync(game.Id); - await importContext.AddAsync(manifest); + lock (addLock) + { + importContext.AddAsync(manifest).GetAwaiter().GetResult(); + } } catch (Exception ex) { Logger?.LogError(ex, "Could not add game with ID {GameId} to import queue", game.Id); } - } + finally + { + semaphore.Release(); + } + }); + + await Task.WhenAll(tasks); await importContext.ImportQueueAsync(); + await importContext.DownloadPendingMediaAsync(); } public async Task ImportGameAsync(Guid gameId)