From bc488c834f780dee3024c55ca44af4c7c7b3a825 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 30 Apr 2026 18:31:21 +0200 Subject: [PATCH 01/13] pass --- .vscode/settings.json | 4 +- .../Containers/CachedPackFileContainer.cs | 88 +++++-- .../CacheDatabase/CacheDbContext.cs | 56 +++++ .../PackFileContainerCacheHelper.cs | 235 ++++++------------ .../Utility/PackFileContainerLoader.cs | 6 +- .../SharedCore/Shared.Core/Shared.Core.csproj | 7 +- .../CachedPackFileContainerTests.cs | 108 +++++++- .../PackFileContainerCacheHelperTests.cs | 206 ++++++--------- .../Shared.CoreTest/Shared.CoreTest.csproj | 1 + .../AssetEditorTests/AssetEditorTests.csproj | 2 +- 10 files changed, 382 insertions(+), 331 deletions(-) create mode 100644 Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs diff --git a/.vscode/settings.json b/.vscode/settings.json index 9a85d1da1..5b47b3ffb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,7 @@ { "chat.tools.terminal.autoApprove": { - "ForEach-Object": true + "ForEach-Object": true, + "dotnet restore": true, + "Set-Content": true } } \ No newline at end of file diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index a9b84215d..a30031e61 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,55 +1,93 @@ -using Shared.Core.Settings; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles.Models.FileSources; +using Shared.Core.PackFiles.Serialization.CacheDatabase; +using Shared.Core.Settings; namespace Shared.Core.PackFiles.Models.Containers { internal class CachedPackFileContainer : IPackFileContainerInternal { + private readonly DbContextOptions _dbOptions; + public string Name { get; set; } public bool IsCaPackFile { get => true; set { } } public string SystemFilePath { get; set; } - public Dictionary FileList { get; set; } = []; public HashSet SourcePackFilePaths { get; set; } = []; - public CachedPackFileContainer(string name) + public CachedPackFileContainer(string name, DbContextOptions dbOptions) { Name = name; SystemFilePath = string.Empty; + _dbOptions = dbOptions; } - public int GetFileCount() => FileList.Count; - public Dictionary GetAllFiles() => FileList; - - public List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention) + public int GetFileCount() { - extention = extention.ToLower(); - var output = new List<(string, PackFile)>(); - foreach (var file in FileList) - { - if (Path.GetExtension(file.Key) == extention) - output.Add((file.Key, file.Value)); - } - return output; + using var db = new CacheDbContext(_dbOptions); + return db.Files.Count(); } public PackFile? FindFile(string path) { var lowerPath = path.Replace('/', '\\').ToLower().Trim(); - return FileList.TryGetValue(lowerPath, out var value) ? value : null; + using var db = new CacheDbContext(_dbOptions); + var entry = db.Files.AsNoTracking().FirstOrDefault(f => f.RelativePath == lowerPath); + return entry != null ? ToPackFile(entry) : null; } public bool ContainsFile(string path) { var lowerPath = path.Replace('/', '\\').ToLower().Trim(); - return FileList.ContainsKey(lowerPath); + using var db = new CacheDbContext(_dbOptions); + return db.Files.Any(f => f.RelativePath == lowerPath); } public string? GetFullPath(PackFile file) { - var res = FileList.FirstOrDefault(x => ReferenceEquals(x.Value, file)).Key; - return string.IsNullOrWhiteSpace(res) ? null : res; + using var db = new CacheDbContext(_dbOptions); + var entry = db.Files.AsNoTracking().FirstOrDefault(f => f.FileName == file.Name); + return entry?.RelativePath; } - public void AddOrUpdateFile(string path, PackFile file) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); + public List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention) + { + extention = extention.ToLower(); + using var db = new CacheDbContext(_dbOptions); + var entries = db.Files.AsNoTracking() + .Where(f => f.Extension == extention) + .ToList(); + + return entries.Select(e => (e.RelativePath, ToPackFile(e))).ToList(); + } + + public Dictionary GetAllFiles() + { + using var db = new CacheDbContext(_dbOptions); + var entries = db.Files.AsNoTracking().ToList(); + var parentCache = new Dictionary(StringComparer.OrdinalIgnoreCase); + var result = new Dictionary(entries.Count); + + foreach (var entry in entries) + { + if (!parentCache.TryGetValue(entry.SourcePackFilePath, out var parent)) + { + parent = new PackedFileSourceParent { FilePath = entry.SourcePackFilePath }; + parentCache[entry.SourcePackFilePath] = parent; + } + + var source = new PackedFileSource( + parent, entry.Offset, entry.Size, + entry.IsEncrypted, entry.IsCompressed, + (Utility.CompressionFormat)entry.CompressionFormat, entry.UncompressedSize); + + result[entry.RelativePath] = new PackFile(entry.FileName, source); + } + + return result; + } + + public void AddOrUpdateFile(string path, PackFile file) => + throw new InvalidOperationException("Cannot modify a cached CA pack file container."); public List AddFiles(List newFiles) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); @@ -74,5 +112,15 @@ public void SaveFileData(PackFile file, byte[] data) => public void SaveToDisk(string path, bool createBackup, GameInformation gameInformation) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); + + private static PackFile ToPackFile(CachedFileEntity entry) + { + var parent = new PackedFileSourceParent { FilePath = entry.SourcePackFilePath }; + var source = new PackedFileSource( + parent, entry.Offset, entry.Size, + entry.IsEncrypted, entry.IsCompressed, + (Utility.CompressionFormat)entry.CompressionFormat, entry.UncompressedSize); + return new PackFile(entry.FileName, source); + } } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs new file mode 100644 index 000000000..a94047d43 --- /dev/null +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore; + +namespace Shared.Core.PackFiles.Serialization.CacheDatabase +{ + internal class CacheDbContext : DbContext + { + public DbSet CacheInfo { get; set; } + public DbSet Files { get; set; } + + public CacheDbContext(DbContextOptions options) : base(options) + { + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("CacheInfo"); + entity.HasKey(e => e.Id); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("FileList"); + entity.HasKey(e => e.Id); + entity.HasIndex(e => e.RelativePath).IsUnique(); + entity.HasIndex(e => e.Extension); + }); + } + } + + internal class CacheInfoEntity + { + public int Id { get; set; } + public int SchemaVersion { get; set; } + public string Fingerprint { get; set; } = ""; + public string ContainerName { get; set; } = ""; + public string SystemFilePath { get; set; } = ""; + public string SourcePackFilePaths { get; set; } = ""; + } + + internal class CachedFileEntity + { + public int Id { get; set; } + public string RelativePath { get; set; } = ""; + public string FileName { get; set; } = ""; + public string Extension { get; set; } = ""; + public string SourcePackFilePath { get; set; } = ""; + public long Offset { get; set; } + public long Size { get; set; } + public bool IsEncrypted { get; set; } + public bool IsCompressed { get; set; } + public int CompressionFormat { get; set; } + public uint UncompressedSize { get; set; } + } +} diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs index 6ef0ae65f..a1d21ed11 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs @@ -1,38 +1,17 @@ using System.Security.Cryptography; using System.Text; +using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; namespace Shared.Core.PackFiles.Serialization { - internal record CachedFileEntry( - string RelativePath, - string FileName, - string SourcePackFilePath, - long Offset, - long Size, - bool IsEncrypted, - bool IsCompressed, - CompressionFormat CompressionFormat, - uint UncompressedSize); - - internal class CachedContainerData - { - public string Fingerprint { get; set; } = ""; - public string ContainerName { get; set; } = ""; - public string SystemFilePath { get; set; } = ""; - public List SourcePackFilePaths { get; set; } = []; - public List Files { get; set; } = []; - } - internal static class PackFileContainerCacheHelper { - private static readonly byte[] s_magic = "AEPC"u8.ToArray(); - private const int CacheVersion = 1; - - public static string GetCacheFilePath(string gameDataFolder, string gameName) + public static string GetCacheFilePath(string gameDataFolder, string gameName, string cacheId) { var cacheDir = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @@ -42,7 +21,7 @@ public static string GetCacheFilePath(string gameDataFolder, string gameName) Directory.CreateDirectory(cacheDir); var safeGameName = string.Join("_", gameName.Split(Path.GetInvalidFileNameChars())); - return Path.Combine(cacheDir, $"ca_pack_cache_{safeGameName}.bin"); + return Path.Combine(cacheDir, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); } public static string ComputeFingerprint(string gameDataFolder, List packFileNames) @@ -79,173 +58,99 @@ public static string ComputeFingerprint(string gameDataFolder, List pack return Convert.ToHexString(hash); } - public static void SaveCache(CachedContainerData data, string cacheFilePath) + public static DbContextOptions CreateDbOptions(string dbFilePath) { - using var stream = File.Create(cacheFilePath); - using var writer = new BinaryWriter(stream, Encoding.UTF8); - - writer.Write(s_magic); - writer.Write(CacheVersion); - - writer.Write(data.Fingerprint); - writer.Write(data.ContainerName); - writer.Write(data.SystemFilePath); - - writer.Write(data.SourcePackFilePaths.Count); - foreach (var path in data.SourcePackFilePaths) - writer.Write(path); - - writer.Write(data.Files.Count); - foreach (var entry in data.Files) - { - writer.Write(entry.RelativePath); - writer.Write(entry.FileName); - writer.Write(entry.SourcePackFilePath); - writer.Write(entry.Offset); - writer.Write(entry.Size); - writer.Write(entry.IsEncrypted); - writer.Write(entry.IsCompressed); - writer.Write((int)entry.CompressionFormat); - writer.Write(entry.UncompressedSize); - } + return new DbContextOptionsBuilder() + .UseSqlite($"Data Source={dbFilePath};Pooling=False") + .Options; } - /// - /// Single-pass optimized load: reads binary cache directly into a CachedPackFileContainer, - /// skipping intermediate CachedContainerData/CachedFileEntry allocations. - /// Returns null if the file is missing, corrupt, or the fingerprint doesn't match. - /// - public static CachedPackFileContainer? LoadContainerFromCache(string cacheFilePath, string expectedFingerprint) - { - if (!File.Exists(cacheFilePath)) - return null; - - using var fileStream = File.OpenRead(cacheFilePath); - using var buffered = new BufferedStream(fileStream, 1024 * 64); - using var reader = new BinaryReader(buffered, Encoding.UTF8); - - var magic = reader.ReadBytes(s_magic.Length); - if (!magic.AsSpan().SequenceEqual(s_magic)) - return null; - - var version = reader.ReadInt32(); - if (version != CacheVersion) - return null; - - var fingerprint = reader.ReadString(); - if (fingerprint != expectedFingerprint) - return null; - - var containerName = reader.ReadString(); - var systemFilePath = reader.ReadString(); - - var container = new CachedPackFileContainer(containerName) - { - SystemFilePath = systemFilePath, - }; - - var sourcePathCount = reader.ReadInt32(); - for (var i = 0; i < sourcePathCount; i++) - container.SourcePackFilePaths.Add(reader.ReadString()); - - var fileCount = reader.ReadInt32(); - var fileList = new Dictionary(fileCount); - var parentCache = new Dictionary(sourcePathCount, StringComparer.OrdinalIgnoreCase); - var stringPool = new Dictionary(sourcePathCount, StringComparer.Ordinal); - - for (var i = 0; i < fileCount; i++) - { - var relativePath = reader.ReadString(); - var fileName = reader.ReadString(); - var sourcePackFilePath = reader.ReadString(); - var offset = reader.ReadInt64(); - var size = reader.ReadInt64(); - var isEncrypted = reader.ReadBoolean(); - var isCompressed = reader.ReadBoolean(); - var compressionFormat = (CompressionFormat)reader.ReadInt32(); - var uncompressedSize = reader.ReadUInt32(); - - // Intern the source pack path string — ~20 unique values across 600K entries - if (!stringPool.TryGetValue(sourcePackFilePath, out var internedPath)) - { - internedPath = sourcePackFilePath; - stringPool[internedPath] = internedPath; - } - - if (!parentCache.TryGetValue(internedPath, out var parent)) - { - parent = new PackedFileSourceParent { FilePath = internedPath }; - parentCache[internedPath] = parent; - } + private const int CurrentSchemaVersion = 2; - var source = new PackedFileSource(parent, offset, size, isEncrypted, isCompressed, compressionFormat, uncompressedSize); - fileList[relativePath] = new PackFile(fileName, source); - } + public static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) + { + using var db = new CacheDbContext(dbOptions); + db.Database.EnsureDeleted(); + db.Database.EnsureCreated(); - container.FileList = fileList; - return container; - } + var sourcePackFilePaths = string.Join("|", container.SourcePackFilePaths); - public static CachedContainerData BuildCacheData(string fingerprint, PackFileContainer container) - { - var data = new CachedContainerData + db.CacheInfo.Add(new CacheInfoEntity { + SchemaVersion = CurrentSchemaVersion, Fingerprint = fingerprint, ContainerName = container.Name, SystemFilePath = container.SystemFilePath, - SourcePackFilePaths = container.SourcePackFilePaths.ToList(), - }; + SourcePackFilePaths = sourcePackFilePaths, + }); foreach (var (relativePath, packFile) in container.GetAllFiles()) { if (packFile.DataSource is PackedFileSource source) { - data.Files.Add(new CachedFileEntry( - relativePath, - packFile.Name, - source.Parent.FilePath, - source.Offset, - source.Size, - source.IsEncrypted, - source.IsCompressed, - source.CompressionFormat, - source.UncompressedSize)); + db.Files.Add(new CachedFileEntity + { + RelativePath = relativePath, + FileName = packFile.Name, + Extension = Path.GetExtension(relativePath).ToLower(), + SourcePackFilePath = source.Parent.FilePath, + Offset = source.Offset, + Size = source.Size, + IsEncrypted = source.IsEncrypted, + IsCompressed = source.IsCompressed, + CompressionFormat = (int)source.CompressionFormat, + UncompressedSize = source.UncompressedSize, + }); } } - return data; + db.SaveChanges(); } - public static CachedPackFileContainer RestoreFromCache(CachedContainerData data) + public static CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) { - var container = new CachedPackFileContainer(data.ContainerName) - { - SystemFilePath = data.SystemFilePath, - }; + if (!File.Exists(dbFilePath)) + return null; - foreach (var sourcePath in data.SourcePackFilePaths) - container.SourcePackFilePaths.Add(sourcePath); + var dbOptions = CreateDbOptions(dbFilePath); + return LoadContainerFromCache(dbOptions, expectedFingerprint); + } - var parentCache = new Dictionary(StringComparer.OrdinalIgnoreCase); + public static CachedPackFileContainer? LoadContainerFromCache(DbContextOptions dbOptions, string expectedFingerprint) + { + using var db = new CacheDbContext(dbOptions); - foreach (var entry in data.Files) + try { - if (!parentCache.TryGetValue(entry.SourcePackFilePath, out var parent)) - { - parent = new PackedFileSourceParent { FilePath = entry.SourcePackFilePath }; - parentCache[entry.SourcePackFilePath] = parent; - } + db.Database.EnsureCreated(); + } + catch + { + return null; + } - var source = new PackedFileSource( - parent, - entry.Offset, - entry.Size, - entry.IsEncrypted, - entry.IsCompressed, - entry.CompressionFormat, - entry.UncompressedSize); + CacheInfoEntity? cacheInfo; + try + { + cacheInfo = db.CacheInfo.FirstOrDefault(); + } + catch + { + return null; + } + + if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) + return null; - container.FileList.Add(entry.RelativePath, new PackFile(entry.FileName, source)); + var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) + { + SystemFilePath = cacheInfo.SystemFilePath, + }; + + if (!string.IsNullOrEmpty(cacheInfo.SourcePackFilePaths)) + { + foreach (var path in cacheInfo.SourcePackFilePaths.Split('|')) + container.SourcePackFilePaths.Add(path); } return container; diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs index 22903ed07..3f96a1d99 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs @@ -103,7 +103,7 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri var allCaPackFiles = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(gameDataFolder, allCaPackFiles); - var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName); + var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName, fingerprint); var cached = TryLoadFromCache(cacheFilePath, fingerprint, gameName); if (cached != null) @@ -113,8 +113,8 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri try { - var cacheData = PackFileContainerCacheHelper.BuildCacheData(fingerprint, container); - PackFileContainerCacheHelper.SaveCache(cacheData, cacheFilePath); + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); + PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); _logger.Here().Information($"Saved CA pack cache for {gameName} to {cacheFilePath}"); } catch (Exception cacheEx) diff --git a/Shared/SharedCore/Shared.Core/Shared.Core.csproj b/Shared/SharedCore/Shared.Core/Shared.Core.csproj index 667ee2128..58e3a872a 100644 --- a/Shared/SharedCore/Shared.Core/Shared.Core.csproj +++ b/Shared/SharedCore/Shared.Core/Shared.Core.csproj @@ -9,13 +9,14 @@ + - + @@ -26,8 +27,8 @@ - - + + diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs index 7987dd1a4..b2c6bce29 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs @@ -1,22 +1,56 @@ -using Shared.Core.PackFiles; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; +using Shared.Core.PackFiles.Models.FileSources; +using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Serialization.CacheDatabase; +using Shared.Core.PackFiles.Utility; using Shared.Core.Settings; namespace Shared.CoreTest.PackFiles.Models.Containers { internal class CachedPackFileContainer_ReadOnly { + private string _tempDir; + private string _dbFilePath; private CachedPackFileContainer _container; [SetUp] public void Setup() { - _container = new CachedPackFileContainer("TestCache") + _tempDir = Path.Combine(Path.GetTempPath(), "CachedContainerTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + _dbFilePath = Path.Combine(_tempDir, "test.db"); + + // Build a container with test data and save to DB + var sourceContainer = new PackFileContainer("TestCache") { + IsCaPackFile = true, SystemFilePath = @"c:\game\data" }; - _container.FileList.Add("folder\\file.txt", new PackFile("file.txt", null)); + sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; + sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", + new PackedFileSource(parent, 100, 200, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", + new PackedFileSource(parent, 300, 400, false, true, CompressionFormat.Lz4, 800))); + sourceContainer.AddOrUpdateFile("audio\\sound.wem", new PackFile("sound.wem", + new PackedFileSource(parent, 700, 500, false, false, CompressionFormat.None, 0))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("test_fp", sourceContainer, dbOptions); + + // Load the cached container (lazy, no files in memory) + _container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp")!; + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, true); } [Test] @@ -32,6 +66,12 @@ public void IsCaPackFile_SetterDoesNotChangeValue() Assert.That(_container.IsCaPackFile, Is.True); } + [Test] + public void GetFileCount_ReturnsCorrectCount() + { + Assert.That(_container.GetFileCount(), Is.EqualTo(3)); + } + [Test] public void FindFile_ReturnsFile() { @@ -54,10 +94,32 @@ public void FindFile_NormalizesPath() Assert.That(result, Is.Not.Null); } + [Test] + public void FindFile_ReturnsCorrectDataSource() + { + var result = _container.FindFile("folder\\file.txt"); + var source = result!.DataSource as PackedFileSource; + Assert.That(source, Is.Not.Null); + Assert.That(source.Offset, Is.EqualTo(100)); + Assert.That(source.Size, Is.EqualTo(200)); + } + + [Test] + public void ContainsFile_ReturnsTrueForExisting() + { + Assert.That(_container.ContainsFile("folder\\file.txt"), Is.True); + } + + [Test] + public void ContainsFile_ReturnsFalseForMissing() + { + Assert.That(_container.ContainsFile("missing.txt"), Is.False); + } + [Test] public void GetFullPath_ReturnsPath() { - var file = _container.FindFile("folder\\file.txt"); + var file = _container.FindFile("folder\\file.txt")!; var path = _container.GetFullPath(file); Assert.That(path, Is.EqualTo("folder\\file.txt")); } @@ -71,9 +133,28 @@ public void GetFullPath_ReturnsNullForMissing() } [Test] - public void AddFiles_AddOrUpdateFile() + public void FindAllWithExtention_ReturnsMatching() + { + var results = _container.FindAllWithExtention(".wem"); + Assert.That(results.Count, Is.EqualTo(1)); + Assert.That(results[0].FileName, Does.EndWith("sound.wem")); + } + + [Test] + public void FindAllWithExtention_ReturnsEmptyForNoMatch() + { + var results = _container.FindAllWithExtention(".xyz"); + Assert.That(results, Is.Empty); + } + + [Test] + public void GetAllFiles_ReturnsAllFiles() { - Assert.Throws(() => _container.AddOrUpdateFile("path", new PackFile("new.txt", null))); + var all = _container.GetAllFiles(); + Assert.That(all.Count, Is.EqualTo(3)); + Assert.That(all.ContainsKey("folder\\file.txt"), Is.True); + Assert.That(all.ContainsKey("other\\data.bin"), Is.True); + Assert.That(all.ContainsKey("audio\\sound.wem"), Is.True); } [Test] @@ -89,7 +170,7 @@ public void AddFiles_Throws() [Test] public void DeleteFile_Throws() { - var file = _container.FindFile("folder\\file.txt"); + var file = _container.FindFile("folder\\file.txt")!; Assert.Throws(() => _container.DeleteFile(file)); } @@ -102,7 +183,7 @@ public void DeleteFolder_Throws() [Test] public void MoveFile_Throws() { - var file = _container.FindFile("folder\\file.txt"); + var file = _container.FindFile("folder\\file.txt")!; Assert.Throws(() => _container.MoveFile(file, "other")); } @@ -115,14 +196,14 @@ public void RenameDirectory_Throws() [Test] public void RenameFile_Throws() { - var file = _container.FindFile("folder\\file.txt"); + var file = _container.FindFile("folder\\file.txt")!; Assert.Throws(() => _container.RenameFile(file, "renamed.txt")); } [Test] public void SaveFileData_Throws() { - var file = _container.FindFile("folder\\file.txt"); + var file = _container.FindFile("folder\\file.txt")!; Assert.Throws(() => _container.SaveFileData(file, [1, 2, 3])); } @@ -132,5 +213,12 @@ public void SaveToDisk_Throws() var gameInfo = GameInformationDatabase.GetGameById(GameTypeEnum.Warhammer3); Assert.Throws(() => _container.SaveToDisk("path", false, gameInfo)); } + + [Test] + public void AddOrUpdateFile_Throws() + { + Assert.Throws(() => + _container.AddOrUpdateFile("test\\new.txt", new PackFile("new.txt", null))); + } } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index 6acfdfc71..50df35c27 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -1,7 +1,9 @@ -using Shared.Core.PackFiles.Models; +using Microsoft.EntityFrameworkCore; +using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; namespace Shared.CoreTest.PackFiles.Serialization @@ -9,14 +11,14 @@ namespace Shared.CoreTest.PackFiles.Serialization internal class PackFileContainerCacheHelperTests { private string _tempDir; - private string _cacheFilePath; + private string _dbFilePath; [SetUp] public void Setup() { _tempDir = Path.Combine(Path.GetTempPath(), "AssetEditorCacheTests_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_tempDir); - _cacheFilePath = Path.Combine(_tempDir, "test_cache.bin"); + _dbFilePath = Path.Combine(_tempDir, "test_cache.db"); } [TearDown] @@ -26,6 +28,11 @@ public void TearDown() Directory.Delete(_tempDir, true); } + private DbContextOptions CreateTestDbOptions() + { + return PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + } + [Test] public void RoundTrip_PreservesMetadata() { @@ -49,9 +56,9 @@ public void RoundTrip_PreservesMetadata() container.AddOrUpdateFile("folder\\file2.bin", new PackFile("file2.bin", source2)); // Act - var cacheData = PackFileContainerCacheHelper.BuildCacheData("fingerprint123", container); - PackFileContainerCacheHelper.SaveCache(cacheData, _cacheFilePath); - var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "fingerprint123"); + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fingerprint123", container, dbOptions); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fingerprint123"); // Assert Assert.That(loaded, Is.Not.Null); @@ -62,52 +69,35 @@ public void RoundTrip_PreservesMetadata() } [Test] - public void RestoreFromCache_CreatesCorrectContainer() + public void LoadCache_ReturnsCorrectFileData() { // Arrange - var cacheData = new CachedContainerData + var container = new PackFileContainer("Test Container") { - Fingerprint = "fp", - ContainerName = "Test Container", - SystemFilePath = @"c:\game\data", - SourcePackFilePaths = [@"c:\game\data\pack1.pack"], - Files = - [ - new CachedFileEntry( - "folder\\file.txt", - "file.txt", - @"c:\game\data\pack1.pack", - Offset: 512, - Size: 1024, - IsEncrypted: false, - IsCompressed: false, - CompressionFormat.None, - UncompressedSize: 0), - new CachedFileEntry( - "other\\data.bin", - "data.bin", - @"c:\game\data\pack1.pack", - Offset: 2048, - Size: 4096, - IsEncrypted: false, - IsCompressed: true, - CompressionFormat.Lz4, - UncompressedSize: 8192) - ] + IsCaPackFile = true, + SystemFilePath = @"c:\game\data" }; + container.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; + container.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", + new PackedFileSource(parent, 512, 1024, false, false, CompressionFormat.None, 0))); + container.AddOrUpdateFile("other\\data.bin", new PackFile("data.bin", + new PackedFileSource(parent, 2048, 4096, false, true, CompressionFormat.Lz4, 8192))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + // Act - var container = PackFileContainerCacheHelper.RestoreFromCache(cacheData); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); // Assert - Assert.That(container, Is.InstanceOf()); - Assert.That(container.Name, Is.EqualTo("Test Container")); - Assert.That(container.IsCaPackFile, Is.True); - Assert.That(container.SystemFilePath, Is.EqualTo(@"c:\game\data")); - Assert.That(container.SourcePackFilePaths.Count, Is.EqualTo(1)); - Assert.That(container.GetFileCount(), Is.EqualTo(2)); - - var file1 = container.FindFile("folder\\file.txt"); + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded.GetFileCount(), Is.EqualTo(2)); + + var file1 = loaded.FindFile("folder\\file.txt"); + Assert.That(file1, Is.Not.Null); Assert.That(file1.Name, Is.EqualTo("file.txt")); var file1Source = file1.DataSource as PackedFileSource; Assert.That(file1Source, Is.Not.Null); @@ -116,7 +106,7 @@ public void RestoreFromCache_CreatesCorrectContainer() Assert.That(file1Source.IsCompressed, Is.False); Assert.That(file1Source.Parent.FilePath, Is.EqualTo(@"c:\game\data\pack1.pack")); - var file2 = container.FindFile("other\\data.bin"); + var file2 = loaded.FindFile("other\\data.bin"); var file2Source = file2.DataSource as PackedFileSource; Assert.That(file2Source, Is.Not.Null); Assert.That(file2Source.Offset, Is.EqualTo(2048)); @@ -127,39 +117,55 @@ public void RestoreFromCache_CreatesCorrectContainer() } [Test] - public void RestoreFromCache_SharesPackedFileSourceParents() + public void LoadCache_PreservesSourcePackFilePath() { - var cacheData = new CachedContainerData + var container = new PackFileContainer("Test") { - Fingerprint = "fp", - ContainerName = "Test", - SystemFilePath = @"c:\game", - Files = - [ - new CachedFileEntry("a.txt", "a.txt", @"c:\pack.pack", 0, 10, false, false, CompressionFormat.None, 0), - new CachedFileEntry("b.txt", "b.txt", @"c:\pack.pack", 10, 20, false, false, CompressionFormat.None, 0), - ] + SystemFilePath = @"c:\game" }; - var container = PackFileContainerCacheHelper.RestoreFromCache(cacheData); + var parent = new PackedFileSourceParent { FilePath = @"c:\pack.pack" }; + container.AddOrUpdateFile("a.txt", new PackFile("a.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + container.AddOrUpdateFile("b.txt", new PackFile("b.txt", + new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); - var sourceA = (PackedFileSource)container.FindFile("a.txt")!.DataSource; - var sourceB = (PackedFileSource)container.FindFile("b.txt")!.DataSource; - Assert.That(ReferenceEquals(sourceA.Parent, sourceB.Parent), Is.True, - "Files from the same pack should share PackedFileSourceParent instances"); + var sourceA = (PackedFileSource)loaded!.FindFile("a.txt")!.DataSource; + var sourceB = (PackedFileSource)loaded!.FindFile("b.txt")!.DataSource; + Assert.That(sourceA.Parent.FilePath, Is.EqualTo(@"c:\pack.pack")); + Assert.That(sourceB.Parent.FilePath, Is.EqualTo(@"c:\pack.pack")); } [Test] public void LoadCache_ReturnsNullForMissingFile() { - var result = PackFileContainerCacheHelper.LoadContainerFromCache(Path.Combine(_tempDir, "nonexistent.bin"), "fp"); + var result = PackFileContainerCacheHelper.LoadContainerFromCache( + Path.Combine(_tempDir, "nonexistent.db"), "fp"); + Assert.That(result, Is.Null); + } + + [Test] + public void LoadCache_ReturnsNullForWrongFingerprint() + { + var container = new PackFileContainer("Test") + { + SystemFilePath = @"c:\game" + }; + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("correct_fp", container, dbOptions); + + var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "wrong_fp"); Assert.That(result, Is.Null); } [Test] public void ComputeFingerprint_DeterministicForSameInputs() { - // Create fake pack files var packDir = Path.Combine(_tempDir, "gamedata"); Directory.CreateDirectory(packDir); File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a"); @@ -182,7 +188,6 @@ public void ComputeFingerprint_ChangesWhenFileChanges() var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); - // Modify the file (change size) File.WriteAllText(Path.Combine(packDir, "a.pack"), "data_a_modified_longer"); var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); @@ -191,7 +196,7 @@ public void ComputeFingerprint_ChangesWhenFileChanges() } [Test] - public void RoundTrip_FullCycle_BuildSaveLoadRestore() + public void RoundTrip_FullCycle() { // Arrange var container = new PackFileContainer("Full Cycle Test") @@ -208,12 +213,13 @@ public void RoundTrip_FullCycle_BuildSaveLoadRestore() container.AddOrUpdateFile("text\\localisation.loc", new PackFile("localisation.loc", new PackedFileSource(parent, 256, 512, false, true, CompressionFormat.Lz4, 1024))); - // Act: build → save → load - var cacheData = PackFileContainerCacheHelper.BuildCacheData("test_fp", container); - PackFileContainerCacheHelper.SaveCache(cacheData, _cacheFilePath); - var restored = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "test_fp"); + // Act: save ? load + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("test_fp", container, dbOptions); + var restored = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "test_fp"); - // Assert: restored container matches original + // Assert + Assert.That(restored, Is.Not.Null); Assert.That(restored.Name, Is.EqualTo("Full Cycle Test")); Assert.That(restored.IsCaPackFile, Is.True); Assert.That(restored.SystemFilePath, Is.EqualTo(@"c:\game\data")); @@ -231,67 +237,11 @@ public void RoundTrip_FullCycle_BuildSaveLoadRestore() } [Test] - public void LoadCache_ReturnsNullForBadMagic() + public void LoadCache_ReturnsNullForCorruptDb() { - File.WriteAllBytes(_cacheFilePath, [0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00]); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "fp"); + File.WriteAllBytes(_dbFilePath, [0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00]); + var result = PackFileContainerCacheHelper.LoadContainerFromCache(_dbFilePath, "fp"); Assert.That(result, Is.Null); } - - [Test] - public void LoadCache_ReturnsNullForWrongVersion() - { - using (var stream = File.Create(_cacheFilePath)) - using (var writer = new BinaryWriter(stream)) - { - writer.Write("AEPC"u8); - writer.Write(999); // wrong version - } - - var result = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "fp"); - Assert.That(result, Is.Null); - } - - [Test] - public void LoadContainerFromCache_ReturnsNullForFingerprintMismatch() - { - var container = new PackFileContainer("Test") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game\data" - }; - var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack.pack" }; - container.AddOrUpdateFile("a.txt", new PackFile("a.txt", - new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - - var cacheData = PackFileContainerCacheHelper.BuildCacheData("original_fp", container); - PackFileContainerCacheHelper.SaveCache(cacheData, _cacheFilePath); - - var result = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "different_fp"); - Assert.That(result, Is.Null); - } - - [Test] - public void LoadContainerFromCache_SharesParentInstances() - { - var container = new PackFileContainer("Test") - { - IsCaPackFile = true, - SystemFilePath = @"c:\game" - }; - var parent = new PackedFileSourceParent { FilePath = @"c:\pack.pack" }; - container.AddOrUpdateFile("a.txt", new PackFile("a.txt", - new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); - container.AddOrUpdateFile("b.txt", new PackFile("b.txt", - new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); - - var cacheData = PackFileContainerCacheHelper.BuildCacheData("fp", container); - PackFileContainerCacheHelper.SaveCache(cacheData, _cacheFilePath); - - var restored = PackFileContainerCacheHelper.LoadContainerFromCache(_cacheFilePath, "fp"); - var sourceA = (PackedFileSource)restored!.FindFile("a.txt")!.DataSource; - var sourceB = (PackedFileSource)restored.FindFile("b.txt")!.DataSource; - Assert.That(ReferenceEquals(sourceA.Parent, sourceB.Parent), Is.True); - } } } diff --git a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj index 32063cc91..27966a8f4 100644 --- a/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj +++ b/Shared/SharedCore/Shared.CoreTest/Shared.CoreTest.csproj @@ -13,6 +13,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/Testing/AssetEditorTests/AssetEditorTests.csproj b/Testing/AssetEditorTests/AssetEditorTests.csproj index eb54ae594..6ea9e6c44 100644 --- a/Testing/AssetEditorTests/AssetEditorTests.csproj +++ b/Testing/AssetEditorTests/AssetEditorTests.csproj @@ -13,7 +13,7 @@ - + From bec3727f892340ede5fd9cf16896e1fdb0e9eee8 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 30 Apr 2026 21:42:26 +0200 Subject: [PATCH 02/13] Code --- .../Shared.Core/Misc/DirectoryHelper.cs | 3 + .../Containers/CachedPackFileContainer.cs | 68 ++++++++++++ .../Models/Containers/PackFileContainer.cs | 35 ++++++ .../PackFiles/Models/IPackFileContainer.cs | 17 ++- .../CacheDatabase/CacheDbContext.cs | 2 + .../PackFileContainerCacheHelper.cs | 50 ++++++--- .../Utility/PackFileContainerLoader.cs | 23 +--- .../PackFileTree/PackFileBrowserViewModel.cs | 103 ++++++------------ .../BaseDialogs/PackFileTree/SearchFilter.cs | 8 ++ .../BaseDialogs/PackFileTree/TreeNode.cs | 9 +- .../PackFileTree/TreeNodeSource.cs | 53 ++++++++- 11 files changed, 267 insertions(+), 104 deletions(-) diff --git a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs index 68a206fed..5d20dc7db 100644 --- a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs +++ b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs @@ -36,10 +36,12 @@ public class DirectoryHelper public static string Applications { get { return ApplicationDirectory + "\\Applications"; } } public static string Temp { get { return ApplicationDirectory + "\\Temp"; } } public static string UpdateDirectory { get { return Temp + "\\Update"; } } + public static string CacheDirectory { get => ApplicationDirectory + "\\Cache"; } public static string AnimationIndexMappingDirectory { get { return ApplicationDirectory + "\\Animation\\BoneIndexMapping"; } } public static void EnsureCreated() { + EnsureCreated(ApplicationDirectory); EnsureCreated(ApplicationDirectory); EnsureCreated(SchemaDirectory); EnsureCreated(LogDirectory); @@ -47,6 +49,7 @@ public static void EnsureCreated() EnsureCreated(Applications); EnsureCreated(Temp); EnsureCreated(AnimationIndexMappingDirectory); + EnsureCreated(CacheDirectory); } public static void EnsureCreated(string? path) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index a30031e61..8c1ef5d2e 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -89,6 +89,74 @@ public Dictionary GetAllFiles() public void AddOrUpdateFile(string path, PackFile file) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); + public DirectoryContent GetDirectoryContent(string directoryPath) + { + using var db = new CacheDbContext(_dbOptions); + + // Get files directly in this folder + var filesInDir = db.Files.AsNoTracking() + .Where(f => f.FolderPath == directoryPath) + .Select(f => new { f.FileName, f.SourcePackFilePath, f.Offset, f.Size, f.IsEncrypted, f.IsCompressed, f.CompressionFormat, f.UncompressedSize }) + .ToList(); + + var files = filesInDir + .Select(f => + { + var parent = new PackedFileSourceParent { FilePath = f.SourcePackFilePath }; + var source = new PackedFileSource(parent, f.Offset, f.Size, f.IsEncrypted, f.IsCompressed, (Utility.CompressionFormat)f.CompressionFormat, f.UncompressedSize); + return (f.FileName, File: new PackFile(f.FileName, source)); + }) + .OrderBy(x => x.FileName, StringComparer.CurrentCultureIgnoreCase) + .ToList(); + + // Get immediate subfolders + var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; + var prefixLen = prefix.Length; + + List subFolders; + if (string.IsNullOrEmpty(directoryPath)) + { + // Root level: get distinct first path segment from all FolderPaths + subFolders = db.Files.AsNoTracking() + .Where(f => f.FolderPath != "") + .Select(f => f.FolderPath) + .Distinct() + .AsEnumerable() + .Select(fp => + { + var sepIdx = fp.IndexOf('\\'); + return sepIdx == -1 ? fp : fp.Substring(0, sepIdx); + }) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) + .ToList(); + } + else + { + // Non-root: get distinct next segment from FolderPaths that start with prefix + subFolders = db.Files.AsNoTracking() + .Where(f => f.FolderPath.StartsWith(prefix)) + .Select(f => f.FolderPath) + .Distinct() + .AsEnumerable() + .Select(fp => + { + var remainder = fp.Substring(prefixLen); + var sepIdx = remainder.IndexOf('\\'); + return sepIdx == -1 ? remainder : remainder.Substring(0, sepIdx); + }) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase) + .ToList(); + } + + return new DirectoryContent + { + SubFolders = subFolders, + Files = files + }; + } + public List AddFiles(List newFiles) => throw new InvalidOperationException("Cannot modify a cached CA pack file container."); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index 425d23638..b77fa162c 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -26,6 +26,41 @@ public void AddOrUpdateFile(string path, PackFile file) public Dictionary GetAllFiles() => FileList; + public DirectoryContent GetDirectoryContent(string directoryPath) + { + var prefix = string.IsNullOrEmpty(directoryPath) ? "" : directoryPath + "\\"; + var prefixLength = prefix.Length; + var subFolders = new HashSet(StringComparer.OrdinalIgnoreCase); + var files = new List<(string FileName, PackFile File)>(); + + foreach (var (path, packFile) in FileList) + { + if (prefixLength > 0 && !path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + continue; + if (prefixLength == 0 && path.Length == 0) + continue; + + var remainder = path.AsSpan(prefixLength); + var separatorIndex = remainder.IndexOf(Path.DirectorySeparatorChar); + + if (separatorIndex == -1) + { + files.Add((packFile.Name, packFile)); + } + else + { + var folderName = remainder.Slice(0, separatorIndex).ToString(); + subFolders.Add(folderName); + } + } + + return new DirectoryContent + { + SubFolders = subFolders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(), + Files = files.OrderBy(x => x.FileName, StringComparer.CurrentCultureIgnoreCase).ToList() + }; + } + public List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention) { extention = extention.ToLower(); diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs index 07c0a7cf5..e20975ffc 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/IPackFileContainer.cs @@ -1,5 +1,19 @@ -namespace Shared.Core.PackFiles.Models +using System.Collections.Generic; + +namespace Shared.Core.PackFiles.Models { + public readonly struct DirectoryContent + { + public IReadOnlyList SubFolders { get; init; } + public IReadOnlyList<(string FileName, PackFile File)> Files { get; init; } + + public static DirectoryContent Empty => new() + { + SubFolders = [], + Files = [] + }; + } + public interface IPackFileContainer { string Name { get; } @@ -14,5 +28,6 @@ public interface IPackFileContainer void AddOrUpdateFile(string path, PackFile file); List<(string FileName, PackFile Pack)> FindAllWithExtention(string extention); Dictionary GetAllFiles(); + DirectoryContent GetDirectoryContent(string directoryPath); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs index a94047d43..03ef10bba 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/CacheDbContext.cs @@ -25,6 +25,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.HasKey(e => e.Id); entity.HasIndex(e => e.RelativePath).IsUnique(); entity.HasIndex(e => e.Extension); + entity.HasIndex(e => e.FolderPath); }); } } @@ -45,6 +46,7 @@ internal class CachedFileEntity public string RelativePath { get; set; } = ""; public string FileName { get; set; } = ""; public string Extension { get; set; } = ""; + public string FolderPath { get; set; } = ""; public string SourcePackFilePath { get; set; } = ""; public long Offset { get; set; } public long Size { get; set; } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs index a1d21ed11..9dee19605 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs @@ -1,27 +1,22 @@ using System.Security.Cryptography; using System.Text; using Microsoft.EntityFrameworkCore; -using Shared.Core.PackFiles.Models; +using Shared.Core.ErrorHandling; +using Shared.Core.Misc; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization.CacheDatabase; -using Shared.Core.PackFiles.Utility; +using SharpDX.Direct3D9; namespace Shared.Core.PackFiles.Serialization { internal static class PackFileContainerCacheHelper { + private static readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerCacheHelper)); public static string GetCacheFilePath(string gameDataFolder, string gameName, string cacheId) { - var cacheDir = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - "AssetEditor", - "Cache"); - - Directory.CreateDirectory(cacheDir); - var safeGameName = string.Join("_", gameName.Split(Path.GetInvalidFileNameChars())); - return Path.Combine(cacheDir, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); + return Path.Combine(DirectoryHelper.CacheDirectory, $"CachedGameFiles_{safeGameName}_{cacheId}.db"); } public static string ComputeFingerprint(string gameDataFolder, List packFileNames) @@ -65,10 +60,11 @@ public static DbContextOptions CreateDbOptions(string dbFilePath .Options; } - private const int CurrentSchemaVersion = 2; + private const int CurrentSchemaVersion = 3; public static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) { + _logger.Here().Information($"Saving cache for '{container.Name}' with {container.GetFileCount()} files"); using var db = new CacheDbContext(dbOptions); db.Database.EnsureDeleted(); db.Database.EnsureCreated(); @@ -88,11 +84,15 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db { if (packFile.DataSource is PackedFileSource source) { + var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); + var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); + db.Files.Add(new CachedFileEntity { RelativePath = relativePath, FileName = packFile.Name, Extension = Path.GetExtension(relativePath).ToLower(), + FolderPath = folderPath, SourcePackFilePath = source.Parent.FilePath, Offset = source.Offset, Size = source.Size, @@ -104,13 +104,18 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db } } + _logger.Here().Information($"Starting to save cache for '{container.Name}'"); db.SaveChanges(); + _logger.Here().Information($"Cache saved successfully for '{container.Name}'"); } public static CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) { if (!File.Exists(dbFilePath)) + { + _logger.Here().Information($"No cache file found at '{dbFilePath}'"); return null; + } var dbOptions = CreateDbOptions(dbFilePath); return LoadContainerFromCache(dbOptions, expectedFingerprint); @@ -124,8 +129,9 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db { db.Database.EnsureCreated(); } - catch + catch (Exception ex) { + _logger.Here().Warning($"Failed to open cache database: {ex.Message}"); return null; } @@ -134,13 +140,17 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db { cacheInfo = db.CacheInfo.FirstOrDefault(); } - catch + catch (Exception ex) { + _logger.Here().Warning($"Failed to read cache info: {ex.Message}"); return null; } if (cacheInfo == null || cacheInfo.SchemaVersion != CurrentSchemaVersion || cacheInfo.Fingerprint != expectedFingerprint) + { + _logger.Here().Information($"Cache invalid - schema:{cacheInfo?.SchemaVersion} (expected {CurrentSchemaVersion}), fingerprint match:{cacheInfo?.Fingerprint == expectedFingerprint}"); return null; + } var container = new CachedPackFileContainer(cacheInfo.ContainerName, dbOptions) { @@ -153,7 +163,21 @@ public static void SaveCache(string fingerprint, PackFileContainer container, Db container.SourcePackFilePaths.Add(path); } + _logger.Here().Information($"Loaded container '{container.Name}' from cache"); return container; } + + public static CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) + { + try + { + return LoadContainerFromCache(cacheFilePath, fingerprint); + } + catch (Exception ex) + { + _logger.Here().Warning($"Failed to load from cache '{cacheFilePath}': {ex.Message}"); + return null; + } + } } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs index 3f96a1d99..2910e781c 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs @@ -105,9 +105,12 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(gameDataFolder, allCaPackFiles); var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName, fingerprint); - var cached = TryLoadFromCache(cacheFilePath, fingerprint, gameName); + var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); if (cached != null) + { + _logger.Here().Information($"Loading CA packs for {gameName} from cache: {cacheFilePath}"); return cached; + } var container = LoadAllCaFilesFromDisk(gameDataFolder, gameName, allCaPackFiles, manifestFileFound); @@ -131,23 +134,7 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri } } - private CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint, string gameName) - { - try - { - var container = PackFileContainerCacheHelper.LoadContainerFromCache(cacheFilePath, fingerprint); - if (container != null) - { - _logger.Here().Information($"Loading CA packs for {gameName} from cache: {cacheFilePath}"); - return container; - } - } - catch (Exception ex) - { - _logger.Here().Warning($"Failed to read CA pack cache for {gameName}, will rebuild: {ex.Message}"); - } - return null; - } + private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string gameName, List allCaPackFiles, bool manifestFileFound) { diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs index fd18c9197..ddbfd589e 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs @@ -351,6 +351,8 @@ private void AddFiles(IPackFileContainer container, List files) if (path.Length == 0) return parent; + parent.EnsureChildrenPopulated(); + var separatorIndex = path.IndexOf(Path.DirectorySeparatorChar); if (separatorIndex == -1) return parent.Children.FirstOrDefault(x => x.Name == path); @@ -368,6 +370,8 @@ private void AddFiles(IPackFileContainer container, List files) if (path.Length == 0) return parent; + parent.EnsureChildrenPopulated(); + var nodeName = path; var remainingStr = ""; @@ -400,6 +404,8 @@ private void AddFiles(IPackFileContainer container, List files) var fullPath = _packFileService.GetFullPath(pf, container); var numSeperators = fullPath.Count(x => x == Path.DirectorySeparatorChar); + root.EnsureChildrenPopulated(); + if (numSeperators == 0) { return root.Children.FirstOrDefault(x => x.Item == pf); @@ -422,79 +428,41 @@ private void ReloadTree(IPackFileContainer container) _treeStates.Remove(container); } - var rootSource = new TreeNodeSource(container.Name, NodeType.Root, container, null); - var directoryMap_new = new Dictionary(container.GetFileCount()) - { - [string.Empty] = rootSource - }; var skipWemFiles = container.IsCaPackFile && _applicationSettingsService.CurrentSettings.ShowCAWemFiles == false; - List<(string FolderName, string FullFolderPath)> stackFileNames = new(10); - foreach (var item in container.GetAllFiles()) - { - ReadOnlySpan pathSpan = item.Key; - var lastTreeNode = rootSource; - - if (skipWemFiles) - { - var isWemFile = pathSpan.EndsWith(".wem", StringComparison.InvariantCultureIgnoreCase); - if (isWemFile) - continue; - } - - stackFileNames.Clear(); - var end = pathSpan.Length - 1; - while (end >= 0) - { - var index = pathSpan.Slice(0, end + 1).LastIndexOf(Path.DirectorySeparatorChar); - if (index == -1) - break; - - var subDirStringSpan = pathSpan.Slice(0, index); - var subDirString = subDirStringSpan.ToString(); + var rootSource = new TreeNodeSource(container.Name, NodeType.Root, container, null); + rootSource.SetChildLoader(node => LoadChildrenFromContainer(node, container, skipWemFiles)); - if (directoryMap_new.TryGetValue(subDirString, out var lookUpNode)) - { - lastTreeNode = lookUpNode; - break; - } - else - { - var subFolderIndex = subDirString.LastIndexOf(Path.DirectorySeparatorChar); - var fullPath = subDirString; - if (subFolderIndex != -1) - subDirString = subDirString.Substring(subFolderIndex + 1, subDirString.Length - 1 - subFolderIndex); - stackFileNames.Add((subDirString, fullPath)); - } + var root = CreateTreeNode(rootSource, null); + root.IsMainEditabelPack = _packFileService.GetEditablePack() == container; - // Move end position backward to continue search - end = index - 1; - } + _treeStates[container] = new PackFileTreeState(rootSource, root); + Files.Add(root); + Filter.Reapply(); + } - // Pop the stack and build the folder structure - for (int i = stackFileNames.Count - 1; i >= 0; i--) - { - var currentInstance = stackFileNames[i]; - var currentNode = new TreeNodeSource(currentInstance.FolderName, NodeType.Directory, container, lastTreeNode); + private bool LoadChildrenFromContainer(TreeNodeSource node, IPackFileContainer container, bool skipWemFiles) + { + var directoryPath = node.GetFullPath(); + var content = container.GetDirectoryContent(directoryPath); - lastTreeNode.AddChild(currentNode); - lastTreeNode = currentNode; + foreach (var folderName in content.SubFolders) + { + var childNode = new TreeNodeSource(folderName, NodeType.Directory, container, node); + childNode.SetChildLoader(n => LoadChildrenFromContainer(n, container, skipWemFiles)); + node.AddChild(childNode); + } - directoryMap_new.Add(currentInstance.FullFolderPath, currentNode); - } + foreach (var (fileName, file) in content.Files) + { + if (skipWemFiles && fileName.EndsWith(".wem", StringComparison.OrdinalIgnoreCase)) + continue; - // Add - var treeNode = new TreeNodeSource(item.Value.Name, NodeType.File, container, lastTreeNode, item.Value); - lastTreeNode.AddChild(treeNode); + var fileNode = new TreeNodeSource(fileName, NodeType.File, container, node, file); + node.AddChild(fileNode); } - SortSourceTree(rootSource); - var root = CreateTreeNode(rootSource, null); - root.IsMainEditabelPack = _packFileService.GetEditablePack() == container; - - _treeStates[container] = new PackFileTreeState(rootSource, root); - Files.Add(root); - Filter.Reapply(); + return true; } private void PackFileContainerRemoved(PackFileContainerRemovedEvent e) @@ -559,21 +527,16 @@ private TreeNode CreateTreeNode(TreeNodeSource source, TreeNode? parent) return StringComparer.CurrentCultureIgnoreCase.Compare(left.Name, right.Name); }; - private static void SortSourceTree(TreeNodeSource node) - { - node.Children.Sort(TreeNodeSourceComparison); - foreach (var child in node.Children) - SortSourceTree(child); - } - private static void InsertChildSorted(TreeNodeSource parent, TreeNodeSource child) { + parent.EnsureChildrenPopulated(); parent.AddChild(child); parent.Children.Sort(TreeNodeSourceComparison); } private static void RemoveExistingFileNode(TreeNodeSource parent, string fileName, PackFile packFile) { + parent.EnsureChildrenPopulated(); var existingFile = parent.Children.FirstOrDefault(node => node.NodeType == NodeType.File && (node.Item == packFile || node.Name == fileName)); if (existingFile == null) return; diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/SearchFilter.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/SearchFilter.cs index 9cbd1c619..31948142b 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/SearchFilter.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/SearchFilter.cs @@ -62,6 +62,14 @@ string Filter(string text) } var rootSources = _sourceRootsFactory().ToList(); + + // Force-populate all source children when an active filter requires full traversal + if (HasActiveFilter) + { + foreach (var item in rootSources) + item.EnsureFullyPopulated(); + } + foreach (var item in rootSources) HasChildWithFilterMatch(item, expression); diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNode.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNode.cs index d6442c014..bec939cc1 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNode.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNode.cs @@ -190,7 +190,9 @@ public TreeNode AddDirectoryChild(string name) return newNode; } + _source.EnsureChildrenPopulated(); var sourceNode = new TreeNodeSource(name, NodeType.Directory, FileOwner, _source); + sourceNode.MarkChildrenLoaded(); _source.AddChild(sourceNode); if (HasMaterializedChildren || IsNodeExpanded || (_isFilterActive?.Invoke() ?? false)) @@ -231,7 +233,12 @@ internal void RefreshLoadedBranch() internal void EnsureChildrenLoaded() { - if (_source == null || !_source.HasChildren || _nodeFactory == null) + if (_source == null || _nodeFactory == null) + return; + + _source.EnsureChildrenPopulated(); + + if (!_source.HasChildren) return; var existingChildren = Children diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNodeSource.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNodeSource.cs index e3ab15fc1..118ed45a4 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNodeSource.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/TreeNodeSource.cs @@ -8,9 +8,13 @@ namespace Shared.Ui.BaseDialogs.PackFileTree /// Represents the lightweight backing data model for the pack-file tree. /// It stores the canonical hierarchy and metadata independent of UI state, and may optionally /// reference a materialized when that branch has been created for display. + /// Children are loaded lazily from the container on first access of a directory/root node. /// internal sealed class TreeNodeSource { + private Func? _childLoader; + private bool _childrenLoaded; + public string Name { get; set; } public NodeType NodeType { get; } public IPackFileContainer FileOwner { get; } @@ -21,7 +25,15 @@ internal sealed class TreeNodeSource public bool UnsavedChanged { get; set; } public TreeNode? MaterializedNode { get; set; } - public bool HasChildren => Children.Count > 0; + /// + /// Returns true if this node has children (loaded or potentially unloaded). + /// + public bool HasChildren => Children.Count > 0 || (!_childrenLoaded && _childLoader != null && NodeType != NodeType.File); + + /// + /// Returns true if children have been loaded from the container. + /// + public bool ChildrenLoaded => _childrenLoaded; public TreeNodeSource(string name, NodeType nodeType, IPackFileContainer fileOwner, TreeNodeSource? parent, PackFile? item = null) { @@ -35,6 +47,44 @@ public TreeNodeSource(string name, NodeType nodeType, IPackFileContainer fileOwn throw new Exception($"Packfile name or folder is empty '{GetFullPath()}', this is not allowed! Please report as a bug if it happens outside of packfile loading! If it happens while loading clean up the packfile in RPFM"); } + /// + /// Sets the child loader delegate used for lazy population of children. + /// The delegate receives this node and returns true if children were loaded. + /// + public void SetChildLoader(Func childLoader) + { + _childLoader = childLoader; + } + + /// + /// Ensures children are populated. If not yet loaded and a child loader is set, loads them now. + /// + public void EnsureChildrenPopulated() + { + if (_childrenLoaded || _childLoader == null || NodeType == NodeType.File) + return; + + _childrenLoaded = _childLoader(this); + } + + /// + /// Recursively ensures all descendants are populated. + /// + public void EnsureFullyPopulated() + { + EnsureChildrenPopulated(); + foreach (var child in Children) + child.EnsureFullyPopulated(); + } + + /// + /// Marks children as already loaded (used when children are added externally). + /// + public void MarkChildrenLoaded() + { + _childrenLoaded = true; + } + public string GetFullPath() { if (NodeType == NodeType.Root) @@ -74,6 +124,7 @@ public IEnumerable EnumerateAllNodesDepthFirst() while (stack.Count > 0) { var current = stack.Pop(); + current.EnsureChildrenPopulated(); yield return current; for (var i = current.Children.Count - 1; i >= 0; i--) From 2d1985e7df17203d6d234cfea0bc14f552172005 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 30 Apr 2026 21:53:16 +0200 Subject: [PATCH 03/13] Code --- .../PackFileContainerCacheHelper.cs | 109 ++++++++++++------ .../Utility/PackFileContainerLoader.cs | 1 + .../CachedPackFileContainerTests.cs | 3 +- .../PackFileContainerCacheHelperTests.cs | 3 +- 4 files changed, 79 insertions(+), 37 deletions(-) rename Shared/SharedCore/Shared.Core/PackFiles/Serialization/{ => CacheDatabase}/PackFileContainerCacheHelper.cs (54%) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs similarity index 54% rename from Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs rename to Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 9dee19605..35281514c 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -1,14 +1,13 @@ using System.Security.Cryptography; using System.Text; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Shared.Core.ErrorHandling; using Shared.Core.Misc; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization.CacheDatabase; -using SharpDX.Direct3D9; -namespace Shared.Core.PackFiles.Serialization +namespace Shared.Core.PackFiles.Serialization.CacheDatabase { internal static class PackFileContainerCacheHelper { @@ -65,50 +64,94 @@ public static DbContextOptions CreateDbOptions(string dbFilePath public static void SaveCache(string fingerprint, PackFileContainer container, DbContextOptions dbOptions) { _logger.Here().Information($"Saving cache for '{container.Name}' with {container.GetFileCount()} files"); - using var db = new CacheDbContext(dbOptions); - db.Database.EnsureDeleted(); - db.Database.EnsureCreated(); - var sourcePackFilePaths = string.Join("|", container.SourcePackFilePaths); + // Use EF only to create the schema, then dispose to release memory + using (var db = new CacheDbContext(dbOptions)) + { + db.Database.EnsureDeleted(); + db.Database.EnsureCreated(); + } + + // Use raw SQLite for bulk insert - avoids EF change tracker memory and perf overhead + var connectionString = $"Data Source={GetDbFilePath(dbOptions)};Pooling=False"; + using var connection = new SqliteConnection(connectionString); + connection.Open(); + + using var transaction = connection.BeginTransaction(); - db.CacheInfo.Add(new CacheInfoEntity + // Insert CacheInfo + using (var cmd = connection.CreateCommand()) { - SchemaVersion = CurrentSchemaVersion, - Fingerprint = fingerprint, - ContainerName = container.Name, - SystemFilePath = container.SystemFilePath, - SourcePackFilePaths = sourcePackFilePaths, - }); - - foreach (var (relativePath, packFile) in container.GetAllFiles()) + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO CacheInfo (SchemaVersion, Fingerprint, ContainerName, SystemFilePath, SourcePackFilePaths) + VALUES ($schemaVersion, $fingerprint, $containerName, $systemFilePath, $sourcePackFilePaths)"; + cmd.Parameters.AddWithValue("$schemaVersion", CurrentSchemaVersion); + cmd.Parameters.AddWithValue("$fingerprint", fingerprint); + cmd.Parameters.AddWithValue("$containerName", container.Name); + cmd.Parameters.AddWithValue("$systemFilePath", container.SystemFilePath); + cmd.Parameters.AddWithValue("$sourcePackFilePaths", string.Join("|", container.SourcePackFilePaths)); + cmd.ExecuteNonQuery(); + } + + // Bulk insert files using a prepared statement + using (var cmd = connection.CreateCommand()) { - if (packFile.DataSource is PackedFileSource source) + cmd.Transaction = transaction; + cmd.CommandText = @"INSERT INTO FileList (RelativePath, FileName, Extension, FolderPath, SourcePackFilePath, Offset, Size, IsEncrypted, IsCompressed, CompressionFormat, UncompressedSize) + VALUES ($relativePath, $fileName, $extension, $folderPath, $sourcePackFilePath, $offset, $size, $isEncrypted, $isCompressed, $compressionFormat, $uncompressedSize)"; + + var pRelativePath = cmd.Parameters.Add("$relativePath", SqliteType.Text); + var pFileName = cmd.Parameters.Add("$fileName", SqliteType.Text); + var pExtension = cmd.Parameters.Add("$extension", SqliteType.Text); + var pFolderPath = cmd.Parameters.Add("$folderPath", SqliteType.Text); + var pSourcePackFilePath = cmd.Parameters.Add("$sourcePackFilePath", SqliteType.Text); + var pOffset = cmd.Parameters.Add("$offset", SqliteType.Integer); + var pSize = cmd.Parameters.Add("$size", SqliteType.Integer); + var pIsEncrypted = cmd.Parameters.Add("$isEncrypted", SqliteType.Integer); + var pIsCompressed = cmd.Parameters.Add("$isCompressed", SqliteType.Integer); + var pCompressionFormat = cmd.Parameters.Add("$compressionFormat", SqliteType.Integer); + var pUncompressedSize = cmd.Parameters.Add("$uncompressedSize", SqliteType.Integer); + + cmd.Prepare(); + + foreach (var (relativePath, packFile) in container.GetAllFiles()) { + if (packFile.DataSource is not PackedFileSource source) + continue; + var lastSep = relativePath.LastIndexOf(Path.DirectorySeparatorChar); var folderPath = lastSep == -1 ? "" : relativePath.Substring(0, lastSep); - db.Files.Add(new CachedFileEntity - { - RelativePath = relativePath, - FileName = packFile.Name, - Extension = Path.GetExtension(relativePath).ToLower(), - FolderPath = folderPath, - SourcePackFilePath = source.Parent.FilePath, - Offset = source.Offset, - Size = source.Size, - IsEncrypted = source.IsEncrypted, - IsCompressed = source.IsCompressed, - CompressionFormat = (int)source.CompressionFormat, - UncompressedSize = source.UncompressedSize, - }); + pRelativePath.Value = relativePath; + pFileName.Value = packFile.Name; + pExtension.Value = Path.GetExtension(relativePath).ToLower(); + pFolderPath.Value = folderPath; + pSourcePackFilePath.Value = source.Parent.FilePath; + pOffset.Value = source.Offset; + pSize.Value = source.Size; + pIsEncrypted.Value = source.IsEncrypted ? 1 : 0; + pIsCompressed.Value = source.IsCompressed ? 1 : 0; + pCompressionFormat.Value = (int)source.CompressionFormat; + pUncompressedSize.Value = (long)source.UncompressedSize; + + cmd.ExecuteNonQuery(); } } - _logger.Here().Information($"Starting to save cache for '{container.Name}'"); - db.SaveChanges(); + transaction.Commit(); _logger.Here().Information($"Cache saved successfully for '{container.Name}'"); } + private static string GetDbFilePath(DbContextOptions dbOptions) + { + var relationalOptions = dbOptions.Extensions + .OfType() + .FirstOrDefault(); + var connStr = relationalOptions?.ConnectionString ?? ""; + var builder = new SqliteConnectionStringBuilder(connStr); + return builder.DataSource; + } + public static CachedPackFileContainer? LoadContainerFromCache(string dbFilePath, string expectedFingerprint) { if (!File.Exists(dbFilePath)) diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs index 2910e781c..ba7ceff94 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs @@ -5,6 +5,7 @@ using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.Settings; namespace Shared.Core.PackFiles.Utility diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs index b2c6bce29..90d9ff8a8 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs @@ -1,9 +1,8 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization; using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; using Shared.Core.Settings; diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index 50df35c27..9e90d60a2 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -1,8 +1,7 @@ -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; -using Shared.Core.PackFiles.Serialization; using Shared.Core.PackFiles.Serialization.CacheDatabase; using Shared.Core.PackFiles.Utility; From 86073c11c01b472ea6a03e39e976742863c2ea6d Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 30 Apr 2026 21:57:11 +0200 Subject: [PATCH 04/13] Code --- .../Models/Containers/CachedPackFileContainer.cs | 5 +++-- .../PackFiles/Models/Containers/PackFileContainer.cs | 7 ++++--- .../Shared.Core/PackFiles/PackFileService.cs | 3 ++- .../Shared.Core/PackFiles/Utility/PathNormalization.cs | 10 ++++++++++ 4 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 Shared/SharedCore/Shared.Core/PackFiles/Utility/PathNormalization.cs diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs index 8c1ef5d2e..ee3bafa20 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/CachedPackFileContainer.cs @@ -1,6 +1,7 @@ using Microsoft.EntityFrameworkCore; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization.CacheDatabase; +using Shared.Core.PackFiles.Utility; using Shared.Core.Settings; namespace Shared.Core.PackFiles.Models.Containers @@ -29,7 +30,7 @@ public int GetFileCount() public PackFile? FindFile(string path) { - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); using var db = new CacheDbContext(_dbOptions); var entry = db.Files.AsNoTracking().FirstOrDefault(f => f.RelativePath == lowerPath); return entry != null ? ToPackFile(entry) : null; @@ -37,7 +38,7 @@ public int GetFileCount() public bool ContainsFile(string path) { - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); using var db = new CacheDbContext(_dbOptions); return db.Files.Any(f => f.RelativePath == lowerPath); } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs index b77fa162c..093fa3087 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Models/Containers/PackFileContainer.cs @@ -1,6 +1,7 @@ using Shared.Core.Misc; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Utility; using Shared.Core.Settings; namespace Shared.Core.PackFiles.Models.Containers @@ -20,7 +21,7 @@ internal class PackFileContainer : IPackFileContainerInternal public void AddOrUpdateFile(string path, PackFile file) { - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); FileList[lowerPath] = file; } @@ -136,13 +137,13 @@ public virtual void DeleteFolder(string folder) public virtual PackFile? FindFile(string path) { - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); return FileList.TryGetValue(lowerPath, out var value) ? value : null; } public virtual bool ContainsFile(string path) { - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); return FileList.ContainsKey(lowerPath); } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs b/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs index a7b65b8da..b1a176ea8 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs @@ -6,6 +6,7 @@ using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Models.FileSources; using Shared.Core.PackFiles.Serialization; +using Shared.Core.PackFiles.Utility; using Shared.Core.Settings; namespace Shared.Core.PackFiles @@ -103,7 +104,7 @@ public void CopyFileFromOtherPackFile(IPackFileContainer source, string path, IP var sourceContainer = CastContainer(source); var targetContainer = CastContainer(target); - var lowerPath = path.Replace('/', '\\').ToLower().Trim(); + var lowerPath = PathNormalization.NormalizeFileName(path); var file = sourceContainer.FindFile(lowerPath); if (file != null) { diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PathNormalization.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PathNormalization.cs new file mode 100644 index 000000000..eb23215e7 --- /dev/null +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PathNormalization.cs @@ -0,0 +1,10 @@ +namespace Shared.Core.PackFiles.Utility +{ + public static class PathNormalization + { + public static string NormalizeFileName(string path) + { + return path.Replace('/', '\\').ToLower().Trim(); + } + } +} From 9e8bb5ae7dbc17d0fc36bc5ea9ed33a5e323521f Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Thu, 30 Apr 2026 22:03:48 +0200 Subject: [PATCH 05/13] tests --- .../PackFileContainerCacheHelperTests.cs | 201 +++++++++++++++++- 1 file changed, 198 insertions(+), 3 deletions(-) diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs index 9e90d60a2..0b1023af0 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Serialization/PackFileContainerCacheHelperTests.cs @@ -236,11 +236,206 @@ public void RoundTrip_FullCycle() } [Test] - public void LoadCache_ReturnsNullForCorruptDb() + public void TryLoadFromCache_ReturnsContainerWhenValid() { - File.WriteAllBytes(_dbFilePath, [0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00]); - var result = PackFileContainerCacheHelper.LoadContainerFromCache(_dbFilePath, "fp"); + var container = new PackFileContainer("TryLoad Test") + { + SystemFilePath = @"c:\game\data" + }; + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack.pack" }; + container.AddOrUpdateFile("test\\file.txt", new PackFile("file.txt", + new PackedFileSource(parent, 0, 100, false, false, CompressionFormat.None, 0))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp_try", container, dbOptions); + + var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp_try"); + Assert.That(result, Is.Not.Null); + Assert.That(result.Name, Is.EqualTo("TryLoad Test")); + } + + [Test] + public void TryLoadFromCache_ReturnsNullForMissingFile() + { + var result = PackFileContainerCacheHelper.TryLoadFromCache( + Path.Combine(_tempDir, "does_not_exist.db"), "fp"); Assert.That(result, Is.Null); } + + [Test] + public void TryLoadFromCache_ReturnsNullForCorruptFile() + { + File.WriteAllBytes(_dbFilePath, [0xFF, 0xFE, 0x00, 0x01]); + var result = PackFileContainerCacheHelper.TryLoadFromCache(_dbFilePath, "fp"); + Assert.That(result, Is.Null); + } + + [Test] + public void SaveCache_PreservesEncryptedFlag() + { + var container = new PackFileContainer("Encrypted Test") + { + SystemFilePath = @"c:\game" + }; + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\encrypted.pack" }; + container.AddOrUpdateFile("secret\\data.bin", new PackFile("data.bin", + new PackedFileSource(parent, 0, 500, isEncrypted: true, isCompressed: false, CompressionFormat.None, 0))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + + var source = (PackedFileSource)loaded!.FindFile("secret\\data.bin")!.DataSource; + Assert.That(source.IsEncrypted, Is.True); + } + + [Test] + public void SaveCache_EmptyContainer_RoundTrips() + { + var container = new PackFileContainer("Empty Pack") + { + SystemFilePath = @"c:\game\empty" + }; + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp_empty", container, dbOptions); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp_empty"); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded.Name, Is.EqualTo("Empty Pack")); + Assert.That(loaded.GetFileCount(), Is.EqualTo(0)); + } + + [Test] + public void SaveCache_OverwritesExistingCache() + { + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + var dbOptions = CreateTestDbOptions(); + + // Save first version + var container1 = new PackFileContainer("Version1") { SystemFilePath = @"c:\game" }; + container1.AddOrUpdateFile("old.txt", new PackFile("old.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + PackFileContainerCacheHelper.SaveCache("fp1", container1, dbOptions); + + // Save second version (same db path) + var container2 = new PackFileContainer("Version2") { SystemFilePath = @"c:\game" }; + container2.AddOrUpdateFile("new.txt", new PackFile("new.txt", + new PackedFileSource(parent, 0, 20, false, false, CompressionFormat.None, 0))); + PackFileContainerCacheHelper.SaveCache("fp2", container2, dbOptions); + + // Old fingerprint should fail + var oldResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp1"); + Assert.That(oldResult, Is.Null); + + // New fingerprint should work + var newResult = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp2"); + Assert.That(newResult, Is.Not.Null); + Assert.That(newResult.Name, Is.EqualTo("Version2")); + Assert.That(newResult.GetFileCount(), Is.EqualTo(1)); + Assert.That(newResult.FindFile("new.txt"), Is.Not.Null); + } + + [Test] + public void SaveCache_StoresFolderPathCorrectly() + { + var container = new PackFileContainer("FolderPath Test") + { + SystemFilePath = @"c:\game" + }; + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + container.AddOrUpdateFile("a\\b\\c\\file.txt", new PackFile("file.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + container.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt", + new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + + // Verify via the CachedPackFileContainer's GetDirectoryContent + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + Assert.That(loaded, Is.Not.Null); + + var rootContent = loaded.GetDirectoryContent(""); + Assert.That(rootContent.Files.Any(f => f.FileName == "root_file.txt"), Is.True); + Assert.That(rootContent.SubFolders, Does.Contain("a")); + + var deepContent = loaded.GetDirectoryContent("a\\b\\c"); + Assert.That(deepContent.Files.Any(f => f.FileName == "file.txt"), Is.True); + } + + [Test] + public void SaveCache_SkipsNonPackedFileSources() + { + var container = new PackFileContainer("Mixed Sources") + { + SystemFilePath = @"c:\game" + }; + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + container.AddOrUpdateFile("packed.txt", new PackFile("packed.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + container.AddOrUpdateFile("memory.txt", new PackFile("memory.txt", + new MemorySource([1, 2, 3]))); + + var dbOptions = CreateTestDbOptions(); + PackFileContainerCacheHelper.SaveCache("fp", container, dbOptions); + var loaded = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp"); + + Assert.That(loaded, Is.Not.Null); + Assert.That(loaded.GetFileCount(), Is.EqualTo(1)); + Assert.That(loaded.FindFile("packed.txt"), Is.Not.Null); + Assert.That(loaded.FindFile("memory.txt"), Is.Null); + } + + [Test] + public void ComputeFingerprint_IgnoresMissingPackFiles() + { + var packDir = Path.Combine(_tempDir, "partial"); + Directory.CreateDirectory(packDir); + File.WriteAllText(Path.Combine(packDir, "exists.pack"), "data"); + + var packFiles = new List { "exists.pack", "missing.pack" }; + var fp = PackFileContainerCacheHelper.ComputeFingerprint(packDir, packFiles); + + Assert.That(fp, Is.Not.Null.And.Not.Empty); + + // Same result regardless of missing file in list + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, new List { "exists.pack", "missing.pack" }); + Assert.That(fp, Is.EqualTo(fp2)); + } + + [Test] + public void ComputeFingerprint_OrderIndependent() + { + var packDir = Path.Combine(_tempDir, "order_test"); + Directory.CreateDirectory(packDir); + File.WriteAllText(Path.Combine(packDir, "alpha.pack"), "aaa"); + File.WriteAllText(Path.Combine(packDir, "beta.pack"), "bbb"); + File.WriteAllText(Path.Combine(packDir, "gamma.pack"), "ccc"); + + var fp1 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, + new List { "gamma.pack", "alpha.pack", "beta.pack" }); + var fp2 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, + new List { "alpha.pack", "beta.pack", "gamma.pack" }); + var fp3 = PackFileContainerCacheHelper.ComputeFingerprint(packDir, + new List { "beta.pack", "gamma.pack", "alpha.pack" }); + + Assert.That(fp1, Is.EqualTo(fp2)); + Assert.That(fp2, Is.EqualTo(fp3)); + } + + [Test] + public void GetCacheFilePath_SanitizesInvalidChars() + { + var path = PackFileContainerCacheHelper.GetCacheFilePath(@"c:\game", "Game:Name/WithChars", "abc123"); + var fileName = Path.GetFileName(path); + + Assert.That(fileName.IndexOfAny(Path.GetInvalidFileNameChars()), Is.EqualTo(-1)); + Assert.That(path.EndsWith(".db"), Is.True); + } } } From 9ea3af0b25c52853769edb77cef154f8c567665e Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sun, 3 May 2026 19:58:52 +0200 Subject: [PATCH 06/13] Code --- .../Components/Selection/SelectionManager.cs | 14 +- .../Events/Global/PackFileSavedEvent.cs | 2 +- .../Shared.Core/PackFiles/PackFileService.cs | 2 +- .../CachedPackFileContainerTests.cs | 267 +++++++++ .../PackFiles/PackFileServiceTest.cs | 512 ++++++++++++++++-- .../PackFileTree/PackFileBrowserViewModel.cs | 31 +- .../PackFileBrowserViewModelTests.cs | 314 ++++++++++- 7 files changed, 1082 insertions(+), 60 deletions(-) diff --git a/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs b/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs index a157d2c04..acbd870f2 100644 --- a/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs +++ b/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs @@ -72,7 +72,6 @@ public override void Initialize() base.Initialize(); } - public ISelectionState CreateSelectionSate(GeometrySelectionMode mode, ISelectable selectedObj, bool sendEvent = true) { if (_currentState != null) @@ -147,6 +146,8 @@ public override void Draw(GameTime gameTime) _cachedEdgeMesh = vertexObject; _cachedEdgeIndices = BuildEdgeIndexCache(geo); _edgeDataDirty = true; + _sampleIdx0 = 0; + _sampleIdx1 = geo.VertexCount() > 1 ? 1 : 0; } if (selectionVertexState.SelectedVertices.Count >= 2) @@ -160,7 +161,14 @@ public override void Draw(GameTime gameTime) _sampleIdx1 = _sampleIdx0 < geo.VertexCount() - 1 ? _sampleIdx0 + 1 : 0; } - if (!_edgeDataDirty && geo.VertexCount() >= 2) + var vertexCount = geo.VertexCount(); + if (vertexCount > 0) + { + _sampleIdx0 = Math.Clamp(_sampleIdx0, 0, vertexCount - 1); + _sampleIdx1 = Math.Clamp(_sampleIdx1, 0, vertexCount - 1); + } + + if (!_edgeDataDirty && vertexCount >= 2) { var p0 = geo.GetVertexById(_sampleIdx0); var p1 = geo.GetVertexById(_sampleIdx1); @@ -173,7 +181,7 @@ public override void Draw(GameTime gameTime) UpdateEdgeQuadData(vertexObject, selectionVertexState); _edgeDataDirty = false; - if (geo.VertexCount() >= 2) + if (vertexCount >= 2) { _samplePos0 = geo.GetVertexById(_sampleIdx0); _samplePos1 = geo.GetVertexById(_sampleIdx1); diff --git a/Shared/SharedCore/Shared.Core/Events/Global/PackFileSavedEvent.cs b/Shared/SharedCore/Shared.Core/Events/Global/PackFileSavedEvent.cs index 760413d6b..57bc733f8 100644 --- a/Shared/SharedCore/Shared.Core/Events/Global/PackFileSavedEvent.cs +++ b/Shared/SharedCore/Shared.Core/Events/Global/PackFileSavedEvent.cs @@ -14,7 +14,7 @@ public record PackFileContainerFilesUpdatedEvent(IPackFileContainer Container, L public record PackFileContainerFilesAddedEvent(IPackFileContainer Container, List AddedFiles) : PackFileContainerManipulationEvent; public record PackFileContainerFilesRemovedEvent(IPackFileContainer Container, List RemovedFiles) : PackFileContainerManipulationEvent; public record PackFileContainerFolderRemovedEvent(IPackFileContainer Container, string Folder) : PackFileContainerManipulationEvent; - public record PackFileContainerFolderRenamedEvent(IPackFileContainer Container, string NewNodePath) : PackFileContainerManipulationEvent; + public record PackFileContainerFolderRenamedEvent(IPackFileContainer Container, string OldNodePath, string NewNodePath) : PackFileContainerManipulationEvent; public class BeforePackFileContainerRemovedEvent(IPackFileContainer removed) { diff --git a/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs b/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs index b1a176ea8..6b6ac60e9 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/PackFileService.cs @@ -197,7 +197,7 @@ public void RenameDirectory(IPackFileContainer pf, string currentNodeName, strin throw new Exception("Name can not be empty"); var newNodePath = container.RenameDirectory(currentNodeName, newName); - _globalEventHub?.PublishGlobalEvent(new PackFileContainerFolderRenamedEvent(container, newNodePath)); + _globalEventHub?.PublishGlobalEvent(new PackFileContainerFolderRenamedEvent(container, currentNodeName, newNodePath)); } public void RenameFile(IPackFileContainer pf, PackFile file, string newName) diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs index 90d9ff8a8..12dc99df7 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Models/Containers/CachedPackFileContainerTests.cs @@ -220,4 +220,271 @@ public void AddOrUpdateFile_Throws() _container.AddOrUpdateFile("test\\new.txt", new PackFile("new.txt", null))); } } + + internal class CachedPackFileContainer_GetDirectoryContent + { + private string _tempDir; + private string _dbFilePath; + private CachedPackFileContainer _container; + + [SetUp] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "CachedDirContentTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + _dbFilePath = Path.Combine(_tempDir, "test.db"); + + var sourceContainer = new PackFileContainer("DirTest") + { + IsCaPackFile = true, + SystemFilePath = @"c:\game\data" + }; + sourceContainer.SourcePackFilePaths.Add(@"c:\game\data\pack1.pack"); + + var parent = new PackedFileSourceParent { FilePath = @"c:\game\data\pack1.pack" }; + + // Root-level files + sourceContainer.AddOrUpdateFile("root_file.txt", new PackFile("root_file.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + + // Single-level folders + sourceContainer.AddOrUpdateFile("models\\unit.model", new PackFile("unit.model", + new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\vehicle.model", new PackFile("vehicle.model", + new PackedFileSource(parent, 30, 40, false, false, CompressionFormat.None, 0))); + + // Nested folders + sourceContainer.AddOrUpdateFile("models\\textures\\diffuse.dds", new PackFile("diffuse.dds", + new PackedFileSource(parent, 70, 50, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\textures\\normal.dds", new PackFile("normal.dds", + new PackedFileSource(parent, 120, 60, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("models\\textures\\specular\\gloss.dds", new PackFile("gloss.dds", + new PackedFileSource(parent, 180, 30, false, false, CompressionFormat.None, 0))); + + // Another top-level folder + sourceContainer.AddOrUpdateFile("audio\\music.wem", new PackFile("music.wem", + new PackedFileSource(parent, 210, 100, false, false, CompressionFormat.None, 0))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + _container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, true); + } + + [Test] + public void Root_ReturnsRootFilesAndTopFolders() + { + var content = _container.GetDirectoryContent(""); + + Assert.That(content.Files.Any(f => f.FileName == "root_file.txt"), Is.True); + Assert.That(content.SubFolders, Does.Contain("models")); + Assert.That(content.SubFolders, Does.Contain("audio")); + // Should NOT contain nested folders at root level + Assert.That(content.SubFolders, Does.Not.Contain("textures")); + } + + [Test] + public void Root_DoesNotIncludeFilesFromSubfolders() + { + var content = _container.GetDirectoryContent(""); + + Assert.That(content.Files.Count, Is.EqualTo(1)); + Assert.That(content.Files[0].FileName, Is.EqualTo("root_file.txt")); + } + + [Test] + public void Subfolder_ReturnsDirectFilesAndImmediateSubfolders() + { + var content = _container.GetDirectoryContent("models"); + + Assert.That(content.Files.Any(f => f.FileName == "unit.model"), Is.True); + Assert.That(content.Files.Any(f => f.FileName == "vehicle.model"), Is.True); + Assert.That(content.SubFolders, Does.Contain("textures")); + // Should NOT include files from deeper levels + Assert.That(content.Files.Any(f => f.FileName == "diffuse.dds"), Is.False); + } + + [Test] + public void NestedSubfolder_ReturnsCorrectContent() + { + var content = _container.GetDirectoryContent("models\\textures"); + + Assert.That(content.Files.Any(f => f.FileName == "diffuse.dds"), Is.True); + Assert.That(content.Files.Any(f => f.FileName == "normal.dds"), Is.True); + Assert.That(content.SubFolders, Does.Contain("specular")); + Assert.That(content.Files.Count, Is.EqualTo(2)); + } + + [Test] + public void DeepNestedFolder_ReturnsOnlyItsFiles() + { + var content = _container.GetDirectoryContent("models\\textures\\specular"); + + Assert.That(content.Files.Count, Is.EqualTo(1)); + Assert.That(content.Files[0].FileName, Is.EqualTo("gloss.dds")); + Assert.That(content.SubFolders, Is.Empty); + } + + [Test] + public void NonexistentFolder_ReturnsEmpty() + { + var content = _container.GetDirectoryContent("nonexistent\\path"); + + Assert.That(content.Files, Is.Empty); + Assert.That(content.SubFolders, Is.Empty); + } + + [Test] + public void SubfoldersAreSorted() + { + var content = _container.GetDirectoryContent(""); + + var sorted = content.SubFolders.OrderBy(x => x, StringComparer.CurrentCultureIgnoreCase).ToList(); + Assert.That(content.SubFolders, Is.EqualTo(sorted)); + } + + [Test] + public void FilesAreSorted() + { + var content = _container.GetDirectoryContent("models"); + + var sorted = content.Files.OrderBy(x => x.FileName, StringComparer.CurrentCultureIgnoreCase).ToList(); + Assert.That(content.Files.Select(f => f.FileName), Is.EqualTo(sorted.Select(f => f.FileName))); + } + + [Test] + public void Files_HaveCorrectDataSource() + { + var content = _container.GetDirectoryContent("models"); + var unitFile = content.Files.First(f => f.FileName == "unit.model"); + var source = unitFile.File.DataSource as PackedFileSource; + + Assert.That(source, Is.Not.Null); + Assert.That(source.Offset, Is.EqualTo(10)); + Assert.That(source.Size, Is.EqualTo(20)); + Assert.That(source.Parent.FilePath, Is.EqualTo(@"c:\game\data\pack1.pack")); + } + } + + internal class CachedPackFileContainer_EdgeCases + { + private string _tempDir; + private string _dbFilePath; + + [SetUp] + public void Setup() + { + _tempDir = Path.Combine(Path.GetTempPath(), "CachedEdgeCaseTests_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempDir); + _dbFilePath = Path.Combine(_tempDir, "test.db"); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_tempDir)) + Directory.Delete(_tempDir, true); + } + + [Test] + public void FindAllWithExtention_IsCaseInsensitive() + { + var sourceContainer = new PackFileContainer("Test") { SystemFilePath = @"c:\game" }; + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + sourceContainer.AddOrUpdateFile("sound.wem", new PackFile("sound.wem", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + var container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + + var upper = container.FindAllWithExtention(".WEM"); + var lower = container.FindAllWithExtention(".wem"); + Assert.That(upper.Count, Is.EqualTo(1)); + Assert.That(lower.Count, Is.EqualTo(1)); + } + + [Test] + public void ContainsFile_NormalizesForwardSlashes() + { + var sourceContainer = new PackFileContainer("Test") { SystemFilePath = @"c:\game" }; + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + sourceContainer.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + var container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + + Assert.That(container.ContainsFile("folder/file.txt"), Is.True); + Assert.That(container.ContainsFile("FOLDER/FILE.TXT"), Is.True); + Assert.That(container.ContainsFile(" folder\\file.txt "), Is.True); + } + + [Test] + public void GetAllFiles_PreservesCompressionMetadata() + { + var sourceContainer = new PackFileContainer("Test") { SystemFilePath = @"c:\game" }; + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + sourceContainer.AddOrUpdateFile("data.bin", new PackFile("data.bin", + new PackedFileSource(parent, 100, 500, true, true, CompressionFormat.Lz4, 2000))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + var container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + + var all = container.GetAllFiles(); + var source = (PackedFileSource)all["data.bin"].DataSource; + + Assert.That(source.IsEncrypted, Is.True); + Assert.That(source.IsCompressed, Is.True); + Assert.That(source.CompressionFormat, Is.EqualTo(CompressionFormat.Lz4)); + Assert.That(source.UncompressedSize, Is.EqualTo(2000)); + Assert.That(source.Offset, Is.EqualTo(100)); + Assert.That(source.Size, Is.EqualTo(500)); + } + + [Test] + public void GetFullPath_WithDuplicateFileNames_ReturnsFirstMatch() + { + var sourceContainer = new PackFileContainer("Test") { SystemFilePath = @"c:\game" }; + var parent = new PackedFileSourceParent { FilePath = @"c:\game\pack.pack" }; + sourceContainer.AddOrUpdateFile("folder_a\\shared.txt", new PackFile("shared.txt", + new PackedFileSource(parent, 0, 10, false, false, CompressionFormat.None, 0))); + sourceContainer.AddOrUpdateFile("folder_b\\shared.txt", new PackFile("shared.txt", + new PackedFileSource(parent, 10, 20, false, false, CompressionFormat.None, 0))); + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + var container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + + // GetFullPath matches by FileName - with duplicates it returns the first DB row + var file = container.FindFile("folder_a\\shared.txt")!; + var path = container.GetFullPath(file); + + // Should return a valid path (either folder_a or folder_b) + Assert.That(path, Does.Contain("shared.txt")); + Assert.That(path, Does.Contain("\\")); + } + + [Test] + public void GetDirectoryContent_EmptyContainer_ReturnsEmpty() + { + var sourceContainer = new PackFileContainer("Empty") { SystemFilePath = @"c:\game" }; + + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(_dbFilePath); + PackFileContainerCacheHelper.SaveCache("fp", sourceContainer, dbOptions); + var container = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, "fp")!; + + var content = container.GetDirectoryContent(""); + Assert.That(content.Files, Is.Empty); + Assert.That(content.SubFolders, Is.Empty); + } + } } diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/PackFileServiceTest.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/PackFileServiceTest.cs index 23f6674eb..5c822cf5e 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/PackFileServiceTest.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/PackFileServiceTest.cs @@ -11,22 +11,34 @@ namespace Shared.CoreTest.PackFiles { internal class PackFileServiceTest { + private static PackFileService CreateService(Mock? eventHub = null) + { + eventHub ??= new Mock(); + var pfs = new PackFileService(eventHub.Object); + pfs.MessageBoxProvider = new Mock().Object; + return pfs; + } + + private static PackFileService CreateServiceWithCaPack(Mock? eventHub = null) + { + var pfs = CreateService(eventHub); + var ca = new PackFileContainer("CaPack") { IsCaPackFile = true, SystemFilePath = "ca_path" }; + pfs.AddContainer(ca); + return pfs; + } + [Test] public void AddContainer_CaPackFile() { - // Arrenge var eventHub = new Mock(); var pfs = new PackFileService(eventHub.Object); var container = new PackFileContainer("MyTest"); container.SystemFilePath = "SystemPath"; container.IsCaPackFile = true; - // Act pfs.AddContainer(container); - // Assert var containers = pfs.GetAllPackfileContainers(); - Assert.That(containers.First(), Is.EqualTo(container)); Assert.That(pfs.GetEditablePack(), Is.Null); eventHub.Verify(m => m.PublishGlobalEvent(new PackFileContainerAddedEvent(container)), Times.Once); @@ -35,7 +47,6 @@ public void AddContainer_CaPackFile() [Test] public void AddContainer_CaPackFileNotSet() { - // Arrenge var eventHub = new Mock(); var dialogProvider = new Mock(); var pfs = new PackFileService(eventHub.Object); @@ -44,12 +55,9 @@ public void AddContainer_CaPackFileNotSet() container.SystemFilePath = "SystemPath"; container.IsCaPackFile = false; - // Act pfs.AddContainer(container); - // Assert var containers = pfs.GetAllPackfileContainers(); - Assert.That(containers.Count, Is.EqualTo(0)); Assert.That(pfs.GetEditablePack(), Is.Null); eventHub.Verify(m => m.PublishGlobalEvent(new PackFileContainerAddedEvent(container)), Times.Never); @@ -59,9 +67,7 @@ public void AddContainer_CaPackFileNotSet() [Test] public void AddContainer_CaPackFileSet() { - // Arrenge var eventHub = new Mock(); - var pfs = new PackFileService(eventHub.Object); var caContainer = new PackFileContainer("MyTest"); caContainer.SystemFilePath = "SystemPath"; @@ -71,13 +77,10 @@ public void AddContainer_CaPackFileSet() customContainer.SystemFilePath = "SystemPath2"; customContainer.IsCaPackFile = false; - // Act pfs.AddContainer(caContainer); pfs.AddContainer(customContainer, true); - // Assert var containers = pfs.GetAllPackfileContainers(); - Assert.That(containers.Count, Is.EqualTo(2)); Assert.That(pfs.GetEditablePack(), Is.EqualTo(customContainer)); eventHub.Verify(m => m.PublishGlobalEvent(new PackFileContainerAddedEvent(caContainer)), Times.Once); @@ -88,12 +91,11 @@ public void AddContainer_CaPackFileSet() [Test] public void AddContainers_Duplicate() { - // Arrenge var eventHub = new Mock(); var dialogProvider = new Mock(); - var pfs = new PackFileService( eventHub.Object); - var caContainer = new PackFileContainer("MyTest"); + var pfs = new PackFileService(eventHub.Object); pfs.MessageBoxProvider = dialogProvider.Object; + var caContainer = new PackFileContainer("MyTest"); caContainer.SystemFilePath = "SystemPath"; caContainer.IsCaPackFile = true; @@ -101,14 +103,11 @@ public void AddContainers_Duplicate() customContainer.SystemFilePath = "SystemPath2"; customContainer.IsCaPackFile = false; - // Act pfs.AddContainer(caContainer); pfs.AddContainer(customContainer, true); pfs.AddContainer(customContainer); - // Assert var containers = pfs.GetAllPackfileContainers(); - Assert.That(containers.Count, Is.EqualTo(2)); Assert.That(pfs.GetEditablePack(), Is.EqualTo(customContainer)); dialogProvider.Verify(m => m.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Once); @@ -117,20 +116,16 @@ public void AddContainers_Duplicate() [Test] public void CreateNewPackFileContainer() { - // Arrenge var eventHub = new Mock(); var pfs = new PackFileService(eventHub.Object); var container = new PackFileContainer("MyTest"); container.SystemFilePath = "SystemPath"; container.IsCaPackFile = true; - // Act pfs.AddContainer(container); var emptyPackFileContainer = pfs.CreateNewPackFileContainer("Custom", PackFileVersion.PFH5, PackFileCAType.MOD, true); - // Assert var containers = pfs.GetAllPackfileContainers(); - Assert.That(containers.Count(), Is.EqualTo(2)); Assert.That(pfs.GetEditablePack(), Is.EqualTo(emptyPackFileContainer)); eventHub.Verify(m => m.PublishGlobalEvent(new PackFileContainerAddedEvent(container)), Times.Once); @@ -138,32 +133,13 @@ public void CreateNewPackFileContainer() } [Test] - public void CreateNewPackFileContainer_Wh3() + public void CreateNewPackFileContainer_EmptyName_Throws() { - var eventHub = new Mock(); - var pfs = new PackFileService(eventHub.Object); - // pfs.CreateNewPackFileContainer("Custom", PackFileCAType.WH3, true); - - + var pfs = CreateServiceWithCaPack(); + Assert.Throws(() => pfs.CreateNewPackFileContainer("", PackFileVersion.PFH5, PackFileCAType.MOD)); + Assert.Throws(() => pfs.CreateNewPackFileContainer(" ", PackFileVersion.PFH5, PackFileCAType.MOD)); } - [Test] - public void CreateNewPackFileContainer_Rome() - { } - - - - - //AddFilesToPack - - - // AddFilesToPack - Ensure valid path! - // CopyFileFromOtherPackFile - // SetEditablePack - Ca file, null, valid - // UnloadPackContainer - // SaveFile - // SavePackContainer - [Test] public void SaveFile_NoEditablePack_ThrowsDescriptiveException() { @@ -193,5 +169,449 @@ public void AddContainer_TwoContainersWithNullSystemFilePath_BothAdded() Assert.That(result, Is.Not.Null); Assert.That(pfs.GetAllPackfileContainers().Count, Is.EqualTo(3)); } + + [Test] + public void AddContainer_EnforceGameFilesDisabled_AllowsNonCaFirst() + { + var pfs = CreateService(); + pfs.EnforceGameFilesMustBeLoaded = false; + + var container = new PackFileContainer("Custom") { IsCaPackFile = false, SystemFilePath = "path" }; + var result = pfs.AddContainer(container); + + Assert.That(result, Is.Not.Null); + Assert.That(pfs.GetAllPackfileContainers().Count, Is.EqualTo(1)); + } + + [Test] + public void SetEditablePack_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + + Assert.Throws(() => pfs.SetEditablePack(caPack)); + } + + [Test] + public void SetEditablePack_Null_ClearsSelection() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var custom = new PackFileContainer("Custom") { SystemFilePath = "custom_path" }; + pfs.AddContainer(custom, true); + + Assert.That(pfs.GetEditablePack(), Is.EqualTo(custom)); + pfs.SetEditablePack(null); + Assert.That(pfs.GetEditablePack(), Is.Null); + } + + [Test] + public void UnloadPackContainer_RemovesContainer() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var custom = new PackFileContainer("Custom") { SystemFilePath = "custom_path" }; + pfs.AddContainer(custom, true); + + pfs.UnloadPackContainer(custom); + + Assert.That(pfs.GetAllPackfileContainers().Count, Is.EqualTo(1)); + Assert.That(pfs.GetEditablePack(), Is.Null); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } + + [Test] + public void UnloadPackContainer_NonEditablePack_DoesNotClearEditable() + { + var pfs = CreateServiceWithCaPack(); + var pack1 = new PackFileContainer("Pack1") { SystemFilePath = "path1" }; + var pack2 = new PackFileContainer("Pack2") { SystemFilePath = "path2" }; + pfs.AddContainer(pack1, true); + pfs.AddContainer(pack2); + + pfs.UnloadPackContainer(pack2); + + Assert.That(pfs.GetEditablePack(), Is.EqualTo(pack1)); + Assert.That(pfs.GetAllPackfileContainers().Count, Is.EqualTo(2)); + } + + [Test] + public void AddFilesToPack_AddsFiles() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + pfs.AddContainer(container, true); + + var newFiles = new List + { + new("folder", new PackFile("test.txt", new MemorySource([1, 2, 3]))) + }; + + pfs.AddFilesToPack(container, newFiles); + + Assert.That(container.FindFile("folder\\test.txt"), Is.Not.Null); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } + + [Test] + public void AddFilesToPack_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + var newFiles = new List + { + new("folder", new PackFile("test.txt", new MemorySource([1]))) + }; + + Assert.Throws(() => pfs.AddFilesToPack(caPack, newFiles)); + } + + [Test] + public void CopyFileFromOtherPackFile_CopiesData() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + + var source = new PackFileContainer("Source") { SystemFilePath = "source_path" }; + source.AddOrUpdateFile("data\\file.bin", new PackFile("file.bin", new MemorySource([10, 20, 30]))); + pfs.AddContainer(source); + + var target = new PackFileContainer("Target") { SystemFilePath = "target_path" }; + pfs.AddContainer(target, true); + + pfs.CopyFileFromOtherPackFile(source, "data\\file.bin", target); + + var copied = target.FindFile("data\\file.bin"); + Assert.That(copied, Is.Not.Null); + Assert.That(copied.Name, Is.EqualTo("file.bin")); + Assert.That(copied.DataSource.ReadData(), Is.EqualTo(new byte[] { 10, 20, 30 })); + } + + [Test] + public void CopyFileFromOtherPackFile_MissingFile_DoesNothing() + { + var pfs = CreateServiceWithCaPack(); + var source = new PackFileContainer("Source") { SystemFilePath = "source_path" }; + pfs.AddContainer(source); + var target = new PackFileContainer("Target") { SystemFilePath = "target_path" }; + pfs.AddContainer(target, true); + + pfs.CopyFileFromOtherPackFile(source, "nonexistent\\file.bin", target); + + Assert.That(target.GetFileCount(), Is.EqualTo(0)); + } + + [Test] + public void CopyFileFromOtherPackFile_TargetIsCaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + var source = new PackFileContainer("Source") { SystemFilePath = "source_path" }; + source.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(source); + + Assert.Throws(() => pfs.CopyFileFromOtherPackFile(source, "file.txt", caPack)); + } + + [Test] + public void DeleteFile_RemovesFile() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("folder\\test.txt", new PackFile("test.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("folder\\test.txt")!; + pfs.DeleteFile(container, file); + + Assert.That(container.FindFile("folder\\test.txt"), Is.Null); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } + + [Test] + public void DeleteFile_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + var file = new PackFile("test.txt", new MemorySource([1])); + + Assert.Throws(() => pfs.DeleteFile(caPack, file)); + } + + [Test] + public void DeleteFolder_RemovesFiles() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("folder\\a.txt", new PackFile("a.txt", new MemorySource([1]))); + container.AddOrUpdateFile("folder\\b.txt", new PackFile("b.txt", new MemorySource([2]))); + container.AddOrUpdateFile("other\\c.txt", new PackFile("c.txt", new MemorySource([3]))); + pfs.AddContainer(container, true); + + pfs.DeleteFolder(container, "folder"); + + Assert.That(container.FindFile("folder\\a.txt"), Is.Null); + Assert.That(container.FindFile("folder\\b.txt"), Is.Null); + Assert.That(container.FindFile("other\\c.txt"), Is.Not.Null); + } + + [Test] + public void DeleteFolder_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + + Assert.Throws(() => pfs.DeleteFolder(caPack, "folder")); + } + + [Test] + public void MoveFile_ChangesPath() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("old\\test.txt", new PackFile("test.txt", new MemorySource([1, 2]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("old\\test.txt")!; + pfs.MoveFile(container, file, "new"); + + Assert.That(container.FindFile("old\\test.txt"), Is.Null); + Assert.That(container.FindFile("new\\test.txt"), Is.Not.Null); + } + + [Test] + public void MoveFile_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + var file = new PackFile("test.txt", new MemorySource([1])); + + Assert.Throws(() => pfs.MoveFile(caPack, file, "new")); + } + + [Test] + public void RenameFile_ChangesName() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("folder\\old.txt", new PackFile("old.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("folder\\old.txt")!; + pfs.RenameFile(container, file, "new.txt"); + + Assert.That(file.Name, Is.EqualTo("new.txt")); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } + + [Test] + public void RenameFile_EmptyName_Throws() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("folder\\file.txt")!; + Assert.Throws(() => pfs.RenameFile(container, file, "")); + Assert.Throws(() => pfs.RenameFile(container, file, " ")); + } + + [Test] + public void RenameFile_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + var file = new PackFile("test.txt", new MemorySource([1])); + + Assert.Throws(() => pfs.RenameFile(caPack, file, "new.txt")); + } + + [Test] + public void RenameDirectory_ChangesPath() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("oldname\\file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + pfs.RenameDirectory(container, "oldname", "newname"); + + Assert.That(container.FindFile("oldname\\file.txt"), Is.Null); + Assert.That(container.FindFile("newname\\file.txt"), Is.Not.Null); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } + + [Test] + public void RenameDirectory_EmptyName_Throws() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + pfs.AddContainer(container, true); + + Assert.Throws(() => pfs.RenameDirectory(container, "folder", "")); + } + + [Test] + public void RenameDirectory_CaPack_Throws() + { + var pfs = CreateServiceWithCaPack(); + var caPack = pfs.GetAllPackfileContainers().First(); + + Assert.Throws(() => pfs.RenameDirectory(caPack, "folder", "new")); + } + + [Test] + public void FindFile_FindsAcrossContainers() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("data\\file.txt", new PackFile("file.txt", new MemorySource([1, 2]))); + pfs.AddContainer(container, true); + + var result = pfs.FindFile("data\\file.txt"); + Assert.That(result, Is.Not.Null); + Assert.That(result.Name, Is.EqualTo("file.txt")); + } + + [Test] + public void FindFile_ReturnsNullForMissing() + { + var pfs = CreateServiceWithCaPack(); + var result = pfs.FindFile("nonexistent\\file.txt"); + Assert.That(result, Is.Null); + } + + [Test] + public void FindFile_SpecificContainer_OnlySearchesThat() + { + var pfs = CreateServiceWithCaPack(); + var container1 = new PackFileContainer("Pack1") { SystemFilePath = "path1" }; + container1.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container1); + + var container2 = new PackFileContainer("Pack2") { SystemFilePath = "path2" }; + pfs.AddContainer(container2); + + var found = pfs.FindFile("file.txt", container2); + Assert.That(found, Is.Null); + + var foundInCorrect = pfs.FindFile("file.txt", container1); + Assert.That(foundInCorrect, Is.Not.Null); + } + + [Test] + public void FindFile_LaterContainerTakesPriority() + { + var pfs = CreateServiceWithCaPack(); + var container1 = new PackFileContainer("Pack1") { SystemFilePath = "path1" }; + container1.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container1); + + var container2 = new PackFileContainer("Pack2") { SystemFilePath = "path2" }; + container2.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([2]))); + pfs.AddContainer(container2); + + var result = pfs.FindFile("file.txt"); + Assert.That(result!.DataSource.ReadData(), Is.EqualTo(new byte[] { 2 })); + } + + [Test] + public void FindAllWithExtention_SearchesAllContainers() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("a.txt", new PackFile("a.txt", new MemorySource([1]))); + container.AddOrUpdateFile("b.txt", new PackFile("b.txt", new MemorySource([2]))); + container.AddOrUpdateFile("c.bin", new PackFile("c.bin", new MemorySource([3]))); + pfs.AddContainer(container, true); + + var results = pfs.FindAllWithExtention(".txt"); + Assert.That(results.Count, Is.EqualTo(2)); + } + + [Test] + public void FindAllWithExtention_SpecificContainer() + { + var pfs = CreateServiceWithCaPack(); + var container1 = new PackFileContainer("Pack1") { SystemFilePath = "path1" }; + container1.AddOrUpdateFile("a.txt", new PackFile("a.txt", new MemorySource([1]))); + pfs.AddContainer(container1); + + var container2 = new PackFileContainer("Pack2") { SystemFilePath = "path2" }; + container2.AddOrUpdateFile("b.txt", new PackFile("b.txt", new MemorySource([2]))); + pfs.AddContainer(container2); + + var results = pfs.FindAllWithExtention(".txt", container1); + Assert.That(results.Count, Is.EqualTo(1)); + } + + [Test] + public void GetFullPath_ReturnsPath() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("folder\\file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("folder\\file.txt")!; + var path = pfs.GetFullPath(file); + Assert.That(path, Is.EqualTo("folder\\file.txt")); + } + + [Test] + public void GetFullPath_UnknownFile_Throws() + { + var pfs = CreateServiceWithCaPack(); + var unknownFile = new PackFile("unknown.txt", new MemorySource([1])); + + Assert.Throws(() => pfs.GetFullPath(unknownFile)); + } + + [Test] + public void GetPackFileContainer_ReturnsCorrectContainer() + { + var pfs = CreateServiceWithCaPack(); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([1]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("file.txt")!; + var result = pfs.GetPackFileContainer(file); + Assert.That(result, Is.EqualTo(container)); + } + + [Test] + public void GetPackFileContainer_UnknownFile_ReturnsNull() + { + var pfs = CreateServiceWithCaPack(); + var unknownFile = new PackFile("unknown.txt", new MemorySource([1])); + + var result = pfs.GetPackFileContainer(unknownFile); + Assert.That(result, Is.Null); + } + + [Test] + public void SaveFile_UpdatesFileData() + { + var eventHub = new Mock(); + var pfs = CreateServiceWithCaPack(eventHub); + var container = new PackFileContainer("Custom") { SystemFilePath = "path" }; + container.AddOrUpdateFile("file.txt", new PackFile("file.txt", new MemorySource([1, 2, 3]))); + pfs.AddContainer(container, true); + + var file = container.FindFile("file.txt")!; + pfs.SaveFile(file, [4, 5, 6]); + + Assert.That(file.DataSource.ReadData(), Is.EqualTo(new byte[] { 4, 5, 6 })); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + eventHub.Verify(m => m.PublishGlobalEvent(It.IsAny()), Times.Once); + } } } diff --git a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs index ddbfd589e..7c1f9475c 100644 --- a/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs +++ b/Shared/SharedUI/Shared.Ui/BaseDialogs/PackFileTree/PackFileBrowserViewModel.cs @@ -69,7 +69,7 @@ public PackFileBrowserViewModel(ApplicationSettingsService applicationSettingsSe _eventHub?.Register(this, x => AddFiles(x.Container, x.AddedFiles)); _eventHub?.Register(this, x => Database_PackFilesRemoved(x.Container, x.RemovedFiles)); _eventHub?.Register(this, x => Database_PackFileFolderRemoved(x.Container, x.Folder)); - _eventHub?.Register(this, x => Database_PackFileFolderRenamed(x.Container, x.NewNodePath)); + _eventHub?.Register(this, x => Database_PackFileFolderRenamed(x.Container, x.OldNodePath, x.NewNodePath)); _eventHub?.Register(this, ContainerSaved); Filter = new SearchFilter(Files, () => _treeStates.Values.Select(x => x.RootSource)); @@ -117,13 +117,19 @@ private void Database_PackFileFolderRemoved(IPackFileContainer container, string Filter.Reapply(); } - private void Database_PackFileFolderRenamed(IPackFileContainer container, string folder) + private void Database_PackFileFolderRenamed(IPackFileContainer container, string oldPath, string newPath) { var state = GetPackFileTreeState(container); - var node = GetNodeFromPath(state.RootSource, folder, false); + var node = GetNodeFromPath(state.RootSource, oldPath, false); if (node == null) return; + var newLeafName = newPath; + var lastSep = newPath.LastIndexOf(Path.DirectorySeparatorChar); + if (lastSep != -1) + newLeafName = newPath.Substring(lastSep + 1); + + node.Name = newLeafName; node.UnsavedChanged = true; Filter.Reapply(); } @@ -133,14 +139,23 @@ private void ContainerSaved(PackFileContainerSavedEvent e) var state = GetPackFileTreeState(e.Container); var root = state.RootNode; - foreach (var node in state.RootSource.EnumerateAllNodesDepthFirst()) - node.UnsavedChanged = false; + ClearUnsavedOnLoadedSourceNodes(state.RootSource); root.UnsavedChanged = false; root.ForeachNode((node) => node.UnsavedChanged = false); Filter.Reapply(); } + private static void ClearUnsavedOnLoadedSourceNodes(TreeNodeSource node) + { + node.UnsavedChanged = false; + if (!node.ChildrenLoaded) + return; + + foreach (var child in node.Children) + ClearUnsavedOnLoadedSourceNodes(child); + } + private void Database_PackFilesRemoved(IPackFileContainer container, List files) { var state = GetPackFileTreeState(container); @@ -255,7 +270,6 @@ private void AddFiles(IPackFileContainer container, List files) var numSeperators = fullPath.Count(x => x == Path.DirectorySeparatorChar); var directoryEnd = fullPath.LastIndexOf(Path.DirectorySeparatorChar); - var fileName = fullPath.Substring(directoryEnd + 1); // Check if alreayd added - this happens moving files. @@ -263,14 +277,14 @@ private void AddFiles(IPackFileContainer container, List files) if (numSeperators == 0) { RemoveExistingFileNode(state.RootSource, item.Name, item); - newNode = new TreeNodeSource(fileName, NodeType.File, container, state.RootSource, item); + newNode = new TreeNodeSource(item.Name, NodeType.File, container, state.RootSource, item); InsertChildSorted(state.RootSource, newNode); } else { var directory = fullPath.Substring(0, directoryEnd); var folder = GetNodeFromPath(state.RootSource, directory)!; - newNode = new TreeNodeSource(fileName, NodeType.File, container, folder, item); + newNode = new TreeNodeSource(item.Name, NodeType.File, container, folder, item); // remove any existing files with same name var existingFile = folder.Children.FirstOrDefault(node => node.Name == item.Name && node.NodeType == NodeType.File); @@ -391,6 +405,7 @@ private void AddFiles(IPackFileContainer container, List files) if (createIfMissing) { var newNode = new TreeNodeSource(nodeName, NodeType.Directory, parent.FileOwner, parent); + newNode.MarkChildrenLoaded(); InsertChildSorted(parent, newNode); return GetNodeFromPath(newNode, remainingStr, createIfMissing); } diff --git a/Shared/SharedUI/Shared.UiTest/PackFileBrowserViewModelTests.cs b/Shared/SharedUI/Shared.UiTest/PackFileBrowserViewModelTests.cs index a9b219224..6eab570f9 100644 --- a/Shared/SharedUI/Shared.UiTest/PackFileBrowserViewModelTests.cs +++ b/Shared/SharedUI/Shared.UiTest/PackFileBrowserViewModelTests.cs @@ -1,6 +1,31 @@ -using System.Windows.Input; +// PackFileBrowserViewModel Lazy Loading Architecture: +// +// The tree is built lazily. When a container is added (via PackFileContainerAddedEvent), only +// the root TreeNodeSource is created with a child-loader delegate. Children are NOT loaded from +// the container's GetDirectoryContent() until the node is expanded or explicitly queried. +// +// TreeNodeSource is the canonical backing model (lightweight, no UI state). +// TreeNode is the materialized WPF-bound node, created on demand when a branch is expanded. +// A placeholder child is used so the WPF TreeView shows the expand arrow for unloaded directories. +// +// Event-driven updates from PackFileService: +// - PackFileContainerAddedEvent → ReloadTree: creates root source + lazy child loader +// - PackFileContainerRemovedEvent → removes container's tree state and TreeNode from Files +// - PackFileContainerFilesAddedEvent → AddFiles: inserts nodes into loaded source branches, creates dirs as needed +// - PackFileContainerFilesRemovedEvent → removes source + materialized nodes for deleted files +// - PackFileContainerFilesUpdatedEvent → updates Name/UnsavedChanged on source nodes (for rename/save) +// - PackFileContainerFolderRemovedEvent → finds and removes folder source node and materialized subtree +// - PackFileContainerFolderRenamedEvent → finds folder by OLD path, renames leaf, marks unsaved +// - PackFileContainerSetAsMainEditableEvent → toggles IsMainEditabelPack on root TreeNodes +// - PackFileContainerSavedEvent → clears UnsavedChanged on loaded nodes only (avoids forcing full population) +// +// SearchFilter: when active, calls EnsureFullyPopulated() to load entire subtree for regex matching. +// When cleared, filter-expanded nodes are absorbed as user expansions and collapsed nodes are unloaded. + +using System.Windows.Input; using AssetEditor.ViewModels; using Microsoft.Extensions.DependencyInjection; +using Shared.Core.Events; using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; using Shared.Core.Settings; @@ -320,6 +345,293 @@ private void CreatePackfiles(params (string Path, string FileName)[] files) _packageFileService.AddContainer(container); } + + [Test] + public void RenameFolderUsingPfs_EnsureTreeViewUpdated() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + Assert.That(folderA, Is.Not.Null); + + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + _packageFileService.RenameDirectory(container, "foldera", "folderb"); + + // Old path gone, new path available + var oldNode = _viewModel.GetFromPath(root, "foldera"); + var newNode = _viewModel.GetFromPath(root, "folderb"); + + Assert.That(oldNode, Is.Null, "Old folder path should not exist after rename"); + Assert.That(newNode, Is.Not.Null, "Renamed folder should be accessible via the new name"); + Assert.That(newNode.UnsavedChanged, Is.True, "Renamed folder should be marked unsaved"); + } + + [Test] + public void RenameFolderUsingPfs_NestedFolder_EnsureTreeViewUpdated() + { + CreatePackfiles(("parent\\child\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var parent = _viewModel.GetFromPath(root, "parent"); + Assert.That(parent, Is.Not.Null); + parent.IsNodeExpanded = true; + + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + _packageFileService.RenameDirectory(container, "parent\\child", "renamed"); + + var newNode = _viewModel.GetFromPath(root, "parent\\renamed"); + Assert.That(newNode, Is.Not.Null, "Nested renamed folder should be accessible"); + Assert.That(newNode.UnsavedChanged, Is.True); + } + + [Test] + public void DeleteFolderUsingPfs_EnsureTreeViewUpdated() + { + CreatePackfiles(("foldera\\file.txt", "file.txt"), ("folderb\\other.txt", "other.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + Assert.That(folderA, Is.Not.Null); + + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + _packageFileService.DeleteFolder(container, "foldera"); + + var deletedNode = _viewModel.GetFromPath(root, "foldera"); + Assert.That(deletedNode, Is.Null, "Deleted folder should not exist in tree"); + + var survivingNode = _viewModel.GetFromPath(root, "folderb"); + Assert.That(survivingNode, Is.Not.Null, "Non-deleted folder should still exist"); + } + + [Test] + public void UnloadPackContainer_RemovesFromTree() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + Assert.That(_viewModel.Files.Count, Is.EqualTo(1)); + + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + _packageFileService.UnloadPackContainer(container); + + Assert.That(_viewModel.Files.Count, Is.EqualTo(0)); + } + + [Test] + public void AddContainer_AppearsInTree() + { + Assert.That(_viewModel.Files.Count, Is.EqualTo(0)); + + var container = _packageFileService.CreateNewPackFileContainer("new.pack", PackFileVersion.PFH5, PackFileCAType.MOD); + _packageFileService.AddContainer(container); + + Assert.That(_viewModel.Files.Count, Is.EqualTo(1)); + Assert.That(_viewModel.Files[0].Name, Is.EqualTo("new.pack")); + Assert.That(_viewModel.Files[0].NodeType, Is.EqualTo(NodeType.Root)); + } + + [Test] + public void MainEditablePackChanged_UpdatesRootNode() + { + var container1 = _packageFileService.CreateNewPackFileContainer("pack1.pack", PackFileVersion.PFH5, PackFileCAType.MOD, true); + _packageFileService.AddContainer(container1); + var container2 = _packageFileService.CreateNewPackFileContainer("pack2.pack", PackFileVersion.PFH5, PackFileCAType.MOD); + _packageFileService.AddContainer(container2); + + var root1 = _viewModel.Files.First(x => x.Name == "pack1.pack"); + var root2 = _viewModel.Files.First(x => x.Name == "pack2.pack"); + + Assert.That(root1.IsMainEditabelPack, Is.True); + Assert.That(root2.IsMainEditabelPack, Is.False); + + _packageFileService.SetEditablePack(container2); + + Assert.That(root1.IsMainEditabelPack, Is.False); + Assert.That(root2.IsMainEditabelPack, Is.True); + } + + [Test] + public void ContainerSaved_ClearsUnsavedFlags() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + + // Add a file to mark things as unsaved + var newFile = PackFile.CreateFromASCII("new.txt", "data"); + _packageFileService.AddFilesToPack(container, [new NewPackFileEntry("foldera", newFile)]); + + root.IsNodeExpanded = true; + var folderA = _viewModel.GetFromPath(root, "foldera"); + Assert.That(folderA, Is.Not.Null); + Assert.That(root.UnsavedChanged, Is.True, "Root should be unsaved after adding file"); + + // Simulate save via PFS — we can't call SavePackContainer without disk I/O, + // so trigger the event directly through the event hub + var eventHub = _runner.ServiceProvider.GetRequiredService(); + eventHub.PublishGlobalEvent(new Shared.Core.Events.Global.PackFileContainerSavedEvent(container)); + + Assert.That(root.UnsavedChanged, Is.False, "Root should be cleared after save"); + } + + [Test] + public void ContainerSaved_DoesNotForcePopulationOfUnloadedBranches() + { + CreatePackfiles(("foldera\\sub\\deep\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + + // Root is collapsed — only root exists as materialized node + Assert.That(root.IsNodeExpanded, Is.False); + + var eventHub = _runner.ServiceProvider.GetRequiredService(); + eventHub.PublishGlobalEvent(new Shared.Core.Events.Global.PackFileContainerSavedEvent(container)); + + // After save, the tree should NOT have expanded/materialized children + Assert.That(root.Children.All(c => c.Name == "" || c.NodeType == NodeType.Root), + Is.True, "Save should not force population of collapsed branches"); + } + + [Test] + public void AddFileToRootLevel_EnsureTreeViewUpdated() + { + var container = _packageFileService.CreateNewPackFileContainer("test.pack", PackFileVersion.PFH5, PackFileCAType.MOD, true); + _packageFileService.AddContainer(container); + + var newFile = PackFile.CreateFromASCII("rootfile.txt", "content"); + _packageFileService.AddFilesToPack(container, [new NewPackFileEntry("", newFile)]); + + var root = _viewModel.Files.First(x => x.Name == "test.pack"); + root.IsNodeExpanded = true; + + var fileNode = _viewModel.GetFromPath(root, "rootfile.txt"); + Assert.That(fileNode, Is.Not.Null, "File added at root level should appear in tree"); + Assert.That(fileNode.NodeType, Is.EqualTo(NodeType.File)); + } + + [Test] + public void DoubleClick_File_InvokesFileOpenEvent() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + folderA.IsNodeExpanded = true; + + var fileNode = _viewModel.GetFromPath(root, "foldera\\file.txt"); + Assert.That(fileNode, Is.Not.Null); + + PackFile? openedFile = null; + _viewModel.FileOpen += f => openedFile = f; + + _viewModel.DoubleClickCommand.Execute(fileNode); + + Assert.That(openedFile, Is.Not.Null, "FileOpen should be invoked on double-click"); + Assert.That(openedFile.Name, Is.EqualTo("file.txt")); + } + + [Test] + public void DoubleClick_Directory_TogglesExpansion() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + Assert.That(folderA, Is.Not.Null); + Assert.That(folderA.IsNodeExpanded, Is.False); + + _viewModel.DoubleClickCommand.Execute(folderA); + Assert.That(folderA.IsNodeExpanded, Is.True); + + _viewModel.DoubleClickCommand.Execute(folderA); + Assert.That(folderA.IsNodeExpanded, Is.False); + } + + [Test] + public void MoveFileUsingPfs_EnsureTreeViewUpdated() + { + CreatePackfiles(("source\\file.txt", "file.txt"), ("target\\other.txt", "other.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + var container = _packageFileService.GetAllPackfileContainers().Last(x => x.Name == "test.pack"); + + var sourceFolder = _viewModel.GetFromPath(root, "source"); + Assert.That(sourceFolder, Is.Not.Null); + sourceFolder.IsNodeExpanded = true; + + var fileNode = _viewModel.GetFromPath(root, "source\\file.txt"); + Assert.That(fileNode, Is.Not.Null); + + var file = _packageFileService.FindFile("source\\file.txt", container); + _packageFileService.MoveFile(container, file, "target"); + + // Old location gone + var oldNode = _viewModel.GetFromPath(root, "source\\file.txt"); + Assert.That(oldNode, Is.Null, "File should not be in old folder after move"); + + // New location has the file + var targetFolder = _viewModel.GetFromPath(root, "target"); + targetFolder.IsNodeExpanded = true; + var newNode = _viewModel.GetFromPath(root, "target\\file.txt"); + Assert.That(newNode, Is.Not.Null, "File should appear in target folder after move"); + } + + [Test] + public void DragDrop_FileOntoFile_NotAllowed() + { + CreatePackfiles(("foldera\\file1.txt", "file1.txt"), ("foldera\\file2.txt", "file2.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + folderA.IsNodeExpanded = true; + + var file1 = _viewModel.GetFromPath(root, "foldera\\file1.txt"); + var file2 = _viewModel.GetFromPath(root, "foldera\\file2.txt"); + + Assert.That(_viewModel.AllowDrop(file1, file2), Is.False, "Should not allow dropping file onto file"); + } + + [Test] + public void DragDrop_FolderDrag_NotAllowed() + { + CreatePackfiles(("foldera\\file.txt", "file.txt"), ("folderb\\file2.txt", "file2.txt")); + var root = _viewModel.Files[0]; + root.IsNodeExpanded = true; + + var folderA = _viewModel.GetFromPath(root, "foldera"); + var folderB = _viewModel.GetFromPath(root, "folderb"); + + Assert.That(_viewModel.AllowDrop(folderA, folderB), Is.False, "Should not allow dragging folders"); + } + + [Test] + public void ShowFoldersOnly_HidesFiles() + { + CreatePackfiles(("foldera\\file.txt", "file.txt")); + var root = _viewModel.Files[0]; + + _viewModel.Filter.ShowFoldersOnly = true; + + root.IsNodeExpanded = true; + var folderA = _viewModel.GetFromPath(root, "foldera"); + Assert.That(folderA, Is.Not.Null, "Folder should be visible with ShowFoldersOnly"); + Assert.That(folderA.IsVisible, Is.True); + + folderA.IsNodeExpanded = true; + + // File nodes are not materialized when ShowFoldersOnly is active + var fileNode = _viewModel.GetFromPath(root, "foldera\\file.txt"); + Assert.That(fileNode, Is.Null, "File should not be materialized with ShowFoldersOnly active"); + + // Only folder children should be present + Assert.That(folderA.Children.All(c => c.NodeType != NodeType.File || !c.IsVisible), Is.True, + "No visible file nodes should exist under folder with ShowFoldersOnly"); + } } } From deefe4d0f2d988bed0ca36b60d55a061ca344cd3 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sun, 3 May 2026 20:00:18 +0200 Subject: [PATCH 07/13] readme --- README.md | 104 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 262d28658..fb006d18b 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,76 @@ -Modding tool for total war games -Features: -- Super easy to use 3d editor which supports saving and loading to all game formats (rmv2) -- Animation editor that easily allows for moving animation between skeletons or making mount animations where there is none. -- Lots of other cool stuff, check it out! - -![image](https://user-images.githubusercontent.com/54080240/143955132-badd843e-b823-4a4d-8326-e64be4e2c877.png) - - -Planned major features: -- Mesh verifications (Show normals, ensure weights) -- Animation transfer tool -- Flip faces tool/Invert normals -- Rendering improvements -- Improve re-rigging -- Auto rig mesh to mesh -- Load anim 4 save rmv2 -- UV selection/display -- Realtime meta editor/full preview -- Animated prop creator -- Split face tool -- Campagin animation bin editor -- Improve animPack editing - -Known bugs: -- Pivot/Normal bug -- Some older vmds not loading -- Some anims for 3k not loading +![AssetEditor](https://user-images.githubusercontent.com/54080240/143955132-badd843e-b823-4a4d-8326-e64be4e2c877.png) + +# The Asset Editor + +[![Build & Test](https://github.com/donkeyProgramming/TheAssetEditor/actions/workflows/pr-test.yml/badge.svg)](https://github.com/donkeyProgramming/TheAssetEditor/actions/workflows/pr-test.yml) +[![Latest release](https://img.shields.io/github/v/release/donkeyProgramming/TheAssetEditor)](https://github.com/donkeyProgramming/TheAssetEditor/releases/latest) +[![Downloads](https://img.shields.io/github/downloads/donkeyProgramming/TheAssetEditor/total)](https://github.com/donkeyProgramming/TheAssetEditor/releases) + +A powerful modding tool for Total War games. Provides a full suite of editors for 3D models, animations, audio, textures, UI components, and more — all in one integrated desktop application. + +**[Documentation](https://donkeyprogramming.github.io/TheAssetEditor/)** · **[Video Tutorial](https://www.youtube.com/watch?v=ZlmG2AVL5-g)** · **[Latest Release](https://github.com/donkeyProgramming/TheAssetEditor/releases/latest)** + +--- + +## Editors + +### Kitbasher (3D Editor) +Full-featured 3D mesh editor for creating composite models by combining parts from multiple meshes. Supports material editing, mesh operations (merge, split, duplicate), vertex/face/object selection modes, and real-time 3D preview with a MonoGame-based viewport. + +### Animation Editor +Keyframe-based animation editing tool for creating and modifying animations. Includes a campaign animation creator and mount animation creator for adding ride animations to units that don't natively support them. + +### Animation Re-Targeting +Retargets animations between different skeletons, allowing you to transfer animations from one character model to another. Provides bone mapping and re-rigging capabilities for cross-skeleton animation reuse. + +### Skeleton Editor +Visual skeleton editor for creating, editing, and rigging skeletal structures. Supports bone manipulation, weight visualization, and reference mesh display for precise rigging workflows. + +### Texture Editor +Texture preview and editing tool supporting all game texture formats. Provides metadata inspection and visual inspection of textures used by game models. + +### Audio Editor +Comprehensive audio project editor with support for Wwise-based audio exploration, project creation, waveform visualization, format conversion, and dialogue event management. + +### Animation Fragment Editor +Manages animation packs (.animpack files) and campaign animation bin files. Includes a batch animation exporter for bulk format conversions across your mod. + +### Import/Export Editor +Handles bidirectional import/export of game asset formats, supporting GLTF/FBX to RMV2 conversions for bringing external 3D assets into the game. + +### TWUI Editor +Editor for Total War UI component files. Allows editing of UI component properties, positioning, docking, anchoring, and nested component structures used in the game's interface. + +### Metadata Editor +Editing interface for animation metadata. Supports manipulation of animation tags, attributes, and timing properties that control how animations blend and trigger in-game. + +### Reports +Generates analysis and diagnostic reports including material reports, mesh info dumps, metadata JSON exports, deep file searches, and dialogue event information. + +--- + +## Getting Started + +1. Download the [latest release](https://github.com/donkeyProgramming/TheAssetEditor/releases/latest) +2. Extract and run `AssetEditor.exe` +3. Load your game's pack files and start editing + +For detailed guides, see the **[Documentation](https://donkeyprogramming.github.io/TheAssetEditor/)**. + +## Building from Source + +Prerequisites: .NET 10 SDK, Windows + +```bash +git clone https://github.com/donkeyProgramming/TheAssetEditor.git +cd TheAssetEditor +dotnet build AssetEditor.sln +``` + +## Contributing + +Pull requests are welcome. The project uses a modular architecture with dependency injection — each editor is a self-contained project under the `Editors/` folder. + +## License + +See the repository for license details. From e991938f2df1f9d84e12248d24198019e28b75cb Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sun, 3 May 2026 20:12:18 +0200 Subject: [PATCH 08/13] Code --- .../Selection/SelectionManagerTests.cs | 139 ++++++++++++++++++ README.md | 18 +-- 2 files changed, 145 insertions(+), 12 deletions(-) create mode 100644 GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs new file mode 100644 index 000000000..753dcd651 --- /dev/null +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs @@ -0,0 +1,139 @@ +using GameWorld.Core.Components.Rendering; +using GameWorld.Core.Components.Selection; +using GameWorld.Core.Rendering; +using GameWorld.Core.Rendering.Geometry; +using GameWorld.Core.Rendering.Materials.Shaders; +using GameWorld.Core.Rendering.RenderItems; +using GameWorld.Core.SceneNodes; +using GameWorld.Core.Test.TestUtility; +using Microsoft.Xna.Framework; +using Moq; +using Shared.Core.Events; +using Shared.GameFormats.RigidModel.MaterialHeaders; +using System.Reflection; +using Test.GameWorld.Core.WsMaterialTemplate; + +namespace GameWorld.Core.Test.Selection +{ + // This test is terrible! Keeping it so that I know what to change later + + [TestFixture] + public class SelectionManagerTests + { + private Mock _eventHubMock; +#pragma warning disable NUnit1032 // RenderEngine constructed with null deps cannot be Disposed safely + private RenderEngineComponent _renderEngine; +#pragma warning restore NUnit1032 + private SelectionManager _selectionManager; + + [TearDown] + public void TearDown() + { + _selectionManager?.Dispose(); + } + + [SetUp] + public void SetUp() + { + _eventHubMock = new Mock(); + + _renderEngine = new RenderEngineComponent(null, null, null, null, null, null, _eventHubMock.Object, null); + + _selectionManager = new SelectionManager( + _eventHubMock.Object, + _renderEngine, + null, + null, + null); + + // Set render items that are normally created in Initialize() (requires GPU) + SetPrivateField(_selectionManager, "_edgeQuadRenderItem", new EdgeQuadRenderItem()); + SetPrivateField(_selectionManager, "_vertexRenderItem", new VertexRenderItem()); + } + + [Test] + public void Draw_SwitchFromLargeMeshToSmallMesh_NoIndexOutOfRange() + { + // Arrange: mesh A with 20 vertices, select vertex 15 + var nodeA = CreateMeshNode(vertexCount: 20); + var stateA = new VertexSelectionState(nodeA, 0); + stateA.ModifySelection(new[] { 15 }, onlyRemove: false); + _selectionManager.SetState(stateA); + _selectionManager.Draw(new GameTime()); + + // Switch to mesh B with only 3 vertices, no selection + var nodeB = CreateMeshNode(vertexCount: 3); + var stateB = new VertexSelectionState(nodeB, 0); + _selectionManager.SetState(stateB); + + // Act & Assert: stale _sampleIdx0=15 must not cause IndexOutOfRangeException + Assert.DoesNotThrow(() => _selectionManager.Draw(new GameTime())); + } + + [Test] + public void Draw_ZeroSelectedVerticesAfterHighIndexSelection_NoIndexOutOfRange() + { + // Arrange: mesh with 20 vertices, select high-index vertices + var node = CreateMeshNode(vertexCount: 20); + var state = new VertexSelectionState(node, 0); + state.ModifySelection(new[] { 15, 16 }, onlyRemove: false); + _selectionManager.SetState(state); + _selectionManager.Draw(new GameTime()); + + // Deselect everything — stale indices persist with same mesh + state.ModifySelection(new[] { 15, 16 }, onlyRemove: true); + + // Act & Assert: stale indices are still valid for same mesh + Assert.DoesNotThrow(() => _selectionManager.Draw(new GameTime())); + } + + [Test] + public void Draw_SwitchFromLargeMeshToSmallMesh_IndicesAreClamped() + { + // Arrange: mesh A with 20 vertices, select vertex 18 + var nodeA = CreateMeshNode(vertexCount: 20); + var stateA = new VertexSelectionState(nodeA, 0); + stateA.ModifySelection(new[] { 18, 19 }, onlyRemove: false); + _selectionManager.SetState(stateA); + _selectionManager.Draw(new GameTime()); + + // Switch to mesh B with only 3 vertices, select vertex 2 + var nodeB = CreateMeshNode(vertexCount: 3); + var stateB = new VertexSelectionState(nodeB, 0); + stateB.ModifySelection(new[] { 2 }, onlyRemove: false); + _selectionManager.SetState(stateB); + + // Act & Assert: should handle the index clamping safely + Assert.DoesNotThrow(() => _selectionManager.Draw(new GameTime())); + } + + private static Rmv2MeshNode CreateMeshNode(int vertexCount) + { + var contextFactory = new TestGeometryGraphicsContextFactory(); + var mesh = new MeshObject(contextFactory.Create(), "test_skeleton"); + mesh.VertexArray = new VertexPositionNormalTextureCustom[vertexCount]; + for (var i = 0; i < vertexCount; i++) + mesh.VertexArray[i] = new VertexPositionNormalTextureCustom { Position = new Vector4(i, 0, 0, 1) }; + + // Build a valid triangle index array + var indices = new List(); + for (var i = 0; i < vertexCount - 2; i++) + indices.AddRange(new ushort[] { 0, (ushort)(i + 1), (ushort)(i + 2) }); + mesh.IndexArray = indices.ToArray(); + + var materialMock = new Mock(); + materialMock.Setup(m => m.ModelName).Returns("test_mesh"); + materialMock.Setup(m => m.PivotPoint).Returns(Vector3.Zero); + + var shader = new CapabilityMaterialMock(CapabilityMaterialsEnum.SpecGlossPbr_Default); + + return new Rmv2MeshNode(mesh, materialMock.Object, shader, null); + } + + private static void SetPrivateField(object obj, string fieldName, object value) + { + var field = obj.GetType().GetField(fieldName, BindingFlags.NonPublic | BindingFlags.Instance); + field!.SetValue(obj, value); + } + } +} diff --git a/README.md b/README.md index fb006d18b..75fed6542 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Latest release](https://img.shields.io/github/v/release/donkeyProgramming/TheAssetEditor)](https://github.com/donkeyProgramming/TheAssetEditor/releases/latest) [![Downloads](https://img.shields.io/github/downloads/donkeyProgramming/TheAssetEditor/total)](https://github.com/donkeyProgramming/TheAssetEditor/releases) -A powerful modding tool for Total War games. Provides a full suite of editors for 3D models, animations, audio, textures, UI components, and more — all in one integrated desktop application. +A powerful modding tool for Total War games. Provides a full suite of editors for 3D models, animations, audio, textures, and more — all in one integrated desktop application. **[Documentation](https://donkeyprogramming.github.io/TheAssetEditor/)** · **[Video Tutorial](https://www.youtube.com/watch?v=ZlmG2AVL5-g)** · **[Latest Release](https://github.com/donkeyProgramming/TheAssetEditor/releases/latest)** @@ -17,8 +17,11 @@ A powerful modding tool for Total War games. Provides a full suite of editors fo ### Kitbasher (3D Editor) Full-featured 3D mesh editor for creating composite models by combining parts from multiple meshes. Supports material editing, mesh operations (merge, split, duplicate), vertex/face/object selection modes, and real-time 3D preview with a MonoGame-based viewport. -### Animation Editor -Keyframe-based animation editing tool for creating and modifying animations. Includes a campaign animation creator and mount animation creator for adding ride animations to units that don't natively support them. +### Visual Metadata Editor +Editing interface for animation metadata. Supports manipulation of animation tags, attributes, and timing properties that control how animations blend and trigger in-game. + +### Audio Editor +Comprehensive audio project editor with support for Wwise-based audio exploration, project creation, waveform visualization, format conversion, and dialogue event management. ### Animation Re-Targeting Retargets animations between different skeletons, allowing you to transfer animations from one character model to another. Provides bone mapping and re-rigging capabilities for cross-skeleton animation reuse. @@ -29,21 +32,12 @@ Visual skeleton editor for creating, editing, and rigging skeletal structures. S ### Texture Editor Texture preview and editing tool supporting all game texture formats. Provides metadata inspection and visual inspection of textures used by game models. -### Audio Editor -Comprehensive audio project editor with support for Wwise-based audio exploration, project creation, waveform visualization, format conversion, and dialogue event management. - ### Animation Fragment Editor Manages animation packs (.animpack files) and campaign animation bin files. Includes a batch animation exporter for bulk format conversions across your mod. ### Import/Export Editor Handles bidirectional import/export of game asset formats, supporting GLTF/FBX to RMV2 conversions for bringing external 3D assets into the game. -### TWUI Editor -Editor for Total War UI component files. Allows editing of UI component properties, positioning, docking, anchoring, and nested component structures used in the game's interface. - -### Metadata Editor -Editing interface for animation metadata. Supports manipulation of animation tags, attributes, and timing properties that control how animations blend and trigger in-game. - ### Reports Generates analysis and diagnostic reports including material reports, mesh info dumps, metadata JSON exports, deep file searches, and dialogue event information. From 1a707611912abd3d13e7c8719363f65a94dc0842 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Sun, 3 May 2026 20:45:06 +0200 Subject: [PATCH 09/13] Code --- AssetEditor/Language_Cn.json | 10 +- AssetEditor/Language_En.json | 10 +- AssetEditor/Language_Fr.json | 10 +- ...letonAnimationLookUpHelperKarlPackTests.cs | 3 +- .../DependencyInjectionContainer.cs | 2 +- .../PackFileContainerCacheHelper.cs | 12 +- .../Utility/PackFileContainerLoader.cs | 63 +++++-- .../Utility/PackFileContainerLoaderTests.cs | 177 ++++++++++++++++++ .../TestUtility/PackFileSerivceTestHelper.cs | 8 +- 9 files changed, 275 insertions(+), 20 deletions(-) create mode 100644 Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs diff --git a/AssetEditor/Language_Cn.json b/AssetEditor/Language_Cn.json index 7596b6afd..7837bdedb 100644 --- a/AssetEditor/Language_Cn.json +++ b/AssetEditor/Language_Cn.json @@ -881,5 +881,13 @@ "AnimationReTarget.BoneSettings.Title": "骨骼设置:", "FbxSettingsDialog.Import": "导入", - "FbxSettingsDialog.Cancel": "取消" + "FbxSettingsDialog.Cancel": "取消", + + "PackFileCache.InvalidReason.Title": "Cache Rebuild Required", + "PackFileCache.InvalidReason.NotFound": "No cache found for {0}. Building cache for the first time - this may take a moment.", + "PackFileCache.InvalidReason.DataChanged": "Game files for {0} have changed since the cache was built. Rebuilding cache.", + "PackFileCache.InvalidReason.Corrupted": "Cache for {0} is corrupted and will be rebuilt.", + "PackFileCache.BuildingCache": "Building pack file cache for {0}. This may take some time, please wait...", + "PackFileCache.BuildingCache.Title": "Building Cache", + "PackFileCache.Description": "The cache allows faster startup times and reduces memory usage by storing pre-processed game file data on disk." } \ No newline at end of file diff --git a/AssetEditor/Language_En.json b/AssetEditor/Language_En.json index a0a16d1a7..29d7c688d 100644 --- a/AssetEditor/Language_En.json +++ b/AssetEditor/Language_En.json @@ -881,6 +881,14 @@ "AnimationReTarget.BoneSettings.Title": "Bone settings:", "FbxSettingsDialog.Import": "Import", - "FbxSettingsDialog.Cancel": "Cancel" + "FbxSettingsDialog.Cancel": "Cancel", + + "PackFileCache.InvalidReason.Title": "Cache Rebuild Required", + "PackFileCache.InvalidReason.NotFound": "No cache found for {0}. Building cache for the first time - this may take a moment.", + "PackFileCache.InvalidReason.DataChanged": "Game files for {0} have changed since the cache was built. Rebuilding cache.", + "PackFileCache.InvalidReason.Corrupted": "Cache for {0} is corrupted and will be rebuilt.", + "PackFileCache.BuildingCache": "Building pack file cache for {0}. This may take some time, please wait...", + "PackFileCache.BuildingCache.Title": "Building Cache", + "PackFileCache.Description": "The cache allows faster startup times and reduces memory usage by storing pre-processed game file data on disk." } diff --git a/AssetEditor/Language_Fr.json b/AssetEditor/Language_Fr.json index 2b8f92667..b905694f0 100644 --- a/AssetEditor/Language_Fr.json +++ b/AssetEditor/Language_Fr.json @@ -27,5 +27,13 @@ "KitbashTool.Debug.Geometry.CreatePlane.Tooltip": "Creer un plan subdivise", "KitbashTool.Debug.Geometry.CreateSphere.Tooltip": "Creer une sphere subdivisee", - "KitbashTool.SaveDialog.DisplayDialogOnNextSave": "Afficher la boîte de dialogue à la prochaine sauvegarde" + "KitbashTool.SaveDialog.DisplayDialogOnNextSave": "Afficher la boîte de dialogue à la prochaine sauvegarde", + + "PackFileCache.InvalidReason.Title": "Cache Rebuild Required", + "PackFileCache.InvalidReason.NotFound": "No cache found for {0}. Building cache for the first time - this may take a moment.", + "PackFileCache.InvalidReason.DataChanged": "Game files for {0} have changed since the cache was built. Rebuilding cache.", + "PackFileCache.InvalidReason.Corrupted": "Cache for {0} is corrupted and will be rebuilt.", + "PackFileCache.BuildingCache": "Building pack file cache for {0}. This may take some time, please wait...", + "PackFileCache.BuildingCache.Title": "Building Cache", + "PackFileCache.Description": "The cache allows faster startup times and reduces memory usage by storing pre-processed game file data on disk." } \ No newline at end of file diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs index e9170bd6d..73a43df35 100644 --- a/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Services/SkeletonAnimationLookUpHelperKarlPackTests.cs @@ -7,6 +7,7 @@ using Shared.Core.PackFiles; using Shared.Core.PackFiles.Models; using Shared.Core.PackFiles.Utility; +using Shared.Core.Services; using Shared.Core.Settings; using Test.TestingUtility.TestUtility; @@ -31,7 +32,7 @@ public void LoadAndUnload_KarlPack_UpdatesSkeletonAndAnimationLookup() .Returns(() => containers.ToList()); var settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); - var loader = new PackFileContainerLoader(settingsService); + var loader = new PackFileContainerLoader(settingsService, new Mock().Object, new LocalizationManager()); var karlPackPath = PathHelper.GetDataFile("Karl_and_celestialgeneral.pack"); var karlContainer = loader.Load(karlPackPath); diff --git a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs index 39bd802a2..34170e869 100644 --- a/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs +++ b/Shared/SharedCore/Shared.Core/DependencyInjectionContainer.cs @@ -37,7 +37,7 @@ public override void Register(IServiceCollection services) services.AddSingleton(); - services.AddSingleton(); + services.AddTransient(); } } diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs index 35281514c..950656807 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Serialization/CacheDatabase/PackFileContainerCacheHelper.cs @@ -212,9 +212,19 @@ private static string GetDbFilePath(DbContextOptions dbOptions) public static CachedPackFileContainer? TryLoadFromCache(string cacheFilePath, string fingerprint) { + if (!File.Exists(cacheFilePath)) + { + _logger.Here().Information($"Cache file does not exist: {cacheFilePath}"); + return null; + } + try { - return LoadContainerFromCache(cacheFilePath, fingerprint); + _logger.Here().Information($"Attempting to load cache from: {cacheFilePath} with fingerprint: {fingerprint}"); + var result = LoadContainerFromCache(cacheFilePath, fingerprint); + if (result == null) + _logger.Here().Information($"Cache load returned null (fingerprint/schema mismatch) for: {cacheFilePath}"); + return result; } catch (Exception ex) { diff --git a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs index ba7ceff94..8352c43f6 100644 --- a/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs +++ b/Shared/SharedCore/Shared.Core/PackFiles/Utility/PackFileContainerLoader.cs @@ -6,6 +6,7 @@ using Shared.Core.PackFiles.Models.Containers; using Shared.Core.PackFiles.Serialization; using Shared.Core.PackFiles.Serialization.CacheDatabase; +using Shared.Core.Services; using Shared.Core.Settings; namespace Shared.Core.PackFiles.Utility @@ -22,10 +23,14 @@ public class PackFileContainerLoader : IPackFileContainerLoader { static private readonly ILogger _logger = Logging.CreateStatic(typeof(PackFileContainerLoader)); private readonly ApplicationSettingsService _settingsService; + private readonly IStandardDialogs _standardDialogs; + private readonly LocalizationManager _localizationManager; - public PackFileContainerLoader(ApplicationSettingsService settingsService) + public PackFileContainerLoader(ApplicationSettingsService settingsService, IStandardDialogs standardDialogs, LocalizationManager localizationManager) { _settingsService = settingsService; + _standardDialogs = standardDialogs; + _localizationManager = localizationManager; } public IPackFileContainer LoadSystemFolderAsPackFileContainer(string packFileSystemPath) @@ -104,26 +109,44 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri var allCaPackFiles = ManifestHelper.GetPackFilesFromManifest(gameDataFolder, out var manifestFileFound); var fingerprint = PackFileContainerCacheHelper.ComputeFingerprint(gameDataFolder, allCaPackFiles); + _logger.Here().Information($"Computed fingerprint for {gameName}: {fingerprint}"); + var cacheFilePath = PackFileContainerCacheHelper.GetCacheFilePath(gameDataFolder, gameName, fingerprint); + _logger.Here().Information($"Cache file path for {gameName}: {cacheFilePath}"); var cached = PackFileContainerCacheHelper.TryLoadFromCache(cacheFilePath, fingerprint); if (cached != null) { - _logger.Here().Information($"Loading CA packs for {gameName} from cache: {cacheFilePath}"); + _logger.Here().Information($"Cache hit for {gameName} - loaded {cached.GetFileCount()} files from: {cacheFilePath}"); return cached; } - var container = LoadAllCaFilesFromDisk(gameDataFolder, gameName, allCaPackFiles, manifestFileFound); + _logger.Here().Information($"Cache miss for {gameName} at: {cacheFilePath}"); - try - { - var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); - PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); - _logger.Here().Information($"Saved CA pack cache for {gameName} to {cacheFilePath}"); - } - catch (Exception cacheEx) + var cacheInvalidReason = GetCacheInvalidReason(cacheFilePath, fingerprint); + _logger.Here().Information($"Cache invalid reason for {gameName}: {cacheInvalidReason}"); + var reasonMessage = string.Format(_localizationManager.Get("PackFileCache.InvalidReason." + cacheInvalidReason), gameName); + var buildingMessage = string.Format(_localizationManager.Get("PackFileCache.BuildingCache"), gameName); + var cacheDescription = _localizationManager.Get("PackFileCache.Description"); + _standardDialogs.ShowDialogBox( + reasonMessage + "\n\n" + buildingMessage + "\n\n" + cacheDescription, + _localizationManager.Get("PackFileCache.InvalidReason.Title")); + + PackFileContainer container; + using (_standardDialogs.ShowWaitCursor()) { - _logger.Here().Warning($"Failed to save CA pack cache for {gameName}: {cacheEx.Message}"); + container = LoadAllCaFilesFromDisk(gameDataFolder, gameName, allCaPackFiles, manifestFileFound); + + try + { + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); + PackFileContainerCacheHelper.SaveCache(fingerprint, container, dbOptions); + _logger.Here().Information($"Saved CA pack cache for {gameName} to {cacheFilePath}"); + } + catch (Exception cacheEx) + { + _logger.Here().Warning($"Failed to save CA pack cache for {gameName}: {cacheEx.Message}"); + } } return container; @@ -135,7 +158,25 @@ private static void AddFolderContentToPackFile(PackFileContainer container, stri } } + private static string GetCacheInvalidReason(string cacheFilePath, string fingerprint) + { + if (!File.Exists(cacheFilePath)) + return "NotFound"; + + try + { + var dbOptions = PackFileContainerCacheHelper.CreateDbOptions(cacheFilePath); + var result = PackFileContainerCacheHelper.LoadContainerFromCache(dbOptions, fingerprint); + if (result == null) + return "DataChanged"; + } + catch + { + return "Corrupted"; + } + return "DataChanged"; + } private PackFileContainer LoadAllCaFilesFromDisk(string gameDataFolder, string gameName, List allCaPackFiles, bool manifestFileFound) { diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs new file mode 100644 index 000000000..e89e03439 --- /dev/null +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -0,0 +1,177 @@ +using Moq; +using Shared.Core.PackFiles.Utility; +using Shared.Core.Services; +using Shared.Core.Settings; +using Test.TestingUtility.TestUtility; + +namespace Shared.CoreTest.PackFiles.Utility +{ + internal class PackFileContainerLoaderTests + { + private string _tempGameDir; + private Mock _dialogs; + private Mock _waitCursor; + private LocalizationManager _localizationManager; + private ApplicationSettingsService _settingsService; + + [SetUp] + public void Setup() + { + _tempGameDir = Path.Combine(Path.GetTempPath(), "AE_LoaderTest_" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_tempGameDir); + + // Copy the Karl pack file into the temp game dir and touch it to get a unique fingerprint + var karlPackSource = PathHelper.GetDataFile("Karl_and_celestialgeneral.pack"); + var destPath = Path.Combine(_tempGameDir, "Karl_and_celestialgeneral.pack"); + File.Copy(karlPackSource, destPath); + File.SetLastWriteTimeUtc(destPath, DateTime.UtcNow); + + _dialogs = new Mock(); + _waitCursor = new Mock(); + _dialogs.Setup(d => d.ShowWaitCursor()).Returns(_waitCursor.Object); + + _localizationManager = new LocalizationManager(); + + _settingsService = new ApplicationSettingsService(GameTypeEnum.Warhammer3); + _settingsService.CurrentSettings.GameDirectories.Add( + new ApplicationSettings.GamePathPair { Game = GameTypeEnum.Warhammer3, Path = _tempGameDir }); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_tempGameDir)) + Directory.Delete(_tempGameDir, true); + } + + private PackFileContainerLoader CreateLoader() + { + return new PackFileContainerLoader(_settingsService, _dialogs.Object, _localizationManager); + } + + [Test] + public void LoadAllCaFiles_NoCacheExists_ShowsNotFoundDialogAndWaitCursor() + { + var loader = CreateLoader(); + + var result = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + + Assert.That(result, Is.Not.Null); + + // Single combined dialog: reason + building message + _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Once); + + // Should show wait cursor + _dialogs.Verify(d => d.ShowWaitCursor(), Times.Once); + _waitCursor.Verify(w => w.Dispose(), Times.Once); + } + + [Test] + public void LoadAllCaFiles_CacheExists_NoDialogShown() + { + var loader = CreateLoader(); + + // First call builds the cache + var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(firstResult, Is.Not.Null); + + _dialogs.Invocations.Clear(); + + // Second call should use cache - no dialogs + var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(secondResult, Is.Not.Null); + + _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Never); + _dialogs.Verify(d => d.ShowWaitCursor(), Times.Never); + } + + [Test] + public void LoadAllCaFiles_CacheCorrupted_ShowsCorruptedDialog() + { + var loader = CreateLoader(); + + // First call builds the cache + var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(firstResult, Is.Not.Null); + + // Find and corrupt the cache file + var cacheDir = Shared.Core.Misc.DirectoryHelper.CacheDirectory; + var cacheFiles = Directory.GetFiles(cacheDir, "*.db", SearchOption.AllDirectories); + Assert.That(cacheFiles.Length, Is.GreaterThan(0), "Cache file should have been created"); + + // Corrupt all matching cache files + foreach (var cacheFile in cacheFiles) + { + if (cacheFile.Contains("Warhammer")) + File.WriteAllText(cacheFile, "CORRUPTED DATA"); + } + + _dialogs.Invocations.Clear(); + + // Load again — should detect corruption + var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(secondResult, Is.Not.Null); + + // Single combined dialog: reason + building message + _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Once); + _dialogs.Verify(d => d.ShowWaitCursor(), Times.Once); + } + + [Test] + public void LoadAllCaFiles_DataChanged_ShowsDialogsAndRebuildsCache() + { + var loader = CreateLoader(); + + // First call builds the cache + var firstResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(firstResult, Is.Not.Null); + + // Add a new pack file to the game dir to change the fingerprint + var newPackPath = Path.Combine(_tempGameDir, "extra.pack"); + File.Copy(Path.Combine(_tempGameDir, "Karl_and_celestialgeneral.pack"), newPackPath); + + _dialogs.Invocations.Clear(); + + // Load again — fingerprint changed, new fingerprint has no cache file + var secondResult = loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + Assert.That(secondResult, Is.Not.Null); + + // Single combined dialog: reason + building message + _dialogs.Verify(d => d.ShowDialogBox(It.IsAny(), It.IsAny()), Times.Once); + _dialogs.Verify(d => d.ShowWaitCursor(), Times.Once); + } + + [Test] + public void LoadAllCaFiles_WaitCursorActive_DuringBuild() + { + var waitCursorCreated = false; + var waitCursorDisposed = false; + var dialogShownBeforeWaitCursor = false; + + _dialogs.Setup(d => d.ShowDialogBox(It.IsAny(), It.IsAny())) + .Callback(() => + { + // Dialog is shown BEFORE wait cursor is created + if (!waitCursorCreated) + dialogShownBeforeWaitCursor = true; + }); + + _dialogs.Setup(d => d.ShowWaitCursor()) + .Returns(() => + { + waitCursorCreated = true; + return _waitCursor.Object; + }); + + _waitCursor.Setup(w => w.Dispose()) + .Callback(() => waitCursorDisposed = true); + + var loader = CreateLoader(); + loader.LoadAllCaFiles(GameTypeEnum.Warhammer3); + + Assert.That(dialogShownBeforeWaitCursor, Is.True, "Dialog should be shown before wait cursor starts"); + Assert.That(waitCursorCreated, Is.True, "Wait cursor should have been created"); + Assert.That(waitCursorDisposed, Is.True, "Wait cursor should have been disposed"); + } + } +} diff --git a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs index 9c87192c7..cf77e6172 100644 --- a/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs +++ b/Testing/Shared/TestUtility/PackFileSerivceTestHelper.cs @@ -1,5 +1,7 @@ -using Shared.Core.PackFiles; +using Moq; +using Shared.Core.PackFiles; using Shared.Core.PackFiles.Utility; +using Shared.Core.Services; using Shared.Core.Settings; using Test.TestingUtility.TestUtility; @@ -10,7 +12,7 @@ public static class PackFileSerivceTestHelper public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = GameTypeEnum.Warhammer3) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum)); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(gameTypeEnum), new Mock().Object, new LocalizationManager()); var container = loader.LoadSystemFolderAsPackFileContainer(path); container.IsCaPackFile = true; pfs.AddContainer(container); @@ -21,7 +23,7 @@ public static IPackFileService Create(string path, GameTypeEnum gameTypeEnum = G public static IPackFileService CreateFromFolder(GameTypeEnum selectedGame, string path ) { var pfs = new PackFileService(null); - var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame)); + var loader = new PackFileContainerLoader(new ApplicationSettingsService(selectedGame), new Mock().Object, new LocalizationManager()); var container = loader.LoadSystemFolderAsPackFileContainer(PathHelper.GetDataFolder(path)); container.IsCaPackFile = true; From e69b775d0a1280c13f991b069324a78a41078851 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 4 May 2026 20:32:23 +0200 Subject: [PATCH 10/13] Code --- AssetEditor/Language_Cn.json | 2 ++ AssetEditor/Language_En.json | 2 ++ AssetEditor/Language_Fr.json | 1 + AssetEditor/ViewModels/SettingsViewModel.cs | 5 +++++ AssetEditor/Views/Settings/SettingsView.xaml | 7 ++++++- .../KitbasherEditor/Core/MenuBarViews/MenuBarViewModel.cs | 7 ++----- .../Components/Selection/SelectionManager.cs | 8 +++++++- .../GameWorld.Core/Rendering/VertexInstanceMesh.cs | 1 + .../GameWorld.CoreTest/Selection/SelectionManagerTests.cs | 4 +++- .../Shared.Core/Settings/ApplicationSettingsService.cs | 2 ++ 10 files changed, 31 insertions(+), 8 deletions(-) diff --git a/AssetEditor/Language_Cn.json b/AssetEditor/Language_Cn.json index 7837bdedb..bd28f99bc 100644 --- a/AssetEditor/Language_Cn.json +++ b/AssetEditor/Language_Cn.json @@ -11,6 +11,7 @@ "SettingsWindow.RenderBackground": "渲染引擎背景颜色", "SettingsWindow.OnlyLoadLod0": "仅为参考 Mesh 加载 Lod0", "SettingsWindow.GridSize": "网格大小", + "SettingsWindow.VertexSelectionColour": "顶点选择颜色", "SettingsWindow.AudioModding": "音频 Mod", "SettingsWindow.SaveButton": "保存", @@ -97,6 +98,7 @@ "SettingsWindow.Title": "设置", "SettingsWindow.OnlyLoadLod0Tooltip": "在 Kitbash 工具中导入参考 Mesh 时，仅导入 Lod0 以节省内存。", "SettingsWindow.GridSizeTooltip": "可视化编辑器中网格的大小", + "SettingsWindow.VertexSelectionColourTooltip": "选中顶点的十六进制颜色（例如 #FF7800）", "SettingsWindow.GameFiles": "游戏文件", "SettingsWindow.GameDataPathHint": "浏览游戏数据位置，例如 \"C:\\SteamLibrary\\steamapps\\common\\Total War WARHAMMER III\\data\"", "SettingsWindow.WwiseCLI": "Wwise CLI", diff --git a/AssetEditor/Language_En.json b/AssetEditor/Language_En.json index 29d7c688d..368db562b 100644 --- a/AssetEditor/Language_En.json +++ b/AssetEditor/Language_En.json @@ -11,6 +11,7 @@ "SettingsWindow.RenderBackground": "Render engine background colour", "SettingsWindow.OnlyLoadLod0": "Only load Lod0 for reference meshes", "SettingsWindow.GridSize": "Grid Size", + "SettingsWindow.VertexSelectionColour": "Vertex Selection Colour", "SettingsWindow.AudioModding": "Audio Modding", "SettingsWindow.SaveButton": "Save", @@ -97,6 +98,7 @@ "SettingsWindow.Title": "Settings", "SettingsWindow.OnlyLoadLod0Tooltip": "When importing a reference mesh in the Kitbash tool, only import Lod0 in order to save memory.", "SettingsWindow.GridSizeTooltip": "Size of the grid in visual editors", + "SettingsWindow.VertexSelectionColourTooltip": "Hex colour for selected vertices (e.g. #FF7800)", "SettingsWindow.GameFiles": "Game Files", "SettingsWindow.GameDataPathHint": "Browse for game data location e.g. \"C:\\SteamLibrary\\steamapps\\common\\Total War WARHAMMER III\\data\"", "SettingsWindow.WwiseCLI": "Wwise CLI", diff --git a/AssetEditor/Language_Fr.json b/AssetEditor/Language_Fr.json index b905694f0..23ccc7b60 100644 --- a/AssetEditor/Language_Fr.json +++ b/AssetEditor/Language_Fr.json @@ -10,6 +10,7 @@ "SettingsWindow.RenderBackground": "Couleur d’arrière-plan du moteur de rendu", "SettingsWindow.OnlyLoadLod0": "Charger uniquement le Lod0 pour les maillages de référence", "SettingsWindow.GridSize": "Taille de la grille", + "SettingsWindow.VertexSelectionColour": "Couleur de sélection des sommets", "SettingsWindow.AudioModding": "Modding audio", "SettingsWindow.SaveButton": "Enregistrer", diff --git a/AssetEditor/ViewModels/SettingsViewModel.cs b/AssetEditor/ViewModels/SettingsViewModel.cs index 07876244f..c3b7a2120 100644 --- a/AssetEditor/ViewModels/SettingsViewModel.cs +++ b/AssetEditor/ViewModels/SettingsViewModel.cs @@ -6,9 +6,11 @@ using System.Windows.Input; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; +using Microsoft.Xna.Framework; using Shared.Core.Misc; using Shared.Core.Services; using Shared.Core.Settings; +using Shared.Ui.BaseDialogs.ColourPickerButton; namespace AssetEditor.ViewModels { @@ -28,6 +30,7 @@ partial class SettingsViewModel : ObservableObject [ObservableProperty] private ThemeType _currentTheme; [ObservableProperty] private BackgroundColour _currentRenderEngineBackgroundColour; [ObservableProperty] private int _visualEditorsGridSize; + [ObservableProperty] private ColourPickerViewModel _vertexSelectionColour; [ObservableProperty] private bool _startMaximised; [ObservableProperty] private GameTypeEnum _currentGame; [ObservableProperty] private bool _loadCaPacksByDefault; @@ -49,6 +52,7 @@ public SettingsViewModel(ApplicationSettingsService settingsService, Localizatio RenderEngineBackgroundColours = new ObservableCollection((BackgroundColour[])Enum.GetValues(typeof(BackgroundColour))); CurrentRenderEngineBackgroundColour = _settingsService.CurrentSettings.RenderEngineBackgroundColour; VisualEditorsGridSize = _settingsService.CurrentSettings.VisualEditorsGridSize; + VertexSelectionColour = new ColourPickerViewModel(_settingsService.CurrentSettings.VertexSelectionColour); CameraModes = new ObservableCollection((CameraControlMode[])Enum.GetValues(typeof(CameraControlMode))); StartMaximised = _settingsService.CurrentSettings.StartMaximised; @@ -79,6 +83,7 @@ private void Save() _settingsService.CurrentSettings.Theme = CurrentTheme; _settingsService.CurrentSettings.RenderEngineBackgroundColour = CurrentRenderEngineBackgroundColour; _settingsService.CurrentSettings.VisualEditorsGridSize = VisualEditorsGridSize; + _settingsService.CurrentSettings.VertexSelectionColour = VertexSelectionColour.SelectedColour; _settingsService.CurrentSettings.StartMaximised = StartMaximised; _settingsService.CurrentSettings.CurrentGame = CurrentGame; _settingsService.CurrentSettings.LoadCaPacksByDefault = LoadCaPacksByDefault; diff --git a/AssetEditor/Views/Settings/SettingsView.xaml b/AssetEditor/Views/Settings/SettingsView.xaml index b3ce1ef27..4d74aa71f 100644 --- a/AssetEditor/Views/Settings/SettingsView.xaml +++ b/AssetEditor/Views/Settings/SettingsView.xaml @@ -7,7 +7,8 @@ xmlns:gameinfo="clr-namespace:Shared.Core.Services;assembly=Shared.Core" xmlns:behaviors="clr-namespace:Shared.Ui.Common.Behaviors;assembly=Shared.Ui" xmlns:loc="clr-namespace:Shared.Ui.Common;assembly=Shared.Ui" - xmlns:local="clr-namespace:AssetEditor.Views.Settings" xmlns:viewmodels="clr-namespace:AssetEditor.ViewModels" + xmlns:local="clr-namespace:AssetEditor.Views.Settings" xmlns:viewmodels="clr-namespace:AssetEditor.ViewModels" + xmlns:colourpickerbutton="clr-namespace:Shared.Ui.BaseDialogs.ColourPickerButton;assembly=Shared.Ui" d:DataContext="{d:DesignInstance Type=viewmodels:SettingsViewModel}" mc:Ignorable="d" > @@ -98,6 +99,7 @@ + @@ -113,6 +115,9 @@ + + + diff --git a/Editors/Kitbashing/KitbasherEditor/Core/MenuBarViews/MenuBarViewModel.cs b/Editors/Kitbashing/KitbasherEditor/Core/MenuBarViews/MenuBarViewModel.cs index 0d19fdbc8..0999442cd 100644 --- a/Editors/Kitbashing/KitbasherEditor/Core/MenuBarViews/MenuBarViewModel.cs +++ b/Editors/Kitbashing/KitbasherEditor/Core/MenuBarViews/MenuBarViewModel.cs @@ -1,5 +1,4 @@ -using System.Collections.ObjectModel; -using System.Diagnostics; +using System.Collections.ObjectModel; using System.Windows.Input; using Editors.KitbasherEditor.ChildEditors.MeshFitter; using Editors.KitbasherEditor.ChildEditors.PhotoStudio; @@ -30,16 +29,14 @@ public class MenuBarViewModel : IKeyboardHandler private readonly CommandExecutor _commandExecutor; private readonly MenuItemVisibilityRuleEngine _menuItemVisibilityRuleEngine; private readonly ActionHotkeyHandler _hotKeyHandler = new ActionHotkeyHandler(); - private readonly IWindowsKeyboard _keyboard; private readonly LocalizationManager _localizationManager; private readonly Dictionary _uiCommands = new(); - public MenuBarViewModel(CommandExecutor commandExecutor, IEventHub eventHub, MenuItemVisibilityRuleEngine menuItemVisibilityRuleEngine, TransformToolViewModel transformToolViewModel, IUiCommandFactory uiCommandFactory, IWindowsKeyboard windowKeyboard, LocalizationManager localizationManager) + public MenuBarViewModel(CommandExecutor commandExecutor, IEventHub eventHub, MenuItemVisibilityRuleEngine menuItemVisibilityRuleEngine, TransformToolViewModel transformToolViewModel, IUiCommandFactory uiCommandFactory, LocalizationManager localizationManager) { _commandExecutor = commandExecutor; _menuItemVisibilityRuleEngine = menuItemVisibilityRuleEngine; _uiCommandFactory = uiCommandFactory; - _keyboard = windowKeyboard; _localizationManager = localizationManager; TransformTool = transformToolViewModel; diff --git a/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs b/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs index acbd870f2..699ac73bb 100644 --- a/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs +++ b/GameWorld/GameWorldCore/GameWorld.Core/Components/Selection/SelectionManager.cs @@ -7,6 +7,7 @@ using GameWorld.Core.Utility; using Microsoft.Xna.Framework; using Shared.Core.Events; +using Shared.Core.Settings; namespace GameWorld.Core.Components.Selection { @@ -31,6 +32,7 @@ public class SelectionManager : BaseComponent, IDisposable private readonly IScopedResourceLibrary _resourceLib; private readonly IDeviceResolver _deviceResolverComponent; private readonly IGraphicsResourceCreator _graphicsResourceCreator; + private readonly ApplicationSettingsService _settingsService; private (int v0, int v1)[] _cachedEdgeIndices; private Rmv2MeshNode _cachedEdgeMesh; @@ -43,13 +45,14 @@ public class SelectionManager : BaseComponent, IDisposable const int MaxRenderEdges = 50000; private readonly EdgeData[] _edgeDataCache = new EdgeData[MaxRenderEdges]; - public SelectionManager(IEventHub eventHub, RenderEngineComponent renderEngine, IScopedResourceLibrary resourceLib, IDeviceResolver deviceResolverComponent, IGraphicsResourceCreator graphicsResourceCreator) + public SelectionManager(IEventHub eventHub, RenderEngineComponent renderEngine, IScopedResourceLibrary resourceLib, IDeviceResolver deviceResolverComponent, IGraphicsResourceCreator graphicsResourceCreator, ApplicationSettingsService settingsService) { _eventHub = eventHub; _renderEngine = renderEngine; _resourceLib = resourceLib; _deviceResolverComponent = deviceResolverComponent; _graphicsResourceCreator = graphicsResourceCreator; + _settingsService = settingsService; } public override void Initialize() @@ -119,6 +122,9 @@ private void SelectionManager_SelectionChanged(ISelectionState state, bool sendE public override void Draw(GameTime gameTime) { + if (_vertexRenderer != null) + _vertexRenderer.SelectionColour = _settingsService.CurrentSettings.VertexSelectionColour; + var selectionState = GetState(); if (selectionState is ObjectSelectionState objectSelectionState) diff --git a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs index 79b30ed73..b4864ced0 100644 --- a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs +++ b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs @@ -52,6 +52,7 @@ public class VertexInstanceMesh : IDisposable Vector3 _selectedColour = new(1.0f, 0.47f, 0.0f); Vector3 _deselectedColour = new(0.0f, 0.0f, 0.0f); + public Vector3 SelectionColour { get => _selectedColour; set => _selectedColour = value; } public float VertexPixelSize { get; set; } = 5.5f; public float SelectedSizeBoost { get; set; } = 2.0f; public float SelectionThresholdMultiplier { get; set; } = 2.0f; diff --git a/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs b/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs index 753dcd651..a7ebed419 100644 --- a/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs +++ b/GameWorld/GameWorldCore/GameWorld.CoreTest/Selection/SelectionManagerTests.cs @@ -9,6 +9,7 @@ using Microsoft.Xna.Framework; using Moq; using Shared.Core.Events; +using Shared.Core.Settings; using Shared.GameFormats.RigidModel.MaterialHeaders; using System.Reflection; using Test.GameWorld.Core.WsMaterialTemplate; @@ -44,7 +45,8 @@ public void SetUp() _renderEngine, null, null, - null); + null, + new ApplicationSettingsService()); // Set render items that are normally created in Initialize() (requires GPU) SetPrivateField(_selectionManager, "_edgeQuadRenderItem", new EdgeQuadRenderItem()); diff --git a/Shared/SharedCore/Shared.Core/Settings/ApplicationSettingsService.cs b/Shared/SharedCore/Shared.Core/Settings/ApplicationSettingsService.cs index 623a97fcb..25fb5125d 100644 --- a/Shared/SharedCore/Shared.Core/Settings/ApplicationSettingsService.cs +++ b/Shared/SharedCore/Shared.Core/Settings/ApplicationSettingsService.cs @@ -1,5 +1,6 @@ using System.Collections.ObjectModel; using System.Text.Json; +using Microsoft.Xna.Framework; using Shared.Core.ErrorHandling; using Shared.Core.Misc; @@ -32,6 +33,7 @@ public class GamePathPair public string WwisePath { get; set; } public bool OnlyLoadLod0ForReferenceMeshes { get; set; } = true; public int VisualEditorsGridSize { get; set; } = 10; + public Vector3 VertexSelectionColour { get; set; } = new Vector3(1.0f, 0.47f, 0.0f); public string SelectedLangauge { get; set; } = "en"; public CameraControlMode CameraControlMode { get; set; } = CameraControlMode.AssetEditorStyle; From 6eb42591433372085d6fb36a3fc80801b90b5d90 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 4 May 2026 20:45:32 +0200 Subject: [PATCH 11/13] Code --- .../GameWorld.Core/Rendering/VertexInstanceMesh.cs | 3 +-- Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs index b4864ced0..c1bcfffcd 100644 --- a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs +++ b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs @@ -49,10 +49,9 @@ public class VertexInstanceMesh : IDisposable readonly int _maxInstanceCount = 50000; int _currentInstanceCount; - Vector3 _selectedColour = new(1.0f, 0.47f, 0.0f); Vector3 _deselectedColour = new(0.0f, 0.0f, 0.0f); - public Vector3 SelectionColour { get => _selectedColour; set => _selectedColour = value; } + public Vector3 SelectionColour { get; set; } = new Vector3(1.0f, 0.47f, 0.0f); public float VertexPixelSize { get; set; } = 5.5f; public float SelectedSizeBoost { get; set; } = 2.0f; public float SelectionThresholdMultiplier { get; set; } = 2.0f; diff --git a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs index 5d20dc7db..8c71f84e9 100644 --- a/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs +++ b/Shared/SharedCore/Shared.Core/Misc/DirectoryHelper.cs @@ -41,7 +41,6 @@ public class DirectoryHelper public static void EnsureCreated() { - EnsureCreated(ApplicationDirectory); EnsureCreated(ApplicationDirectory); EnsureCreated(SchemaDirectory); EnsureCreated(LogDirectory); From f45267ce6a9b472779dc41d7a8a6c8114a97629e Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 4 May 2026 20:53:38 +0200 Subject: [PATCH 12/13] Code fixes --- AssetEditor/AssetEditor.csproj | 2 +- .../Test.KitbashEditor/PinTool/SkinWrapRiggingCommandTests.cs | 2 +- .../GameWorld.Core/Rendering/VertexInstanceMesh.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AssetEditor/AssetEditor.csproj b/AssetEditor/AssetEditor.csproj index 5771acb38..8aafb3bba 100644 --- a/AssetEditor/AssetEditor.csproj +++ b/AssetEditor/AssetEditor.csproj @@ -48,7 +48,7 @@ https://github.com/donkeyProgramming/TheAssetEditor https://github.com/donkeyProgramming/TheAssetEditor AssetEditor - 0.71.0 + 0.72.0 6.0 diff --git a/Editors/Kitbashing/Test.KitbashEditor/PinTool/SkinWrapRiggingCommandTests.cs b/Editors/Kitbashing/Test.KitbashEditor/PinTool/SkinWrapRiggingCommandTests.cs index f695e13e4..9843abe32 100644 --- a/Editors/Kitbashing/Test.KitbashEditor/PinTool/SkinWrapRiggingCommandTests.cs +++ b/Editors/Kitbashing/Test.KitbashEditor/PinTool/SkinWrapRiggingCommandTests.cs @@ -41,7 +41,7 @@ public void Setup() // SelectionManager has complex constructor and non-virtual methods; // use reflection to set _currentState so GetStateCopy() works. _selectionManagerMock = new Mock(MockBehavior.Loose, - null!, null!, null!, null!, null!); + null!, null!, null!, null!, null!, null!); var selectionState = Mock.Of(s => s.Clone() == Mock.Of()); var stateField = typeof(SelectionManager).GetField("_currentState", BindingFlags.NonPublic | BindingFlags.Instance)!; diff --git a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs index c1bcfffcd..0a7b7123b 100644 --- a/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs +++ b/GameWorld/GameWorldCore/GameWorld.Core/Rendering/VertexInstanceMesh.cs @@ -108,7 +108,7 @@ public void Update(MeshObject geo, Matrix modelMatrix, Vector3 cameraPos, var distance = (cameraPos - vertPos).Length(); var weight = selectedVertexes.VertexWeights[i]; - var color = Vector3.Lerp(_deselectedColour, _selectedColour, weight); + var color = Vector3.Lerp(_deselectedColour, SelectionColour, weight); var effectivePixelSize = VertexPixelSize + weight * SelectedSizeBoost; var worldScale = effectivePixelSize * distance * fovScale; From 978d343e8f7463ec82a58120db4fc10cdaf589b1 Mon Sep 17 00:00:00 2001 From: ole kristian homelien Date: Mon, 4 May 2026 21:06:15 +0200 Subject: [PATCH 13/13] unit test fixes --- .../PackFiles/Utility/PackFileContainerLoaderTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs index e89e03439..1f0c835db 100644 --- a/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs +++ b/Shared/SharedCore/Shared.CoreTest/PackFiles/Utility/PackFileContainerLoaderTests.cs @@ -1,4 +1,5 @@ using Moq; +using Shared.Core.Misc; using Shared.Core.PackFiles.Utility; using Shared.Core.Services; using Shared.Core.Settings; @@ -17,6 +18,8 @@ internal class PackFileContainerLoaderTests [SetUp] public void Setup() { + DirectoryHelper.EnsureCreated(); + _tempGameDir = Path.Combine(Path.GetTempPath(), "AE_LoaderTest_" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_tempGameDir);