VFS: Support Encodingless CASC + A bunch of TVFS fixups

TACTLib/Client/ClientCreateArgs.cs

@@ -116,13 +116,14 @@ public bool Online {
 
         public ICDNClient? CustomCDNClient { get; set; } = null;
         public NGDPClientBase? CustomNGDPClient { get; set; } = null;
-        
+
         public bool LoadCDNIndices { get; set; } = true;
         public ClientHandler? TryShareCDNIndexWithHandler { get; set; } = null;
         public bool ParallelCDNIndexLoading { get; set; } = false;
         public int MaxCDNIndexLoadingParallelism { get; set; } = 4;
-        
+
         public bool LoadRoot { get; set; } = true;
         public bool LoadVFS { get; set; } = true;
+        public bool LoadAllVFS { get; set; } = false;
     }
-}
+}

TACTLib/Client/ClientHandler.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
 using TACTLib.Agent;
 using TACTLib.Agent.Protobuf;
@@ -110,6 +111,11 @@
 
             var staticBuildConfigPath = Path.Combine(BasePath, "data", ".build.config"); // todo: um
             IsStaticContainer = File.Exists(staticBuildConfigPath);
+            if (!IsStaticContainer) {
+                staticBuildConfigPath = Path.Combine(BasePath, "Data", ".build.config"); // todo: um x2 (thx fenris.)
+                IsStaticContainer = File.Exists(staticBuildConfigPath);
+            }
+
             if (IsStaticContainer) {
                 if (CreateArgs.VersionSource != ClientCreateArgs.InstallMode.Local) throw new Exception("only local version sources are supported for static containers (steam)");
                 CreateArgs.Online = false;
@@ -140,7 +146,7 @@
                 } else if (CreateArgs.OnlineRootHost.StartsWith("https:")) {
                     ngdpClient = new RibbitHttpClient(CreateArgs.OnlineRootHost);
                 } else if (CreateArgs.OnlineRootHost.StartsWith("ribbit:")) {
                     ngdpClient = new RibbitTcpClient(CreateArgs.OnlineRootHost);
                 } else {
                     ngdpClient = new NGDPClient(CreateArgs.OnlineRootHost);
                 }
@@ -162,7 +168,7 @@
             if (CreateArgs.OverrideVersionName != null) {
                 InstallationInfo.Values["Version"] = CreateArgs.OverrideVersionName;
             }
-            
+
             if (CreateArgs.Online) {
                 CDNClient = createArgs.CustomCDNClient ?? new HttpCDNClient(null);
                 CDNClient.SetClientHandler(this);
@@ -199,7 +205,7 @@
             if (CreateArgs.UseContainer) {
                 Logger.Info("CASC", "Initializing...");
                 if (IsStaticContainer) {
-                    ContainerHandler = new StaticContainerHandler(this);
+                    ContainerHandler = CreateStaticContainerHandler() ?? throw new NotImplementedException($"Product \"{Product}\" as static container is not supported.");
                 } else {
                     using var _ = new PerfCounter("ContainerHandler::ctor`ClientHandler");
                     ContainerHandler = new ContainerHandler(this);
@@ -224,13 +230,28 @@
             // for testing local cdn index init but remote data:
             //ContainerHandler = null;
 
-            using (var _ = new PerfCounter("EncodingHandler::ctor`ClientHandler"))
+            if (ConfigHandler.BuildConfig.Encoding != null) {
+                using var _ = new PerfCounter("EncodingHandler::ctor`ClientHandler");
                 EncodingHandler = new EncodingHandler(this);
+            }
 
             if (ConfigHandler.BuildConfig.VFSRoot != null && CreateArgs.LoadVFS) {
-                using var _ = new PerfCounter("VFSFileTree::ctor`ClientHandler");
-                using var vfsStream = OpenCKey(ConfigHandler.BuildConfig.VFSRoot!.ContentKey)!;
-                VFS = new VFSFileTree(this, vfsStream);
+                using (new PerfCounter("ClientHandler::ctor``VFS")) {
+                    VFS = new VFSFileTree(this);
+                    foreach (var vfs in ConfigHandler.BuildConfig.VFS) {
+                        using var _ = new PerfCounter("VFSFileTree::ctor`ClientHandler");
+                        using var vfsStream =
+                            OpenCKey(vfs.File.ContentKey) ??
+                            OpenEKey(vfs.File.EncodingKey, vfs.Size?.EncodedSize ?? 0, vfs.Spec);
+                        if (vfsStream != null) {
+                            VFS.Load(vfsStream);
+                        }
+
+                        if (!CreateArgs.LoadAllVFS) {
+                            break;
+                        }
+                    }
+                }
             }
 
             if (CreateArgs.LoadRoot)
@@ -243,7 +264,12 @@
 
         public IProductHandler? CreateProductHandler() {
             using var _ = new PerfCounter("ProductHandlerFactory::GetHandler`TACTProduct`ClientHandler`Stream");
-            return ProductHandlerFactory.GetHandler(Product, this, OpenCKey(ConfigHandler.BuildConfig.Root.ContentKey));
+            return ProductHandlerFactory.GetHandler(Product, this, ConfigHandler.BuildConfig.Root is {} root ? OpenCKey(root.ContentKey) : null);
+        }
+
+        public IContainerHandler? CreateStaticContainerHandler() {
+            using var _ = new PerfCounter("StaticContainerHandlerFactory::GetHandler`TACTProduct`ClientHandler");
+            return StaticContainerHandlerFactory.GetHandler(Product, this);
         }
 
         private bool CanShareCDNData([NotNullWhen(true)] ClientHandler? other) {
@@ -270,8 +296,8 @@
             return true;
         }
 
-        public Stream? OpenEKey(FullEKey fullEKey, int eSize) {  // ekey = value of ckey in encoding table
-            var fromContainer = TryOpenEKeyFromContainer(fullEKey, eSize);
+        public Stream? OpenEKey(FullEKey fullEKey, int eSize, string? espec) {  // ekey = value of ckey in encoding table
+            var fromContainer = TryOpenEKeyFromContainer(fullEKey, eSize, espec);
             if (fromContainer != null) return fromContainer;
 
             return TryOpenEKeyFromRemote(fullEKey, eSize);
@@ -301,17 +327,17 @@
             if (EncodingHandler == null) return null; // cant get here but okay
 
             foreach (var ekey in eKeys) {
-                var fromContainer = TryOpenEKeyFromContainer(ekey, EncodingHandler.GetEncodedSize(ekey));
+                var fromContainer = TryOpenEKeyFromContainer(ekey, EncodingHandler.GetEncodedSize(ekey), null);
                 if (fromContainer != null) return fromContainer;
             }
             return null;
         }
 
-        private Stream? TryOpenEKeyFromContainer(FullEKey fullEKey, int eSize) {
+        private Stream? TryOpenEKeyFromContainer(FullEKey fullEKey, int eSize, string? espec) {
             if (ContainerHandler == null) return null;
             if (!ContainerHandler.CheckResidency(fullEKey)) return null;
             try {
-                var cascBlte = OpenEKeyFromContainer(fullEKey, eSize);
+                var cascBlte = OpenEKeyFromContainer(fullEKey, eSize, espec);
                 if (cascBlte != null) return cascBlte;
             } catch (Exception e) {
                 if (e is BLTEKeyException) throw;
@@ -321,11 +347,11 @@
             return null;
         }
 
-        private Stream? OpenEKeyFromContainer(FullEKey fullEKey, int eSize) {  // ekey = value of ckey in encoding table
+        private Stream? OpenEKeyFromContainer(FullEKey fullEKey, int eSize, string? espec) {  // ekey = value of ckey in encoding table
             if (ContainerHandler == null) return null;
             var fromContainer = ContainerHandler.OpenEKey(fullEKey, eSize);
             if (fromContainer == null) throw new Exception($"failed to load local file {fullEKey.ToHexString()} (it was marked resident)");
-            return TryDecodeToStream(fromContainer);
+            return TryDecodeToStream(fromContainer, espec);
         }
 
         private Stream? TryOpenEKeyFromRemote(FullEKey fullEKey, int eSize) {
@@ -346,20 +372,41 @@
             if (!CDNIndex.TryGetIndexEntry(fullEKey, out var cdnIdx)) return null;
             var encodedData = CDNIndex.OpenIndexEntry(cdnIdx);
             if (encodedData == null) throw new Exception($"failed to fetch archived cdn file {fullEKey.ToHexString()}");
-            return TryDecodeToStream(encodedData);
+            return TryDecodeToStream(encodedData, null);
         }
 
         private Stream? TryOpenRemoteLooseFile(FullEKey fullKey) {
             if (CDNIndex == null) return null;
             if (!CDNIndex.IsLooseFile(fullKey)) return null;
             var encodedData = CDNClient!.FetchLooseData(fullKey);
             if (encodedData == null) throw new Exception($"failed to fetch loose cdn file {fullKey.ToHexString()}");
-            return TryDecodeToStream(encodedData);
+            return TryDecodeToStream(encodedData, null);
         }
 
-        private MemoryStream? TryDecodeToStream(ArraySegment<byte>? data) {
-            if (data == null) return null;
-            return new MemoryStream(BLTEDecoder.Decode(this, data.Value.AsSpan()), false);
+        private MemoryStream? TryDecodeToStream(ArraySegment<byte>? data, string? espec) {
+            if (data == null) {
+                return null;
+            }
+
+            if (espec is null or "b" || espec.StartsWith("b:")) {
+                return new MemoryStream(BLTEDecoder.Decode(this, data.Value.AsSpan()), false);
+            }
+
+            if (espec == "z" || espec.StartsWith("z:")) {
+                var mem = data.Value.AsMemory();
+                using var pin = mem.Pin();
+                unsafe {
+                    using var ums = new UnmanagedMemoryStream((byte*) pin.Pointer, mem.Length);
+                    using var z = new ZLibStream(ums, CompressionMode.Decompress);
+                    var stream = new MemoryStream();
+                    z.CopyTo(stream);
+                    stream.Position = 0;
+                    return stream;
+                }
+            }
+
+            // todo: whag
+            return null;
         }
 
         public Stream? OpenConfigKey(string key) {

TACTLib/Config/BuildConfig.cs

@@ -1,76 +1,105 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Globalization;
 using System.IO;
+using System.Linq;
 
 namespace TACTLib.Config {
+
     public class BuildConfig : Config {
-        public FileRecord Root;
+        public FileRecord? Root;
         public FileRecord? Install;
         public FileRecord? Patch;
         public FileRecord? Download;
-        public FileRecord Encoding;
+        public FileRecord? Encoding;
         public SizeRecord? EncodingSize;
-        public FileRecord? VFSRoot;
-
-        public string GetBuildName() => Values["build-name"][0];
-
+        public VFSRecord? VFSRoot;
+        public List<VFSRecord> VFS;
+
+        public string GetBuildName() => (Values.TryGetValue("build-name", out var buildName) ? buildName.FirstOrDefault() : null) ?? "Unknown";
+
         public BuildConfig(Stream? stream) : base(stream) {
-            GetFileRecord("root", out var root);
-            GetFileRecord("install", out Install);
-            GetFileRecord("patch", out Patch);
-            GetFileRecord("download", out Download);
-            GetFileRecord("encoding", out var encoding);
-            GetSizeRecord("encoding-size", out EncodingSize);
-            GetFileRecord("vfs-root", out VFSRoot);
-
-            if (root == null) throw new NullReferenceException(nameof(root));
-            Root = root;
-
-            if (encoding == null) throw new NullReferenceException(nameof(encoding));
-            Encoding = encoding;
+            TryGetFileRecord("root", out Root);
+            TryGetFileRecord("install", out Install);
+            TryGetFileRecord("patch", out Patch);
+            TryGetFileRecord("download", out Download);
+            TryGetFileRecord("encoding", out Encoding);
+            TryGetSizeRecord("encoding-size", out EncodingSize);
+            BuildVFS();
         }
 
-        private void GetFileRecord(string key, out FileRecord? @out) {
-            if (!Values.TryGetValue(key, out var list)) {
-                @out = null;
+        private void BuildVFS() {
+            VFS = [];
+
+            if (!TryGetVFSRecord("vfs-root", out VFSRoot)) {
                 return;
             }
-            @out = GetFileRecord(list);
+
+            VFS.Add(VFSRoot);
+            var index = 1;
+            while (TryGetVFSRecord("vfs-" + index++, out var vfs)) {
+                VFS.Add(vfs);
+            }
         }
-        
-        private void GetSizeRecord(string key, out SizeRecord? @out) {
+
+        private bool TryGetFileRecord(string key, [MaybeNullWhen(false)] out FileRecord record) {
             if (!Values.TryGetValue(key, out var list)) {
-                @out = null;
-                return;
+                record = null;
+                return false;
             }
-            @out = new SizeRecord {
-                ContentSize = int.Parse(list[0]),
-                EncodedSize = int.Parse(list[1])
-            };
-        }
 
-        private static FileRecord GetFileRecord(IReadOnlyList<string> vals) {
-            FileRecord record = new FileRecord();
+            record = GetFileRecord(list);
+            return true;
+        }
 
-            if (vals.Count > 0) {
-                record.ContentKey = CKey.FromString(vals[0]);
+        private bool TryGetSizeRecord(string key, [MaybeNullWhen(false)] out SizeRecord record) {
+            if (!Values.TryGetValue(key, out var list)) {
+                record = null;
+                return false;
             }
 
-            if (vals.Count > 1) {
-                record.EncodingKey = FullEKey.FromString(vals[1]);
+            record = GetSizeRecord(list);
+            return true;
+        }
+
+        private bool TryGetVFSRecord(string key, [MaybeNullWhen(false)] out VFSRecord record) {
+            if (!TryGetFileRecord(key, out var file)) {
+                record = null;
+                return false;
             }
-
-            return record;
+
+            record = new VFSRecord {
+                File = file,
+                Size = TryGetSizeRecord(key + "-size", out var size) ? size : default,
+                Spec = Values.TryGetValue(key + "-espec", out var list) ? string.Join(" ", list) : default,
+            };
+            return true;
+        }
+
+        private static FileRecord GetFileRecord(List<string> vals) => new() {
+            ContentKey = vals.Count > 0 ? CKey.FromString(vals[0]) : default,
+            EncodingKey = vals.Count > 1 ? CKey.FromString(vals[1]) : default,
+        };
+
+        private static SizeRecord GetSizeRecord(List<string> vals) => new() {
+            ContentSize = vals.Count > 0 ? int.Parse(vals[0], NumberStyles.Integer, CultureInfo.InvariantCulture) : default,
+            EncodedSize = vals.Count > 1 ? int.Parse(vals[1], NumberStyles.Integer, CultureInfo.InvariantCulture) : default,
+        };
+
+        public record FileRecord {
+            public CKey ContentKey { get; init; }
+            public FullEKey EncodingKey { get; init; }
         }
 
-        public class FileRecord {
-            public CKey ContentKey;
-            public FullEKey EncodingKey;
+        public record SizeRecord {
+            public int ContentSize { get; init; }
+            public int EncodedSize { get; init; }
         }
 
-        public class SizeRecord {
-            public int ContentSize;
-            public int EncodedSize;
+        public record VFSRecord {
+            public required FileRecord File { get; init; }
+            public SizeRecord? Size { get; init; }
+            public string? Spec { get; init; }
         }
     }
-}
+}