From 63fd4add2e7f4d9e02a668ed5aefe054b0f7b4df Mon Sep 17 00:00:00 2001 From: SurfSam Date: Thu, 17 Sep 2026 09:50:24 +0200 Subject: [PATCH 01/13] move from materials-per-map to global, hash validated materials folder with per-map fallback --- SourceUtils.WebExport/Bsp/Materials.cs | 20 +- SourceUtils.WebExport/Material.cs | 15 +- SourceUtils.WebExport/Program.cs | 1 + .../SourceUtils.WebExport.csproj | 1 + SourceUtils.WebExport/Texture.cs | 85 ++---- SourceUtils.WebExport/TextureSource.cs | 268 ++++++++++++++++++ SourceUtils/ValveBsp/GameLump.cs | 5 +- 7 files changed, 320 insertions(+), 75 deletions(-) create mode 100644 SourceUtils.WebExport/TextureSource.cs diff --git a/SourceUtils.WebExport/Bsp/Materials.cs b/SourceUtils.WebExport/Bsp/Materials.cs index 303fa33c..b39edc8a 100644 --- a/SourceUtils.WebExport/Bsp/Materials.cs +++ b/SourceUtils.WebExport/Bsp/Materials.cs @@ -58,22 +58,22 @@ public MaterialPage GetPage( [Url] string map, [Url] int index ) prop.Type = MaterialPropertyType.TextureIndex; var texUrl = (Url) prop.Value; - int texIndex; - if ( texDict.TryGetValue( texUrl, out texIndex ) ) - { - prop.Value = texIndex; - continue; - } + if (texDict.TryGetValue(texUrl, out int texIndex)) + { + prop.Value = texIndex; + continue; + } - prop.Value = texIndex = page.Textures.Count; + prop.Value = texIndex = page.Textures.Count; - var texPath = TextureController.GetTexturePath( texUrl ); - var tex = Texture.Get( bsp, texPath ); + var tex = TextureSource.TryParseUrl( texUrl, out var texPath, out _, out _ ) + ? Texture.Get( bsp, texPath ) + : null; if ( tex == null ) { Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine($"Missing texture '{texPath}'!"); + Console.WriteLine($"Missing texture '{texPath ?? (string) texUrl}'!"); Console.ResetColor(); } diff --git a/SourceUtils.WebExport/Material.cs b/SourceUtils.WebExport/Material.cs index a812d305..78979648 100644 --- a/SourceUtils.WebExport/Material.cs +++ b/SourceUtils.WebExport/Material.cs @@ -121,13 +121,14 @@ private static Url GetTextureUrl(string path, string vmtPath, ValveBspFile bsp) if (!path.EndsWith(".vtf")) path = $"{path}.vtf"; path = !path.Contains('/') ? $"{Path.GetDirectoryName(vmtPath)}/{path}" : $"materials/{path}"; + path = TextureSource.NormalizePath(path); + if ( path == null ) return null; - if (bsp != null && bsp.PakFile.ContainsFile(path)) - { - return $"/maps/{bsp.Name}/{path}.json"; - } + var info = TextureSource.Resolve(bsp, path); - return $"/{path}.json"; + return info == null + ? TextureSource.BuildUrl(path, null, null) + : TextureSource.BuildUrl(info.Path, info.Hash, null); } private static void AddMaterialProperties(Material mat, ValveMaterialFile vmt, string vmtPath, ValveBspFile bsp) @@ -324,7 +325,9 @@ public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) { hdrCompressed = texProp.Name == "hdrcompressedtexture"; - var tex = Texture.Get( bsp, TextureController.GetTexturePath( (Url) texProp.Value ) ); + var tex = TextureSource.TryParseUrl( (Url) texProp.Value, out var texPath, out _, out _ ) + ? Texture.Get( bsp, texPath ) + : null; aspect = tex == null ? 1f : (float) tex.Width / tex.Height; } diff --git a/SourceUtils.WebExport/Program.cs b/SourceUtils.WebExport/Program.cs index dabbd650..8397b15b 100644 --- a/SourceUtils.WebExport/Program.cs +++ b/SourceUtils.WebExport/Program.cs @@ -112,6 +112,7 @@ public static void UnloadMap( string name ) if ( !_sOpenMaps.TryGetValue( name, out var map ) ) return; _sOpenMaps.Remove( name ); + TextureSource.ForgetMap( map.Name ); map.Dispose(); } diff --git a/SourceUtils.WebExport/SourceUtils.WebExport.csproj b/SourceUtils.WebExport/SourceUtils.WebExport.csproj index 67a7db1a..81f0eb78 100644 --- a/SourceUtils.WebExport/SourceUtils.WebExport.csproj +++ b/SourceUtils.WebExport/SourceUtils.WebExport.csproj @@ -122,6 +122,7 @@ + diff --git a/SourceUtils.WebExport/Texture.cs b/SourceUtils.WebExport/Texture.cs index d422b50b..9bf8d728 100644 --- a/SourceUtils.WebExport/Texture.cs +++ b/SourceUtils.WebExport/Texture.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text.RegularExpressions; -using System.Web; using ImageMagick; using Newtonsoft.Json; using Ziks.WebServer; @@ -48,13 +46,16 @@ public partial class Texture { public static Texture Get( ValveBspFile bsp, string path ) { - var inBsp = bsp != null && bsp.PakFile.ContainsFile( path ); - var res = inBsp ? bsp.PakFile : Program.Resources; + var info = TextureSource.Resolve( bsp, path ); + return info == null ? null : Get( info ); + } - if ( !res.ContainsFile( path ) ) return null; + public static Texture Get( TextureSourceInfo info ) + { + var path = info.Path; ValveTextureFile vtf; - using (var stream = res.OpenFile(path)) + using (var stream = info.Provider.OpenFile(path)) { vtf = new ValveTextureFile(stream); } @@ -64,7 +65,7 @@ public static Texture Get( ValveBspFile bsp, string path ) var tex = new Texture { - Path = path.ToLower(), + Path = path, Target = isCube ? TextureTarget.TextureCubeMap : TextureTarget.Texture2D, Width = untextured ? 1 : vtf.Header.Width, Height = untextured ? 1 : vtf.Header.Height, @@ -128,7 +129,7 @@ public static Texture Get( ValveBspFile bsp, string path ) } else { - var fileName = $"{path}/mip{mip}"; + var fileName = $"mip{mip}"; if (vtf.FrameCount > 1) { @@ -142,7 +143,7 @@ public static Texture Get( ValveBspFile bsp, string path ) fileName = $"{fileName}.png"; - elem.Url = inBsp ? $"/maps/{bsp.Name}/{fileName}" : $"/{fileName}"; + elem.Url = TextureSource.BuildUrl(path, info.Hash, fileName); } tex.Elements.Add(elem); @@ -181,26 +182,30 @@ public static Texture Get( ValveBspFile bsp, string path ) } [Prefix("/materials")] - [Prefix("/maps/{map}/materials")] class TextureController : ResourceController { - public static string GetTexturePath(string url) + private TextureSourceInfo LookupRequest( bool image, out string file ) { - var path = url.Substring(url.IndexOf("/materials") + 1); - return HttpUtility.UrlDecode(path.Substring(0, path.Length - ".json".Length)); + if ( !TextureSource.TryParseUrl( Request.Url.AbsolutePath, out var path, out var hash, out file ) + || hash == null || (file != null) != image ) + { + throw NotFoundException(); + } + + var info = TextureSource.Lookup( path, hash ); + if ( info == null ) throw NotFoundException(); + + return info; } [Get( MatchAllUrl = false, Extension = ".vtf.json" )] - public Texture GetInfo( [Url] string map ) + public Texture GetInfo() { - var bsp = map == null ? null : Program.GetMap( map ); - return Texture.Get( bsp, GetTexturePath( Request.Url.AbsolutePath ) ); + return Texture.Get( LookupRequest( false, out _ ) ); } - private static readonly Regex _sFileNameRegex = new Regex( @"^((?mip|face|frame)(?[0-9]+)\.)*(?png)$", RegexOptions.IgnoreCase | RegexOptions.Compiled ); - [Get(MatchAllUrl = false, Extension = ".png")] - public void GetImage( [Url] string map ) + public void GetImage() { if ( Skip ) { @@ -208,49 +213,17 @@ public void GetImage( [Url] string map ) return; } - var absolute = Request.Url.AbsolutePath; - var pathStart = absolute.IndexOf( "/materials" ) + 1; - var pathEnd = absolute.IndexOf( ".vtf", pathStart ) + ".vtf".Length; - var path = HttpUtility.UrlDecode( absolute.Substring( pathStart, pathEnd - pathStart ) ); - - var fileName = Path.GetFileName( absolute ); - var match = _sFileNameRegex.Match( fileName ); - - if ( !match.Success ) throw NotFoundException(); - - var mip = 0; - var face = 0; - var frame = 0; - - var index = 0; - foreach ( Capture capture in match.Groups["param"].Captures ) - { - var param = capture.Value.ToLower(); - var value = int.Parse( match.Groups["value"].Captures[index++].Value ); - - switch (param) - { - case "mip": - mip = value; - break; - case "face": - face = value; - break; - case "frame": - frame = value; - break; - } - } - - var bsp = map == null ? null : Program.GetMap(map); - var res = bsp == null ? Program.Resources : bsp.PakFile; + var info = LookupRequest( true, out var fileName ); ValveTextureFile vtf; - using (var stream = res.OpenFile(path)) + using (var stream = info.Provider.OpenFile(info.Path)) { vtf = new ValveTextureFile(stream); } + if ( !TextureSource.TryParseImageFileName( fileName, vtf.MipmapCount, vtf.FrameCount, vtf.FaceCount, + out var mip, out var frame, out var face ) ) throw NotFoundException(); + using ( var image = Texture.DecodeImage( vtf, mip, frame, face, 0 ) ) { image.Write( Response.OutputStream, MagickFormat.Png ); diff --git a/SourceUtils.WebExport/TextureSource.cs b/SourceUtils.WebExport/TextureSource.cs new file mode 100644 index 00000000..5f647e11 --- /dev/null +++ b/SourceUtils.WebExport/TextureSource.cs @@ -0,0 +1,268 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace SourceUtils.WebExport +{ + public static partial class TextureSource + { + private const string UrlPrefix = "/materials/"; + private const string VtfSegment = ".vtf/"; + private const string InfoSuffix = ".vtf.json"; + private const string MissingSuffix = ".json"; + private const int HashLength = 16; + private static readonly Regex _sImageFileNameRegex = new Regex( @"^((?mip|face|frame)(?[0-9]+)\.)*(?png)$", RegexOptions.IgnoreCase | RegexOptions.Compiled ); + + public static string NormalizePath( string path ) + { + if ( path == null ) return null; + + path = NormalizeSlashes( path.ToLowerInvariant() ).TrimStart( '/' ); + if ( !path.StartsWith( "materials/" ) ) path = $"materials/{path}"; + if ( !path.EndsWith( ".vtf" ) ) path = $"{path}.vtf"; + + return IsSafePath( path ) ? path : null; + } + + public static string ComputeHash( Stream vtf ) + { + using ( var sha1 = SHA1.Create() ) + { + var bytes = sha1.ComputeHash( vtf ); + var builder = new StringBuilder( HashLength ); + for ( var i = 0; i < HashLength / 2; ++i ) builder.Append( bytes[i].ToString( "x2" ) ); + return builder.ToString(); + } + } + + public static string BuildUrl( string path, string hash, string file ) + { + if ( hash == null ) return $"/{path}{MissingSuffix}"; + if ( file == null ) return $"/{path}/{hash}{InfoSuffix}"; + return $"/{path}/{hash}/{file}"; + } + + public static bool TryParseUrl( string absolutePath, out string path, out string hash, out string file ) + { + path = null; hash = null; file = null; + if ( absolutePath == null ) return false; + var url = NormalizeSlashes( Uri.UnescapeDataString( absolutePath ) ); + if ( !url.StartsWith( UrlPrefix, StringComparison.Ordinal ) ) return false; + var vtfIndex = url.LastIndexOf( VtfSegment, StringComparison.Ordinal ); + if ( vtfIndex < 0 ) + { + if ( !url.EndsWith( InfoSuffix, StringComparison.Ordinal ) ) return false; + path = url.Substring( 1, url.Length - 1 - MissingSuffix.Length ); + return IsSafePath( path ); + } + var rest = url.Substring( vtfIndex + VtfSegment.Length ); + string parsedHash; + string parsedFile = null; + if ( rest.EndsWith( InfoSuffix, StringComparison.Ordinal ) && rest.IndexOf( '/' ) < 0 ) + parsedHash = rest.Substring( 0, rest.Length - InfoSuffix.Length ); + else + { + var slash = rest.IndexOf( '/' ); + if ( slash < 0 ) return false; + parsedHash = rest.Substring( 0, slash ); + parsedFile = rest.Substring( slash + 1 ); + if ( parsedFile.Length == 0 || parsedFile.IndexOf( '/' ) >= 0 ) return false; + } + if ( !IsValidHash( parsedHash ) ) return false; + path = url.Substring( 1, vtfIndex + ".vtf".Length - 1 ); + if ( !IsSafePath( path ) ) return false; + hash = parsedHash; file = parsedFile; + return true; + } + + public static bool TryParseImageFileName( string fileName, int mipCount, int frameCount, int faceCount, + out int mip, out int frame, out int face ) + { + mip = 0; + frame = 0; + face = 0; + + var match = _sImageFileNameRegex.Match( fileName ?? "" ); + if ( !match.Success ) return false; + + var index = 0; + foreach ( Capture capture in match.Groups["param"].Captures ) + { + if ( !int.TryParse( match.Groups["value"].Captures[index++].Value, NumberStyles.None, + CultureInfo.InvariantCulture, out var value ) ) return false; + + switch ( capture.Value.ToLowerInvariant() ) + { + case "mip": + mip = value; + break; + case "face": + face = value; + break; + case "frame": + frame = value; + break; + } + } + + return mip >= 0 && mip < mipCount + && frame >= 0 && frame < frameCount + && face >= 0 && face < faceCount; + } + + private static bool IsValidHash( string hash ) + { + if ( hash.Length != HashLength ) return false; + foreach ( var c in hash ) + if ( (c < '0' || c > '9') && (c < 'a' || c > 'f') ) return false; + return true; + } + + private static string NormalizeSlashes( string path ) + { + path = path.Replace( '\\', '/' ); + while ( path.Contains( "//" ) ) path = path.Replace( "//", "/" ); + return path; + } + + private static bool IsSafePath( string path ) + { + if ( path == null || !path.StartsWith( "materials/", StringComparison.Ordinal ) ) return false; + + foreach ( var segment in path.Split( '/' ) ) + { + if ( segment == "." || segment == ".." ) return false; + } + + return true; + } + } + + public sealed class TextureSourceInfo + { + public TextureSourceInfo( IResourceProvider provider, string mapName, string path, string hash ) + { + Provider = provider; + MapName = mapName; + Path = path; + Hash = hash; + } + + public IResourceProvider Provider { get; } + + /// + /// Name of the map whose pakfile contains this texture, or null for game resources. + /// + public string MapName { get; } + + public string Path { get; } + + public string Hash { get; } + } + + public static partial class TextureSource + { + private static readonly Dictionary _sHashCache = new Dictionary(); + private static readonly Dictionary _sRegistry = new Dictionary(); + + private static string GetCacheKey( string mapName, string path ) + { + return $"{mapName ?? ""}|{path}"; + } + + private static string GetRegistryKey( string path, string hash ) + { + return $"{path}|{hash}"; + } + + internal static TextureSourceInfo Resolve( ValveBspFile bsp, string path ) + { + path = NormalizePath( path ); + if ( path == null ) return null; + + IResourceProvider provider; + string mapName; + + if ( bsp != null && bsp.PakFile.ContainsFile( path ) ) + { + provider = bsp.PakFile; + mapName = bsp.Name; + } + else if ( Program.Resources.ContainsFile( path ) ) + { + provider = Program.Resources; + mapName = null; + } + else + { + return null; + } + + var cacheKey = GetCacheKey( mapName, path ); + string hash; + + lock ( _sHashCache ) + { + _sHashCache.TryGetValue( cacheKey, out hash ); + } + + if ( hash == null ) + { + using ( var stream = provider.OpenFile( path ) ) + { + hash = ComputeHash( stream ); + } + + lock ( _sHashCache ) + { + _sHashCache[cacheKey] = hash; + } + } + + lock ( _sRegistry ) + { + // Last registration wins, so host mode prefers the most recently viewed map. + _sRegistry[GetRegistryKey( path, hash )] = mapName; + } + + return new TextureSourceInfo( provider, mapName, path, hash ); + } + + internal static TextureSourceInfo Lookup( string path, string hash ) + { + path = NormalizePath( path ); + if ( path == null ) return null; + + bool registered; + string mapName; + + lock ( _sRegistry ) + { + registered = _sRegistry.TryGetValue( GetRegistryKey( path, hash ), out mapName ); + } + + var bsp = registered && mapName != null ? Program.GetMap( mapName ) : null; + var info = Resolve( bsp, path ); + + return info != null && info.Hash == hash ? info : null; + } + + internal static void ForgetMap( string mapName ) + { + var prefix = GetCacheKey( mapName, "" ); + + lock ( _sHashCache ) + { + foreach ( var key in _sHashCache.Keys.Where( x => x.StartsWith( prefix ) ).ToArray() ) + { + _sHashCache.Remove( key ); + } + } + } + } +} diff --git a/SourceUtils/ValveBsp/GameLump.cs b/SourceUtils/ValveBsp/GameLump.cs index 2d340822..aeae1c17 100644 --- a/SourceUtils/ValveBsp/GameLump.cs +++ b/SourceUtils/ValveBsp/GameLump.cs @@ -112,13 +112,12 @@ private void EnsureLoaded() // lump's offset with that of the next entry. For this reason, when game lumps are compressed // the last game lump is always an empty dummy which only contains the offset. - for ( var i = 0; i < count; i++ ) + for ( var i = 0; i < items.Length - 1; i++ ) { var item = items[i]; - var length = items[i + 1].FileOffset - item.FileOffset; - if ( item.Id == 0 ) break; + var length = items[i + 1].FileOffset - item.FileOffset; _items.Add( GetIdString( item.Id ), item.WithLength( length ) ); } } From 9fdff97a074e65e20db318cefe67fce68c3c6b2d Mon Sep 17 00:00:00 2001 From: SurfSam Date: Thu, 17 Sep 2026 09:50:33 +0200 Subject: [PATCH 02/13] tests --- SourceUtils.Test/GameLumpTest.cs | 72 +++++++++ SourceUtils.Test/SourceUtils.Test.csproj | 6 + SourceUtils.Test/TextureSourceTest.cs | 185 +++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 SourceUtils.Test/GameLumpTest.cs create mode 100644 SourceUtils.Test/TextureSourceTest.cs diff --git a/SourceUtils.Test/GameLumpTest.cs b/SourceUtils.Test/GameLumpTest.cs new file mode 100644 index 00000000..fe28e8de --- /dev/null +++ b/SourceUtils.Test/GameLumpTest.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class GameLumpTest + { + private const int Sprp = 0x73707270; + private const int Dprp = 0x64707270; + + [TestMethod] + public void CompressedGameLumpIgnoresFinalSentinelWhenReadingItemVersion() + { + WithGameLump( new[] { Sprp, Dprp, 0 }, bsp => + { + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) ); + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) ); + } ); + } + + [TestMethod] + public void CompressedGameLumpStopsAtFirstZeroIdEntry() + { + WithGameLump( new[] { Sprp, Dprp, 0, 0, 0 }, bsp => + { + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) ); + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) ); + Assert.ThrowsExactly( () => bsp.GameData.GetItemVersion( "" ) ); + } ); + } + + private static void WithGameLump( int[] itemIds, Action action ) + { + var path = Path.Combine( Path.GetTempPath(), $"sourceutils-gamelump-{Guid.NewGuid():N}.bsp" ); + + try + { + const int gameLumpOffset = 8 + 64 * 16 + 4; + const int gameLumpIndex = 35; + const int itemSize = 16; + + var gameLumpLength = 4 + itemIds.Length * itemSize; + var data = new byte[gameLumpOffset + gameLumpLength]; + + Buffer.BlockCopy( BitConverter.GetBytes( gameLumpOffset ), 0, data, 8 + gameLumpIndex * 16, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( gameLumpLength ), 0, data, 8 + gameLumpIndex * 16 + 4, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( itemIds.Length ), 0, data, gameLumpOffset, 4 ); + + for ( var i = 0; i < itemIds.Length; i++ ) + { + var itemOffset = gameLumpOffset + 4 + i * itemSize; + Buffer.BlockCopy( BitConverter.GetBytes( itemIds[i] ), 0, data, itemOffset, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( (ushort) 10 ), 0, data, itemOffset + 6, 2 ); // version + } + + File.WriteAllBytes( path, data ); + + using ( var bsp = new ValveBspFile( path ) ) + { + action( bsp ); + } + } + finally + { + if ( File.Exists( path ) ) File.Delete( path ); + } + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index bfedb304..f37d67a2 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -119,7 +119,9 @@ + + True @@ -132,6 +134,10 @@ {2c5fefa1-39ba-458e-b95c-2d8414958820} SourceUtils + + {171622AD-547B-496C-A230-16FCA08A0112} + SourceUtils.WebExport + diff --git a/SourceUtils.Test/TextureSourceTest.cs b/SourceUtils.Test/TextureSourceTest.cs new file mode 100644 index 00000000..a60d9054 --- /dev/null +++ b/SourceUtils.Test/TextureSourceTest.cs @@ -0,0 +1,185 @@ +using System.IO; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class TextureSourceTest + { + private const string Hash = "0123456789abcdef"; + + [TestMethod] + public void NormalizePathAddsPrefixAndExtension() + { + Assert.AreEqual( "materials/concrete/wall.vtf", TextureSource.NormalizePath( "concrete/wall" ) ); + } + + [TestMethod] + public void NormalizePathFixesSlashesAndCase() + { + Assert.AreEqual( "materials/concrete/wall.vtf", TextureSource.NormalizePath( "/Materials\\Concrete//Wall.VTF" ) ); + } + + [TestMethod] + public void NormalizePathIsIdempotent() + { + const string path = "materials/skybox/sky_bk.vtf"; + Assert.AreEqual( path, TextureSource.NormalizePath( path ) ); + } + + [TestMethod] + public void ComputeHashIsSixteenLowerHexChars() + { + using ( var stream = new MemoryStream( Encoding.ASCII.GetBytes( "VTF\0hello" ) ) ) + { + var hash = TextureSource.ComputeHash( stream ); + Assert.AreEqual( 16, hash.Length ); + StringAssert.Matches( hash, new System.Text.RegularExpressions.Regex( "^[0-9a-f]{16}$" ) ); + } + } + + [TestMethod] + public void ComputeHashIsStableAndContentSensitive() + { + string HashOf( string content ) + { + using ( var stream = new MemoryStream( Encoding.ASCII.GetBytes( content ) ) ) + { + return TextureSource.ComputeHash( stream ); + } + } + + Assert.AreEqual( HashOf( "abc" ), HashOf( "abc" ) ); + Assert.AreNotEqual( HashOf( "abc" ), HashOf( "abd" ) ); + } + + [TestMethod] + public void BuildUrlImage() + { + Assert.AreEqual( "/materials/concrete/wall.vtf/0123456789abcdef/mip0.png", TextureSource.BuildUrl( "materials/concrete/wall.vtf", Hash, "mip0.png" ) ); + } + + [TestMethod] + public void BuildUrlInfo() + { + Assert.AreEqual( "/materials/skybox/sky_bk.vtf/0123456789abcdef.vtf.json", TextureSource.BuildUrl( "materials/skybox/sky_bk.vtf", Hash, null ) ); + } + + [TestMethod] + public void BuildUrlMissing() + { + Assert.AreEqual( "/materials/concrete/wall.vtf.json", TextureSource.BuildUrl( "materials/concrete/wall.vtf", null, null ) ); + } + + private static void AssertRoundTrip( string path, string hash, string file ) + { + var url = TextureSource.BuildUrl( path, hash, file ); + Assert.IsTrue( TextureSource.TryParseUrl( url, out var parsedPath, out var parsedHash, out var parsedFile ), url ); + Assert.AreEqual( path, parsedPath, url ); + Assert.AreEqual( hash, parsedHash, url ); + Assert.AreEqual( file, parsedFile, url ); + } + + [TestMethod] + public void RoundTripImage() + { + AssertRoundTrip( "materials/concrete/wall.vtf", Hash, "mip0.png" ); + } + + [TestMethod] + public void RoundTripFrameFaceImage() + { + AssertRoundTrip( "materials/water/anim.vtf", Hash, "mip2.frame3.face1.png" ); + } + + [TestMethod] + public void RoundTripInfo() + { + AssertRoundTrip( "materials/skybox/sky_bk.vtf", Hash, null ); + } + + [TestMethod] + public void RoundTripMissing() + { + AssertRoundTrip( "materials/concrete/wall.vtf", null, null ); + } + + [TestMethod] + public void ParseUnescapesEncodedPath() + { + Assert.IsTrue( TextureSource.TryParseUrl( "/materials/my%20dir/wall.vtf/0123456789abcdef/mip0.png", out var path, out var hash, out var file ) ); + Assert.AreEqual( "materials/my dir/wall.vtf", path ); + Assert.AreEqual( Hash, hash ); + Assert.AreEqual( "mip0.png", file ); + } + + [TestMethod] + public void ParseUsesRightmostVtfSegment() + { + Assert.IsTrue( TextureSource.TryParseUrl( "/materials/dir.vtf/inner/wall.vtf/0123456789abcdef/mip0.png", + out var path, out var hash, out var file ) ); + Assert.AreEqual( "materials/dir.vtf/inner/wall.vtf", path ); + Assert.AreEqual( Hash, hash ); + Assert.AreEqual( "mip0.png", file ); + } + + [TestMethod] + public void ParseRejectsDecodedTraversal() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete%2f..%2fsecret.vtf/0123456789abcdef/mip0.png", + out _, out _, out _ ) ); + Assert.IsNull( TextureSource.NormalizePath( "materials\\concrete\\..\\secret" ) ); + } + + [TestMethod] + public void ParseRejectsOutsideMaterials() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/maps/surf_x/materials/wall.vtf/0123456789abcdef/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsNoVtfSegment() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vmt.json", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsBadHash() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/NOTAHASH/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsNestedFile() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/0123456789abcdef/a/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseImageFileNameRejectsMalformedAndOverflowingIndices() + { + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip.png", 2, 2, 2, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip999999999999999999999.png", 2, 2, 2, out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseImageFileNameRejectsOutOfRangeIndices() + { + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip2.png", 2, 1, 1, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "frame1.png", 1, 1, 1, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "face1.png", 1, 1, 1, out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseImageFileNameReturnsIndicesInsideTextureBounds() + { + Assert.IsTrue( TextureSource.TryParseImageFileName( "mip1.frame2.face5.png", 2, 3, 6, + out var mip, out var frame, out var face ) ); + Assert.AreEqual( 1, mip ); + Assert.AreEqual( 2, frame ); + Assert.AreEqual( 5, face ); + } + } +} From 88e647411de473bc4b4ad16f535652e3e63c10f7 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 03/13] Read the pre-Orange Box per-leaf ambient lighting lump Maps compiled before the Orange Box store one bare ambient cube per leaf and write no index lump, so reading the lump through the index gave every leaf zero samples and left props and the world unlit. The layout has to be recognised by size, because those maps write the lump length into its version field. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/BspBuilder.cs | 89 ++++++++++ SourceUtils.Test/LeafAmbientTest.cs | 198 ++++++++++++++++++++++ SourceUtils.Test/SourceUtils.Test.csproj | 2 + SourceUtils.WebExport/Bsp/AmbientCubes.cs | 12 +- SourceUtils/SourceUtils.csproj | 1 + SourceUtils/ValveBsp/LeafAmbient.cs | 81 +++++++++ SourceUtils/ValveBsp/Structures.cs | 9 + 7 files changed, 384 insertions(+), 8 deletions(-) create mode 100644 SourceUtils.Test/BspBuilder.cs create mode 100644 SourceUtils.Test/LeafAmbientTest.cs create mode 100644 SourceUtils/ValveBsp/LeafAmbient.cs diff --git a/SourceUtils.Test/BspBuilder.cs b/SourceUtils.Test/BspBuilder.cs new file mode 100644 index 00000000..232a0599 --- /dev/null +++ b/SourceUtils.Test/BspBuilder.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Writes a minimal version 20 BSP file holding only the lumps a test gives it. + /// + internal class BspBuilder + { + private const int LumpCount = 64; + private const int HeaderSize = 4 + 4 + LumpCount * 16 + 4; + + private readonly Dictionary> _lumps + = new Dictionary>(); + + public BspBuilder Lump( LumpType type, byte[] data, int version = 0 ) + { + _lumps[type] = new KeyValuePair( data, version ); + return this; + } + + public string Write() + { + var path = Path.Combine( Path.GetTempPath(), $"sourceutils-test-{Guid.NewGuid():N}.bsp" ); + + using ( var writer = new BinaryWriter( File.Create( path ) ) ) + { + writer.Write( new[] { (byte) 'V', (byte) 'B', (byte) 'S', (byte) 'P' } ); + writer.Write( 20 ); + + var offset = HeaderSize; + + for ( var i = 0; i < LumpCount; ++i ) + { + KeyValuePair lump; + if ( !_lumps.TryGetValue( (LumpType) i, out lump ) ) + { + writer.Write( offset ); + writer.Write( 0 ); + writer.Write( 0 ); + writer.Write( new byte[4] ); + continue; + } + + writer.Write( offset ); + writer.Write( lump.Key.Length ); + writer.Write( lump.Value ); + writer.Write( new byte[4] ); + + offset += lump.Key.Length; + } + + writer.Write( 0 ); + + for ( var i = 0; i < LumpCount; ++i ) + { + KeyValuePair lump; + if ( _lumps.TryGetValue( (LumpType) i, out lump ) ) writer.Write( lump.Key ); + } + } + + return path; + } + + /// + /// Writes the file, opens it for and deletes it afterwards. + /// + public void With( Action action ) + { + var path = Write(); + + try + { + using ( var bsp = new ValveBspFile( path ) ) + { + action( bsp ); + } + } + finally + { + File.Delete( path ); + } + } + } +} diff --git a/SourceUtils.Test/LeafAmbientTest.cs b/SourceUtils.Test/LeafAmbientTest.cs new file mode 100644 index 00000000..7cfe5856 --- /dev/null +++ b/SourceUtils.Test/LeafAmbientTest.cs @@ -0,0 +1,198 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Maps compiled by pre-Orange Box tools store one ambient cube per leaf and write no leaf + /// ambient index lump at all. These tests build minimal BSP files covering both layouts. + /// + [TestClass] + public class LeafAmbientTest + { + private const int LeafSize = 32; + private const int CubeSize = 24; + private const int SampleSize = 28; + private const int IndexSize = 4; + + private static byte[] Leaves( int count ) + { + var data = new byte[count * LeafSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < count; ++i ) + { + writer.Write( 0 ); // Contents + writer.Write( (short) i ); // Cluster + writer.Write( (short) 0 ); // AreaFlags + writer.Write( (short) (i * 10) ); // Min.X + writer.Write( (short) 0 ); // Min.Y + writer.Write( (short) 0 ); // Min.Z + writer.Write( (short) (i * 10 + 8) ); // Max.X + writer.Write( (short) 8 ); // Max.Y + writer.Write( (short) 8 ); // Max.Z + writer.Write( (ushort) 0 ); // FirstLeafFace + writer.Write( (ushort) 0 ); // NumLeafFaces + writer.Write( (ushort) 0 ); // FirstLeafBrush + writer.Write( (ushort) 0 ); // NumLeafBrushes + writer.Write( (short) -1 ); // LeafWaterDataId + writer.Write( (short) 0 ); // Padding + } + } + + return data; + } + + /// Red channel of face 0 identifies each cube. + private static void WriteCube( BinaryWriter writer, byte id ) + { + writer.Write( id ); + writer.Write( (byte) 0 ); + writer.Write( (byte) 0 ); + writer.Write( (byte) 0 ); + + for ( var face = 1; face < 6; ++face ) writer.Write( new byte[4] ); + } + + private static byte[] LegacyAmbient( int leafCount ) + { + var data = new byte[leafCount * CubeSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < leafCount; ++i ) WriteCube( writer, (byte) (i + 1) ); + } + + return data; + } + + private static byte[] IndexedAmbient( params byte[] ids ) + { + var data = new byte[ids.Length * SampleSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + foreach ( var id in ids ) + { + WriteCube( writer, id ); + writer.Write( (byte) (id * 2) ); // X + writer.Write( (byte) 0 ); // Y + writer.Write( (byte) 0 ); // Z + writer.Write( (byte) 0 ); // Padding + } + } + + return data; + } + + private static byte[] AmbientIndices( params int[] countsAndFirsts ) + { + var data = new byte[countsAndFirsts.Length / 2 * IndexSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < countsAndFirsts.Length; i += 2 ) + { + writer.Write( (ushort) countsAndFirsts[i] ); + writer.Write( (ushort) countsAndFirsts[i + 1] ); + } + } + + return data; + } + + private static void WithBsp( BspBuilder builder, Action action ) + { + builder.With( action ); + } + + [TestMethod] + public void LegacyLayoutGivesOneCentredSamplePerLeaf() + { + var ambient = LegacyAmbient( 3 ); + + // Pre-Orange Box tools write the lump length into the version field as well. + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, ambient, ambient.Length ); + + WithBsp( builder, bsp => + { + Assert.AreEqual( 3, bsp.Leaves.Length ); + + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 1, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + + var sample = bsp.GetLeafAmbientSample( leaf, 0 ); + + Assert.AreEqual( leaf + 1, sample.Cube[0].R, $"leaf {leaf} cube" ); + Assert.AreEqual( 128, sample.X, $"leaf {leaf} x" ); + Assert.AreEqual( 128, sample.Y, $"leaf {leaf} y" ); + Assert.AreEqual( 128, sample.Z, $"leaf {leaf} z" ); + } + } ); + } + + [TestMethod] + public void IndexedLayoutReadsSamplesThroughTheIndex() + { + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_INDEX, AmbientIndices( 0, 0, 2, 0, 1, 2 ) ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, IndexedAmbient( 10, 20, 30 ), 1 ); + + WithBsp( builder, bsp => + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( 0 ) ); + Assert.AreEqual( 2, bsp.GetLeafAmbientSampleCount( 1 ) ); + Assert.AreEqual( 1, bsp.GetLeafAmbientSampleCount( 2 ) ); + + Assert.AreEqual( 10, bsp.GetLeafAmbientSample( 1, 0 ).Cube[0].R ); + Assert.AreEqual( 20, bsp.GetLeafAmbientSample( 1, 1 ).Cube[0].R ); + Assert.AreEqual( 30, bsp.GetLeafAmbientSample( 2, 0 ).Cube[0].R ); + + // Indexed samples keep their own position within the leaf. + Assert.AreEqual( 40, bsp.GetLeafAmbientSample( 1, 1 ).X ); + } ); + } + + [TestMethod] + public void MissingAmbientLumpsGiveNoSamples() + { + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ); + + WithBsp( builder, bsp => + { + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + } + } ); + } + + [TestMethod] + public void AmbientLumpWithoutOnePerLeafIsIgnored() + { + // An index-less ambient lump that isn't one cube per leaf can't be mapped to leaves. + var ambient = LegacyAmbient( 2 ); + + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, ambient, ambient.Length ); + + WithBsp( builder, bsp => + { + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + } + } ); + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index f37d67a2..1e3ed7a3 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -119,6 +119,8 @@ + + diff --git a/SourceUtils.WebExport/Bsp/AmbientCubes.cs b/SourceUtils.WebExport/Bsp/AmbientCubes.cs index b7b54fc6..d5cb59de 100644 --- a/SourceUtils.WebExport/Bsp/AmbientCubes.cs +++ b/SourceUtils.WebExport/Bsp/AmbientCubes.cs @@ -41,24 +41,20 @@ public AmbientPage Get( [Url] string map, [Url] int page ) count = 0; } - var hdr = bsp.LeafAmbientLightingHdr.Length > bsp.LeafAmbientLighting.Length; - var indices = hdr ? bsp.LeafAmbientIndicesHdr : bsp.LeafAmbientIndices; - var ambients = hdr ? bsp.LeafAmbientLightingHdr : bsp.LeafAmbientLighting; - return new AmbientPage { Values = Enumerable.Range( first, count ).Select( x => { var leaf = bsp.Leaves[x]; - var index = indices[x]; - var list = new List(index.AmbientSampleCount); + var sampleCount = bsp.GetLeafAmbientSampleCount( x ); + var list = new List(sampleCount); var min = new SourceUtils.Vector3(leaf.Min.X, leaf.Min.Y, leaf.Min.Z); var max = new SourceUtils.Vector3(leaf.Max.X, leaf.Max.Y, leaf.Max.Z); - for (var i = 0; i < index.AmbientSampleCount; ++i) + for (var i = 0; i < sampleCount; ++i) { - var ambient = ambients[index.FirstAmbientSample + i]; + var ambient = bsp.GetLeafAmbientSample( x, i ); var samples = new int[6]; var relPos = new SourceUtils.Vector3(ambient.X, ambient.Y, ambient.Z) * (1f / 255f); diff --git a/SourceUtils/SourceUtils.csproj b/SourceUtils/SourceUtils.csproj index 21fde398..119a3891 100644 --- a/SourceUtils/SourceUtils.csproj +++ b/SourceUtils/SourceUtils.csproj @@ -94,6 +94,7 @@ + diff --git a/SourceUtils/ValveBsp/LeafAmbient.cs b/SourceUtils/ValveBsp/LeafAmbient.cs new file mode 100644 index 00000000..0e6750a2 --- /dev/null +++ b/SourceUtils/ValveBsp/LeafAmbient.cs @@ -0,0 +1,81 @@ +using System; +using SourceUtils.ValveBsp; + +namespace SourceUtils +{ + partial class ValveBspFile + { + /// + /// Centre of a leaf's bounds, where the single sample of a legacy ambient lump sits. + /// + private const byte LegacyAmbientPosition = 128; + + /// + /// The same lump as , read as the legacy layout of one + /// bare cube per leaf. + /// + [BspLump( LumpType.LEAF_AMBIENT_LIGHTING )] + public StructArrayLump LeafAmbientCubes { get; private set; } + + [BspLump( LumpType.LEAF_AMBIENT_LIGHTING_HDR )] + public StructArrayLump LeafAmbientCubesHdr { get; private set; } + + private bool UseHdrAmbient => LeafAmbientLightingHdr.Length > LeafAmbientLighting.Length; + + /// + /// Maps compiled before the Orange Box store one ambient cube per leaf and write no + /// index lump. They also write the lump length into its version field, so the layout has + /// to be recognised by size rather than by version. + /// + private bool IsLegacyLeafAmbient + { + get + { + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + var cubes = UseHdrAmbient ? LeafAmbientCubesHdr : LeafAmbientCubes; + + return Leaves.Length > 0 && indices.Length == 0 && cubes.Length == Leaves.Length; + } + } + + /// + /// Number of ambient light samples stored for the given leaf, which is 0 when the map + /// has no usable ambient lighting at all. + /// + public int GetLeafAmbientSampleCount( int leafIndex ) + { + if ( leafIndex < 0 || leafIndex >= Leaves.Length ) return 0; + if ( IsLegacyLeafAmbient ) return 1; + + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + + return leafIndex >= indices.Length ? 0 : indices[leafIndex].AmbientSampleCount; + } + + public LeafAmbientLighting GetLeafAmbientSample( int leafIndex, int sampleIndex ) + { + var count = GetLeafAmbientSampleCount( leafIndex ); + + if ( sampleIndex < 0 || sampleIndex >= count ) + { + throw new IndexOutOfRangeException( + $"Leaf {leafIndex} has {count} ambient samples, so {sampleIndex} is out of range." ); + } + + if ( IsLegacyLeafAmbient ) + { + var cubes = UseHdrAmbient ? LeafAmbientCubesHdr : LeafAmbientCubes; + + return new LeafAmbientLighting( cubes[leafIndex], + LegacyAmbientPosition, LegacyAmbientPosition, LegacyAmbientPosition ); + } + + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + var samples = UseHdrAmbient ? LeafAmbientLightingHdr : LeafAmbientLighting; + + var first = indices[leafIndex].FirstAmbientSample + sampleIndex; + + return first >= samples.Length ? default( LeafAmbientLighting ) : samples[first]; + } + } +} diff --git a/SourceUtils/ValveBsp/Structures.cs b/SourceUtils/ValveBsp/Structures.cs index de422f36..5bab8a85 100644 --- a/SourceUtils/ValveBsp/Structures.cs +++ b/SourceUtils/ValveBsp/Structures.cs @@ -534,6 +534,15 @@ public struct CubemapSample [StructLayout( LayoutKind.Sequential, Pack = 1 )] public struct LeafAmbientLighting { + public LeafAmbientLighting( CompressedLightCube cube, byte x, byte y, byte z ) + { + Cube = cube; + X = x; + Y = y; + Z = z; + _padding = 0; + } + public readonly CompressedLightCube Cube; public readonly byte X; public readonly byte Y; From 99274ca3699b40c1869cec4187e8065e706cb267 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 04/13] Don't light props from leaves that have no ambient samples A prop whose origin lands inside a solid brush resolves to a leaf with no samples. Using it anyway asked for an ambient cube that never arrives, so the prop stayed unlit forever. Co-Authored-By: Claude Opus 5 --- SourceUtils.WebExport/Resources/js/sourceutils.js | 3 ++- SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/SourceUtils.WebExport/Resources/js/sourceutils.js b/SourceUtils.WebExport/Resources/js/sourceutils.js index 74d71198..8bad094f 100644 --- a/SourceUtils.WebExport/Resources/js/sourceutils.js +++ b/SourceUtils.WebExport/Resources/js/sourceutils.js @@ -1881,7 +1881,8 @@ var SourceUtils; } else { leaf.getAmbientCube(null, null, function (success) { - _this.lighting = leaf; + // Origins inside solid brushes land in leaves with no ambient samples. + _this.lighting = success ? leaf : null; _this.checkLoaded(); }); } diff --git a/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts b/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts index bd41a6c1..4a36c4fd 100644 --- a/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts +++ b/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts @@ -37,7 +37,8 @@ namespace SourceUtils { this.checkLoaded(); } else { leaf.getAmbientCube(null, null, success => { - this.lighting = leaf; + // Origins inside solid brushes land in leaves with no ambient samples. + this.lighting = success ? leaf : null; this.checkLoaded(); }); } From 0c8560141e07950192c8de9f1f5ef2ba2a0b6394 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 05/13] Serve empty pages rather than null for skipped vis and ambient pages Returning null writes no response body at all, which drops the connection and has the exporter record the file as failed. Co-Authored-By: Claude Opus 5 --- SourceUtils.WebExport/Bsp/AmbientCubes.cs | 4 +++- SourceUtils.WebExport/Bsp/Visibility.cs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/SourceUtils.WebExport/Bsp/AmbientCubes.cs b/SourceUtils.WebExport/Bsp/AmbientCubes.cs index d5cb59de..ec3548ed 100644 --- a/SourceUtils.WebExport/Bsp/AmbientCubes.cs +++ b/SourceUtils.WebExport/Bsp/AmbientCubes.cs @@ -29,7 +29,9 @@ class AmbientController : ResourceController [Get("/ambientpage{page}.json")] public AmbientPage Get( [Url] string map, [Url] int page ) { - if ( Skip ) return null; + // A skipped page still has to be a valid response. Returning null writes nothing at + // all, which drops the connection and has the exporter count the file as failed. + if ( Skip ) return new AmbientPage { Values = Enumerable.Empty>() }; var bsp = Program.GetMap(map); var first = page * AmbientPage.LeavesPerPage; diff --git a/SourceUtils.WebExport/Bsp/Visibility.cs b/SourceUtils.WebExport/Bsp/Visibility.cs index 88efcce6..03b2229d 100644 --- a/SourceUtils.WebExport/Bsp/Visibility.cs +++ b/SourceUtils.WebExport/Bsp/Visibility.cs @@ -20,7 +20,9 @@ class VisController : ResourceController [Get("/vispage{page}.json")] public VisPage Get( [Url] string map, [Url] int page ) { - if ( Skip ) return null; + // A skipped page still has to be a valid response. Returning null writes nothing at + // all, which drops the connection and has the exporter count the file as failed. + if ( Skip ) return new VisPage { Values = Enumerable.Empty>() }; var bsp = Program.GetMap(map); var first = page * VisPage.ClustersPerPage; From 56e99adac79b541cc33bf8609e8e1f22fe1299aa Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 06/13] Add in the lightmap of every light style that starts switched on Named lights get their own switchable style, and VRAD writes their samples in pages after style 0 rather than into it. Only the first page was being read, so every named light was missing from the exported lightmap. Styles are summed in linear space, and a style is left out when every light using it starts dark. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/LightStyleTest.cs | 262 +++++++++++++++++++++++ SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.WebExport/Bsp/Lightmap.cs | 5 +- SourceUtils/SourceUtils.csproj | 1 + SourceUtils/ValveBsp/LightStyles.cs | 174 +++++++++++++++ SourceUtils/ValveBsp/Structures.cs | 8 + 6 files changed, 447 insertions(+), 4 deletions(-) create mode 100644 SourceUtils.Test/LightStyleTest.cs create mode 100644 SourceUtils/ValveBsp/LightStyles.cs diff --git a/SourceUtils.Test/LightStyleTest.cs b/SourceUtils.Test/LightStyleTest.cs new file mode 100644 index 00000000..ef98a8d5 --- /dev/null +++ b/SourceUtils.Test/LightStyleTest.cs @@ -0,0 +1,262 @@ +using System; +using System.IO; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.ValveBsp; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Named lights are compiled into their own light styles, which VRAD stores as extra + /// lightmap pages after a face's style 0 page. The engine adds together every style that + /// is switched on, so these tests check the pages are combined the same way. + /// + [TestClass] + public class LightStyleTest + { + private const int TexInfoSize = 72; + private const int FaceSize = 56; + private const int SampleSize = 4; + + private const int FaceWidth = 2; + private const int FaceHeight = 1; + private const int SampleCount = FaceWidth * FaceHeight; + + private static byte[] TexInfos( params SurfFlags[] flags ) + { + var data = new byte[flags.Length * TexInfoSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + foreach ( var flag in flags ) + { + writer.Write( new byte[64] ); // Texture and lightmap axes + writer.Write( (int) flag ); + writer.Write( 0 ); // TexData + } + } + + return data; + } + + private static byte[] Face( int texInfo, int lightOffset, params byte[] styles ) + { + var packedStyles = new byte[] { 255, 255, 255, 255 }; + Array.Copy( styles, packedStyles, styles.Length ); + + var data = new byte[FaceSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( (ushort) 0 ); // PlaneNum + writer.Write( (byte) 0 ); // Side + writer.Write( (byte) 0 ); // OnNode + writer.Write( 0 ); // FirstEdge + writer.Write( (short) 0 ); // NumEdges + writer.Write( (short) texInfo ); + writer.Write( (short) -1 ); // DispInfo + writer.Write( (short) 0 ); // FogVolumeId + writer.Write( packedStyles ); + writer.Write( lightOffset ); + writer.Write( 0f ); // Area + writer.Write( 0 ); // LightMapOffsetX + writer.Write( 0 ); // LightMapOffsetY + writer.Write( FaceWidth - 1 ); // LightMapSizeX + writer.Write( FaceHeight - 1 ); // LightMapSizeY + writer.Write( 0 ); // OriginalFace + writer.Write( (ushort) 0 ); // NumPrimitives + writer.Write( (ushort) 0 ); // FirstPrimitive + writer.Write( 0u ); // SmoothingGroups + } + + return data; + } + + /// One page holds a sample per luxel, all with the same colour here. + private static byte[] Page( byte r, byte g, byte b, sbyte exponent ) + { + var data = new byte[SampleCount * SampleSize]; + + for ( var i = 0; i < SampleCount; ++i ) + { + data[i * SampleSize + 0] = r; + data[i * SampleSize + 1] = g; + data[i * SampleSize + 2] = b; + data[i * SampleSize + 3] = (byte) exponent; + } + + return data; + } + + private static byte[] Concat( params byte[][] parts ) + { + using ( var stream = new MemoryStream() ) + { + foreach ( var part in parts ) stream.Write( part, 0, part.Length ); + return stream.ToArray(); + } + } + + private static byte[] Entities( params string[] entities ) + { + return Encoding.ASCII.GetBytes( string.Concat( entities ) + "\0" ); + } + + private static string Light( int style, int spawnFlags ) + { + return "{\n\"classname\" \"light\"\n\"targetname\" \"lamp\"\n" + + $"\"style\" \"{style}\"\n\"spawnflags\" \"{spawnFlags}\"\n}}\n"; + } + + private static ColorRGBExp32[] ReadFace( ValveBspFile bsp ) + { + var output = new ColorRGBExp32[SampleCount]; + + using ( var lighting = bsp.GetLumpStream( LumpType.LIGHTING ) ) + { + bsp.ReadLightmapSamples( lighting, bsp.Faces[0], output ); + } + + return output; + } + + private static void AssertLinear( float r, float g, float b, ColorRGBExp32 sample ) + { + var scale = (float) Math.Pow( 2, sample.Exponent ); + + // Re-encoding keeps 8 bits of precision on the brightest channel. + var tolerance = Math.Max( Math.Max( r, g ), b ) / 128f; + + Assert.AreEqual( r, sample.R * scale, tolerance, "red" ); + Assert.AreEqual( g, sample.G * scale, tolerance, "green" ); + Assert.AreEqual( b, sample.B * scale, tolerance, "blue" ); + } + + private static BspBuilder Map( SurfFlags flags, byte[] face, byte[] lighting, params string[] entities ) + { + return new BspBuilder() + .Lump( LumpType.ENTITIES, Entities( entities ) ) + .Lump( LumpType.TEXINFO, TexInfos( flags ) ) + .Lump( LumpType.FACES, face, 1 ) + .Lump( LumpType.LIGHTING, lighting, 1 ); + } + + [TestMethod] + public void SwitchedOnStyleIsAddedToStyleZero() + { + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void StylesWithDifferentExponentsAreAddedInLinearSpace() + { + // 64 * 2^-6 = 1 and 16 * 2^2 = 64. + var lighting = Concat( Page( 64, 64, 64, -6 ), Page( 16, 0, 0, 2 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 65, 1, 1, sample ); + } ); + } + + [TestMethod] + public void InitiallyDarkStyleIsLeftOut() + { + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 1 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 10, sample.R ); + Assert.AreEqual( 20, sample.G ); + Assert.AreEqual( 30, sample.B ); + Assert.AreEqual( 0, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void BumpedFacesStoreFourPagesPerStyle() + { + // Pages 1 to 3 of each style hold the bump directions, which must be skipped. + var bump = Page( 200, 200, 200, 3 ); + var lighting = Concat( + Page( 10, 20, 30, 0 ), bump, bump, bump, + Page( 100, 50, 0, 0 ), bump, bump, bump ); + + Map( SurfFlags.BUMPLIGHT, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void SingleStyleIsCopiedUnchanged() + { + // Exponents outside what re-encoding would produce must survive untouched. + var lighting = Page( 3, 5, 7, -20 ); + + Map( 0, Face( 0, 0, 0 ), lighting ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 3, sample.R ); + Assert.AreEqual( 5, sample.G ); + Assert.AreEqual( 7, sample.B ); + Assert.AreEqual( -20, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void MapWithoutLightingIsFullbright() + { + // Maps compiled without VRAD have no lighting at all but still list style 0 in + // every slot. The engine draws them fully lit. + Map( 0, Face( 0, 0, 0, 0, 0, 0 ), new byte[0] ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 255, sample.R ); + Assert.AreEqual( 255, sample.G ); + Assert.AreEqual( 255, sample.B ); + Assert.AreEqual( 0, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void StylesWithoutDataAreIgnored() + { + // Only the pages that fit inside the lump are added together. + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32, 33 ), lighting, Light( 32, 0 ), Light( 33, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void LightStylesDefaultToOn() + { + Map( 0, Face( 0, 0, 0 ), Page( 0, 0, 0, 0 ), Light( 32, 0 ), Light( 33, 1 ) ).With( bsp => + { + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 0 ), "style 0" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 5 ), "built-in animated style" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 32 ), "named light that starts on" ); + Assert.IsFalse( bsp.IsLightStyleOnByDefault( 33 ), "named light that starts dark" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 34 ), "style with no light entity" ); + Assert.IsFalse( bsp.IsLightStyleOnByDefault( 255 ), "unused style slot" ); + } ); + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 1e3ed7a3..15230fca 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -121,6 +121,7 @@ + diff --git a/SourceUtils.WebExport/Bsp/Lightmap.cs b/SourceUtils.WebExport/Bsp/Lightmap.cs index cf8046dc..57a088e6 100644 --- a/SourceUtils.WebExport/Bsp/Lightmap.cs +++ b/SourceUtils.WebExport/Bsp/Lightmap.cs @@ -70,11 +70,8 @@ public void Get( [Url] string map ) if (face.LightOffset == -1) continue; var rect = lightmap.GetLightmapRegion(i); - var sampleCount = rect.Width * rect.Height; - sampleStream.Seek(face.LightOffset, SeekOrigin.Begin); - - LumpReader.ReadLumpFromStream(sampleStream, sampleCount, sampleBuffer); + bsp.ReadLightmapSamples(sampleStream, face, sampleBuffer); for (var y = 0; y < rect.Height; ++y) for (var x = 0; x < rect.Width; ++x) diff --git a/SourceUtils/SourceUtils.csproj b/SourceUtils/SourceUtils.csproj index 119a3891..79a2b65e 100644 --- a/SourceUtils/SourceUtils.csproj +++ b/SourceUtils/SourceUtils.csproj @@ -96,6 +96,7 @@ + diff --git a/SourceUtils/ValveBsp/LightStyles.cs b/SourceUtils/ValveBsp/LightStyles.cs new file mode 100644 index 00000000..c59e900a --- /dev/null +++ b/SourceUtils/ValveBsp/LightStyles.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SourceUtils.ValveBsp; + +namespace SourceUtils +{ + partial class ValveBspFile + { + /// + /// Marks an unused light style slot on a face. + /// + public const byte NoLightStyle = 255; + + /// + /// VRAD gives each group of named lights its own style from here on, so they can be + /// switched at runtime. Styles below this are the built-in animated patterns. + /// + private const int FirstSwitchableLightStyle = 32; + + private const int MaxLightStylesPerFace = 4; + + /// + /// Bumped faces store a flat lightmap followed by one for each of the three bump basis + /// directions, for every style. + /// + private const int BumpLightmapPages = 4; + + private const int SpawnFlagInitiallyDark = 1; + + private static readonly ColorRGBExp32 Fullbright = new ColorRGBExp32( 255, 255, 255, 0 ); + + private HashSet _initiallyDarkLightStyles; + + /// + /// Whether the given light style is lit when the map starts. Switchable styles are lit + /// unless every light using them is flagged to start dark. + /// + public bool IsLightStyleOnByDefault( byte style ) + { + if ( style == NoLightStyle ) return false; + if ( style < FirstSwitchableLightStyle ) return true; + + return !GetInitiallyDarkLightStyles().Contains( style ); + } + + private HashSet GetInitiallyDarkLightStyles() + { + lock ( this ) + { + if ( _initiallyDarkLightStyles != null ) return _initiallyDarkLightStyles; + + var dark = new HashSet(); + var lit = new HashSet(); + + foreach ( var entity in Entities ) + { + if ( entity.ClassName == null || !entity.ClassName.StartsWith( "light" ) ) continue; + + int style = entity["style"]; + if ( style < FirstSwitchableLightStyle ) continue; + + int spawnFlags = entity["spawnflags"]; + (( spawnFlags & SpawnFlagInitiallyDark ) != 0 ? dark : lit).Add( style ); + } + + dark.ExceptWith( lit ); + + return _initiallyDarkLightStyles = dark; + } + } + + /// + /// Reads the flat lightmap of a face into , adding together + /// every light style that is lit when the map starts. + /// + /// Stream of the lighting lump that matches . + public void ReadLightmapSamples( Stream lighting, Face face, ColorRGBExp32[] output ) + { + var size = face.LightMapSize; + var sampleCount = size.X * size.Y; + + // Maps compiled without VRAD have no lighting at all, and the engine draws them + // fully lit. + if ( lighting.Length == 0 ) + { + for ( var i = 0; i < sampleCount; ++i ) output[i] = Fullbright; + return; + } + + var bumped = (TextureInfos[face.TexInfo].Flags & SurfFlags.BUMPLIGHT) != 0; + var styleStride = sampleCount * (bumped ? BumpLightmapPages : 1); + + var litStyles = new List( MaxLightStylesPerFace ); + + for ( var i = 0; i < MaxLightStylesPerFace; ++i ) + { + var style = face.GetLightStyle( i ); + if ( style == NoLightStyle ) break; + if ( !IsLightStyleOnByDefault( style ) ) continue; + + // Maps compiled without VRAD list style 0 in every slot with no lighting behind it. + var pageEnd = face.LightOffset + ((long) i * styleStride + sampleCount) * 4; + if ( pageEnd > lighting.Length ) continue; + + litStyles.Add( i ); + } + + if ( litStyles.Count == 0 ) + { + Array.Clear( output, 0, sampleCount ); + return; + } + + // A single style is copied as-is, so maps without named lights export unchanged. + if ( litStyles.Count == 1 ) + { + ReadLightmapPage( lighting, face, litStyles[0] * styleStride, sampleCount, output ); + return; + } + + var sum = new float[sampleCount * 3]; + var page = new ColorRGBExp32[sampleCount]; + + foreach ( var styleIndex in litStyles ) + { + ReadLightmapPage( lighting, face, styleIndex * styleStride, sampleCount, page ); + + for ( var i = 0; i < sampleCount; ++i ) + { + var sample = page[i]; + var scale = (float) Math.Pow( 2, sample.Exponent ); + + sum[i * 3 + 0] += sample.R * scale; + sum[i * 3 + 1] += sample.G * scale; + sum[i * 3 + 2] += sample.B * scale; + } + } + + for ( var i = 0; i < sampleCount; ++i ) + { + output[i] = EncodeRGBExp32( sum[i * 3 + 0], sum[i * 3 + 1], sum[i * 3 + 2] ); + } + } + + private static void ReadLightmapPage( Stream lighting, Face face, int sampleOffset, int sampleCount, ColorRGBExp32[] output ) + { + lighting.Seek( face.LightOffset + (long) sampleOffset * 4, SeekOrigin.Begin ); + LumpReader.ReadLumpFromStream( lighting, sampleCount, output ); + } + + /// + /// Picks the exponent that keeps the brightest channel in the top half of a byte, the + /// same precision VRAD writes with. + /// + private static ColorRGBExp32 EncodeRGBExp32( float r, float g, float b ) + { + var max = Math.Max( r, Math.Max( g, b ) ); + if ( max <= 0f ) return default( ColorRGBExp32 ); + + var exponent = (int) Math.Floor( Math.Log( max, 2 ) ) - 7; + exponent = Math.Max( sbyte.MinValue, Math.Min( sbyte.MaxValue, exponent ) ); + + var scale = (float) Math.Pow( 2, -exponent ); + + return new ColorRGBExp32( ToChannel( r * scale ), ToChannel( g * scale ), ToChannel( b * scale ), (sbyte) exponent ); + } + + private static byte ToChannel( float value ) + { + return (byte) Math.Max( 0, Math.Min( 255, (int) Math.Round( value ) ) ); + } + } +} diff --git a/SourceUtils/ValveBsp/Structures.cs b/SourceUtils/ValveBsp/Structures.cs index 5bab8a85..33b525df 100644 --- a/SourceUtils/ValveBsp/Structures.cs +++ b/SourceUtils/ValveBsp/Structures.cs @@ -516,6 +516,14 @@ public static implicit operator int(ColorRGBExp32 color) return color.R | (color.G << 8) | (color.B << 16) | ((color.Exponent + 128) << 24); } + public ColorRGBExp32( byte r, byte g, byte b, sbyte exponent ) + { + R = r; + G = g; + B = b; + Exponent = exponent; + } + public readonly byte R; public readonly byte G; public readonly byte B; From f3e845a79fae123c3ad6f834068b41edf1b364b7 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 07/13] Fix pitch black props in version 10 static prop lumps StaticPropV10Bsp20 declared ColorModulation at byte 64, but in that layout those bytes are a widened copy of the flags field. The exporter wrote the value out as albedoModulation and the shader multiplies prop albedo by it, so 0x100 decoded to R=0, G=1, B=0 and rendered every prop black. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.Test/StaticPropTest.cs | 119 +++++++++++++++++++++++ SourceUtils/ValveBsp/StaticProps.cs | 13 ++- 3 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 SourceUtils.Test/StaticPropTest.cs diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 15230fca..0f911fda 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -123,6 +123,7 @@ + diff --git a/SourceUtils.Test/StaticPropTest.cs b/SourceUtils.Test/StaticPropTest.cs new file mode 100644 index 00000000..59643090 --- /dev/null +++ b/SourceUtils.Test/StaticPropTest.cs @@ -0,0 +1,119 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.ValveBsp; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + [TestClass] + public class StaticPropTest + { + private const int Sprp = 0x73707270; + + /// + /// Offset of the game lump's data within a file written by , + /// which puts the first lump straight after the header. + /// + private const int GameLumpOffset = 8 + 64 * 16 + 4; + + /// + /// Version 10 props in a version 20 BSP hold a second flags field where later layouts + /// keep a diffuse colour, so reading one as a colour modulates the prop towards black. + /// + [TestMethod] + public void V10Bsp20PropsAreNotColourModulated() + { + const uint noPerTexelLighting = 0x100; + + WithStaticProp( noPerTexelLighting, bsp => + { + StaticPropFlags flags; + bool solid; + uint diffuseModulation; + + bsp.StaticProps.GetPropInfo( 0, out flags, out solid, out diffuseModulation ); + + Assert.AreEqual( 0xffffffff, diffuseModulation ); + } ); + } + + /// + /// Guards the fields either side of the flags against being shifted along to make the + /// colour read give the right answer. + /// + [TestMethod] + public void V10Bsp20PropsKeepTheirTransform() + { + WithStaticProp( 0x100, bsp => + { + Vector3 origin; + Vector3 angles; + float scale; + + bsp.StaticProps.GetPropTransform( 0, out origin, out angles, out scale ); + + Assert.AreEqual( -14592f, origin.X ); + Assert.AreEqual( 7904f, origin.Z ); + Assert.AreEqual( 180f, angles.Y ); + } ); + } + + /// + /// Writes a BSP holding a single version 10 static prop, with + /// in the four bytes that follow the CPU and GPU levels. + /// + private static void WithStaticProp( uint flagsEx, Action action ) + { + const int propSize = 72; + + byte[] sprp; + + using ( var stream = new MemoryStream() ) + using ( var writer = new BinaryWriter( stream ) ) + { + writer.Write( 1 ); // Model dictionary entries + var name = new byte[128]; + Buffer.BlockCopy( System.Text.Encoding.ASCII.GetBytes( "models/test/prop.mdl" ), 0, name, 0, 20 ); + writer.Write( name ); + + writer.Write( 0 ); // Leaf entries + writer.Write( 1 ); // Props + + var prop = new byte[propSize]; + + Buffer.BlockCopy( BitConverter.GetBytes( -14592f ), 0, prop, 0, 4 ); // Origin + Buffer.BlockCopy( BitConverter.GetBytes( 7904f ), 0, prop, 8, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( 180f ), 0, prop, 16, 4 ); // Angles + prop[30] = 6; // Solid + Buffer.BlockCopy( BitConverter.GetBytes( 1f ), 0, prop, 56, 4 ); // ForcedFadeScale + Buffer.BlockCopy( BitConverter.GetBytes( flagsEx ), 0, prop, 64, 4 ); + + writer.Write( prop ); + + sprp = stream.ToArray(); + } + + byte[] gameLump; + + using ( var stream = new MemoryStream() ) + using ( var writer = new BinaryWriter( stream ) ) + { + writer.Write( 1 ); // Game lump items + writer.Write( Sprp ); + writer.Write( (ushort) 0 ); // Flags + writer.Write( (ushort) 10 ); // Version + writer.Write( GameLumpOffset + 4 + 16 ); // Offset of the sprp data + writer.Write( sprp.Length ); + writer.Write( sprp ); + + gameLump = stream.ToArray(); + } + + new BspBuilder() + .Lump( LumpType.GAME_LUMP, gameLump ) + .With( action ); + } + } +} diff --git a/SourceUtils/ValveBsp/StaticProps.cs b/SourceUtils/ValveBsp/StaticProps.cs index 3792088d..d01dc1ff 100644 --- a/SourceUtils/ValveBsp/StaticProps.cs +++ b/SourceUtils/ValveBsp/StaticProps.cs @@ -133,9 +133,14 @@ public struct StaticPropV10Bsp20 : IStaticProp public readonly byte MinGpuLevel; public readonly byte MaxGpuLevel; - public readonly uint ColorModulation; - [MarshalAs(UnmanagedType.U1)] - public readonly bool DisableX360; + /// + /// A widened copy of rather than the diffuse colour that later + /// layouts keep here: across every version 10 prop in the map library these bytes only + /// ever hold flag combinations (0x100 for no per texel lighting, 0x40 for no per vertex + /// lighting, 0x10 for no shadow) and never the 0xffffffff an unmodulated prop needs. + /// Reading them as a colour leaves every prop multiplied towards black. + /// + public readonly uint FlagsEx; Vector3 IStaticProp.Origin => Origin; Vector3 IStaticProp.Angles => Angles; @@ -145,7 +150,7 @@ public struct StaticPropV10Bsp20 : IStaticProp ushort IStaticProp.LeafCount => LeafCount; StaticPropFlags IStaticProp.Flags => Flags; bool IStaticProp.Solid => Solid; - uint IStaticProp.ColorModulation => ColorModulation; + uint IStaticProp.ColorModulation => 0xffffffff; float IStaticProp.FadeMinDist => FadeMinDist; float IStaticProp.FadeMaxDist => FadeMaxDist; float IStaticProp.ForcedFadeScale => ForcedFadeScale; From 4c2d73143edb860cc5576d73b4d318afa53de8d2 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 08/13] Support A8 and BGRA4444 textures Both threw from the size and conversion switches. A8 widens to white with the stored alpha, BGRA4444 widens each 4 bit channel so 0xf becomes 0xff. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.Test/TextureFormatTest.cs | 39 ++++++++++++++++++++++++ SourceUtils.WebExport/Texture.Convert.cs | 32 +++++++++++++++++++ SourceUtils/ValveTextureFile.cs | 4 +++ 4 files changed, 76 insertions(+) create mode 100644 SourceUtils.Test/TextureFormatTest.cs diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 0f911fda..2c750e49 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -122,6 +122,7 @@ + diff --git a/SourceUtils.Test/TextureFormatTest.cs b/SourceUtils.Test/TextureFormatTest.cs new file mode 100644 index 00000000..e9fc37a5 --- /dev/null +++ b/SourceUtils.Test/TextureFormatTest.cs @@ -0,0 +1,39 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class TextureFormatTest + { + private const int Width = 64; + private const int Height = 32; + + private static int SizeOf( TextureFormat format ) + { + return ValveTextureFile.GetImageDataSize( Width, Height, 1, 1, format ); + } + + [TestMethod] + public void SingleChannelFormatsAreOneBytePerPixel() + { + Assert.AreEqual( Width * Height, SizeOf( TextureFormat.A8 ) ); + Assert.AreEqual( Width * Height, SizeOf( TextureFormat.I8 ) ); + } + + [TestMethod] + public void PackedSixteenBitFormatsAreTwoBytesPerPixel() + { + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.BGRA4444 ) ); + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.BGR565 ) ); + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.IA88 ) ); + } + + [TestMethod] + public void FullColourFormatsKeepTheirWidths() + { + Assert.AreEqual( Width * Height * 3, SizeOf( TextureFormat.BGR888 ) ); + Assert.AreEqual( Width * Height * 4, SizeOf( TextureFormat.BGRA8888 ) ); + Assert.AreEqual( Width * Height * 8, SizeOf( TextureFormat.RGBA16161616F ) ); + } + } +} diff --git a/SourceUtils.WebExport/Texture.Convert.cs b/SourceUtils.WebExport/Texture.Convert.cs index 63fe86e4..c1af4ec1 100644 --- a/SourceUtils.WebExport/Texture.Convert.cs +++ b/SourceUtils.WebExport/Texture.Convert.cs @@ -130,6 +130,12 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, case TextureFormat.BGR565: dataLength = dataLength * 3 / 2; break; + case TextureFormat.BGRA4444: + dataLength *= 2; + break; + case TextureFormat.A8: + dataLength *= 4; + break; } var totalLength = dataLength + 128; @@ -183,6 +189,8 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, break; case TextureFormat.BGRA8888: case TextureFormat.BGRX8888: + case TextureFormat.BGRA4444: + case TextureFormat.A8: readSettings.PixelStorage = new PixelStorageSettings(StorageType.Char, "BGRA"); break; case TextureFormat.RGBA8888: @@ -208,6 +216,30 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, buffer[i * 3 + 2] = (byte)(((pixel >> 11) & 31) / 31f * 255f); } break; + case TextureFormat.BGRA4444: + // Each 4 bit channel is widened to 8 bits, so 0xf becomes 0xff. + for (var i = width * height - 1; i >= 0; --i) + { + var pixel = (ushort)(buffer[i * 2] | (buffer[i * 2 + 1] << 8)); + + buffer[i * 4] = (byte) ((pixel & 0xf) * 0x11); + buffer[i * 4 + 1] = (byte) (((pixel >> 4) & 0xf) * 0x11); + buffer[i * 4 + 2] = (byte) (((pixel >> 8) & 0xf) * 0x11); + buffer[i * 4 + 3] = (byte) (((pixel >> 12) & 0xf) * 0x11); + } + break; + case TextureFormat.A8: + // Alpha only, so the colour channels are left white. + for (var i = width * height - 1; i >= 0; --i) + { + var alpha = buffer[i]; + + buffer[i * 4] = 0xff; + buffer[i * 4 + 1] = 0xff; + buffer[i * 4 + 2] = 0xff; + buffer[i * 4 + 3] = alpha; + } + break; } var img = new MagickImage( buffer, readSettings ); diff --git a/SourceUtils/ValveTextureFile.cs b/SourceUtils/ValveTextureFile.cs index d6eb432d..56071ca1 100644 --- a/SourceUtils/ValveTextureFile.cs +++ b/SourceUtils/ValveTextureFile.cs @@ -164,6 +164,7 @@ public static int GetImageDataSize( int width, int height, int depth, int mipCou case TextureFormat.DXT3: case TextureFormat.DXT5: return toAdd + width * height * depth; + case TextureFormat.A8: case TextureFormat.I8: return toAdd + width * height * depth; case TextureFormat.IA88: @@ -181,6 +182,7 @@ public static int GetImageDataSize( int width, int height, int depth, int mipCou return toAdd + width * height * depth * 3; case TextureFormat.BGR565: case TextureFormat.RGB565: + case TextureFormat.BGRA4444: return toAdd + width * height * depth * 2; default: throw new NotImplementedException(); @@ -288,8 +290,10 @@ public ValveTextureFile( Stream stream, bool onlyHeader = false ) case TextureFormat.DXT1: case TextureFormat.DXT3: case TextureFormat.DXT5: + case TextureFormat.A8: case TextureFormat.I8: case TextureFormat.IA88: + case TextureFormat.BGRA4444: case TextureFormat.BGR565: case TextureFormat.RGB565: case TextureFormat.BGR888: From 334f560d5e822da54508ea5c429939b58324aa9e Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 09/13] Check vertex lighting headers against the size of the file Some maps pack unrelated files under .vhv names, so a header's mesh and vertex counts can point far outside the file. Placeholder files holding no meshes at all are treated as having no vertex lighting, and a model packed without its .vvd or .vtx is skipped with a warning instead of throwing. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.Test/VertexLightingTest.cs | 155 +++++++++++++++++++++++ SourceUtils.WebExport/Bsp/Geometry.cs | 20 ++- SourceUtils/ValveVertexLightingFile.cs | 40 +++++- 4 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 SourceUtils.Test/VertexLightingTest.cs diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 2c750e49..5747a4e1 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -122,6 +122,7 @@ + diff --git a/SourceUtils.Test/VertexLightingTest.cs b/SourceUtils.Test/VertexLightingTest.cs new file mode 100644 index 00000000..ea9cccd9 --- /dev/null +++ b/SourceUtils.Test/VertexLightingTest.cs @@ -0,0 +1,155 @@ +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class VertexLightingTest + { + private const int HeaderSize = 40; + private const int MeshCountOffset = 20; + + /// + /// VRAD writes padded placeholder .vhv files holding no meshes at all for props it + /// didn't light per vertex. + /// + private static Stream EmptyVhv( int length = 512 ) + { + var data = new byte[length]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); // Version + } + + return new MemoryStream( data ); + } + + private const int MeshHeaderSize = 28; + + private static Stream VhvWithMeshes( int meshCount ) + { + var vertOffset = HeaderSize + meshCount * MeshHeaderSize; + var data = new byte[vertOffset]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( meshCount ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + + for ( var i = 0; i < meshCount; ++i ) + { + writer.Write( 0 ); // Lod + writer.Write( 0 ); // VertCount + writer.Write( vertOffset ); // VertOffset + writer.Write( 0 ); // Unused0 + writer.Write( 0 ); // Unused1 + writer.Write( 0 ); // Unused2 + writer.Write( 0 ); // Unused3 + } + } + + return new MemoryStream( data ); + } + + [TestMethod] + public void EmptyFileParsesWithNoMeshes() + { + using ( var stream = EmptyVhv() ) + { + var vhv = ValveVertexLightingFile.FromStream( stream ); + + Assert.IsNotNull( vhv ); + Assert.AreEqual( 0, vhv.GetMeshCount( 0 ) ); + Assert.AreEqual( 0, vhv.GetMeshCount( 3 ) ); + } + } + + private static void AssertRejected( byte[] data ) + { + try + { + ValveVertexLightingFile.FromStream( new MemoryStream( data ) ); + } + catch ( System.NotSupportedException ) + { + return; + } + + Assert.Fail( "Expected the file to be rejected as unsupported." ); + } + + /// + /// Some maps pack unrelated files under .vhv names. Their headers claim absurd counts, + /// which used to read past the end of the file or exhaust memory. + /// + [TestMethod] + public void HeaderClaimingMoreMeshesThanTheFileHoldsIsRejected() + { + var data = new byte[512]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1634533376 ); + } + + AssertRejected( data ); + } + + [TestMethod] + public void MeshPointingOutsideTheFileIsRejected() + { + var vertOffset = HeaderSize + MeshHeaderSize; + var data = new byte[vertOffset + 16]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1 ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + writer.Write( 0 ); // Lod + writer.Write( 1000000 ); // VertCount, way past the end + writer.Write( vertOffset ); // VertOffset + } + + AssertRejected( data ); + } + + [TestMethod] + public void AbsurdLodIsRejected() + { + var vertOffset = HeaderSize + MeshHeaderSize; + var data = new byte[vertOffset]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1 ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + writer.Write( 1634533376 ); // Lod + writer.Write( 0 ); // VertCount + writer.Write( vertOffset ); // VertOffset + } + + AssertRejected( data ); + } + + [TestMethod] + public void FileWithMeshesReportsThem() + { + using ( var stream = VhvWithMeshes( 2 ) ) + { + var vhv = ValveVertexLightingFile.FromStream( stream ); + + Assert.AreEqual( 2, vhv.GetMeshCount( 0 ) ); + Assert.AreEqual( 0, vhv.GetMeshCount( 1 ) ); + } + } + } +} diff --git a/SourceUtils.WebExport/Bsp/Geometry.cs b/SourceUtils.WebExport/Bsp/Geometry.cs index 40b32e52..63a2b238 100644 --- a/SourceUtils.WebExport/Bsp/Geometry.cs +++ b/SourceUtils.WebExport/Bsp/Geometry.cs @@ -764,6 +764,16 @@ public StudioModelPage GetStudioModelPage( [Url] string map, [Url] int index ) StudioModel mdl; page.Models.Add( mdl = new StudioModel() ); + // Some maps pack a model without the vertex or triangle files that go with it, + // leaving nothing to draw. The empty model keeps the page indices lined up. + if ( mdlFile == null || vvdFile == null || vtxFile == null ) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine( $"Missing mesh data for model '{mdlPath}'!" ); + Console.ResetColor(); + continue; + } + for ( var j = 0; j < mdlFile.BodyPartCount; ++j ) { SmdBodyPart smdBodyPart; @@ -884,9 +894,17 @@ public VertexLightingPage GetVertexLightingPage( [Url] string map, [Url] int ind continue; } - var meshList = new List>(); var meshCount = vhvFile.GetMeshCount( 0 ); + if ( meshCount == 0 ) + { + // An empty lighting file means the same as having none at all. + page.Props.Add( null ); + continue; + } + + var meshList = new List>(); + for ( var j = 0; j < meshCount; ++j ) { var vertices = new CompressedList(); diff --git a/SourceUtils/ValveVertexLightingFile.cs b/SourceUtils/ValveVertexLightingFile.cs index 44a3727b..4a1fc9b6 100644 --- a/SourceUtils/ValveVertexLightingFile.cs +++ b/SourceUtils/ValveVertexLightingFile.cs @@ -9,6 +9,15 @@ namespace SourceUtils { public class ValveVertexLightingFile { + private const int MeshHeaderSize = 28; + private const int VertexData4Size = 4; + private const int VertexData2Size = 12; + + /// + /// Source supports up to 8 levels of detail, so anything beyond that is a bad header. + /// + private const int MaxLodCount = 8; + [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct VhvMeshHeader { @@ -122,10 +131,39 @@ public ValveVertexLightingFile( Stream stream ) var unused0 = reader.ReadInt64(); // Unused var unused1 = reader.ReadInt64(); // Unused + // Some maps pack unrelated files under .vhv names, so nothing in the header can + // be trusted until it has been checked against the size of the file. + if ( meshCount < 0 || stream.Position + (long) meshCount * MeshHeaderSize > stream.Length ) + { + throw new NotSupportedException( + $"Vertex lighting file claims {meshCount} meshes, which don't fit in {stream.Length} bytes." ); + } + var meshHeaders = new List(); LumpReader.ReadLumpFromStream(stream, meshCount, meshHeaders); - _samples = new VertexData4[meshHeaders.Max( x => x.Lod ) + 1][][]; + var sampleSize = vertFlags == 2 ? VertexData2Size : VertexData4Size; + + foreach ( var meshHeader in meshHeaders ) + { + var start = (long) meshHeader.VertOffset + baseOffset; + + if ( meshHeader.Lod < 0 || meshHeader.Lod >= MaxLodCount ) + { + throw new NotSupportedException( $"Vertex lighting mesh has an out of range LOD of {meshHeader.Lod}." ); + } + + if ( meshHeader.VertCount < 0 || start < 0 + || start + (long) meshHeader.VertCount * sampleSize > stream.Length ) + { + throw new NotSupportedException( + $"Vertex lighting mesh with {meshHeader.VertCount} vertices at {start} doesn't fit in {stream.Length} bytes." ); + } + } + + // VRAD writes placeholder files holding no meshes at all for props it didn't + // light per vertex. + _samples = new VertexData4[meshHeaders.Count == 0 ? 0 : meshHeaders.Max( x => x.Lod ) + 1][][]; for ( var i = 0; i < _samples.Length; ++i ) { From 5d86822fcc993b7f52fad4bc840eba02805f4e28 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 10/13] Fall back to the base texture for sky faces without an HDR one Plenty of maps name an hdrcompressedtexture without packing it, and skies built from an environment cubemap have no base texture either. Each candidate now has to resolve before it is used, and a sky with nothing to draw with is dropped. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/SkyMaterialTest.cs | 55 ++++++++++++++++++++++++ SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.WebExport/Material.cs | 50 ++++++++++++++++++--- 3 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 SourceUtils.Test/SkyMaterialTest.cs diff --git a/SourceUtils.Test/SkyMaterialTest.cs b/SourceUtils.Test/SkyMaterialTest.cs new file mode 100644 index 00000000..fe627750 --- /dev/null +++ b/SourceUtils.Test/SkyMaterialTest.cs @@ -0,0 +1,55 @@ +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class SkyMaterialTest + { + private const string BaseUrl = "/materials/skybox/sky_bk.vtf/0123456789abcdef.vtf.json"; + private const string HdrUrl = "/materials/skybox/sky_bk_hdr.vtf/fedcba9876543210.vtf.json"; + + private static string[] Names( Material mat ) + { + return Material.GetSkyFaceTextures( mat ).Select( x => x.Name ).ToArray(); + } + + [TestMethod] + public void BaseTextureIsUsedWhenPresent() + { + var mat = new Material(); + mat.SetTextureUrl( "basetexture", BaseUrl ); + + CollectionAssert.AreEqual( new[] { "basetexture" }, Names( mat ) ); + } + + /// + /// Maps often name an HDR texture without packing it, so the base texture has to stay + /// available as a fallback rather than being dropped. + /// + [TestMethod] + public void HdrTextureComesFirstButBaseTextureRemains() + { + var mat = new Material(); + mat.SetTextureUrl( "basetexture", BaseUrl ); + mat.SetTextureUrl( "hdrcompressedtexture", HdrUrl ); + + CollectionAssert.AreEqual( new[] { "hdrcompressedtexture", "basetexture" }, Names( mat ) ); + } + + /// + /// Some maps build their sky from an environment cubemap instead, with a shader like + /// WindowImposter and no $basetexture at all. That used to throw, failing the whole + /// map's index.json and leaving the map unexported. + /// + [TestMethod] + public void MaterialWithoutTexturesGivesNothing() + { + var mat = new Material(); + mat.SetBoolean( "translucent", true ); + + Assert.AreEqual( 0, Names( mat ).Length ); + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 5747a4e1..2f21fb14 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -123,6 +123,7 @@ + diff --git a/SourceUtils.WebExport/Material.cs b/SourceUtils.WebExport/Material.cs index 78979648..431bd9ca 100644 --- a/SourceUtils.WebExport/Material.cs +++ b/SourceUtils.WebExport/Material.cs @@ -292,6 +292,20 @@ public static Material Get(ValveBspFile bsp, string path) return mat; } + /// + /// Textures a sky face could be drawn with, best first. The list is empty for skies + /// built from an environment cubemap (WindowImposter and friends), which have no base + /// texture at all. + /// + public static IEnumerable GetSkyFaceTextures( Material mat ) + { + var hdr = mat.Properties.FirstOrDefault( x => x.Name == "hdrcompressedtexture" ); + var ldr = mat.Properties.FirstOrDefault( x => x.Name == "basetexture" ); + + if ( hdr != null ) yield return hdr; + if ( ldr != null ) yield return ldr; + } + public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) { var postfixes = new[] @@ -310,6 +324,7 @@ public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) var hdrCompressed = false; var aspect = 1f; + var faceCount = 0; for ( var face = 0; face < 6; ++face ) { @@ -318,22 +333,43 @@ public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) if ( mat == null ) continue; - var texProp = mat.Properties.FirstOrDefault( x => x.Name == "hdrcompressedtexture") - ?? mat.Properties.First(x => x.Name == "basetexture"); + // The HDR texture is preferred, but plenty of maps name one without packing it, + // so each candidate has to actually resolve before it can be used. + MaterialProperty texProp = null; + Texture faceTex = null; - if ( face == 0 ) + foreach ( var candidate in GetSkyFaceTextures( mat ) ) { - hdrCompressed = texProp.Name == "hdrcompressedtexture"; - - var tex = TextureSource.TryParseUrl( (Url) texProp.Value, out var texPath, out _, out _ ) + faceTex = TextureSource.TryParseUrl( (Url) candidate.Value, out var texPath, out _, out _ ) ? Texture.Get( bsp, texPath ) : null; - aspect = tex == null ? 1f : (float) tex.Width / tex.Height; + + if ( faceTex == null ) continue; + + texProp = candidate; + break; + } + + if ( texProp == null ) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine( $"Sky material '{matPath}' has no texture to draw with!" ); + Console.ResetColor(); + continue; + } + + if ( faceCount++ == 0 ) + { + hdrCompressed = texProp.Name == "hdrcompressedtexture"; + aspect = (float) faceTex.Width / faceTex.Height; } skyMaterial.SetTextureUrl($"face{names[face]}", (Url)texProp.Value); } + // Nothing to draw the sky with, so the map is better off without one. + if ( faceCount == 0 ) return null; + if ( hdrCompressed ) { skyMaterial.SetBoolean( "hdrCompressed", true ); From d395629ea352b1af393c47f0aaabf37353b568aa Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 11/13] Link materials to the global textures instead of inlining them per map Material pages kept rewriting texture urls into per-map indices, which pulled every frame and mip level of a shared texture into each map that used it. Urls into the hash addressed materials folder are now left alone, so a texture is fetched and exported once. Hash addressed urls also skip the timestamp query and, once exported, are not requested again: they cannot go stale while their file exists, and neither can anything below them. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/TextureSourceTest.cs | 21 +++++++++++++ SourceUtils.WebExport/Bsp/Materials.cs | 34 ++++++++------------- SourceUtils.WebExport/Export.cs | 7 +++++ SourceUtils.WebExport/ResourceController.cs | 5 +++ SourceUtils.WebExport/TextureSource.cs | 9 ++++++ 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/SourceUtils.Test/TextureSourceTest.cs b/SourceUtils.Test/TextureSourceTest.cs index a60d9054..876c43a6 100644 --- a/SourceUtils.Test/TextureSourceTest.cs +++ b/SourceUtils.Test/TextureSourceTest.cs @@ -157,6 +157,27 @@ public void ParseRejectsNestedFile() Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/0123456789abcdef/a/mip0.png", out _, out _, out _ ) ); } + [TestMethod] + public void ContentAddressedForHashedInfoAndImageUrls() + { + Assert.IsTrue( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, null ) ) ); + Assert.IsTrue( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, "mip0.frame7.png" ) ) ); + } + + [TestMethod] + public void NotContentAddressedWithoutHash() + { + Assert.IsFalse( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", null, null ) ) ); + } + + [TestMethod] + public void NotContentAddressedForOtherResources() + { + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/geom/vispage0.json" ) ); + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/materials/matpage0.json" ) ); + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/lightmap.png" ) ); + } + [TestMethod] public void ParseImageFileNameRejectsMalformedAndOverflowingIndices() { diff --git a/SourceUtils.WebExport/Bsp/Materials.cs b/SourceUtils.WebExport/Bsp/Materials.cs index b39edc8a..167df17a 100644 --- a/SourceUtils.WebExport/Bsp/Materials.cs +++ b/SourceUtils.WebExport/Bsp/Materials.cs @@ -33,8 +33,6 @@ public MaterialPage GetPage( [Url] string map, [Url] int index ) count = 0; } - var texDict = new Dictionary(); - var page = new MaterialPage(); for ( var i = 0; i < count; ++i ) @@ -55,30 +53,22 @@ public MaterialPage GetPage( [Url] string map, [Url] int index ) { if ( prop.Type != MaterialPropertyType.TextureUrl ) continue; - prop.Type = MaterialPropertyType.TextureIndex; - + // Textures are left as urls into the global, hash addressed materials folder so + // that each one is fetched (and exported) once as a whole, rather than having + // every frame and mip level inlined into each map that happens to use it. var texUrl = (Url) prop.Value; - if (texDict.TryGetValue(texUrl, out int texIndex)) - { - prop.Value = texIndex; - continue; - } - - prop.Value = texIndex = page.Textures.Count; - var tex = TextureSource.TryParseUrl( texUrl, out var texPath, out _, out _ ) - ? Texture.Get( bsp, texPath ) - : null; + if ( TextureSource.TryParseUrl( texUrl, out var texPath, out var texHash, out _ ) + && texHash != null ) continue; - if ( tex == null ) - { - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine($"Missing texture '{texPath ?? (string) texUrl}'!"); - Console.ResetColor(); - } + // Unresolved textures have no hash to address them by, so there is nothing to + // link to. Null it out rather than emitting a url that can only ever 404. + prop.Type = MaterialPropertyType.TextureInfo; + prop.Value = null; - texDict.Add( texUrl, texIndex ); - page.Textures.Add( tex ); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine($"Missing texture '{texPath ?? (string) texUrl}'!"); + Console.ResetColor(); } } diff --git a/SourceUtils.WebExport/Export.cs b/SourceUtils.WebExport/Export.cs index b0e044a8..d1cdebd1 100644 --- a/SourceUtils.WebExport/Export.cs +++ b/SourceUtils.WebExport/Export.cs @@ -154,6 +154,13 @@ static void ExportMap( string mapName, ExportOptions args ) else Console.Write($"[{exported + skipped + failed + 1}/{_sExportUrls.Count - oldCount}] Exporting '{url}' ... "); } + // Skipped urls are normally still requested, because serving them is what + // discovers the urls nested inside them. A content addressed url can't have + // gone stale while its file exists, and neither can anything below it, so + // there is nothing left to discover. One check skips a whole texture, + // however many frames and mip levels it turns out to have. + if ( skip && TextureSource.IsContentAddressed( url ) ) continue; + var dir = Path.GetDirectoryName(path); if ( !args.DryRun && !Directory.Exists( dir ) ) { diff --git a/SourceUtils.WebExport/ResourceController.cs b/SourceUtils.WebExport/ResourceController.cs index 3e258ccb..007816b5 100644 --- a/SourceUtils.WebExport/ResourceController.cs +++ b/SourceUtils.WebExport/ResourceController.cs @@ -79,6 +79,11 @@ private static bool ShouldAppendVersionSuffix( Url url ) { case ".png": return false; + case ".json": + // Hash addressed texture urls are already versioned by their contents. A + // timestamp would only break caching, and would make the same texture + // referenced either side of a second boundary look like two textures. + return !TextureSource.IsContentAddressed( url ); default: return true; } diff --git a/SourceUtils.WebExport/TextureSource.cs b/SourceUtils.WebExport/TextureSource.cs index 5f647e11..c56b9d94 100644 --- a/SourceUtils.WebExport/TextureSource.cs +++ b/SourceUtils.WebExport/TextureSource.cs @@ -80,6 +80,15 @@ public static bool TryParseUrl( string absolutePath, out string path, out string return true; } + /// + /// True if the url names a specific version of a texture by the hash of its contents, and + /// so can never go stale while the file it points at exists. + /// + public static bool IsContentAddressed( string url ) + { + return TryParseUrl( url, out _, out var hash, out _ ) && hash != null; + } + public static bool TryParseImageFileName( string fileName, int mipCount, int frameCount, int faceCount, out int mip, out int frame, out int face ) { From f1272fb68107ace8b346ac702b82d58edef4c1b8 Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:54 +0200 Subject: [PATCH 12/13] Add --overwrite-maps and --overwrite-materials Re-exporting a map meant re-exporting every texture it touches, and picking up a change to how textures are converted meant re-exporting every map. The content hash of a source .vtf cannot see a change in the converter, so overwriting materials stays a deliberate choice. --overwrite still implies both, and static viewer files are always replaced. Co-Authored-By: Claude Opus 5 --- SourceUtils.Test/ExportOverwriteTest.cs | 99 ++++++++++++++++++++++++ SourceUtils.Test/SourceUtils.Test.csproj | 1 + SourceUtils.Test/TextureSourceTest.cs | 17 ++++ SourceUtils.WebExport/Export.cs | 31 +++++++- SourceUtils.WebExport/Program.cs | 2 +- SourceUtils.WebExport/TextureSource.cs | 9 +++ 6 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 SourceUtils.Test/ExportOverwriteTest.cs diff --git a/SourceUtils.Test/ExportOverwriteTest.cs b/SourceUtils.Test/ExportOverwriteTest.cs new file mode 100644 index 00000000..fda44948 --- /dev/null +++ b/SourceUtils.Test/ExportOverwriteTest.cs @@ -0,0 +1,99 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class ExportOverwriteTest + { + private const string MaterialInfo = "/materials/concrete/wall.vtf/0123456789abcdef.vtf.json"; + private const string MaterialImage = "/materials/concrete/wall.vtf/0123456789abcdef/mip0.frame7.png"; + private const string MissingMaterial = "/materials/concrete/wall.vtf.json"; + private const string MapPage = "/maps/surf_x/materials/matpage0.json"; + private const string MapIndex = "/maps/surf_x/index.html"; + private const string StaticScript = "/js/sourceutils.js"; + private const string StaticStyle = "/styles/main.css"; + + private static ExportOptions Options( bool all = false, bool maps = false, bool materials = false ) + { + return new ExportOptions + { + Overwrite = all, + OverwriteMaps = maps, + OverwriteMaterials = materials + }; + } + + [TestMethod] + public void StaticFilesAreAlwaysOverwritten() + { + var none = Options(); + Assert.IsTrue( none.ShouldOverwrite( StaticScript ) ); + Assert.IsTrue( none.ShouldOverwrite( StaticStyle ) ); + } + + [TestMethod] + public void NoFlagsLeavesMapsAndMaterialsAlone() + { + var none = Options(); + Assert.IsFalse( none.ShouldOverwrite( MaterialInfo ) ); + Assert.IsFalse( none.ShouldOverwrite( MaterialImage ) ); + Assert.IsFalse( none.ShouldOverwrite( MapPage ) ); + Assert.IsFalse( none.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void LegacyOverwriteCoversMapsAndMaterials() + { + var all = Options( all: true ); + Assert.IsTrue( all.ShouldOverwrite( MaterialInfo ) ); + Assert.IsTrue( all.ShouldOverwrite( MaterialImage ) ); + Assert.IsTrue( all.ShouldOverwrite( MapPage ) ); + Assert.IsTrue( all.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void OverwriteMaterialsLeavesMapsAlone() + { + var materials = Options( materials: true ); + Assert.IsTrue( materials.ShouldOverwrite( MaterialInfo ) ); + Assert.IsTrue( materials.ShouldOverwrite( MaterialImage ) ); + Assert.IsTrue( materials.ShouldOverwrite( MissingMaterial ) ); + Assert.IsFalse( materials.ShouldOverwrite( MapPage ) ); + Assert.IsFalse( materials.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void OverwriteMapsLeavesMaterialsAlone() + { + var maps = Options( maps: true ); + Assert.IsTrue( maps.ShouldOverwrite( MapPage ) ); + Assert.IsTrue( maps.ShouldOverwrite( MapIndex ) ); + Assert.IsFalse( maps.ShouldOverwrite( MaterialInfo ) ); + Assert.IsFalse( maps.ShouldOverwrite( MaterialImage ) ); + } + + /// + /// A map's own material page lives under /maps, so it must follow the maps flag rather + /// than being caught by the word "materials" appearing in its url. + /// + [TestMethod] + public void MapMaterialPageCountsAsAMapFile() + { + Assert.IsTrue( Options( maps: true ).ShouldOverwrite( MapPage ) ); + Assert.IsFalse( Options( materials: true ).ShouldOverwrite( MapPage ) ); + } + + [TestMethod] + public void BothNewFlagsMatchLegacyOverwrite() + { + var split = Options( maps: true, materials: true ); + var legacy = Options( all: true ); + + foreach ( var url in new[] { MaterialInfo, MaterialImage, MapPage, MapIndex, StaticScript } ) + { + Assert.AreEqual( legacy.ShouldOverwrite( url ), split.ShouldOverwrite( url ), url ); + } + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index 2f21fb14..1b3327a8 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -119,6 +119,7 @@ + diff --git a/SourceUtils.Test/TextureSourceTest.cs b/SourceUtils.Test/TextureSourceTest.cs index 876c43a6..8418bdce 100644 --- a/SourceUtils.Test/TextureSourceTest.cs +++ b/SourceUtils.Test/TextureSourceTest.cs @@ -157,6 +157,23 @@ public void ParseRejectsNestedFile() Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/0123456789abcdef/a/mip0.png", out _, out _, out _ ) ); } + [TestMethod] + public void MaterialUrlCoversEveryFormUnderTheMaterialsFolder() + { + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, null ) ) ); + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, "mip0.png" ) ) ); + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", null, null ) ) ); + } + + [TestMethod] + public void MaterialUrlExcludesMapsAndStaticFiles() + { + Assert.IsFalse( TextureSource.IsMaterialUrl( "/maps/surf_x/materials/matpage0.json" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( "/maps/surf_x/index.html" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( "/js/sourceutils.js" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( null ) ); + } + [TestMethod] public void ContentAddressedForHashedInfoAndImageUrls() { diff --git a/SourceUtils.WebExport/Export.cs b/SourceUtils.WebExport/Export.cs index d1cdebd1..0b4c8e82 100644 --- a/SourceUtils.WebExport/Export.cs +++ b/SourceUtils.WebExport/Export.cs @@ -11,14 +11,39 @@ namespace SourceUtils.WebExport { [Verb("export", HelpText = "Export one or more maps for serving over HTTP.")] - class ExportOptions : BaseOptions + public class ExportOptions : BaseOptions { [Option('o', "outdir", HelpText = "Output directory.", Required = true)] public string OutDir { get; set; } - [Option('r', "overwrite", HelpText = "Overwrite existing exported files.")] + [Option('r', "overwrite", HelpText = "Overwrite every existing exported file (implies --overwrite-maps and --overwrite-materials).")] public bool Overwrite { get; set; } + [Option("overwrite-maps", HelpText = "Overwrite existing exported map files.")] + public bool OverwriteMaps { get; set; } + + [Option("overwrite-materials", HelpText = "Overwrite existing exported material files. Needed to pick up changes to how textures are converted, which the content hash of a source .vtf can't see.")] + public bool OverwriteMaterials { get; set; } + + /// + /// Which of the exported files this run is willing to replace. Static viewer files are + /// always replaced: they belong to the build rather than to any map or texture, and a + /// stale one is a broken viewer for the sake of a few bytes. + /// + public bool ShouldOverwrite( string url ) + { + if ( Overwrite ) return true; + if ( TextureSource.IsMaterialUrl( url ) ) return OverwriteMaterials; + if ( IsMapUrl( url ) ) return OverwriteMaps; + + return true; + } + + private static bool IsMapUrl( string url ) + { + return url != null && url.StartsWith( "/maps/", StringComparison.Ordinal ); + } + [Option('p', "url-prefix", HelpText = "Prefix to prepend to each exported URL." )] public string UrlPrefix { get; set; } = ""; @@ -140,7 +165,7 @@ static void ExportMap( string mapName, ExportOptions args ) var url = _sToExport.Dequeue(); var path = Path.Combine(args.OutDir, url.Value.Substring(1)); - var skip = !args.Overwrite && File.Exists(path); + var skip = !args.ShouldOverwrite(url) && File.Exists(path); if ( skip ) { diff --git a/SourceUtils.WebExport/Program.cs b/SourceUtils.WebExport/Program.cs index 8397b15b..7a8cd6af 100644 --- a/SourceUtils.WebExport/Program.cs +++ b/SourceUtils.WebExport/Program.cs @@ -14,7 +14,7 @@ namespace SourceUtils.WebExport { - class BaseOptions + public class BaseOptions { [Option('g', "gamedir", HelpText = "Game directory to export from.", Required = true)] public string GameDir { get; set; } diff --git a/SourceUtils.WebExport/TextureSource.cs b/SourceUtils.WebExport/TextureSource.cs index c56b9d94..7f4fa162 100644 --- a/SourceUtils.WebExport/TextureSource.cs +++ b/SourceUtils.WebExport/TextureSource.cs @@ -80,6 +80,15 @@ public static bool TryParseUrl( string absolutePath, out string path, out string return true; } + /// + /// True if the url points into the global materials folder, whatever form it takes. Note + /// that a map's own material page lives under /maps and is not one of these. + /// + public static bool IsMaterialUrl( string url ) + { + return url != null && url.StartsWith( UrlPrefix, StringComparison.Ordinal ); + } + /// /// True if the url names a specific version of a texture by the hash of its contents, and /// so can never go stale while the file it points at exists. From 2f114b1172bbb739f7ed98255865be90458171da Mon Sep 17 00:00:00 2001 From: SurfSam Date: Wed, 23 Sep 2026 13:20:55 +0200 Subject: [PATCH 13/13] Ignore the local test export directory Co-Authored-By: Claude Opus 5 --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6c27658d..a0d7c5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,5 @@ paket-files/ *.sln.iml MapViewServer/Scripts/typings/ + +test/ \ No newline at end of file