diff --git a/.gitignore b/.gitignore index 6c27658d..a0d7c5ca 100644 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,5 @@ paket-files/ *.sln.iml MapViewServer/Scripts/typings/ + +test/ \ No newline at end of file diff --git a/SourceUtils.Test/BspBuilder.cs b/SourceUtils.Test/BspBuilder.cs new file mode 100644 index 00000000..232a0599 --- /dev/null +++ b/SourceUtils.Test/BspBuilder.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Writes a minimal version 20 BSP file holding only the lumps a test gives it. + /// + internal class BspBuilder + { + private const int LumpCount = 64; + private const int HeaderSize = 4 + 4 + LumpCount * 16 + 4; + + private readonly Dictionary> _lumps + = new Dictionary>(); + + public BspBuilder Lump( LumpType type, byte[] data, int version = 0 ) + { + _lumps[type] = new KeyValuePair( data, version ); + return this; + } + + public string Write() + { + var path = Path.Combine( Path.GetTempPath(), $"sourceutils-test-{Guid.NewGuid():N}.bsp" ); + + using ( var writer = new BinaryWriter( File.Create( path ) ) ) + { + writer.Write( new[] { (byte) 'V', (byte) 'B', (byte) 'S', (byte) 'P' } ); + writer.Write( 20 ); + + var offset = HeaderSize; + + for ( var i = 0; i < LumpCount; ++i ) + { + KeyValuePair lump; + if ( !_lumps.TryGetValue( (LumpType) i, out lump ) ) + { + writer.Write( offset ); + writer.Write( 0 ); + writer.Write( 0 ); + writer.Write( new byte[4] ); + continue; + } + + writer.Write( offset ); + writer.Write( lump.Key.Length ); + writer.Write( lump.Value ); + writer.Write( new byte[4] ); + + offset += lump.Key.Length; + } + + writer.Write( 0 ); + + for ( var i = 0; i < LumpCount; ++i ) + { + KeyValuePair lump; + if ( _lumps.TryGetValue( (LumpType) i, out lump ) ) writer.Write( lump.Key ); + } + } + + return path; + } + + /// + /// Writes the file, opens it for and deletes it afterwards. + /// + public void With( Action action ) + { + var path = Write(); + + try + { + using ( var bsp = new ValveBspFile( path ) ) + { + action( bsp ); + } + } + finally + { + File.Delete( path ); + } + } + } +} diff --git a/SourceUtils.Test/ExportOverwriteTest.cs b/SourceUtils.Test/ExportOverwriteTest.cs new file mode 100644 index 00000000..fda44948 --- /dev/null +++ b/SourceUtils.Test/ExportOverwriteTest.cs @@ -0,0 +1,99 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class ExportOverwriteTest + { + private const string MaterialInfo = "/materials/concrete/wall.vtf/0123456789abcdef.vtf.json"; + private const string MaterialImage = "/materials/concrete/wall.vtf/0123456789abcdef/mip0.frame7.png"; + private const string MissingMaterial = "/materials/concrete/wall.vtf.json"; + private const string MapPage = "/maps/surf_x/materials/matpage0.json"; + private const string MapIndex = "/maps/surf_x/index.html"; + private const string StaticScript = "/js/sourceutils.js"; + private const string StaticStyle = "/styles/main.css"; + + private static ExportOptions Options( bool all = false, bool maps = false, bool materials = false ) + { + return new ExportOptions + { + Overwrite = all, + OverwriteMaps = maps, + OverwriteMaterials = materials + }; + } + + [TestMethod] + public void StaticFilesAreAlwaysOverwritten() + { + var none = Options(); + Assert.IsTrue( none.ShouldOverwrite( StaticScript ) ); + Assert.IsTrue( none.ShouldOverwrite( StaticStyle ) ); + } + + [TestMethod] + public void NoFlagsLeavesMapsAndMaterialsAlone() + { + var none = Options(); + Assert.IsFalse( none.ShouldOverwrite( MaterialInfo ) ); + Assert.IsFalse( none.ShouldOverwrite( MaterialImage ) ); + Assert.IsFalse( none.ShouldOverwrite( MapPage ) ); + Assert.IsFalse( none.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void LegacyOverwriteCoversMapsAndMaterials() + { + var all = Options( all: true ); + Assert.IsTrue( all.ShouldOverwrite( MaterialInfo ) ); + Assert.IsTrue( all.ShouldOverwrite( MaterialImage ) ); + Assert.IsTrue( all.ShouldOverwrite( MapPage ) ); + Assert.IsTrue( all.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void OverwriteMaterialsLeavesMapsAlone() + { + var materials = Options( materials: true ); + Assert.IsTrue( materials.ShouldOverwrite( MaterialInfo ) ); + Assert.IsTrue( materials.ShouldOverwrite( MaterialImage ) ); + Assert.IsTrue( materials.ShouldOverwrite( MissingMaterial ) ); + Assert.IsFalse( materials.ShouldOverwrite( MapPage ) ); + Assert.IsFalse( materials.ShouldOverwrite( MapIndex ) ); + } + + [TestMethod] + public void OverwriteMapsLeavesMaterialsAlone() + { + var maps = Options( maps: true ); + Assert.IsTrue( maps.ShouldOverwrite( MapPage ) ); + Assert.IsTrue( maps.ShouldOverwrite( MapIndex ) ); + Assert.IsFalse( maps.ShouldOverwrite( MaterialInfo ) ); + Assert.IsFalse( maps.ShouldOverwrite( MaterialImage ) ); + } + + /// + /// A map's own material page lives under /maps, so it must follow the maps flag rather + /// than being caught by the word "materials" appearing in its url. + /// + [TestMethod] + public void MapMaterialPageCountsAsAMapFile() + { + Assert.IsTrue( Options( maps: true ).ShouldOverwrite( MapPage ) ); + Assert.IsFalse( Options( materials: true ).ShouldOverwrite( MapPage ) ); + } + + [TestMethod] + public void BothNewFlagsMatchLegacyOverwrite() + { + var split = Options( maps: true, materials: true ); + var legacy = Options( all: true ); + + foreach ( var url in new[] { MaterialInfo, MaterialImage, MapPage, MapIndex, StaticScript } ) + { + Assert.AreEqual( legacy.ShouldOverwrite( url ), split.ShouldOverwrite( url ), url ); + } + } + } +} diff --git a/SourceUtils.Test/GameLumpTest.cs b/SourceUtils.Test/GameLumpTest.cs new file mode 100644 index 00000000..fe28e8de --- /dev/null +++ b/SourceUtils.Test/GameLumpTest.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class GameLumpTest + { + private const int Sprp = 0x73707270; + private const int Dprp = 0x64707270; + + [TestMethod] + public void CompressedGameLumpIgnoresFinalSentinelWhenReadingItemVersion() + { + WithGameLump( new[] { Sprp, Dprp, 0 }, bsp => + { + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) ); + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) ); + } ); + } + + [TestMethod] + public void CompressedGameLumpStopsAtFirstZeroIdEntry() + { + WithGameLump( new[] { Sprp, Dprp, 0, 0, 0 }, bsp => + { + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) ); + Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) ); + Assert.ThrowsExactly( () => bsp.GameData.GetItemVersion( "" ) ); + } ); + } + + private static void WithGameLump( int[] itemIds, Action action ) + { + var path = Path.Combine( Path.GetTempPath(), $"sourceutils-gamelump-{Guid.NewGuid():N}.bsp" ); + + try + { + const int gameLumpOffset = 8 + 64 * 16 + 4; + const int gameLumpIndex = 35; + const int itemSize = 16; + + var gameLumpLength = 4 + itemIds.Length * itemSize; + var data = new byte[gameLumpOffset + gameLumpLength]; + + Buffer.BlockCopy( BitConverter.GetBytes( gameLumpOffset ), 0, data, 8 + gameLumpIndex * 16, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( gameLumpLength ), 0, data, 8 + gameLumpIndex * 16 + 4, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( itemIds.Length ), 0, data, gameLumpOffset, 4 ); + + for ( var i = 0; i < itemIds.Length; i++ ) + { + var itemOffset = gameLumpOffset + 4 + i * itemSize; + Buffer.BlockCopy( BitConverter.GetBytes( itemIds[i] ), 0, data, itemOffset, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( (ushort) 10 ), 0, data, itemOffset + 6, 2 ); // version + } + + File.WriteAllBytes( path, data ); + + using ( var bsp = new ValveBspFile( path ) ) + { + action( bsp ); + } + } + finally + { + if ( File.Exists( path ) ) File.Delete( path ); + } + } + } +} diff --git a/SourceUtils.Test/LeafAmbientTest.cs b/SourceUtils.Test/LeafAmbientTest.cs new file mode 100644 index 00000000..7cfe5856 --- /dev/null +++ b/SourceUtils.Test/LeafAmbientTest.cs @@ -0,0 +1,198 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Maps compiled by pre-Orange Box tools store one ambient cube per leaf and write no leaf + /// ambient index lump at all. These tests build minimal BSP files covering both layouts. + /// + [TestClass] + public class LeafAmbientTest + { + private const int LeafSize = 32; + private const int CubeSize = 24; + private const int SampleSize = 28; + private const int IndexSize = 4; + + private static byte[] Leaves( int count ) + { + var data = new byte[count * LeafSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < count; ++i ) + { + writer.Write( 0 ); // Contents + writer.Write( (short) i ); // Cluster + writer.Write( (short) 0 ); // AreaFlags + writer.Write( (short) (i * 10) ); // Min.X + writer.Write( (short) 0 ); // Min.Y + writer.Write( (short) 0 ); // Min.Z + writer.Write( (short) (i * 10 + 8) ); // Max.X + writer.Write( (short) 8 ); // Max.Y + writer.Write( (short) 8 ); // Max.Z + writer.Write( (ushort) 0 ); // FirstLeafFace + writer.Write( (ushort) 0 ); // NumLeafFaces + writer.Write( (ushort) 0 ); // FirstLeafBrush + writer.Write( (ushort) 0 ); // NumLeafBrushes + writer.Write( (short) -1 ); // LeafWaterDataId + writer.Write( (short) 0 ); // Padding + } + } + + return data; + } + + /// Red channel of face 0 identifies each cube. + private static void WriteCube( BinaryWriter writer, byte id ) + { + writer.Write( id ); + writer.Write( (byte) 0 ); + writer.Write( (byte) 0 ); + writer.Write( (byte) 0 ); + + for ( var face = 1; face < 6; ++face ) writer.Write( new byte[4] ); + } + + private static byte[] LegacyAmbient( int leafCount ) + { + var data = new byte[leafCount * CubeSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < leafCount; ++i ) WriteCube( writer, (byte) (i + 1) ); + } + + return data; + } + + private static byte[] IndexedAmbient( params byte[] ids ) + { + var data = new byte[ids.Length * SampleSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + foreach ( var id in ids ) + { + WriteCube( writer, id ); + writer.Write( (byte) (id * 2) ); // X + writer.Write( (byte) 0 ); // Y + writer.Write( (byte) 0 ); // Z + writer.Write( (byte) 0 ); // Padding + } + } + + return data; + } + + private static byte[] AmbientIndices( params int[] countsAndFirsts ) + { + var data = new byte[countsAndFirsts.Length / 2 * IndexSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + for ( var i = 0; i < countsAndFirsts.Length; i += 2 ) + { + writer.Write( (ushort) countsAndFirsts[i] ); + writer.Write( (ushort) countsAndFirsts[i + 1] ); + } + } + + return data; + } + + private static void WithBsp( BspBuilder builder, Action action ) + { + builder.With( action ); + } + + [TestMethod] + public void LegacyLayoutGivesOneCentredSamplePerLeaf() + { + var ambient = LegacyAmbient( 3 ); + + // Pre-Orange Box tools write the lump length into the version field as well. + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, ambient, ambient.Length ); + + WithBsp( builder, bsp => + { + Assert.AreEqual( 3, bsp.Leaves.Length ); + + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 1, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + + var sample = bsp.GetLeafAmbientSample( leaf, 0 ); + + Assert.AreEqual( leaf + 1, sample.Cube[0].R, $"leaf {leaf} cube" ); + Assert.AreEqual( 128, sample.X, $"leaf {leaf} x" ); + Assert.AreEqual( 128, sample.Y, $"leaf {leaf} y" ); + Assert.AreEqual( 128, sample.Z, $"leaf {leaf} z" ); + } + } ); + } + + [TestMethod] + public void IndexedLayoutReadsSamplesThroughTheIndex() + { + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_INDEX, AmbientIndices( 0, 0, 2, 0, 1, 2 ) ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, IndexedAmbient( 10, 20, 30 ), 1 ); + + WithBsp( builder, bsp => + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( 0 ) ); + Assert.AreEqual( 2, bsp.GetLeafAmbientSampleCount( 1 ) ); + Assert.AreEqual( 1, bsp.GetLeafAmbientSampleCount( 2 ) ); + + Assert.AreEqual( 10, bsp.GetLeafAmbientSample( 1, 0 ).Cube[0].R ); + Assert.AreEqual( 20, bsp.GetLeafAmbientSample( 1, 1 ).Cube[0].R ); + Assert.AreEqual( 30, bsp.GetLeafAmbientSample( 2, 0 ).Cube[0].R ); + + // Indexed samples keep their own position within the leaf. + Assert.AreEqual( 40, bsp.GetLeafAmbientSample( 1, 1 ).X ); + } ); + } + + [TestMethod] + public void MissingAmbientLumpsGiveNoSamples() + { + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ); + + WithBsp( builder, bsp => + { + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + } + } ); + } + + [TestMethod] + public void AmbientLumpWithoutOnePerLeafIsIgnored() + { + // An index-less ambient lump that isn't one cube per leaf can't be mapped to leaves. + var ambient = LegacyAmbient( 2 ); + + var builder = new BspBuilder() + .Lump( LumpType.LEAFS, Leaves( 3 ), 1 ) + .Lump( LumpType.LEAF_AMBIENT_LIGHTING, ambient, ambient.Length ); + + WithBsp( builder, bsp => + { + for ( var leaf = 0; leaf < 3; ++leaf ) + { + Assert.AreEqual( 0, bsp.GetLeafAmbientSampleCount( leaf ), $"leaf {leaf}" ); + } + } ); + } + } +} diff --git a/SourceUtils.Test/LightStyleTest.cs b/SourceUtils.Test/LightStyleTest.cs new file mode 100644 index 00000000..ef98a8d5 --- /dev/null +++ b/SourceUtils.Test/LightStyleTest.cs @@ -0,0 +1,262 @@ +using System; +using System.IO; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.ValveBsp; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + /// + /// Named lights are compiled into their own light styles, which VRAD stores as extra + /// lightmap pages after a face's style 0 page. The engine adds together every style that + /// is switched on, so these tests check the pages are combined the same way. + /// + [TestClass] + public class LightStyleTest + { + private const int TexInfoSize = 72; + private const int FaceSize = 56; + private const int SampleSize = 4; + + private const int FaceWidth = 2; + private const int FaceHeight = 1; + private const int SampleCount = FaceWidth * FaceHeight; + + private static byte[] TexInfos( params SurfFlags[] flags ) + { + var data = new byte[flags.Length * TexInfoSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + foreach ( var flag in flags ) + { + writer.Write( new byte[64] ); // Texture and lightmap axes + writer.Write( (int) flag ); + writer.Write( 0 ); // TexData + } + } + + return data; + } + + private static byte[] Face( int texInfo, int lightOffset, params byte[] styles ) + { + var packedStyles = new byte[] { 255, 255, 255, 255 }; + Array.Copy( styles, packedStyles, styles.Length ); + + var data = new byte[FaceSize]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( (ushort) 0 ); // PlaneNum + writer.Write( (byte) 0 ); // Side + writer.Write( (byte) 0 ); // OnNode + writer.Write( 0 ); // FirstEdge + writer.Write( (short) 0 ); // NumEdges + writer.Write( (short) texInfo ); + writer.Write( (short) -1 ); // DispInfo + writer.Write( (short) 0 ); // FogVolumeId + writer.Write( packedStyles ); + writer.Write( lightOffset ); + writer.Write( 0f ); // Area + writer.Write( 0 ); // LightMapOffsetX + writer.Write( 0 ); // LightMapOffsetY + writer.Write( FaceWidth - 1 ); // LightMapSizeX + writer.Write( FaceHeight - 1 ); // LightMapSizeY + writer.Write( 0 ); // OriginalFace + writer.Write( (ushort) 0 ); // NumPrimitives + writer.Write( (ushort) 0 ); // FirstPrimitive + writer.Write( 0u ); // SmoothingGroups + } + + return data; + } + + /// One page holds a sample per luxel, all with the same colour here. + private static byte[] Page( byte r, byte g, byte b, sbyte exponent ) + { + var data = new byte[SampleCount * SampleSize]; + + for ( var i = 0; i < SampleCount; ++i ) + { + data[i * SampleSize + 0] = r; + data[i * SampleSize + 1] = g; + data[i * SampleSize + 2] = b; + data[i * SampleSize + 3] = (byte) exponent; + } + + return data; + } + + private static byte[] Concat( params byte[][] parts ) + { + using ( var stream = new MemoryStream() ) + { + foreach ( var part in parts ) stream.Write( part, 0, part.Length ); + return stream.ToArray(); + } + } + + private static byte[] Entities( params string[] entities ) + { + return Encoding.ASCII.GetBytes( string.Concat( entities ) + "\0" ); + } + + private static string Light( int style, int spawnFlags ) + { + return "{\n\"classname\" \"light\"\n\"targetname\" \"lamp\"\n" + + $"\"style\" \"{style}\"\n\"spawnflags\" \"{spawnFlags}\"\n}}\n"; + } + + private static ColorRGBExp32[] ReadFace( ValveBspFile bsp ) + { + var output = new ColorRGBExp32[SampleCount]; + + using ( var lighting = bsp.GetLumpStream( LumpType.LIGHTING ) ) + { + bsp.ReadLightmapSamples( lighting, bsp.Faces[0], output ); + } + + return output; + } + + private static void AssertLinear( float r, float g, float b, ColorRGBExp32 sample ) + { + var scale = (float) Math.Pow( 2, sample.Exponent ); + + // Re-encoding keeps 8 bits of precision on the brightest channel. + var tolerance = Math.Max( Math.Max( r, g ), b ) / 128f; + + Assert.AreEqual( r, sample.R * scale, tolerance, "red" ); + Assert.AreEqual( g, sample.G * scale, tolerance, "green" ); + Assert.AreEqual( b, sample.B * scale, tolerance, "blue" ); + } + + private static BspBuilder Map( SurfFlags flags, byte[] face, byte[] lighting, params string[] entities ) + { + return new BspBuilder() + .Lump( LumpType.ENTITIES, Entities( entities ) ) + .Lump( LumpType.TEXINFO, TexInfos( flags ) ) + .Lump( LumpType.FACES, face, 1 ) + .Lump( LumpType.LIGHTING, lighting, 1 ); + } + + [TestMethod] + public void SwitchedOnStyleIsAddedToStyleZero() + { + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void StylesWithDifferentExponentsAreAddedInLinearSpace() + { + // 64 * 2^-6 = 1 and 16 * 2^2 = 64. + var lighting = Concat( Page( 64, 64, 64, -6 ), Page( 16, 0, 0, 2 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 65, 1, 1, sample ); + } ); + } + + [TestMethod] + public void InitiallyDarkStyleIsLeftOut() + { + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32 ), lighting, Light( 32, 1 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 10, sample.R ); + Assert.AreEqual( 20, sample.G ); + Assert.AreEqual( 30, sample.B ); + Assert.AreEqual( 0, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void BumpedFacesStoreFourPagesPerStyle() + { + // Pages 1 to 3 of each style hold the bump directions, which must be skipped. + var bump = Page( 200, 200, 200, 3 ); + var lighting = Concat( + Page( 10, 20, 30, 0 ), bump, bump, bump, + Page( 100, 50, 0, 0 ), bump, bump, bump ); + + Map( SurfFlags.BUMPLIGHT, Face( 0, 0, 0, 32 ), lighting, Light( 32, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void SingleStyleIsCopiedUnchanged() + { + // Exponents outside what re-encoding would produce must survive untouched. + var lighting = Page( 3, 5, 7, -20 ); + + Map( 0, Face( 0, 0, 0 ), lighting ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 3, sample.R ); + Assert.AreEqual( 5, sample.G ); + Assert.AreEqual( 7, sample.B ); + Assert.AreEqual( -20, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void MapWithoutLightingIsFullbright() + { + // Maps compiled without VRAD have no lighting at all but still list style 0 in + // every slot. The engine draws them fully lit. + Map( 0, Face( 0, 0, 0, 0, 0, 0 ), new byte[0] ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) + { + Assert.AreEqual( 255, sample.R ); + Assert.AreEqual( 255, sample.G ); + Assert.AreEqual( 255, sample.B ); + Assert.AreEqual( 0, sample.Exponent ); + } + } ); + } + + [TestMethod] + public void StylesWithoutDataAreIgnored() + { + // Only the pages that fit inside the lump are added together. + var lighting = Concat( Page( 10, 20, 30, 0 ), Page( 100, 50, 0, 0 ) ); + + Map( 0, Face( 0, 0, 0, 32, 33 ), lighting, Light( 32, 0 ), Light( 33, 0 ) ).With( bsp => + { + foreach ( var sample in ReadFace( bsp ) ) AssertLinear( 110, 70, 30, sample ); + } ); + } + + [TestMethod] + public void LightStylesDefaultToOn() + { + Map( 0, Face( 0, 0, 0 ), Page( 0, 0, 0, 0 ), Light( 32, 0 ), Light( 33, 1 ) ).With( bsp => + { + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 0 ), "style 0" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 5 ), "built-in animated style" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 32 ), "named light that starts on" ); + Assert.IsFalse( bsp.IsLightStyleOnByDefault( 33 ), "named light that starts dark" ); + Assert.IsTrue( bsp.IsLightStyleOnByDefault( 34 ), "style with no light entity" ); + Assert.IsFalse( bsp.IsLightStyleOnByDefault( 255 ), "unused style slot" ); + } ); + } + } +} diff --git a/SourceUtils.Test/SkyMaterialTest.cs b/SourceUtils.Test/SkyMaterialTest.cs new file mode 100644 index 00000000..fe627750 --- /dev/null +++ b/SourceUtils.Test/SkyMaterialTest.cs @@ -0,0 +1,55 @@ +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class SkyMaterialTest + { + private const string BaseUrl = "/materials/skybox/sky_bk.vtf/0123456789abcdef.vtf.json"; + private const string HdrUrl = "/materials/skybox/sky_bk_hdr.vtf/fedcba9876543210.vtf.json"; + + private static string[] Names( Material mat ) + { + return Material.GetSkyFaceTextures( mat ).Select( x => x.Name ).ToArray(); + } + + [TestMethod] + public void BaseTextureIsUsedWhenPresent() + { + var mat = new Material(); + mat.SetTextureUrl( "basetexture", BaseUrl ); + + CollectionAssert.AreEqual( new[] { "basetexture" }, Names( mat ) ); + } + + /// + /// Maps often name an HDR texture without packing it, so the base texture has to stay + /// available as a fallback rather than being dropped. + /// + [TestMethod] + public void HdrTextureComesFirstButBaseTextureRemains() + { + var mat = new Material(); + mat.SetTextureUrl( "basetexture", BaseUrl ); + mat.SetTextureUrl( "hdrcompressedtexture", HdrUrl ); + + CollectionAssert.AreEqual( new[] { "hdrcompressedtexture", "basetexture" }, Names( mat ) ); + } + + /// + /// Some maps build their sky from an environment cubemap instead, with a shader like + /// WindowImposter and no $basetexture at all. That used to throw, failing the whole + /// map's index.json and leaving the map unexported. + /// + [TestMethod] + public void MaterialWithoutTexturesGivesNothing() + { + var mat = new Material(); + mat.SetBoolean( "translucent", true ); + + Assert.AreEqual( 0, Names( mat ).Length ); + } + } +} diff --git a/SourceUtils.Test/SourceUtils.Test.csproj b/SourceUtils.Test/SourceUtils.Test.csproj index bfedb304..1b3327a8 100644 --- a/SourceUtils.Test/SourceUtils.Test.csproj +++ b/SourceUtils.Test/SourceUtils.Test.csproj @@ -119,7 +119,17 @@ + + + + + + + + + + True @@ -132,6 +142,10 @@ {2c5fefa1-39ba-458e-b95c-2d8414958820} SourceUtils + + {171622AD-547B-496C-A230-16FCA08A0112} + SourceUtils.WebExport + diff --git a/SourceUtils.Test/StaticPropTest.cs b/SourceUtils.Test/StaticPropTest.cs new file mode 100644 index 00000000..59643090 --- /dev/null +++ b/SourceUtils.Test/StaticPropTest.cs @@ -0,0 +1,119 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.ValveBsp; + +namespace SourceUtils.Test +{ + using LumpType = ValveBspFile.LumpType; + + [TestClass] + public class StaticPropTest + { + private const int Sprp = 0x73707270; + + /// + /// Offset of the game lump's data within a file written by , + /// which puts the first lump straight after the header. + /// + private const int GameLumpOffset = 8 + 64 * 16 + 4; + + /// + /// Version 10 props in a version 20 BSP hold a second flags field where later layouts + /// keep a diffuse colour, so reading one as a colour modulates the prop towards black. + /// + [TestMethod] + public void V10Bsp20PropsAreNotColourModulated() + { + const uint noPerTexelLighting = 0x100; + + WithStaticProp( noPerTexelLighting, bsp => + { + StaticPropFlags flags; + bool solid; + uint diffuseModulation; + + bsp.StaticProps.GetPropInfo( 0, out flags, out solid, out diffuseModulation ); + + Assert.AreEqual( 0xffffffff, diffuseModulation ); + } ); + } + + /// + /// Guards the fields either side of the flags against being shifted along to make the + /// colour read give the right answer. + /// + [TestMethod] + public void V10Bsp20PropsKeepTheirTransform() + { + WithStaticProp( 0x100, bsp => + { + Vector3 origin; + Vector3 angles; + float scale; + + bsp.StaticProps.GetPropTransform( 0, out origin, out angles, out scale ); + + Assert.AreEqual( -14592f, origin.X ); + Assert.AreEqual( 7904f, origin.Z ); + Assert.AreEqual( 180f, angles.Y ); + } ); + } + + /// + /// Writes a BSP holding a single version 10 static prop, with + /// in the four bytes that follow the CPU and GPU levels. + /// + private static void WithStaticProp( uint flagsEx, Action action ) + { + const int propSize = 72; + + byte[] sprp; + + using ( var stream = new MemoryStream() ) + using ( var writer = new BinaryWriter( stream ) ) + { + writer.Write( 1 ); // Model dictionary entries + var name = new byte[128]; + Buffer.BlockCopy( System.Text.Encoding.ASCII.GetBytes( "models/test/prop.mdl" ), 0, name, 0, 20 ); + writer.Write( name ); + + writer.Write( 0 ); // Leaf entries + writer.Write( 1 ); // Props + + var prop = new byte[propSize]; + + Buffer.BlockCopy( BitConverter.GetBytes( -14592f ), 0, prop, 0, 4 ); // Origin + Buffer.BlockCopy( BitConverter.GetBytes( 7904f ), 0, prop, 8, 4 ); + Buffer.BlockCopy( BitConverter.GetBytes( 180f ), 0, prop, 16, 4 ); // Angles + prop[30] = 6; // Solid + Buffer.BlockCopy( BitConverter.GetBytes( 1f ), 0, prop, 56, 4 ); // ForcedFadeScale + Buffer.BlockCopy( BitConverter.GetBytes( flagsEx ), 0, prop, 64, 4 ); + + writer.Write( prop ); + + sprp = stream.ToArray(); + } + + byte[] gameLump; + + using ( var stream = new MemoryStream() ) + using ( var writer = new BinaryWriter( stream ) ) + { + writer.Write( 1 ); // Game lump items + writer.Write( Sprp ); + writer.Write( (ushort) 0 ); // Flags + writer.Write( (ushort) 10 ); // Version + writer.Write( GameLumpOffset + 4 + 16 ); // Offset of the sprp data + writer.Write( sprp.Length ); + writer.Write( sprp ); + + gameLump = stream.ToArray(); + } + + new BspBuilder() + .Lump( LumpType.GAME_LUMP, gameLump ) + .With( action ); + } + } +} diff --git a/SourceUtils.Test/TextureFormatTest.cs b/SourceUtils.Test/TextureFormatTest.cs new file mode 100644 index 00000000..e9fc37a5 --- /dev/null +++ b/SourceUtils.Test/TextureFormatTest.cs @@ -0,0 +1,39 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class TextureFormatTest + { + private const int Width = 64; + private const int Height = 32; + + private static int SizeOf( TextureFormat format ) + { + return ValveTextureFile.GetImageDataSize( Width, Height, 1, 1, format ); + } + + [TestMethod] + public void SingleChannelFormatsAreOneBytePerPixel() + { + Assert.AreEqual( Width * Height, SizeOf( TextureFormat.A8 ) ); + Assert.AreEqual( Width * Height, SizeOf( TextureFormat.I8 ) ); + } + + [TestMethod] + public void PackedSixteenBitFormatsAreTwoBytesPerPixel() + { + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.BGRA4444 ) ); + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.BGR565 ) ); + Assert.AreEqual( Width * Height * 2, SizeOf( TextureFormat.IA88 ) ); + } + + [TestMethod] + public void FullColourFormatsKeepTheirWidths() + { + Assert.AreEqual( Width * Height * 3, SizeOf( TextureFormat.BGR888 ) ); + Assert.AreEqual( Width * Height * 4, SizeOf( TextureFormat.BGRA8888 ) ); + Assert.AreEqual( Width * Height * 8, SizeOf( TextureFormat.RGBA16161616F ) ); + } + } +} diff --git a/SourceUtils.Test/TextureSourceTest.cs b/SourceUtils.Test/TextureSourceTest.cs new file mode 100644 index 00000000..8418bdce --- /dev/null +++ b/SourceUtils.Test/TextureSourceTest.cs @@ -0,0 +1,223 @@ +using System.IO; +using System.Text; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SourceUtils.WebExport; + +namespace SourceUtils.Test +{ + [TestClass] + public class TextureSourceTest + { + private const string Hash = "0123456789abcdef"; + + [TestMethod] + public void NormalizePathAddsPrefixAndExtension() + { + Assert.AreEqual( "materials/concrete/wall.vtf", TextureSource.NormalizePath( "concrete/wall" ) ); + } + + [TestMethod] + public void NormalizePathFixesSlashesAndCase() + { + Assert.AreEqual( "materials/concrete/wall.vtf", TextureSource.NormalizePath( "/Materials\\Concrete//Wall.VTF" ) ); + } + + [TestMethod] + public void NormalizePathIsIdempotent() + { + const string path = "materials/skybox/sky_bk.vtf"; + Assert.AreEqual( path, TextureSource.NormalizePath( path ) ); + } + + [TestMethod] + public void ComputeHashIsSixteenLowerHexChars() + { + using ( var stream = new MemoryStream( Encoding.ASCII.GetBytes( "VTF\0hello" ) ) ) + { + var hash = TextureSource.ComputeHash( stream ); + Assert.AreEqual( 16, hash.Length ); + StringAssert.Matches( hash, new System.Text.RegularExpressions.Regex( "^[0-9a-f]{16}$" ) ); + } + } + + [TestMethod] + public void ComputeHashIsStableAndContentSensitive() + { + string HashOf( string content ) + { + using ( var stream = new MemoryStream( Encoding.ASCII.GetBytes( content ) ) ) + { + return TextureSource.ComputeHash( stream ); + } + } + + Assert.AreEqual( HashOf( "abc" ), HashOf( "abc" ) ); + Assert.AreNotEqual( HashOf( "abc" ), HashOf( "abd" ) ); + } + + [TestMethod] + public void BuildUrlImage() + { + Assert.AreEqual( "/materials/concrete/wall.vtf/0123456789abcdef/mip0.png", TextureSource.BuildUrl( "materials/concrete/wall.vtf", Hash, "mip0.png" ) ); + } + + [TestMethod] + public void BuildUrlInfo() + { + Assert.AreEqual( "/materials/skybox/sky_bk.vtf/0123456789abcdef.vtf.json", TextureSource.BuildUrl( "materials/skybox/sky_bk.vtf", Hash, null ) ); + } + + [TestMethod] + public void BuildUrlMissing() + { + Assert.AreEqual( "/materials/concrete/wall.vtf.json", TextureSource.BuildUrl( "materials/concrete/wall.vtf", null, null ) ); + } + + private static void AssertRoundTrip( string path, string hash, string file ) + { + var url = TextureSource.BuildUrl( path, hash, file ); + Assert.IsTrue( TextureSource.TryParseUrl( url, out var parsedPath, out var parsedHash, out var parsedFile ), url ); + Assert.AreEqual( path, parsedPath, url ); + Assert.AreEqual( hash, parsedHash, url ); + Assert.AreEqual( file, parsedFile, url ); + } + + [TestMethod] + public void RoundTripImage() + { + AssertRoundTrip( "materials/concrete/wall.vtf", Hash, "mip0.png" ); + } + + [TestMethod] + public void RoundTripFrameFaceImage() + { + AssertRoundTrip( "materials/water/anim.vtf", Hash, "mip2.frame3.face1.png" ); + } + + [TestMethod] + public void RoundTripInfo() + { + AssertRoundTrip( "materials/skybox/sky_bk.vtf", Hash, null ); + } + + [TestMethod] + public void RoundTripMissing() + { + AssertRoundTrip( "materials/concrete/wall.vtf", null, null ); + } + + [TestMethod] + public void ParseUnescapesEncodedPath() + { + Assert.IsTrue( TextureSource.TryParseUrl( "/materials/my%20dir/wall.vtf/0123456789abcdef/mip0.png", out var path, out var hash, out var file ) ); + Assert.AreEqual( "materials/my dir/wall.vtf", path ); + Assert.AreEqual( Hash, hash ); + Assert.AreEqual( "mip0.png", file ); + } + + [TestMethod] + public void ParseUsesRightmostVtfSegment() + { + Assert.IsTrue( TextureSource.TryParseUrl( "/materials/dir.vtf/inner/wall.vtf/0123456789abcdef/mip0.png", + out var path, out var hash, out var file ) ); + Assert.AreEqual( "materials/dir.vtf/inner/wall.vtf", path ); + Assert.AreEqual( Hash, hash ); + Assert.AreEqual( "mip0.png", file ); + } + + [TestMethod] + public void ParseRejectsDecodedTraversal() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete%2f..%2fsecret.vtf/0123456789abcdef/mip0.png", + out _, out _, out _ ) ); + Assert.IsNull( TextureSource.NormalizePath( "materials\\concrete\\..\\secret" ) ); + } + + [TestMethod] + public void ParseRejectsOutsideMaterials() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/maps/surf_x/materials/wall.vtf/0123456789abcdef/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsNoVtfSegment() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vmt.json", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsBadHash() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/NOTAHASH/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseRejectsNestedFile() + { + Assert.IsFalse( TextureSource.TryParseUrl( "/materials/concrete/wall.vtf/0123456789abcdef/a/mip0.png", out _, out _, out _ ) ); + } + + [TestMethod] + public void MaterialUrlCoversEveryFormUnderTheMaterialsFolder() + { + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, null ) ) ); + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, "mip0.png" ) ) ); + Assert.IsTrue( TextureSource.IsMaterialUrl( TextureSource.BuildUrl( "materials/water/anim.vtf", null, null ) ) ); + } + + [TestMethod] + public void MaterialUrlExcludesMapsAndStaticFiles() + { + Assert.IsFalse( TextureSource.IsMaterialUrl( "/maps/surf_x/materials/matpage0.json" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( "/maps/surf_x/index.html" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( "/js/sourceutils.js" ) ); + Assert.IsFalse( TextureSource.IsMaterialUrl( null ) ); + } + + [TestMethod] + public void ContentAddressedForHashedInfoAndImageUrls() + { + Assert.IsTrue( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, null ) ) ); + Assert.IsTrue( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", Hash, "mip0.frame7.png" ) ) ); + } + + [TestMethod] + public void NotContentAddressedWithoutHash() + { + Assert.IsFalse( TextureSource.IsContentAddressed( TextureSource.BuildUrl( "materials/water/anim.vtf", null, null ) ) ); + } + + [TestMethod] + public void NotContentAddressedForOtherResources() + { + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/geom/vispage0.json" ) ); + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/materials/matpage0.json" ) ); + Assert.IsFalse( TextureSource.IsContentAddressed( "/maps/surf_x/lightmap.png" ) ); + } + + [TestMethod] + public void ParseImageFileNameRejectsMalformedAndOverflowingIndices() + { + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip.png", 2, 2, 2, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip999999999999999999999.png", 2, 2, 2, out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseImageFileNameRejectsOutOfRangeIndices() + { + Assert.IsFalse( TextureSource.TryParseImageFileName( "mip2.png", 2, 1, 1, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "frame1.png", 1, 1, 1, out _, out _, out _ ) ); + Assert.IsFalse( TextureSource.TryParseImageFileName( "face1.png", 1, 1, 1, out _, out _, out _ ) ); + } + + [TestMethod] + public void ParseImageFileNameReturnsIndicesInsideTextureBounds() + { + Assert.IsTrue( TextureSource.TryParseImageFileName( "mip1.frame2.face5.png", 2, 3, 6, + out var mip, out var frame, out var face ) ); + Assert.AreEqual( 1, mip ); + Assert.AreEqual( 2, frame ); + Assert.AreEqual( 5, face ); + } + } +} diff --git a/SourceUtils.Test/VertexLightingTest.cs b/SourceUtils.Test/VertexLightingTest.cs new file mode 100644 index 00000000..ea9cccd9 --- /dev/null +++ b/SourceUtils.Test/VertexLightingTest.cs @@ -0,0 +1,155 @@ +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace SourceUtils.Test +{ + [TestClass] + public class VertexLightingTest + { + private const int HeaderSize = 40; + private const int MeshCountOffset = 20; + + /// + /// VRAD writes padded placeholder .vhv files holding no meshes at all for props it + /// didn't light per vertex. + /// + private static Stream EmptyVhv( int length = 512 ) + { + var data = new byte[length]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); // Version + } + + return new MemoryStream( data ); + } + + private const int MeshHeaderSize = 28; + + private static Stream VhvWithMeshes( int meshCount ) + { + var vertOffset = HeaderSize + meshCount * MeshHeaderSize; + var data = new byte[vertOffset]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( meshCount ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + + for ( var i = 0; i < meshCount; ++i ) + { + writer.Write( 0 ); // Lod + writer.Write( 0 ); // VertCount + writer.Write( vertOffset ); // VertOffset + writer.Write( 0 ); // Unused0 + writer.Write( 0 ); // Unused1 + writer.Write( 0 ); // Unused2 + writer.Write( 0 ); // Unused3 + } + } + + return new MemoryStream( data ); + } + + [TestMethod] + public void EmptyFileParsesWithNoMeshes() + { + using ( var stream = EmptyVhv() ) + { + var vhv = ValveVertexLightingFile.FromStream( stream ); + + Assert.IsNotNull( vhv ); + Assert.AreEqual( 0, vhv.GetMeshCount( 0 ) ); + Assert.AreEqual( 0, vhv.GetMeshCount( 3 ) ); + } + } + + private static void AssertRejected( byte[] data ) + { + try + { + ValveVertexLightingFile.FromStream( new MemoryStream( data ) ); + } + catch ( System.NotSupportedException ) + { + return; + } + + Assert.Fail( "Expected the file to be rejected as unsupported." ); + } + + /// + /// Some maps pack unrelated files under .vhv names. Their headers claim absurd counts, + /// which used to read past the end of the file or exhaust memory. + /// + [TestMethod] + public void HeaderClaimingMoreMeshesThanTheFileHoldsIsRejected() + { + var data = new byte[512]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1634533376 ); + } + + AssertRejected( data ); + } + + [TestMethod] + public void MeshPointingOutsideTheFileIsRejected() + { + var vertOffset = HeaderSize + MeshHeaderSize; + var data = new byte[vertOffset + 16]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1 ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + writer.Write( 0 ); // Lod + writer.Write( 1000000 ); // VertCount, way past the end + writer.Write( vertOffset ); // VertOffset + } + + AssertRejected( data ); + } + + [TestMethod] + public void AbsurdLodIsRejected() + { + var vertOffset = HeaderSize + MeshHeaderSize; + var data = new byte[vertOffset]; + + using ( var writer = new BinaryWriter( new MemoryStream( data ) ) ) + { + writer.Write( 2 ); + writer.Seek( MeshCountOffset, SeekOrigin.Begin ); + writer.Write( 1 ); + writer.Seek( HeaderSize, SeekOrigin.Begin ); + writer.Write( 1634533376 ); // Lod + writer.Write( 0 ); // VertCount + writer.Write( vertOffset ); // VertOffset + } + + AssertRejected( data ); + } + + [TestMethod] + public void FileWithMeshesReportsThem() + { + using ( var stream = VhvWithMeshes( 2 ) ) + { + var vhv = ValveVertexLightingFile.FromStream( stream ); + + Assert.AreEqual( 2, vhv.GetMeshCount( 0 ) ); + Assert.AreEqual( 0, vhv.GetMeshCount( 1 ) ); + } + } + } +} diff --git a/SourceUtils.WebExport/Bsp/AmbientCubes.cs b/SourceUtils.WebExport/Bsp/AmbientCubes.cs index b7b54fc6..ec3548ed 100644 --- a/SourceUtils.WebExport/Bsp/AmbientCubes.cs +++ b/SourceUtils.WebExport/Bsp/AmbientCubes.cs @@ -29,7 +29,9 @@ class AmbientController : ResourceController [Get("/ambientpage{page}.json")] public AmbientPage Get( [Url] string map, [Url] int page ) { - if ( Skip ) return null; + // A skipped page still has to be a valid response. Returning null writes nothing at + // all, which drops the connection and has the exporter count the file as failed. + if ( Skip ) return new AmbientPage { Values = Enumerable.Empty>() }; var bsp = Program.GetMap(map); var first = page * AmbientPage.LeavesPerPage; @@ -41,24 +43,20 @@ public AmbientPage Get( [Url] string map, [Url] int page ) count = 0; } - var hdr = bsp.LeafAmbientLightingHdr.Length > bsp.LeafAmbientLighting.Length; - var indices = hdr ? bsp.LeafAmbientIndicesHdr : bsp.LeafAmbientIndices; - var ambients = hdr ? bsp.LeafAmbientLightingHdr : bsp.LeafAmbientLighting; - return new AmbientPage { Values = Enumerable.Range( first, count ).Select( x => { var leaf = bsp.Leaves[x]; - var index = indices[x]; - var list = new List(index.AmbientSampleCount); + var sampleCount = bsp.GetLeafAmbientSampleCount( x ); + var list = new List(sampleCount); var min = new SourceUtils.Vector3(leaf.Min.X, leaf.Min.Y, leaf.Min.Z); var max = new SourceUtils.Vector3(leaf.Max.X, leaf.Max.Y, leaf.Max.Z); - for (var i = 0; i < index.AmbientSampleCount; ++i) + for (var i = 0; i < sampleCount; ++i) { - var ambient = ambients[index.FirstAmbientSample + i]; + var ambient = bsp.GetLeafAmbientSample( x, i ); var samples = new int[6]; var relPos = new SourceUtils.Vector3(ambient.X, ambient.Y, ambient.Z) * (1f / 255f); diff --git a/SourceUtils.WebExport/Bsp/Geometry.cs b/SourceUtils.WebExport/Bsp/Geometry.cs index 40b32e52..63a2b238 100644 --- a/SourceUtils.WebExport/Bsp/Geometry.cs +++ b/SourceUtils.WebExport/Bsp/Geometry.cs @@ -764,6 +764,16 @@ public StudioModelPage GetStudioModelPage( [Url] string map, [Url] int index ) StudioModel mdl; page.Models.Add( mdl = new StudioModel() ); + // Some maps pack a model without the vertex or triangle files that go with it, + // leaving nothing to draw. The empty model keeps the page indices lined up. + if ( mdlFile == null || vvdFile == null || vtxFile == null ) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine( $"Missing mesh data for model '{mdlPath}'!" ); + Console.ResetColor(); + continue; + } + for ( var j = 0; j < mdlFile.BodyPartCount; ++j ) { SmdBodyPart smdBodyPart; @@ -884,9 +894,17 @@ public VertexLightingPage GetVertexLightingPage( [Url] string map, [Url] int ind continue; } - var meshList = new List>(); var meshCount = vhvFile.GetMeshCount( 0 ); + if ( meshCount == 0 ) + { + // An empty lighting file means the same as having none at all. + page.Props.Add( null ); + continue; + } + + var meshList = new List>(); + for ( var j = 0; j < meshCount; ++j ) { var vertices = new CompressedList(); diff --git a/SourceUtils.WebExport/Bsp/Lightmap.cs b/SourceUtils.WebExport/Bsp/Lightmap.cs index cf8046dc..57a088e6 100644 --- a/SourceUtils.WebExport/Bsp/Lightmap.cs +++ b/SourceUtils.WebExport/Bsp/Lightmap.cs @@ -70,11 +70,8 @@ public void Get( [Url] string map ) if (face.LightOffset == -1) continue; var rect = lightmap.GetLightmapRegion(i); - var sampleCount = rect.Width * rect.Height; - sampleStream.Seek(face.LightOffset, SeekOrigin.Begin); - - LumpReader.ReadLumpFromStream(sampleStream, sampleCount, sampleBuffer); + bsp.ReadLightmapSamples(sampleStream, face, sampleBuffer); for (var y = 0; y < rect.Height; ++y) for (var x = 0; x < rect.Width; ++x) diff --git a/SourceUtils.WebExport/Bsp/Materials.cs b/SourceUtils.WebExport/Bsp/Materials.cs index 303fa33c..167df17a 100644 --- a/SourceUtils.WebExport/Bsp/Materials.cs +++ b/SourceUtils.WebExport/Bsp/Materials.cs @@ -33,8 +33,6 @@ public MaterialPage GetPage( [Url] string map, [Url] int index ) count = 0; } - var texDict = new Dictionary(); - var page = new MaterialPage(); for ( var i = 0; i < count; ++i ) @@ -55,30 +53,22 @@ public MaterialPage GetPage( [Url] string map, [Url] int index ) { if ( prop.Type != MaterialPropertyType.TextureUrl ) continue; - prop.Type = MaterialPropertyType.TextureIndex; - + // Textures are left as urls into the global, hash addressed materials folder so + // that each one is fetched (and exported) once as a whole, rather than having + // every frame and mip level inlined into each map that happens to use it. var texUrl = (Url) prop.Value; - int texIndex; - if ( texDict.TryGetValue( texUrl, out texIndex ) ) - { - prop.Value = texIndex; - continue; - } - - prop.Value = texIndex = page.Textures.Count; - var texPath = TextureController.GetTexturePath( texUrl ); - var tex = Texture.Get( bsp, texPath ); + if ( TextureSource.TryParseUrl( texUrl, out var texPath, out var texHash, out _ ) + && texHash != null ) continue; - if ( tex == null ) - { - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine($"Missing texture '{texPath}'!"); - Console.ResetColor(); - } + // Unresolved textures have no hash to address them by, so there is nothing to + // link to. Null it out rather than emitting a url that can only ever 404. + prop.Type = MaterialPropertyType.TextureInfo; + prop.Value = null; - texDict.Add( texUrl, texIndex ); - page.Textures.Add( tex ); + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine($"Missing texture '{texPath ?? (string) texUrl}'!"); + Console.ResetColor(); } } diff --git a/SourceUtils.WebExport/Bsp/Visibility.cs b/SourceUtils.WebExport/Bsp/Visibility.cs index 88efcce6..03b2229d 100644 --- a/SourceUtils.WebExport/Bsp/Visibility.cs +++ b/SourceUtils.WebExport/Bsp/Visibility.cs @@ -20,7 +20,9 @@ class VisController : ResourceController [Get("/vispage{page}.json")] public VisPage Get( [Url] string map, [Url] int page ) { - if ( Skip ) return null; + // A skipped page still has to be a valid response. Returning null writes nothing at + // all, which drops the connection and has the exporter count the file as failed. + if ( Skip ) return new VisPage { Values = Enumerable.Empty>() }; var bsp = Program.GetMap(map); var first = page * VisPage.ClustersPerPage; diff --git a/SourceUtils.WebExport/Export.cs b/SourceUtils.WebExport/Export.cs index b0e044a8..0b4c8e82 100644 --- a/SourceUtils.WebExport/Export.cs +++ b/SourceUtils.WebExport/Export.cs @@ -11,14 +11,39 @@ namespace SourceUtils.WebExport { [Verb("export", HelpText = "Export one or more maps for serving over HTTP.")] - class ExportOptions : BaseOptions + public class ExportOptions : BaseOptions { [Option('o', "outdir", HelpText = "Output directory.", Required = true)] public string OutDir { get; set; } - [Option('r', "overwrite", HelpText = "Overwrite existing exported files.")] + [Option('r', "overwrite", HelpText = "Overwrite every existing exported file (implies --overwrite-maps and --overwrite-materials).")] public bool Overwrite { get; set; } + [Option("overwrite-maps", HelpText = "Overwrite existing exported map files.")] + public bool OverwriteMaps { get; set; } + + [Option("overwrite-materials", HelpText = "Overwrite existing exported material files. Needed to pick up changes to how textures are converted, which the content hash of a source .vtf can't see.")] + public bool OverwriteMaterials { get; set; } + + /// + /// Which of the exported files this run is willing to replace. Static viewer files are + /// always replaced: they belong to the build rather than to any map or texture, and a + /// stale one is a broken viewer for the sake of a few bytes. + /// + public bool ShouldOverwrite( string url ) + { + if ( Overwrite ) return true; + if ( TextureSource.IsMaterialUrl( url ) ) return OverwriteMaterials; + if ( IsMapUrl( url ) ) return OverwriteMaps; + + return true; + } + + private static bool IsMapUrl( string url ) + { + return url != null && url.StartsWith( "/maps/", StringComparison.Ordinal ); + } + [Option('p', "url-prefix", HelpText = "Prefix to prepend to each exported URL." )] public string UrlPrefix { get; set; } = ""; @@ -140,7 +165,7 @@ static void ExportMap( string mapName, ExportOptions args ) var url = _sToExport.Dequeue(); var path = Path.Combine(args.OutDir, url.Value.Substring(1)); - var skip = !args.Overwrite && File.Exists(path); + var skip = !args.ShouldOverwrite(url) && File.Exists(path); if ( skip ) { @@ -154,6 +179,13 @@ static void ExportMap( string mapName, ExportOptions args ) else Console.Write($"[{exported + skipped + failed + 1}/{_sExportUrls.Count - oldCount}] Exporting '{url}' ... "); } + // Skipped urls are normally still requested, because serving them is what + // discovers the urls nested inside them. A content addressed url can't have + // gone stale while its file exists, and neither can anything below it, so + // there is nothing left to discover. One check skips a whole texture, + // however many frames and mip levels it turns out to have. + if ( skip && TextureSource.IsContentAddressed( url ) ) continue; + var dir = Path.GetDirectoryName(path); if ( !args.DryRun && !Directory.Exists( dir ) ) { diff --git a/SourceUtils.WebExport/Material.cs b/SourceUtils.WebExport/Material.cs index a812d305..431bd9ca 100644 --- a/SourceUtils.WebExport/Material.cs +++ b/SourceUtils.WebExport/Material.cs @@ -121,13 +121,14 @@ private static Url GetTextureUrl(string path, string vmtPath, ValveBspFile bsp) if (!path.EndsWith(".vtf")) path = $"{path}.vtf"; path = !path.Contains('/') ? $"{Path.GetDirectoryName(vmtPath)}/{path}" : $"materials/{path}"; + path = TextureSource.NormalizePath(path); + if ( path == null ) return null; - if (bsp != null && bsp.PakFile.ContainsFile(path)) - { - return $"/maps/{bsp.Name}/{path}.json"; - } + var info = TextureSource.Resolve(bsp, path); - return $"/{path}.json"; + return info == null + ? TextureSource.BuildUrl(path, null, null) + : TextureSource.BuildUrl(info.Path, info.Hash, null); } private static void AddMaterialProperties(Material mat, ValveMaterialFile vmt, string vmtPath, ValveBspFile bsp) @@ -291,6 +292,20 @@ public static Material Get(ValveBspFile bsp, string path) return mat; } + /// + /// Textures a sky face could be drawn with, best first. The list is empty for skies + /// built from an environment cubemap (WindowImposter and friends), which have no base + /// texture at all. + /// + public static IEnumerable GetSkyFaceTextures( Material mat ) + { + var hdr = mat.Properties.FirstOrDefault( x => x.Name == "hdrcompressedtexture" ); + var ldr = mat.Properties.FirstOrDefault( x => x.Name == "basetexture" ); + + if ( hdr != null ) yield return hdr; + if ( ldr != null ) yield return ldr; + } + public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) { var postfixes = new[] @@ -309,6 +324,7 @@ public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) var hdrCompressed = false; var aspect = 1f; + var faceCount = 0; for ( var face = 0; face < 6; ++face ) { @@ -317,20 +333,43 @@ public static Material CreateSkyMaterial( ValveBspFile bsp, string skyName ) if ( mat == null ) continue; - var texProp = mat.Properties.FirstOrDefault( x => x.Name == "hdrcompressedtexture") - ?? mat.Properties.First(x => x.Name == "basetexture"); + // The HDR texture is preferred, but plenty of maps name one without packing it, + // so each candidate has to actually resolve before it can be used. + MaterialProperty texProp = null; + Texture faceTex = null; - if ( face == 0 ) + foreach ( var candidate in GetSkyFaceTextures( mat ) ) { - hdrCompressed = texProp.Name == "hdrcompressedtexture"; + faceTex = TextureSource.TryParseUrl( (Url) candidate.Value, out var texPath, out _, out _ ) + ? Texture.Get( bsp, texPath ) + : null; + + if ( faceTex == null ) continue; - var tex = Texture.Get( bsp, TextureController.GetTexturePath( (Url) texProp.Value ) ); - aspect = tex == null ? 1f : (float) tex.Width / tex.Height; + texProp = candidate; + break; + } + + if ( texProp == null ) + { + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine( $"Sky material '{matPath}' has no texture to draw with!" ); + Console.ResetColor(); + continue; + } + + if ( faceCount++ == 0 ) + { + hdrCompressed = texProp.Name == "hdrcompressedtexture"; + aspect = (float) faceTex.Width / faceTex.Height; } skyMaterial.SetTextureUrl($"face{names[face]}", (Url)texProp.Value); } + // Nothing to draw the sky with, so the map is better off without one. + if ( faceCount == 0 ) return null; + if ( hdrCompressed ) { skyMaterial.SetBoolean( "hdrCompressed", true ); diff --git a/SourceUtils.WebExport/Program.cs b/SourceUtils.WebExport/Program.cs index dabbd650..7a8cd6af 100644 --- a/SourceUtils.WebExport/Program.cs +++ b/SourceUtils.WebExport/Program.cs @@ -14,7 +14,7 @@ namespace SourceUtils.WebExport { - class BaseOptions + public class BaseOptions { [Option('g', "gamedir", HelpText = "Game directory to export from.", Required = true)] public string GameDir { get; set; } @@ -112,6 +112,7 @@ public static void UnloadMap( string name ) if ( !_sOpenMaps.TryGetValue( name, out var map ) ) return; _sOpenMaps.Remove( name ); + TextureSource.ForgetMap( map.Name ); map.Dispose(); } diff --git a/SourceUtils.WebExport/ResourceController.cs b/SourceUtils.WebExport/ResourceController.cs index 3e258ccb..007816b5 100644 --- a/SourceUtils.WebExport/ResourceController.cs +++ b/SourceUtils.WebExport/ResourceController.cs @@ -79,6 +79,11 @@ private static bool ShouldAppendVersionSuffix( Url url ) { case ".png": return false; + case ".json": + // Hash addressed texture urls are already versioned by their contents. A + // timestamp would only break caching, and would make the same texture + // referenced either side of a second boundary look like two textures. + return !TextureSource.IsContentAddressed( url ); default: return true; } diff --git a/SourceUtils.WebExport/Resources/js/sourceutils.js b/SourceUtils.WebExport/Resources/js/sourceutils.js index 74d71198..8bad094f 100644 --- a/SourceUtils.WebExport/Resources/js/sourceutils.js +++ b/SourceUtils.WebExport/Resources/js/sourceutils.js @@ -1881,7 +1881,8 @@ var SourceUtils; } else { leaf.getAmbientCube(null, null, function (success) { - _this.lighting = leaf; + // Origins inside solid brushes land in leaves with no ambient samples. + _this.lighting = success ? leaf : null; _this.checkLoaded(); }); } diff --git a/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts b/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts index bd41a6c1..4a36c4fd 100644 --- a/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts +++ b/SourceUtils.WebExport/Resources/src/Entities/StaticProp.ts @@ -37,7 +37,8 @@ namespace SourceUtils { this.checkLoaded(); } else { leaf.getAmbientCube(null, null, success => { - this.lighting = leaf; + // Origins inside solid brushes land in leaves with no ambient samples. + this.lighting = success ? leaf : null; this.checkLoaded(); }); } diff --git a/SourceUtils.WebExport/SourceUtils.WebExport.csproj b/SourceUtils.WebExport/SourceUtils.WebExport.csproj index 67a7db1a..81f0eb78 100644 --- a/SourceUtils.WebExport/SourceUtils.WebExport.csproj +++ b/SourceUtils.WebExport/SourceUtils.WebExport.csproj @@ -122,6 +122,7 @@ + diff --git a/SourceUtils.WebExport/Texture.Convert.cs b/SourceUtils.WebExport/Texture.Convert.cs index 63fe86e4..c1af4ec1 100644 --- a/SourceUtils.WebExport/Texture.Convert.cs +++ b/SourceUtils.WebExport/Texture.Convert.cs @@ -130,6 +130,12 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, case TextureFormat.BGR565: dataLength = dataLength * 3 / 2; break; + case TextureFormat.BGRA4444: + dataLength *= 2; + break; + case TextureFormat.A8: + dataLength *= 4; + break; } var totalLength = dataLength + 128; @@ -183,6 +189,8 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, break; case TextureFormat.BGRA8888: case TextureFormat.BGRX8888: + case TextureFormat.BGRA4444: + case TextureFormat.A8: readSettings.PixelStorage = new PixelStorageSettings(StorageType.Char, "BGRA"); break; case TextureFormat.RGBA8888: @@ -208,6 +216,30 @@ public static MagickImage DecodeImage(ValveTextureFile vtf, int mip, int frame, buffer[i * 3 + 2] = (byte)(((pixel >> 11) & 31) / 31f * 255f); } break; + case TextureFormat.BGRA4444: + // Each 4 bit channel is widened to 8 bits, so 0xf becomes 0xff. + for (var i = width * height - 1; i >= 0; --i) + { + var pixel = (ushort)(buffer[i * 2] | (buffer[i * 2 + 1] << 8)); + + buffer[i * 4] = (byte) ((pixel & 0xf) * 0x11); + buffer[i * 4 + 1] = (byte) (((pixel >> 4) & 0xf) * 0x11); + buffer[i * 4 + 2] = (byte) (((pixel >> 8) & 0xf) * 0x11); + buffer[i * 4 + 3] = (byte) (((pixel >> 12) & 0xf) * 0x11); + } + break; + case TextureFormat.A8: + // Alpha only, so the colour channels are left white. + for (var i = width * height - 1; i >= 0; --i) + { + var alpha = buffer[i]; + + buffer[i * 4] = 0xff; + buffer[i * 4 + 1] = 0xff; + buffer[i * 4 + 2] = 0xff; + buffer[i * 4 + 3] = alpha; + } + break; } var img = new MagickImage( buffer, readSettings ); diff --git a/SourceUtils.WebExport/Texture.cs b/SourceUtils.WebExport/Texture.cs index d422b50b..9bf8d728 100644 --- a/SourceUtils.WebExport/Texture.cs +++ b/SourceUtils.WebExport/Texture.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text.RegularExpressions; -using System.Web; using ImageMagick; using Newtonsoft.Json; using Ziks.WebServer; @@ -48,13 +46,16 @@ public partial class Texture { public static Texture Get( ValveBspFile bsp, string path ) { - var inBsp = bsp != null && bsp.PakFile.ContainsFile( path ); - var res = inBsp ? bsp.PakFile : Program.Resources; + var info = TextureSource.Resolve( bsp, path ); + return info == null ? null : Get( info ); + } - if ( !res.ContainsFile( path ) ) return null; + public static Texture Get( TextureSourceInfo info ) + { + var path = info.Path; ValveTextureFile vtf; - using (var stream = res.OpenFile(path)) + using (var stream = info.Provider.OpenFile(path)) { vtf = new ValveTextureFile(stream); } @@ -64,7 +65,7 @@ public static Texture Get( ValveBspFile bsp, string path ) var tex = new Texture { - Path = path.ToLower(), + Path = path, Target = isCube ? TextureTarget.TextureCubeMap : TextureTarget.Texture2D, Width = untextured ? 1 : vtf.Header.Width, Height = untextured ? 1 : vtf.Header.Height, @@ -128,7 +129,7 @@ public static Texture Get( ValveBspFile bsp, string path ) } else { - var fileName = $"{path}/mip{mip}"; + var fileName = $"mip{mip}"; if (vtf.FrameCount > 1) { @@ -142,7 +143,7 @@ public static Texture Get( ValveBspFile bsp, string path ) fileName = $"{fileName}.png"; - elem.Url = inBsp ? $"/maps/{bsp.Name}/{fileName}" : $"/{fileName}"; + elem.Url = TextureSource.BuildUrl(path, info.Hash, fileName); } tex.Elements.Add(elem); @@ -181,26 +182,30 @@ public static Texture Get( ValveBspFile bsp, string path ) } [Prefix("/materials")] - [Prefix("/maps/{map}/materials")] class TextureController : ResourceController { - public static string GetTexturePath(string url) + private TextureSourceInfo LookupRequest( bool image, out string file ) { - var path = url.Substring(url.IndexOf("/materials") + 1); - return HttpUtility.UrlDecode(path.Substring(0, path.Length - ".json".Length)); + if ( !TextureSource.TryParseUrl( Request.Url.AbsolutePath, out var path, out var hash, out file ) + || hash == null || (file != null) != image ) + { + throw NotFoundException(); + } + + var info = TextureSource.Lookup( path, hash ); + if ( info == null ) throw NotFoundException(); + + return info; } [Get( MatchAllUrl = false, Extension = ".vtf.json" )] - public Texture GetInfo( [Url] string map ) + public Texture GetInfo() { - var bsp = map == null ? null : Program.GetMap( map ); - return Texture.Get( bsp, GetTexturePath( Request.Url.AbsolutePath ) ); + return Texture.Get( LookupRequest( false, out _ ) ); } - private static readonly Regex _sFileNameRegex = new Regex( @"^((?mip|face|frame)(?[0-9]+)\.)*(?png)$", RegexOptions.IgnoreCase | RegexOptions.Compiled ); - [Get(MatchAllUrl = false, Extension = ".png")] - public void GetImage( [Url] string map ) + public void GetImage() { if ( Skip ) { @@ -208,49 +213,17 @@ public void GetImage( [Url] string map ) return; } - var absolute = Request.Url.AbsolutePath; - var pathStart = absolute.IndexOf( "/materials" ) + 1; - var pathEnd = absolute.IndexOf( ".vtf", pathStart ) + ".vtf".Length; - var path = HttpUtility.UrlDecode( absolute.Substring( pathStart, pathEnd - pathStart ) ); - - var fileName = Path.GetFileName( absolute ); - var match = _sFileNameRegex.Match( fileName ); - - if ( !match.Success ) throw NotFoundException(); - - var mip = 0; - var face = 0; - var frame = 0; - - var index = 0; - foreach ( Capture capture in match.Groups["param"].Captures ) - { - var param = capture.Value.ToLower(); - var value = int.Parse( match.Groups["value"].Captures[index++].Value ); - - switch (param) - { - case "mip": - mip = value; - break; - case "face": - face = value; - break; - case "frame": - frame = value; - break; - } - } - - var bsp = map == null ? null : Program.GetMap(map); - var res = bsp == null ? Program.Resources : bsp.PakFile; + var info = LookupRequest( true, out var fileName ); ValveTextureFile vtf; - using (var stream = res.OpenFile(path)) + using (var stream = info.Provider.OpenFile(info.Path)) { vtf = new ValveTextureFile(stream); } + if ( !TextureSource.TryParseImageFileName( fileName, vtf.MipmapCount, vtf.FrameCount, vtf.FaceCount, + out var mip, out var frame, out var face ) ) throw NotFoundException(); + using ( var image = Texture.DecodeImage( vtf, mip, frame, face, 0 ) ) { image.Write( Response.OutputStream, MagickFormat.Png ); diff --git a/SourceUtils.WebExport/TextureSource.cs b/SourceUtils.WebExport/TextureSource.cs new file mode 100644 index 00000000..7f4fa162 --- /dev/null +++ b/SourceUtils.WebExport/TextureSource.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace SourceUtils.WebExport +{ + public static partial class TextureSource + { + private const string UrlPrefix = "/materials/"; + private const string VtfSegment = ".vtf/"; + private const string InfoSuffix = ".vtf.json"; + private const string MissingSuffix = ".json"; + private const int HashLength = 16; + private static readonly Regex _sImageFileNameRegex = new Regex( @"^((?mip|face|frame)(?[0-9]+)\.)*(?png)$", RegexOptions.IgnoreCase | RegexOptions.Compiled ); + + public static string NormalizePath( string path ) + { + if ( path == null ) return null; + + path = NormalizeSlashes( path.ToLowerInvariant() ).TrimStart( '/' ); + if ( !path.StartsWith( "materials/" ) ) path = $"materials/{path}"; + if ( !path.EndsWith( ".vtf" ) ) path = $"{path}.vtf"; + + return IsSafePath( path ) ? path : null; + } + + public static string ComputeHash( Stream vtf ) + { + using ( var sha1 = SHA1.Create() ) + { + var bytes = sha1.ComputeHash( vtf ); + var builder = new StringBuilder( HashLength ); + for ( var i = 0; i < HashLength / 2; ++i ) builder.Append( bytes[i].ToString( "x2" ) ); + return builder.ToString(); + } + } + + public static string BuildUrl( string path, string hash, string file ) + { + if ( hash == null ) return $"/{path}{MissingSuffix}"; + if ( file == null ) return $"/{path}/{hash}{InfoSuffix}"; + return $"/{path}/{hash}/{file}"; + } + + public static bool TryParseUrl( string absolutePath, out string path, out string hash, out string file ) + { + path = null; hash = null; file = null; + if ( absolutePath == null ) return false; + var url = NormalizeSlashes( Uri.UnescapeDataString( absolutePath ) ); + if ( !url.StartsWith( UrlPrefix, StringComparison.Ordinal ) ) return false; + var vtfIndex = url.LastIndexOf( VtfSegment, StringComparison.Ordinal ); + if ( vtfIndex < 0 ) + { + if ( !url.EndsWith( InfoSuffix, StringComparison.Ordinal ) ) return false; + path = url.Substring( 1, url.Length - 1 - MissingSuffix.Length ); + return IsSafePath( path ); + } + var rest = url.Substring( vtfIndex + VtfSegment.Length ); + string parsedHash; + string parsedFile = null; + if ( rest.EndsWith( InfoSuffix, StringComparison.Ordinal ) && rest.IndexOf( '/' ) < 0 ) + parsedHash = rest.Substring( 0, rest.Length - InfoSuffix.Length ); + else + { + var slash = rest.IndexOf( '/' ); + if ( slash < 0 ) return false; + parsedHash = rest.Substring( 0, slash ); + parsedFile = rest.Substring( slash + 1 ); + if ( parsedFile.Length == 0 || parsedFile.IndexOf( '/' ) >= 0 ) return false; + } + if ( !IsValidHash( parsedHash ) ) return false; + path = url.Substring( 1, vtfIndex + ".vtf".Length - 1 ); + if ( !IsSafePath( path ) ) return false; + hash = parsedHash; file = parsedFile; + return true; + } + + /// + /// True if the url points into the global materials folder, whatever form it takes. Note + /// that a map's own material page lives under /maps and is not one of these. + /// + public static bool IsMaterialUrl( string url ) + { + return url != null && url.StartsWith( UrlPrefix, StringComparison.Ordinal ); + } + + /// + /// True if the url names a specific version of a texture by the hash of its contents, and + /// so can never go stale while the file it points at exists. + /// + public static bool IsContentAddressed( string url ) + { + return TryParseUrl( url, out _, out var hash, out _ ) && hash != null; + } + + public static bool TryParseImageFileName( string fileName, int mipCount, int frameCount, int faceCount, + out int mip, out int frame, out int face ) + { + mip = 0; + frame = 0; + face = 0; + + var match = _sImageFileNameRegex.Match( fileName ?? "" ); + if ( !match.Success ) return false; + + var index = 0; + foreach ( Capture capture in match.Groups["param"].Captures ) + { + if ( !int.TryParse( match.Groups["value"].Captures[index++].Value, NumberStyles.None, + CultureInfo.InvariantCulture, out var value ) ) return false; + + switch ( capture.Value.ToLowerInvariant() ) + { + case "mip": + mip = value; + break; + case "face": + face = value; + break; + case "frame": + frame = value; + break; + } + } + + return mip >= 0 && mip < mipCount + && frame >= 0 && frame < frameCount + && face >= 0 && face < faceCount; + } + + private static bool IsValidHash( string hash ) + { + if ( hash.Length != HashLength ) return false; + foreach ( var c in hash ) + if ( (c < '0' || c > '9') && (c < 'a' || c > 'f') ) return false; + return true; + } + + private static string NormalizeSlashes( string path ) + { + path = path.Replace( '\\', '/' ); + while ( path.Contains( "//" ) ) path = path.Replace( "//", "/" ); + return path; + } + + private static bool IsSafePath( string path ) + { + if ( path == null || !path.StartsWith( "materials/", StringComparison.Ordinal ) ) return false; + + foreach ( var segment in path.Split( '/' ) ) + { + if ( segment == "." || segment == ".." ) return false; + } + + return true; + } + } + + public sealed class TextureSourceInfo + { + public TextureSourceInfo( IResourceProvider provider, string mapName, string path, string hash ) + { + Provider = provider; + MapName = mapName; + Path = path; + Hash = hash; + } + + public IResourceProvider Provider { get; } + + /// + /// Name of the map whose pakfile contains this texture, or null for game resources. + /// + public string MapName { get; } + + public string Path { get; } + + public string Hash { get; } + } + + public static partial class TextureSource + { + private static readonly Dictionary _sHashCache = new Dictionary(); + private static readonly Dictionary _sRegistry = new Dictionary(); + + private static string GetCacheKey( string mapName, string path ) + { + return $"{mapName ?? ""}|{path}"; + } + + private static string GetRegistryKey( string path, string hash ) + { + return $"{path}|{hash}"; + } + + internal static TextureSourceInfo Resolve( ValveBspFile bsp, string path ) + { + path = NormalizePath( path ); + if ( path == null ) return null; + + IResourceProvider provider; + string mapName; + + if ( bsp != null && bsp.PakFile.ContainsFile( path ) ) + { + provider = bsp.PakFile; + mapName = bsp.Name; + } + else if ( Program.Resources.ContainsFile( path ) ) + { + provider = Program.Resources; + mapName = null; + } + else + { + return null; + } + + var cacheKey = GetCacheKey( mapName, path ); + string hash; + + lock ( _sHashCache ) + { + _sHashCache.TryGetValue( cacheKey, out hash ); + } + + if ( hash == null ) + { + using ( var stream = provider.OpenFile( path ) ) + { + hash = ComputeHash( stream ); + } + + lock ( _sHashCache ) + { + _sHashCache[cacheKey] = hash; + } + } + + lock ( _sRegistry ) + { + // Last registration wins, so host mode prefers the most recently viewed map. + _sRegistry[GetRegistryKey( path, hash )] = mapName; + } + + return new TextureSourceInfo( provider, mapName, path, hash ); + } + + internal static TextureSourceInfo Lookup( string path, string hash ) + { + path = NormalizePath( path ); + if ( path == null ) return null; + + bool registered; + string mapName; + + lock ( _sRegistry ) + { + registered = _sRegistry.TryGetValue( GetRegistryKey( path, hash ), out mapName ); + } + + var bsp = registered && mapName != null ? Program.GetMap( mapName ) : null; + var info = Resolve( bsp, path ); + + return info != null && info.Hash == hash ? info : null; + } + + internal static void ForgetMap( string mapName ) + { + var prefix = GetCacheKey( mapName, "" ); + + lock ( _sHashCache ) + { + foreach ( var key in _sHashCache.Keys.Where( x => x.StartsWith( prefix ) ).ToArray() ) + { + _sHashCache.Remove( key ); + } + } + } + } +} diff --git a/SourceUtils/SourceUtils.csproj b/SourceUtils/SourceUtils.csproj index 21fde398..79a2b65e 100644 --- a/SourceUtils/SourceUtils.csproj +++ b/SourceUtils/SourceUtils.csproj @@ -94,7 +94,9 @@ + + diff --git a/SourceUtils/ValveBsp/GameLump.cs b/SourceUtils/ValveBsp/GameLump.cs index ecffd1fe..0baa4676 100644 --- a/SourceUtils/ValveBsp/GameLump.cs +++ b/SourceUtils/ValveBsp/GameLump.cs @@ -114,7 +114,7 @@ private void EnsureLoaded() count -= 1; - for ( var i = 0; i < count; i++ ) + for ( var i = 0; i < items.Length - 1; i++ ) { var item = items[i]; diff --git a/SourceUtils/ValveBsp/LeafAmbient.cs b/SourceUtils/ValveBsp/LeafAmbient.cs new file mode 100644 index 00000000..0e6750a2 --- /dev/null +++ b/SourceUtils/ValveBsp/LeafAmbient.cs @@ -0,0 +1,81 @@ +using System; +using SourceUtils.ValveBsp; + +namespace SourceUtils +{ + partial class ValveBspFile + { + /// + /// Centre of a leaf's bounds, where the single sample of a legacy ambient lump sits. + /// + private const byte LegacyAmbientPosition = 128; + + /// + /// The same lump as , read as the legacy layout of one + /// bare cube per leaf. + /// + [BspLump( LumpType.LEAF_AMBIENT_LIGHTING )] + public StructArrayLump LeafAmbientCubes { get; private set; } + + [BspLump( LumpType.LEAF_AMBIENT_LIGHTING_HDR )] + public StructArrayLump LeafAmbientCubesHdr { get; private set; } + + private bool UseHdrAmbient => LeafAmbientLightingHdr.Length > LeafAmbientLighting.Length; + + /// + /// Maps compiled before the Orange Box store one ambient cube per leaf and write no + /// index lump. They also write the lump length into its version field, so the layout has + /// to be recognised by size rather than by version. + /// + private bool IsLegacyLeafAmbient + { + get + { + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + var cubes = UseHdrAmbient ? LeafAmbientCubesHdr : LeafAmbientCubes; + + return Leaves.Length > 0 && indices.Length == 0 && cubes.Length == Leaves.Length; + } + } + + /// + /// Number of ambient light samples stored for the given leaf, which is 0 when the map + /// has no usable ambient lighting at all. + /// + public int GetLeafAmbientSampleCount( int leafIndex ) + { + if ( leafIndex < 0 || leafIndex >= Leaves.Length ) return 0; + if ( IsLegacyLeafAmbient ) return 1; + + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + + return leafIndex >= indices.Length ? 0 : indices[leafIndex].AmbientSampleCount; + } + + public LeafAmbientLighting GetLeafAmbientSample( int leafIndex, int sampleIndex ) + { + var count = GetLeafAmbientSampleCount( leafIndex ); + + if ( sampleIndex < 0 || sampleIndex >= count ) + { + throw new IndexOutOfRangeException( + $"Leaf {leafIndex} has {count} ambient samples, so {sampleIndex} is out of range." ); + } + + if ( IsLegacyLeafAmbient ) + { + var cubes = UseHdrAmbient ? LeafAmbientCubesHdr : LeafAmbientCubes; + + return new LeafAmbientLighting( cubes[leafIndex], + LegacyAmbientPosition, LegacyAmbientPosition, LegacyAmbientPosition ); + } + + var indices = UseHdrAmbient ? LeafAmbientIndicesHdr : LeafAmbientIndices; + var samples = UseHdrAmbient ? LeafAmbientLightingHdr : LeafAmbientLighting; + + var first = indices[leafIndex].FirstAmbientSample + sampleIndex; + + return first >= samples.Length ? default( LeafAmbientLighting ) : samples[first]; + } + } +} diff --git a/SourceUtils/ValveBsp/LightStyles.cs b/SourceUtils/ValveBsp/LightStyles.cs new file mode 100644 index 00000000..c59e900a --- /dev/null +++ b/SourceUtils/ValveBsp/LightStyles.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.IO; +using SourceUtils.ValveBsp; + +namespace SourceUtils +{ + partial class ValveBspFile + { + /// + /// Marks an unused light style slot on a face. + /// + public const byte NoLightStyle = 255; + + /// + /// VRAD gives each group of named lights its own style from here on, so they can be + /// switched at runtime. Styles below this are the built-in animated patterns. + /// + private const int FirstSwitchableLightStyle = 32; + + private const int MaxLightStylesPerFace = 4; + + /// + /// Bumped faces store a flat lightmap followed by one for each of the three bump basis + /// directions, for every style. + /// + private const int BumpLightmapPages = 4; + + private const int SpawnFlagInitiallyDark = 1; + + private static readonly ColorRGBExp32 Fullbright = new ColorRGBExp32( 255, 255, 255, 0 ); + + private HashSet _initiallyDarkLightStyles; + + /// + /// Whether the given light style is lit when the map starts. Switchable styles are lit + /// unless every light using them is flagged to start dark. + /// + public bool IsLightStyleOnByDefault( byte style ) + { + if ( style == NoLightStyle ) return false; + if ( style < FirstSwitchableLightStyle ) return true; + + return !GetInitiallyDarkLightStyles().Contains( style ); + } + + private HashSet GetInitiallyDarkLightStyles() + { + lock ( this ) + { + if ( _initiallyDarkLightStyles != null ) return _initiallyDarkLightStyles; + + var dark = new HashSet(); + var lit = new HashSet(); + + foreach ( var entity in Entities ) + { + if ( entity.ClassName == null || !entity.ClassName.StartsWith( "light" ) ) continue; + + int style = entity["style"]; + if ( style < FirstSwitchableLightStyle ) continue; + + int spawnFlags = entity["spawnflags"]; + (( spawnFlags & SpawnFlagInitiallyDark ) != 0 ? dark : lit).Add( style ); + } + + dark.ExceptWith( lit ); + + return _initiallyDarkLightStyles = dark; + } + } + + /// + /// Reads the flat lightmap of a face into , adding together + /// every light style that is lit when the map starts. + /// + /// Stream of the lighting lump that matches . + public void ReadLightmapSamples( Stream lighting, Face face, ColorRGBExp32[] output ) + { + var size = face.LightMapSize; + var sampleCount = size.X * size.Y; + + // Maps compiled without VRAD have no lighting at all, and the engine draws them + // fully lit. + if ( lighting.Length == 0 ) + { + for ( var i = 0; i < sampleCount; ++i ) output[i] = Fullbright; + return; + } + + var bumped = (TextureInfos[face.TexInfo].Flags & SurfFlags.BUMPLIGHT) != 0; + var styleStride = sampleCount * (bumped ? BumpLightmapPages : 1); + + var litStyles = new List( MaxLightStylesPerFace ); + + for ( var i = 0; i < MaxLightStylesPerFace; ++i ) + { + var style = face.GetLightStyle( i ); + if ( style == NoLightStyle ) break; + if ( !IsLightStyleOnByDefault( style ) ) continue; + + // Maps compiled without VRAD list style 0 in every slot with no lighting behind it. + var pageEnd = face.LightOffset + ((long) i * styleStride + sampleCount) * 4; + if ( pageEnd > lighting.Length ) continue; + + litStyles.Add( i ); + } + + if ( litStyles.Count == 0 ) + { + Array.Clear( output, 0, sampleCount ); + return; + } + + // A single style is copied as-is, so maps without named lights export unchanged. + if ( litStyles.Count == 1 ) + { + ReadLightmapPage( lighting, face, litStyles[0] * styleStride, sampleCount, output ); + return; + } + + var sum = new float[sampleCount * 3]; + var page = new ColorRGBExp32[sampleCount]; + + foreach ( var styleIndex in litStyles ) + { + ReadLightmapPage( lighting, face, styleIndex * styleStride, sampleCount, page ); + + for ( var i = 0; i < sampleCount; ++i ) + { + var sample = page[i]; + var scale = (float) Math.Pow( 2, sample.Exponent ); + + sum[i * 3 + 0] += sample.R * scale; + sum[i * 3 + 1] += sample.G * scale; + sum[i * 3 + 2] += sample.B * scale; + } + } + + for ( var i = 0; i < sampleCount; ++i ) + { + output[i] = EncodeRGBExp32( sum[i * 3 + 0], sum[i * 3 + 1], sum[i * 3 + 2] ); + } + } + + private static void ReadLightmapPage( Stream lighting, Face face, int sampleOffset, int sampleCount, ColorRGBExp32[] output ) + { + lighting.Seek( face.LightOffset + (long) sampleOffset * 4, SeekOrigin.Begin ); + LumpReader.ReadLumpFromStream( lighting, sampleCount, output ); + } + + /// + /// Picks the exponent that keeps the brightest channel in the top half of a byte, the + /// same precision VRAD writes with. + /// + private static ColorRGBExp32 EncodeRGBExp32( float r, float g, float b ) + { + var max = Math.Max( r, Math.Max( g, b ) ); + if ( max <= 0f ) return default( ColorRGBExp32 ); + + var exponent = (int) Math.Floor( Math.Log( max, 2 ) ) - 7; + exponent = Math.Max( sbyte.MinValue, Math.Min( sbyte.MaxValue, exponent ) ); + + var scale = (float) Math.Pow( 2, -exponent ); + + return new ColorRGBExp32( ToChannel( r * scale ), ToChannel( g * scale ), ToChannel( b * scale ), (sbyte) exponent ); + } + + private static byte ToChannel( float value ) + { + return (byte) Math.Max( 0, Math.Min( 255, (int) Math.Round( value ) ) ); + } + } +} diff --git a/SourceUtils/ValveBsp/StaticProps.cs b/SourceUtils/ValveBsp/StaticProps.cs index 3792088d..d01dc1ff 100644 --- a/SourceUtils/ValveBsp/StaticProps.cs +++ b/SourceUtils/ValveBsp/StaticProps.cs @@ -133,9 +133,14 @@ public struct StaticPropV10Bsp20 : IStaticProp public readonly byte MinGpuLevel; public readonly byte MaxGpuLevel; - public readonly uint ColorModulation; - [MarshalAs(UnmanagedType.U1)] - public readonly bool DisableX360; + /// + /// A widened copy of rather than the diffuse colour that later + /// layouts keep here: across every version 10 prop in the map library these bytes only + /// ever hold flag combinations (0x100 for no per texel lighting, 0x40 for no per vertex + /// lighting, 0x10 for no shadow) and never the 0xffffffff an unmodulated prop needs. + /// Reading them as a colour leaves every prop multiplied towards black. + /// + public readonly uint FlagsEx; Vector3 IStaticProp.Origin => Origin; Vector3 IStaticProp.Angles => Angles; @@ -145,7 +150,7 @@ public struct StaticPropV10Bsp20 : IStaticProp ushort IStaticProp.LeafCount => LeafCount; StaticPropFlags IStaticProp.Flags => Flags; bool IStaticProp.Solid => Solid; - uint IStaticProp.ColorModulation => ColorModulation; + uint IStaticProp.ColorModulation => 0xffffffff; float IStaticProp.FadeMinDist => FadeMinDist; float IStaticProp.FadeMaxDist => FadeMaxDist; float IStaticProp.ForcedFadeScale => ForcedFadeScale; diff --git a/SourceUtils/ValveBsp/Structures.cs b/SourceUtils/ValveBsp/Structures.cs index de422f36..33b525df 100644 --- a/SourceUtils/ValveBsp/Structures.cs +++ b/SourceUtils/ValveBsp/Structures.cs @@ -516,6 +516,14 @@ public static implicit operator int(ColorRGBExp32 color) return color.R | (color.G << 8) | (color.B << 16) | ((color.Exponent + 128) << 24); } + public ColorRGBExp32( byte r, byte g, byte b, sbyte exponent ) + { + R = r; + G = g; + B = b; + Exponent = exponent; + } + public readonly byte R; public readonly byte G; public readonly byte B; @@ -534,6 +542,15 @@ public struct CubemapSample [StructLayout( LayoutKind.Sequential, Pack = 1 )] public struct LeafAmbientLighting { + public LeafAmbientLighting( CompressedLightCube cube, byte x, byte y, byte z ) + { + Cube = cube; + X = x; + Y = y; + Z = z; + _padding = 0; + } + public readonly CompressedLightCube Cube; public readonly byte X; public readonly byte Y; diff --git a/SourceUtils/ValveTextureFile.cs b/SourceUtils/ValveTextureFile.cs index d6eb432d..56071ca1 100644 --- a/SourceUtils/ValveTextureFile.cs +++ b/SourceUtils/ValveTextureFile.cs @@ -164,6 +164,7 @@ public static int GetImageDataSize( int width, int height, int depth, int mipCou case TextureFormat.DXT3: case TextureFormat.DXT5: return toAdd + width * height * depth; + case TextureFormat.A8: case TextureFormat.I8: return toAdd + width * height * depth; case TextureFormat.IA88: @@ -181,6 +182,7 @@ public static int GetImageDataSize( int width, int height, int depth, int mipCou return toAdd + width * height * depth * 3; case TextureFormat.BGR565: case TextureFormat.RGB565: + case TextureFormat.BGRA4444: return toAdd + width * height * depth * 2; default: throw new NotImplementedException(); @@ -288,8 +290,10 @@ public ValveTextureFile( Stream stream, bool onlyHeader = false ) case TextureFormat.DXT1: case TextureFormat.DXT3: case TextureFormat.DXT5: + case TextureFormat.A8: case TextureFormat.I8: case TextureFormat.IA88: + case TextureFormat.BGRA4444: case TextureFormat.BGR565: case TextureFormat.RGB565: case TextureFormat.BGR888: diff --git a/SourceUtils/ValveVertexLightingFile.cs b/SourceUtils/ValveVertexLightingFile.cs index 44a3727b..4a1fc9b6 100644 --- a/SourceUtils/ValveVertexLightingFile.cs +++ b/SourceUtils/ValveVertexLightingFile.cs @@ -9,6 +9,15 @@ namespace SourceUtils { public class ValveVertexLightingFile { + private const int MeshHeaderSize = 28; + private const int VertexData4Size = 4; + private const int VertexData2Size = 12; + + /// + /// Source supports up to 8 levels of detail, so anything beyond that is a bad header. + /// + private const int MaxLodCount = 8; + [StructLayout(LayoutKind.Sequential, Pack = 1)] private struct VhvMeshHeader { @@ -122,10 +131,39 @@ public ValveVertexLightingFile( Stream stream ) var unused0 = reader.ReadInt64(); // Unused var unused1 = reader.ReadInt64(); // Unused + // Some maps pack unrelated files under .vhv names, so nothing in the header can + // be trusted until it has been checked against the size of the file. + if ( meshCount < 0 || stream.Position + (long) meshCount * MeshHeaderSize > stream.Length ) + { + throw new NotSupportedException( + $"Vertex lighting file claims {meshCount} meshes, which don't fit in {stream.Length} bytes." ); + } + var meshHeaders = new List(); LumpReader.ReadLumpFromStream(stream, meshCount, meshHeaders); - _samples = new VertexData4[meshHeaders.Max( x => x.Lod ) + 1][][]; + var sampleSize = vertFlags == 2 ? VertexData2Size : VertexData4Size; + + foreach ( var meshHeader in meshHeaders ) + { + var start = (long) meshHeader.VertOffset + baseOffset; + + if ( meshHeader.Lod < 0 || meshHeader.Lod >= MaxLodCount ) + { + throw new NotSupportedException( $"Vertex lighting mesh has an out of range LOD of {meshHeader.Lod}." ); + } + + if ( meshHeader.VertCount < 0 || start < 0 + || start + (long) meshHeader.VertCount * sampleSize > stream.Length ) + { + throw new NotSupportedException( + $"Vertex lighting mesh with {meshHeader.VertCount} vertices at {start} doesn't fit in {stream.Length} bytes." ); + } + } + + // VRAD writes placeholder files holding no meshes at all for props it didn't + // light per vertex. + _samples = new VertexData4[meshHeaders.Count == 0 ? 0 : meshHeaders.Max( x => x.Lod ) + 1][][]; for ( var i = 0; i < _samples.Length; ++i ) {