move from materials-per-map to global, hash validated materials folder with per-map fallback

.gitignore

@@ -255,3 +255,5 @@ paket-files/
 *.sln.iml
 
 MapViewServer/Scripts/typings/
+
+test/

SourceUtils.Test/BspBuilder.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace SourceUtils.Test
+{
+    using LumpType = ValveBspFile.LumpType;
+
+    /// <summary>
+    /// Writes a minimal version 20 BSP file holding only the lumps a test gives it.
+    /// </summary>
+    internal class BspBuilder
+    {
+        private const int LumpCount = 64;
+        private const int HeaderSize = 4 + 4 + LumpCount * 16 + 4;
+
+        private readonly Dictionary<LumpType, KeyValuePair<byte[], int>> _lumps
+            = new Dictionary<LumpType, KeyValuePair<byte[], int>>();
+
+        public BspBuilder Lump( LumpType type, byte[] data, int version = 0 )
+        {
+            _lumps[type] = new KeyValuePair<byte[], int>( data, version );
+            return this;
+        }
+
+        public string Write()
+        {
+            var path = Path.Combine( Path.GetTempPath(), $"sourceutils-test-{Guid.NewGuid():N}.bsp" );
+
+            using ( var writer = new BinaryWriter( File.Create( path ) ) )
+            {
+                writer.Write( new[] { (byte) 'V', (byte) 'B', (byte) 'S', (byte) 'P' } );
+                writer.Write( 20 );
+
+                var offset = HeaderSize;
+
+                for ( var i = 0; i < LumpCount; ++i )
+                {
+                    KeyValuePair<byte[], int> lump;
+                    if ( !_lumps.TryGetValue( (LumpType) i, out lump ) )
+                    {
+                        writer.Write( offset );
+                        writer.Write( 0 );
+                        writer.Write( 0 );
+                        writer.Write( new byte[4] );
+                        continue;
+                    }
+
+                    writer.Write( offset );
+                    writer.Write( lump.Key.Length );
+                    writer.Write( lump.Value );
+                    writer.Write( new byte[4] );
+
+                    offset += lump.Key.Length;
+                }
+
+                writer.Write( 0 );
+
+                for ( var i = 0; i < LumpCount; ++i )
+                {
+                    KeyValuePair<byte[], int> lump;
+                    if ( _lumps.TryGetValue( (LumpType) i, out lump ) ) writer.Write( lump.Key );
+                }
+            }
+
+            return path;
+        }
+
+        /// <summary>
+        /// Writes the file, opens it for <paramref name="action"/> and deletes it afterwards.
+        /// </summary>
+        public void With( Action<ValveBspFile> action )
+        {
+            var path = Write();
+
+            try
+            {
+                using ( var bsp = new ValveBspFile( path ) )
+                {
+                    action( bsp );
+                }
+            }
+            finally
+            {
+                File.Delete( path );
+            }
+        }
+    }
+}

SourceUtils.Test/ExportOverwriteTest.cs

@@ -0,0 +1,99 @@
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using SourceUtils.WebExport;
+
+namespace SourceUtils.Test
+{
+    [TestClass]
+    public class ExportOverwriteTest
+    {
+        private const string MaterialInfo = "/materials/concrete/wall.vtf/0123456789abcdef.vtf.json";
+        private const string MaterialImage = "/materials/concrete/wall.vtf/0123456789abcdef/mip0.frame7.png";
+        private const string MissingMaterial = "/materials/concrete/wall.vtf.json";
+        private const string MapPage = "/maps/surf_x/materials/matpage0.json";
+        private const string MapIndex = "/maps/surf_x/index.html";
+        private const string StaticScript = "/js/sourceutils.js";
+        private const string StaticStyle = "/styles/main.css";
+
+        private static ExportOptions Options( bool all = false, bool maps = false, bool materials = false )
+        {
+            return new ExportOptions
+            {
+                Overwrite = all,
+                OverwriteMaps = maps,
+                OverwriteMaterials = materials
+            };
+        }
+
+        [TestMethod]
+        public void StaticFilesAreAlwaysOverwritten()
+        {
+            var none = Options();
+            Assert.IsTrue( none.ShouldOverwrite( StaticScript ) );
+            Assert.IsTrue( none.ShouldOverwrite( StaticStyle ) );
+        }
+
+        [TestMethod]
+        public void NoFlagsLeavesMapsAndMaterialsAlone()
+        {
+            var none = Options();
+            Assert.IsFalse( none.ShouldOverwrite( MaterialInfo ) );
+            Assert.IsFalse( none.ShouldOverwrite( MaterialImage ) );
+            Assert.IsFalse( none.ShouldOverwrite( MapPage ) );
+            Assert.IsFalse( none.ShouldOverwrite( MapIndex ) );
+        }
+
+        [TestMethod]
+        public void LegacyOverwriteCoversMapsAndMaterials()
+        {
+            var all = Options( all: true );
+            Assert.IsTrue( all.ShouldOverwrite( MaterialInfo ) );
+            Assert.IsTrue( all.ShouldOverwrite( MaterialImage ) );
+            Assert.IsTrue( all.ShouldOverwrite( MapPage ) );
+            Assert.IsTrue( all.ShouldOverwrite( MapIndex ) );
+        }
+
+        [TestMethod]
+        public void OverwriteMaterialsLeavesMapsAlone()
+        {
+            var materials = Options( materials: true );
+            Assert.IsTrue( materials.ShouldOverwrite( MaterialInfo ) );
+            Assert.IsTrue( materials.ShouldOverwrite( MaterialImage ) );
+            Assert.IsTrue( materials.ShouldOverwrite( MissingMaterial ) );
+            Assert.IsFalse( materials.ShouldOverwrite( MapPage ) );
+            Assert.IsFalse( materials.ShouldOverwrite( MapIndex ) );
+        }
+
+        [TestMethod]
+        public void OverwriteMapsLeavesMaterialsAlone()
+        {
+            var maps = Options( maps: true );
+            Assert.IsTrue( maps.ShouldOverwrite( MapPage ) );
+            Assert.IsTrue( maps.ShouldOverwrite( MapIndex ) );
+            Assert.IsFalse( maps.ShouldOverwrite( MaterialInfo ) );
+            Assert.IsFalse( maps.ShouldOverwrite( MaterialImage ) );
+        }
+
+        /// <remarks>
+        /// A map's own material page lives under /maps, so it must follow the maps flag rather
+        /// than being caught by the word "materials" appearing in its url.
+        /// </remarks>
+        [TestMethod]
+        public void MapMaterialPageCountsAsAMapFile()
+        {
+            Assert.IsTrue( Options( maps: true ).ShouldOverwrite( MapPage ) );
+            Assert.IsFalse( Options( materials: true ).ShouldOverwrite( MapPage ) );
+        }
+
+        [TestMethod]
+        public void BothNewFlagsMatchLegacyOverwrite()
+        {
+            var split = Options( maps: true, materials: true );
+            var legacy = Options( all: true );
+
+            foreach ( var url in new[] { MaterialInfo, MaterialImage, MapPage, MapIndex, StaticScript } )
+            {
+                Assert.AreEqual( legacy.ShouldOverwrite( url ), split.ShouldOverwrite( url ), url );
+            }
+        }
+    }
+}

SourceUtils.Test/GameLumpTest.cs

@@ -0,0 +1,72 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace SourceUtils.Test
+{
+    [TestClass]
+    public class GameLumpTest
+    {
+        private const int Sprp = 0x73707270;
+        private const int Dprp = 0x64707270;
+
+        [TestMethod]
+        public void CompressedGameLumpIgnoresFinalSentinelWhenReadingItemVersion()
+        {
+            WithGameLump( new[] { Sprp, Dprp, 0 }, bsp =>
+            {
+                Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) );
+                Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) );
+            } );
+        }
+
+        [TestMethod]
+        public void CompressedGameLumpStopsAtFirstZeroIdEntry()
+        {
+            WithGameLump( new[] { Sprp, Dprp, 0, 0, 0 }, bsp =>
+            {
+                Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "sprp" ) );
+                Assert.AreEqual( (ushort) 10, bsp.GameData.GetItemVersion( "dprp" ) );
+                Assert.ThrowsExactly<KeyNotFoundException>( () => bsp.GameData.GetItemVersion( "" ) );
+            } );
+        }
+
+        private static void WithGameLump( int[] itemIds, Action<ValveBspFile> action )
+        {
+            var path = Path.Combine( Path.GetTempPath(), $"sourceutils-gamelump-{Guid.NewGuid():N}.bsp" );
+
+            try
+            {
+                const int gameLumpOffset = 8 + 64 * 16 + 4;
+                const int gameLumpIndex = 35;
+                const int itemSize = 16;
+
+                var gameLumpLength = 4 + itemIds.Length * itemSize;
+                var data = new byte[gameLumpOffset + gameLumpLength];
+
+                Buffer.BlockCopy( BitConverter.GetBytes( gameLumpOffset ), 0, data, 8 + gameLumpIndex * 16, 4 );
+                Buffer.BlockCopy( BitConverter.GetBytes( gameLumpLength ), 0, data, 8 + gameLumpIndex * 16 + 4, 4 );
+                Buffer.BlockCopy( BitConverter.GetBytes( itemIds.Length ), 0, data, gameLumpOffset, 4 );
+
+                for ( var i = 0; i < itemIds.Length; i++ )
+                {
+                    var itemOffset = gameLumpOffset + 4 + i * itemSize;
+                    Buffer.BlockCopy( BitConverter.GetBytes( itemIds[i] ), 0, data, itemOffset, 4 );
+                    Buffer.BlockCopy( BitConverter.GetBytes( (ushort) 10 ), 0, data, itemOffset + 6, 2 ); // version
+                }
+
+                File.WriteAllBytes( path, data );
+
+                using ( var bsp = new ValveBspFile( path ) )
+                {
+                    action( bsp );
+                }
+            }
+            finally
+            {
+                if ( File.Exists( path ) ) File.Delete( path );
+            }
+        }
+    }
+}