From 3d0ae6d36340d76933c2a0fda61dc0b02a66dd0d Mon Sep 17 00:00:00 2001 From: rosebyte Date: Thu, 17 Sep 2026 12:44:32 +0200 Subject: [PATCH] remove GenAi experiment --- .github/copilot-instructions.md | 9 +- .github/workflows/issue-triage.agent.md | 2 +- Directory.Packages.props | 10 - Microsoft.ML.sln | 99 -- THIRD-PARTY-NOTICES.TXT | 26 - docs/README.md | 1 - docs/gen-ai/Benchmark && Evaluation.md | 14 - docs/gen-ai/CausalLMPipeline.md | 118 -- docs/gen-ai/CausalLanguageModel.md | 75 -- docs/gen-ai/DynamicLoading.md | 11 - docs/gen-ai/DynamicLoadingReport.md | 64 - docs/gen-ai/Package Structure.md | 7 - docs/gen-ai/README.md | 16 - docs/gen-ai/Tokenizer.md | 6 - docs/gen-ai/Usage.md | 77 -- .../components-and-dependencies.md | 28 +- .../Llama/LlamaSample.cs | 50 - .../Llama/SFT_Llama_3_2_1B.cs | 118 -- .../MEAI/Llama3_1.cs | 54 - .../Microsoft.ML.GenAI.Samples/MEAI/Phi3.cs | 44 - .../Microsoft.ML.GenAI.Samples.csproj | 26 - .../Mistral/Mistral_7B_Instruct.cs | 156 --- .../Phi3Mini/AutoGenSample.cs | 43 - .../Phi3Mini/SemanticKernelSample.cs | 69 - .../Microsoft.ML.GenAI.Samples/Program.cs | 6 - eng/Versions.props | 6 - .../CausalLMPipelineChatClient.cs | 90 -- .../Extension/ModuleExtension.cs | 287 ----- .../Extension/TensorExtension.cs | 33 - .../Microsoft.ML.GenAI.Core.csproj | 36 - .../Module/Attention.cs | 219 ---- .../Module/DynamicLoadingModule.cs | 54 - .../Module/GenAILinear.cs | 53 - .../Module/IDynamicLoadModule.cs | 19 - .../Module/IQuantizeModule.cs | 12 - .../Module/NewGELUActivation.cs | 28 - .../Module/QuantizedLinear.cs | 209 --- src/Microsoft.ML.GenAI.Core/Module/RMSNorm.cs | 59 - .../Module/RotaryEmbedding.cs | 125 -- .../Pipeline/CausalLMModelInput.cs | 69 - .../Pipeline/CausalLMModelOutput.cs | 50 - .../Pipeline/CausalLMPipeline.cs | 325 ----- .../Trainer/CausalLMDataset.cs | 112 -- .../CausalLMSupervisedFineTuningTrainer.cs | 105 -- .../Utility/AttentionMaskConverter.cs | 179 --- src/Microsoft.ML.GenAI.Core/Utility/Cache.cs | 89 -- .../Utility/IChatTemplateBuilder.cs | 40 - src/Microsoft.ML.GenAI.Core/Utils.cs | 180 --- .../Llama3CausalLMChatClient.cs | 57 - .../Llama3_1ChatTemplateBuilder.cs | 130 -- .../LlamaCausalLMAgent.cs | 89 -- .../LlamaChatCompletionService.cs | 55 - src/Microsoft.ML.GenAI.LLaMA/LlamaConfig.cs | 138 -- .../LlamaForCausalLM.cs | 185 --- .../LlamaTextCompletionService.cs | 77 -- .../LlamaTokenizerHelper.cs | 55 - .../Microsoft.ML.GenAI.LLaMA.csproj | 32 - .../Module/LlamaDecoderLayer.cs | 154 --- .../Module/LlamaMLP.cs | 61 - .../Module/LlamaModel.cs | 154 --- .../Config/meta-llama-3.1-405B-Instruct.json | 32 - .../Config/meta-llama-3.1-70B-Instruct.json | 32 - .../Config/meta-llama-3.1-8B-Instruct.json | 33 - .../Config/meta-llama-3.2-1B-Instruct.json | 35 - .../Config/meta-llama-3.2-3B-Instruct.json | 35 - .../Microsoft.ML.GenAI.Mistral.csproj | 28 - .../MistralCausalLMAgent.cs | 166 --- .../MistralConfig.cs | 112 -- .../MistralDecoderLayer.cs | 148 --- .../MistralForCausalLM.cs | 130 -- src/Microsoft.ML.GenAI.Mistral/MistralMLP.cs | 45 - .../MistralModel.cs | 148 --- .../MistralTokenizerHelper.cs | 107 -- .../Mistral_7B_0_3ChatTemplateBuilder.cs | 202 --- .../Config/mistral-7B-instruct-v0.3.json | 21 - .../Extension/SemanticKernelExtension.cs | 33 - .../Microsoft.ML.GenAI.Phi.csproj | 32 - .../Module/Phi2Attention.cs | 160 --- .../Module/Phi2DecoderLayer.cs | 67 - src/Microsoft.ML.GenAI.Phi/Module/Phi2MLP.cs | 38 - .../Module/Phi2Model.cs | 158 --- .../Module/Phi2RotaryEmbedding.cs | 50 - .../Module/Phi3DecoderLayer.cs | 157 --- src/Microsoft.ML.GenAI.Phi/Module/Phi3MLP.cs | 51 - .../Module/Phi3Model.cs | 147 --- .../Module/Phi3SuScaledRotaryEmbedding.cs | 80 -- src/Microsoft.ML.GenAI.Phi/Phi2/Config.cs | 118 -- .../Phi2/Phi2ForCausalLM.cs | 68 - .../Phi2/Phi2TokenizerHelper.cs | 30 - src/Microsoft.ML.GenAI.Phi/Phi3/Config.cs | 152 --- .../Phi3/Phi3CausalLMAgent.cs | 89 -- .../Phi3/Phi3CausalLMChatClient.cs | 62 - .../Phi3/Phi3CausalLMChatCompletionService.cs | 57 - .../Phi3/Phi3CausalLMTextGenerationService.cs | 74 -- .../Phi3/Phi3ChatTemplateBuilder.cs | 131 -- .../Phi3/Phi3ForCausalLM.cs | 123 -- .../Phi3/Phi3TokenizerHelper.cs | 50 - src/Microsoft.ML.GenAI.Phi/README.md | 119 -- .../Resource/Config/phi-2-config.json | 31 - .../phi-3-medium-128k-instruct-config.json | 170 --- .../phi-3-medium-4k-instruct-config.json | 36 - .../phi-3-mini-128k-instruct-config.json | 140 -- .../Config/phi-3-mini-4k-instruct-config.json | 36 - src/Microsoft.ML.GenAI.Phi/Resource/README.md | 8 - src/Microsoft.ML.GenAI.Phi/Utils.cs | 107 -- .../CausalLMDatasetTest.cs | 103 -- .../Microsoft.ML.GenAI.Core.Tests.csproj | 37 - .../QuantizedLinearTests.cs | 127 -- ...emplateFromAutoGenChatHistory.approved.txt | 7 - ...atTemplateFromMEAIChatHistory.approved.txt | 7 - ...FromSemanticKernelChatHistory.approved.txt | 7 - ...ests.Llama_3_1_405b_ShapeTest.approved.txt | 1137 ----------------- ...Tests.Llama_3_1_70b_ShapeTest.approved.txt | 723 ----------- ...1Tests.Llama_3_1_8b_ShapeTest.approved.txt | 291 ----- .../LLaMA3_1Tests.TokenizerTest.approved.txt | 8 - ...2Tests.Llama_3_2_1b_ShapeTest.approved.txt | 146 --- ...2Tests.Llama_3_2_3b_ShapeTest.approved.txt | 254 ---- .../LLaMA3_1Tests.TokenizerTest.received.txt | 6 - .../LLaMA3_1Tests.cs | 143 --- .../LLaMA3_2Tests.cs | 46 - .../Microsoft.ML.GenAI.LLaMA.Tests.csproj | 47 - ...emplateFromAutoGenChatHistory.approved.txt | 3 - ...thToolsFromAutoGenChatHistory.approved.txt | 3 - ...al_7B_Instruct_V0_3_ShapeTest.approved.txt | 291 ----- ...truct_V0_3Tests.TokenizerTest.approved.txt | 2 - ...stral_V0_3Tests.TokenizerTest.approved.txt | 2 - .../Microsoft.ML.GenAI.Mistral.Tests.csproj | 45 - .../Mistral_7B_Instruct_V0_3Tests.cs | 137 -- ...2Test.LoadSafeTensorShapeTest.approved.txt | 453 ------- .../Phi2Test.TokenizerTest.approved.txt | 3 - ...Tests.LoadSafeTensorShapeTest.approved.txt | 453 ------- .../Phi2Tests.TokenizerTest.approved.txt | 3 - ...Tests.Phi3Medium128KShapeTest.approved.txt | 243 ---- ...i3Tests.Phi3Medium4KShapeTest.approved.txt | 243 ---- ...sts.Phi3Mini128KLayerSizeTest.approved.txt | 34 - ...i3Tests.Phi3Mini128KShapeTest.approved.txt | 195 --- ...i3Mini4KInt4QuantizeShapeTest.approved.txt | 451 ------- ...i3Mini4KInt8QuantizeShapeTest.approved.txt | 451 ------- ...Phi3Tests.Phi3Mini4KShapeTest.approved.txt | 195 --- .../Phi3Tests.TokenizerTest.approved.txt | 20 - .../AutoGenTests.cs | 40 - .../Microsoft.ML.GenAI.Phi.Tests.csproj | 46 - .../Microsoft.ML.GenAI.Phi.Tests/Phi2Tests.cs | 63 - .../Microsoft.ML.GenAI.Phi.Tests/Phi3Tests.cs | 151 --- .../SemanticKernelTests.cs | 79 -- .../Microsoft.ML.TorchSharp.Tests.csproj | 1 - 146 files changed, 5 insertions(+), 15234 deletions(-) delete mode 100644 docs/gen-ai/Benchmark && Evaluation.md delete mode 100644 docs/gen-ai/CausalLMPipeline.md delete mode 100644 docs/gen-ai/CausalLanguageModel.md delete mode 100644 docs/gen-ai/DynamicLoading.md delete mode 100644 docs/gen-ai/DynamicLoadingReport.md delete mode 100644 docs/gen-ai/Package Structure.md delete mode 100644 docs/gen-ai/README.md delete mode 100644 docs/gen-ai/Tokenizer.md delete mode 100644 docs/gen-ai/Usage.md delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Llama/LlamaSample.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Llama/SFT_Llama_3_2_1B.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Llama3_1.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Phi3.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Mistral/Mistral_7B_Instruct.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/SemanticKernelSample.cs delete mode 100644 docs/samples/Microsoft.ML.GenAI.Samples/Program.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/CausalLMPipelineChatClient.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Extension/TensorExtension.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/Attention.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/DynamicLoadingModule.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/GenAILinear.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/IDynamicLoadModule.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/IQuantizeModule.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/NewGELUActivation.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/QuantizedLinear.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/RMSNorm.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Module/RotaryEmbedding.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelInput.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelOutput.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMPipeline.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Trainer/CausalLMDataset.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Trainer/CausalLMSupervisedFineTuningTrainer.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Utility/AttentionMaskConverter.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Utility/Cache.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Utility/IChatTemplateBuilder.cs delete mode 100644 src/Microsoft.ML.GenAI.Core/Utils.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Llama3CausalLMChatClient.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Llama3_1ChatTemplateBuilder.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaCausalLMAgent.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaChatCompletionService.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaConfig.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaForCausalLM.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaTextCompletionService.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/LlamaTokenizerHelper.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Microsoft.ML.GenAI.LLaMA.csproj delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Module/LlamaDecoderLayer.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Module/LlamaMLP.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Module/LlamaModel.cs delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-405B-Instruct.json delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-70B-Instruct.json delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-8B-Instruct.json delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-1B-Instruct.json delete mode 100644 src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-3B-Instruct.json delete mode 100644 src/Microsoft.ML.GenAI.Mistral/Microsoft.ML.GenAI.Mistral.csproj delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralCausalLMAgent.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralConfig.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralDecoderLayer.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralForCausalLM.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralMLP.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralModel.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/MistralTokenizerHelper.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/Mistral_7B_0_3ChatTemplateBuilder.cs delete mode 100644 src/Microsoft.ML.GenAI.Mistral/Resource/Config/mistral-7B-instruct-v0.3.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Extension/SemanticKernelExtension.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Microsoft.ML.GenAI.Phi.csproj delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi2Attention.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi2DecoderLayer.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi2MLP.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi2Model.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi2RotaryEmbedding.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi3DecoderLayer.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi3MLP.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi3Model.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Module/Phi3SuScaledRotaryEmbedding.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi2/Config.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi2/Phi2ForCausalLM.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi2/Phi2TokenizerHelper.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Config.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMAgent.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatClient.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatCompletionService.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMTextGenerationService.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ChatTemplateBuilder.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ForCausalLM.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/Phi3/Phi3TokenizerHelper.cs delete mode 100644 src/Microsoft.ML.GenAI.Phi/README.md delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-2-config.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-128k-instruct-config.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-4k-instruct-config.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-128k-instruct-config.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-4k-instruct-config.json delete mode 100644 src/Microsoft.ML.GenAI.Phi/Resource/README.md delete mode 100644 src/Microsoft.ML.GenAI.Phi/Utils.cs delete mode 100644 test/Microsoft.ML.GenAI.Core.Tests/CausalLMDatasetTest.cs delete mode 100644 test/Microsoft.ML.GenAI.Core.Tests/Microsoft.ML.GenAI.Core.Tests.csproj delete mode 100644 test/Microsoft.ML.GenAI.Core.Tests/QuantizedLinearTests.cs delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromMEAIChatHistory.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromSemanticKernelChatHistory.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_405b_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_70b_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_8b_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_1b_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_3b_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.TokenizerTest.received.txt delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.cs delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_2Tests.cs delete mode 100644 test/Microsoft.ML.GenAI.LLaMA.Tests/Microsoft.ML.GenAI.LLaMA.Tests.csproj delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateWithToolsFromAutoGenChatHistory.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.Mistral_7B_Instruct_V0_3_ShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_V0_3Tests.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Microsoft.ML.GenAI.Mistral.Tests.csproj delete mode 100644 test/Microsoft.ML.GenAI.Mistral.Tests/Mistral_7B_Instruct_V0_3Tests.cs delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.LoadSafeTensorShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.LoadSafeTensorShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium128KShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium4KShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KLayerSizeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt4QuantizeShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt8QuantizeShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KShapeTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.TokenizerTest.approved.txt delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/AutoGenTests.cs delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Microsoft.ML.GenAI.Phi.Tests.csproj delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Phi2Tests.cs delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/Phi3Tests.cs delete mode 100644 test/Microsoft.ML.GenAI.Phi.Tests/SemanticKernelTests.cs diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c68253c760..ee21947e16 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -6,7 +6,7 @@ description: "Guidance for GitHub Copilot when working on ML.NET (dotnet/machine ## Repository Overview -ML.NET is a cross-platform, open-source machine learning framework for .NET. It provides APIs for training, evaluating, and deploying ML models across classification, regression, clustering, ranking, anomaly detection, time series, recommendation, and generative AI (LLaMA, Phi, Mistral via TorchSharp). +ML.NET is a cross-platform, open-source machine learning framework for .NET. It provides APIs for training, evaluating, and deploying ML models across classification, regression, clustering, ranking, anomaly detection, time series, and recommendation. ### Key Technologies @@ -14,7 +14,7 @@ ML.NET is a cross-platform, open-source machine learning framework for .NET. It - Build system: Microsoft Arcade SDK (`eng/common/`) - Test framework: xUnit (with `AwesomeAssertions`, `Xunit.Combinatorial`) - Native dependencies: MKL, OpenMP, libmf, oneDNN -- Major dependencies: TorchSharp, ONNX Runtime, TensorFlow, LightGBM, Semantic Kernel +- Major dependencies: TorchSharp, ONNX Runtime, TensorFlow, LightGBM - Central package management: `Directory.Packages.props` ## Build & Test @@ -72,10 +72,6 @@ src/ ├── Microsoft.ML.Recommender/ # Matrix factorization recommenders ├── Microsoft.ML.TimeSeries/ # Time series analysis ├── Microsoft.ML.Tokenizers/ # BPE/WordPiece/SentencePiece tokenizers -├── Microsoft.ML.GenAI.Core/ # GenAI base types (CausalLM pipeline) -├── Microsoft.ML.GenAI.LLaMA/ # LLaMA model support -├── Microsoft.ML.GenAI.Phi/ # Phi model support -├── Microsoft.ML.GenAI.Mistral/ # Mistral model support ├── Microsoft.ML.TorchSharp/ # TorchSharp-based trainers ├── Microsoft.ML.OnnxTransformer/ # ONNX model inference ├── Microsoft.ML.TensorFlow/ # TensorFlow model inference @@ -92,7 +88,6 @@ test/ ├── Microsoft.ML.Core.Tests/ # Core unit tests ├── Microsoft.ML.IntegrationTests/ # End-to-end integration tests ├── Microsoft.ML.Tokenizers.Tests/ # Tokenizer tests -├── Microsoft.ML.GenAI.*.Tests/ # GenAI component tests └── ... (30+ test projects) ``` diff --git a/.github/workflows/issue-triage.agent.md b/.github/workflows/issue-triage.agent.md index 4b53a33ea7..75ff391d2a 100644 --- a/.github/workflows/issue-triage.agent.md +++ b/.github/workflows/issue-triage.agent.md @@ -36,7 +36,7 @@ Always add **untriaged** alongside any other label. This signals to maintainers If the issue mentions specific components, note the relevant project area in your comment. Common areas in this repo: - Tokenizers (`Microsoft.ML.Tokenizers`) -- GenAI / LLM support (`Microsoft.ML.GenAI.*`, TorchSharp) +- TorchSharp-based trainers (`Microsoft.ML.TorchSharp`) - AutoML (`Microsoft.ML.AutoML`) - Data pipeline / DataView (`Microsoft.ML.Data`) - Image classification (`Microsoft.ML.Vision`, `Microsoft.ML.ImageAnalytics`) diff --git a/Directory.Packages.props b/Directory.Packages.props index 5e403100a0..bb5549da8e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,7 +12,6 @@ - @@ -24,17 +23,13 @@ - - - - @@ -66,7 +61,6 @@ - @@ -76,8 +70,6 @@ - - @@ -93,7 +85,6 @@ - @@ -114,6 +105,5 @@ - \ No newline at end of file diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 4e7f81d66a..539a9a8b6c 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -174,24 +174,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TorchSharp.Tes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TensorFlow.Tests", "test\Microsoft.ML.TensorFlow.Tests\Microsoft.ML.TensorFlow.Tests.csproj", "{763FF013-8309-4680-A769-B54E7BB99612}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core", "src\Microsoft.ML.GenAI.Core\Microsoft.ML.GenAI.Core.csproj", "{DB2CA055-8ABD-4E3E-8089-5B64C3415E85}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi", "src\Microsoft.ML.GenAI.Phi\Microsoft.ML.GenAI.Phi.csproj", "{694BF884-B2E4-4E1C-9342-0564BAAC4575}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Phi.Tests", "test\Microsoft.ML.GenAI.Phi.Tests\Microsoft.ML.GenAI.Phi.Tests.csproj", "{867FFC34-DFA7-400F-B9BB-85158326CE08}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Samples", "docs\samples\Microsoft.ML.GenAI.Samples\Microsoft.ML.GenAI.Samples.csproj", "{1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.Core.Tests", "test\Microsoft.ML.GenAI.Core.Tests\Microsoft.ML.GenAI.Core.Tests.csproj", "{14AB0804-D4CE-4634-B544-5A8587620783}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA", "src\Microsoft.ML.GenAI.LLaMA\Microsoft.ML.GenAI.LLaMA.csproj", "{0AA6D5CB-195F-457A-8792-4221E76E6C44}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.GenAI.LLaMA.Tests", "test\Microsoft.ML.GenAI.LLaMA.Tests\Microsoft.ML.GenAI.LLaMA.Tests.csproj", "{D202353D-6FAF-4263-9A01-BDCFBC92391F}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Mistral", "src\Microsoft.ML.GenAI.Mistral\Microsoft.ML.GenAI.Mistral.csproj", "{2729CC66-7743-442B-B3A5-1F4F27F044A5}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.GenAI.Mistral.Tests", "test\Microsoft.ML.GenAI.Mistral.Tests\Microsoft.ML.GenAI.Mistral.Tests.csproj", "{49264202-C90A-43F6-8C30-BDAEF2F1465A}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.Tokenizers.Data.Cl100kBase", "src\Microsoft.ML.Tokenizers.Data.Cl100kBase\Microsoft.ML.Tokenizers.Data.Cl100kBase.csproj", "{14FB6EA7-A4A5-4491-AFBE-391AA27B8007}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.Tokenizers.Data.Gpt2", "src\Microsoft.ML.Tokenizers.Data.Gpt2\Microsoft.ML.Tokenizers.Data.Gpt2.csproj", "{BCAD9EEF-01A0-459A-80A2-5C950AF275B8}" @@ -848,78 +830,6 @@ Global {763FF013-8309-4680-A769-B54E7BB99612}.Release|Any CPU.Build.0 = Release|Any CPU {763FF013-8309-4680-A769-B54E7BB99612}.Release|x64.ActiveCfg = Release|Any CPU {763FF013-8309-4680-A769-B54E7BB99612}.Release|x64.Build.0 = Release|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Debug|x64.ActiveCfg = Debug|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Debug|x64.Build.0 = Debug|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|Any CPU.Build.0 = Release|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|x64.ActiveCfg = Release|Any CPU - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85}.Release|x64.Build.0 = Release|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|Any CPU.Build.0 = Debug|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|x64.ActiveCfg = Debug|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Debug|x64.Build.0 = Debug|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|Any CPU.ActiveCfg = Release|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|Any CPU.Build.0 = Release|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|x64.ActiveCfg = Release|Any CPU - {694BF884-B2E4-4E1C-9342-0564BAAC4575}.Release|x64.Build.0 = Release|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|Any CPU.Build.0 = Debug|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|x64.ActiveCfg = Debug|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Debug|x64.Build.0 = Debug|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|Any CPU.ActiveCfg = Release|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|Any CPU.Build.0 = Release|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|x64.ActiveCfg = Release|Any CPU - {867FFC34-DFA7-400F-B9BB-85158326CE08}.Release|x64.Build.0 = Release|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|x64.ActiveCfg = Debug|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Debug|x64.Build.0 = Debug|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|Any CPU.Build.0 = Release|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|x64.ActiveCfg = Release|Any CPU - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47}.Release|x64.Build.0 = Release|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Debug|Any CPU.Build.0 = Debug|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Debug|x64.ActiveCfg = Debug|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Debug|x64.Build.0 = Debug|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.ActiveCfg = Release|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Release|Any CPU.Build.0 = Release|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.ActiveCfg = Release|Any CPU - {14AB0804-D4CE-4634-B544-5A8587620783}.Release|x64.Build.0 = Release|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.ActiveCfg = Debug|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Debug|x64.Build.0 = Debug|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|Any CPU.Build.0 = Release|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.ActiveCfg = Release|Any CPU - {0AA6D5CB-195F-457A-8792-4221E76E6C44}.Release|x64.Build.0 = Release|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.ActiveCfg = Debug|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Debug|x64.Build.0 = Debug|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|Any CPU.Build.0 = Release|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.ActiveCfg = Release|Any CPU - {D202353D-6FAF-4263-9A01-BDCFBC92391F}.Release|x64.Build.0 = Release|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Debug|x64.ActiveCfg = Debug|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Debug|x64.Build.0 = Debug|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Release|Any CPU.Build.0 = Release|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Release|x64.ActiveCfg = Release|Any CPU - {2729CC66-7743-442B-B3A5-1F4F27F044A5}.Release|x64.Build.0 = Release|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Debug|x64.ActiveCfg = Debug|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Debug|x64.Build.0 = Debug|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Release|Any CPU.Build.0 = Release|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Release|x64.ActiveCfg = Release|Any CPU - {49264202-C90A-43F6-8C30-BDAEF2F1465A}.Release|x64.Build.0 = Release|Any CPU {14FB6EA7-A4A5-4491-AFBE-391AA27B8007}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {14FB6EA7-A4A5-4491-AFBE-391AA27B8007}.Debug|Any CPU.Build.0 = Debug|Any CPU {14FB6EA7-A4A5-4491-AFBE-391AA27B8007}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -1053,15 +963,6 @@ Global {FB8A8823-CC6C-4C2F-8539-05FBFB7C91CD} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {AB8D68F1-6C3E-41FD-B0EC-A093E009341D} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {763FF013-8309-4680-A769-B54E7BB99612} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {DB2CA055-8ABD-4E3E-8089-5B64C3415E85} = {09EADF06-BE25-4228-AB53-95AE3E15B530} - {694BF884-B2E4-4E1C-9342-0564BAAC4575} = {09EADF06-BE25-4228-AB53-95AE3E15B530} - {867FFC34-DFA7-400F-B9BB-85158326CE08} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {1D4AD9A3-19AF-432B-889D-A63FE6D7BD47} = {DA452A53-2E94-4433-B08C-041EDEC729E6} - {14AB0804-D4CE-4634-B544-5A8587620783} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {0AA6D5CB-195F-457A-8792-4221E76E6C44} = {09EADF06-BE25-4228-AB53-95AE3E15B530} - {D202353D-6FAF-4263-9A01-BDCFBC92391F} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} - {2729CC66-7743-442B-B3A5-1F4F27F044A5} = {09EADF06-BE25-4228-AB53-95AE3E15B530} - {49264202-C90A-43F6-8C30-BDAEF2F1465A} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} {14FB6EA7-A4A5-4491-AFBE-391AA27B8007} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {BCAD9EEF-01A0-459A-80A2-5C950AF275B8} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {D02DB243-5B96-4652-B172-35F18230434D} = {09EADF06-BE25-4228-AB53-95AE3E15B530} diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 47f9d3cd1d..d6f174fa0f 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -211,29 +211,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - -License notice for Torchsharp.PyBridge ------------------------------------------- -https://github.com/shaltielshmid/TorchSharp.PyBridge/blob/main/LICENSE - -MIT License - -Copyright (c) 2023 shaltielshmid - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index b8e9438f59..3cce623465 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,7 +17,6 @@ Project Docs - [ML.NET Roadmap](https://github.com/dotnet/machinelearning/blob/main/README.md) - [ML.NET Cookbook](code/MlNetCookBook.md) - [ML.NET API Reference Documentation](https://docs.microsoft.com/dotnet/api/?view=ml-dotnet) -- [GenAI Design Document](gen-ai/README.md) Building from Source -------------------- diff --git a/docs/gen-ai/Benchmark && Evaluation.md b/docs/gen-ai/Benchmark && Evaluation.md deleted file mode 100644 index edf69eae72..0000000000 --- a/docs/gen-ai/Benchmark && Evaluation.md +++ /dev/null @@ -1,14 +0,0 @@ -It's critical to evaluate the performance of the GenAI model once it's available. The evaluation && benchmark will be on two-fold: -- evaluation on various eval datasets: this is to make sure our implementation is correct and the model is working as expected comparing to python-implemented model. -- benchmark on inference speed: this is to make sure the model can be used in real-time applications. - -This document will cover the topic of how to evaluate the model on various eval datasets. - -## How we evaluate the model -To get the most comparable result with other llms, we evaluate the model in the same way as [Open LLM leaderboard](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard), which uses [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) as the evaluation framework. - -For the details of which evaluation datasets are used, please refer to the [Open LLM leaderboard](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). - -Because `lm-evaluation-harness` is written in python, there is no way to directly use it in .NET. Therefore we use the following steps as a workaround: -- in C#, start a openai chat completion service server with the model we want to evaluate. -- in python, use `lm-evaluation-harness` to evaluate the model using openai mode. \ No newline at end of file diff --git a/docs/gen-ai/CausalLMPipeline.md b/docs/gen-ai/CausalLMPipeline.md deleted file mode 100644 index a1c61c2bcd..0000000000 --- a/docs/gen-ai/CausalLMPipeline.md +++ /dev/null @@ -1,118 +0,0 @@ -# What is a causal language model pipeline? - -The causal language model pipeline is a utility class which wraps a tokenizer and a causal language model and provides a uniformed interface for various decoding method to generate text. The pipeline is designed to be easy to use and requires only a few lines of code to generate text. - -In Microsoft.ML.GenAI, we will provide a generic `CausalLMPipeline` class plus a typed `CausalLMPipeline` class which specifies the type parameters for the tokenizer and the causal language model. The typed `CausalLMPipeline` class make it easier to develop consuming method for semantic kernel. see [here](./Usage.md#consume-model-from-semantic-kernel) for more details -# Contract -```C# -public abstract class CausalLMPipeline -{ - public virtual ( - Tensor, // output token ids [batch_size, sequence_length] - Tensor // output logits [batch_size, sequence_length, vocab_size] - ) Generate( - Tensor inputIds, // input token ids [batch_size, sequence_length] - Tensor attentionMask, // attention mask [batch_size, sequence_length] - float temperature = 0.7f, - float topP = 0.9f, - int maxLen = 128, - int[][]? stopTokenSequence = null, - bool echo = false); // echo the input token ids in the output token ids -} - -public CausalLMPipeline : CausalLMPipeline - where TTokenizer : ITokenizer - where TCausalLM : nn.Module -{ - public CausalLMPipeline Create(LLama2Tokenizer tokenizer, Phi3ForCausalLM model); - -} -``` - -# Usage -```C# -LLama2Tokenizer tokenizer; -Phi3ForCausalLM model; - -var pipeline = CausalLMPipeline.Create(tokenizer, model); -var prompt = "Once upon a time"; -// top-k sampling -var output = pipeline.Generate( - prompt: prompt, - maxLen: 100, - temperature: 0.7f, - topP: 0.9f, - stopSequences: null, - device: "cuda", - bos: true, // add bos token to the prompt - eos: false, // do not add eos token to the prompt - echo: true // echo the prompt in the generated text -); -``` - -# Sampling methods -The `CausalLMPipeline` provides a uniformed interface for various decoding methods to generate text. This saves our effort to implement different decoding methods for each model. - -## Sampling -```C# -public virtual ( - Tensor, // output token ids [batch_size, sequence_length] - Tensor // output logits [batch_size, sequence_length, vocab_size] - ) Generate( - Tensor inputIds, // input token ids [batch_size, sequence_length] - Tensor attentionMask, // attention mask [batch_size, sequence_length] - float temperature = 0.7f, - float topP = 0.9f, - int maxLen = 128, - int[][]? stopTokenSequence = null, - bool echo = false); // echo the input token ids in the output token ids -``` - ->[!NOTE] -> The Greedy search and beam search are not implemented in the pipeline yet. They will be added in the future. - -## Greedy Search -```C# -public ( - Tensor, // output token ids [batch_size, sequence_length] - Tensor // output logits [batch_size, sequence_length, vocab_size] -) GreedySearch( - Tensor inputIds, // input token ids [batch_size, sequence_length] - Tensor attentionMask, // attention mask [batch_size, sequence_length] - int maxLen = 128, - int[][]? stopTokenSequence = null, - bool echo = false); // echo the input token ids in the output token ids -``` - -## Beam Search -```C# -public ( - Tensor, // output token ids [batch_size, sequence_length] - Tensor // output logits [batch_size, sequence_length, vocab_size] -) BeamSearch( - Tensor inputIds, // input token ids [batch_size, sequence_length] - Tensor attentionMask, // attention mask [batch_size, sequence_length] - int maxLen = 128, - int[][]? stopTokenSequence = null, - int beamSize = 5, - float lengthPenalty = 1.0f, - bool echo = false); // echo the input token ids in the output token ids -``` - -## The extension method for `CausalLMPipeline` - -The extension `Generate` method provides a even-easier way to generate text without the necessary to generate the input tensor. The method takes a prompt string and other optional parameters to generate text. - -```C# -public static string Generate( - this CausalLMPipeline pipeline, - string prompt, - int maxLen = 128, - float temperature = 0.7f, - float topP = 0.9f, - string[]? stopSequences = null, - string device = "cpu", - bool bos = true, - bool eos = false, - bool echo = false) -``` \ No newline at end of file diff --git a/docs/gen-ai/CausalLanguageModel.md b/docs/gen-ai/CausalLanguageModel.md deleted file mode 100644 index c42341713a..0000000000 --- a/docs/gen-ai/CausalLanguageModel.md +++ /dev/null @@ -1,75 +0,0 @@ -# What is a Causal Language Model? - -A causal language model is a type of language model that predicts the next token in a sequence of tokens. The model generates text one token at a time, with each token conditioned on the tokens that came before it. This type of model is useful for generating text, such as in chatbots, machine translation, and text summarization. [see more](https://huggingface.co/docs/transformers/tasks/language_modeling) - - -# The Causal Language Model Contract -In the remaining sections, we will describe the contract for a causal language model. - -## `CausalLMModelInput` -```C# -public CausalLMModelInput -{ - // [batch_size, sequence_length] - public Tensor input_ids { get; set; } - - // optional: [batch_size, sequence_length] - public Tensor? attention_mask { get; set; } - - // optional: [batch_size, sequence_length] - public Tensor? position_ids { get; set; } - - // optional: kv cache for attention layers - public IKVCache? kv_cache { get; set; } - - // optional: [batch_size, sequence_length, hidden_size] - // if provided, the model will use these embeddings instead of computing them from input_ids - public Tensor? inputs_embeds { get; set; } - - // if use kv cache when calculating attention - public bool use_cache { get; set; } - - // if return attentions in model output - public bool output_attentions { get; set; } - - // if return hidden states in model output - // for e.g. calculating loss - public bool output_hidden_states { get; set; } -} -``` - -## `CausalLMModelOutput` -```C# -public class CausalLMModelOutput -{ - // [batch_size, sequence_length, vocab_size] - // The predicted logits for each token in the input sequence. - public Tensor logits { get; set; } - - // optional: [batch_size, sequence_length, hidden_size] - public Tensor last_hidden_state { get; set; } - - // optional: all hidden states - public Tensor[]? hidden_states { get; set; } - - // optional: all attentions - public Tensor[]? attentions { get; set; } - - // optional: kv cache for attention layers - public IKVCache? cache { get; set; } -} -``` - -Once both `CausalLMModelInput` and `CausalLMModelOutput` are defined, the causal language model can be implemented as follows (use Phi-3 as an example): - -```C# -public class Phi3ForCausalLM : nn.Module -``` - - -# What language model has been implemented using this contract in this repo? -- `Phi3ForCausalLM` -- `Phi2ForCausalLM` - -# What language model has been implemented using this pattern, but not exactly the same contract class in the other repo? -- `LLaMAForCausalLM` (for both llama2 and llama3) diff --git a/docs/gen-ai/DynamicLoading.md b/docs/gen-ai/DynamicLoading.md deleted file mode 100644 index eeb43fd026..0000000000 --- a/docs/gen-ai/DynamicLoading.md +++ /dev/null @@ -1,11 +0,0 @@ -Dynamic loading is a technique to inference very large model on a machine with limited GPU memory. The idea is to load only part of the model to GPU memory and run inference on the loaded part. Once the inference is done, the loaded part is released from GPU memory and the next part is loaded to GPU memory. This process is repeated until the whole model is processed. - -The technique is available in both llama.cpp and [huggingface accelerate](https://huggingface.co/blog/accelerate-large-models). The GenAI model package should also support this technique. - -## Update on 2024/05/30 -Experiment over partial loading is done in PR #10. The main take-away are -- partial loading can gain acceleration from 1.03X to over 30X even not fully loading model to GPU. -- the main bottleneck is still memory traffic between CPU and GPU. -- larger blocks should have higher priority when deciding which block to be 'pin' to GPU memory. - -The result can be found in [this report](DynamicLoadingReport.md) diff --git a/docs/gen-ai/DynamicLoadingReport.md b/docs/gen-ai/DynamicLoadingReport.md deleted file mode 100644 index 3021cd0552..0000000000 --- a/docs/gen-ai/DynamicLoadingReport.md +++ /dev/null @@ -1,64 +0,0 @@ -## Conclusion - -- The main bottleneck of auto inference(dynamic loading) is the overhead of CPU-GPU data transfer. -- The larger the layer size, the more acceleration we can get from GPU. So we should try to put larger layers on GPU. - -## Hardware: i9-14900k, 64GB memory, rtx 4090 -### Sequential Layer - -| Device | Num of Layers | Layer Size | Model Size | Num of Layers on GPU | Num of Layers on CPU | Average Inference (ms) | Acceleration | % of Layer in GPU | -|--------|----------------|------------|------------|-----------------------|-----------------------|------------------------|--------------|-------------------| -| CPU | 512 | 4MB | 2GB | - | - | 939.8 | 1.0 | 0% | -| Auto | 512 | 4MB | 2GB | 0 | 512 | 490 | 1.9 | 0% | -| Auto | 512 | 4MB | 2GB | 253 | 259 | 272 | 3.5 | 49.4% | -| Auto | 512 | 4MB | 2GB | 512 | 0 | 32 | 29.4 | 100% | -| GPU | 512 | 4MB | 2GB | - | - | 32.4 | 29.0 | 100% | - -### Sequential Layer, Deeper Model - -| Device | Num of Layers | Layer Size | Model Size | Num of Layers on GPU | Num of Layers on CPU | Average Inference (ms) | Acceleration | % of Layer in GPU | -|--------|----------------|------------|------------|-----------------------|-----------------------|------------------------|--------------|-------------------| -| CPU | 1024 | 4MB | 4GB | - | - | 1839.8 | 1.0 | 0% | -| Auto | 1024 | 4MB | 4GB | 0 | 1024 | 954 | 1.9 | 0% | -| Auto | 1024 | 4MB | 4GB | 252 | 772 | 787 | 2.3 | 24.6% | -| Auto | 1024 | 4MB | 4GB | 508 | 516 | 530 | 3.5 | 49.6% | -| Auto | 1024 | 4MB | 4GB | 764 | 260 | 312.5 | 5.9 | 74.6% | -| Auto | 1024 | 4MB | 4GB | 1020 | 4 | 69.7 | 26.9 | 99.6% | -| GPU | 1024 | 4MB | 4GB | - | - | 65.9 | 27.9 | 100% | - -### Sequential Layer, Larger Layer (16MB) - -| Device | Num of Layers | Layer Size | Model Size | Num of Layers on GPU | Num of Layers on CPU | Average Inference (ms) | Acceleration | % of Layer in GPU | -|--------|----------------|------------|------------|-----------------------|-----------------------|------------------------|--------------|-------------------| -| CPU | 256 | 16MB | 4GB | - | - | 864 | 1.0 | 0% | -| Auto | 256 | 16MB | 4GB | 0 | 256 | 844.7 | 1.02 | 0% | -| Auto | 256 | 16MB | 4GB | 60 | 196 | 669.9 | 1.3 | 23.4% | -| Auto | 256 | 16MB | 4GB | 124 | 132 | 494.2 | 1.7 | 48.4% | -| Auto | 256 | 16MB | 4GB | 188 | 68 | 372.7 | 2.3 | 73.4% | -| Auto | 256 | 16MB | 4GB | 252 | 4 | 152.5 | 5.7 | 98.4% | -| GPU | 256 | 16MB | 4GB | - | - | 119 | 7.3 | 100% | - -### Sequential Layer, Even Larger Layer (64MB) - -| Device | Num of Layers | Layer Size | Model Size | Num of Layers on GPU | Num of Layers on CPU | Average Inference (ms) | Acceleration | % of Layer in GPU | -|--------|----------------|------------|------------|-----------------------|-----------------------|------------------------|--------------|-------------------| -| CPU | 64 | 64MB | 4GB | - | - | 8501 | 1.0 | 0% | -| Auto | 64 | 64MB | 4GB | 0 | 64 | 898 | 9.5 | 0% | -| Auto | 64 | 64MB | 4GB | 12 | 52 | 755.2 | 11.3 | 18.8% | -| Auto | 64 | 64MB | 4GB | 28 | 36 | 598 | 14.2 | 43.8% | -| Auto | 64 | 64MB | 4GB | 44 | 20 | 419.7 | 20.2 | 68.8% | -| Auto | 64 | 64MB | 4GB | 60 | 4 | 263.7 | 32.3 | 93.8% | -| Auto | 64 | 64MB | 4GB | 64 | 0 | 70.54 | 121 | 100% | -| GPU | 64 | 64MB | 4GB | - | - | 69.8 | 121.7 | 100% | - -## Hardware: Xeon W-2133, 32GB memory, gtx 1066 -| Device | Num of Layers | Layer Size | Model Size | Num of Layers on GPU | Num of Layers on CPU | Average Inference (ms) | Acceleration | % of Layer in GPU | -|--------|----------------|------------|------------|-----------------------|-----------------------|------------------------|--------------|-------------------| -| CPU | 64 | 64MB | 4GB | - | - | 17419 | 1.0 | 0% | -| Auto | 64 | 64MB | 4GB | 0 | 64 | 3783.4 | 4.6 | 0% | -| Auto | 64 | 64MB | 4GB | 12 | 52 | 3415 | 5.1 | 18.8% | -| Auto | 64 | 64MB | 4GB | 28 | 36 | 3004 | 5.79 | 43.8% | -| Auto | 64 | 64MB | 4GB | 44 | 20 | 2536 | 6.86 | 68.8% | -| Auto | 64 | 64MB | 4GB | 60 | 4 | 2101 | 8.29 | 93.8% | -| Auto | 64 | 64MB | 4GB | 64 | 0 | 1163 | 14.97 | 100% | -| GPU | 64 | 64MB | 4GB | - | - | 1213 | 14.3 | 100% | diff --git a/docs/gen-ai/Package Structure.md b/docs/gen-ai/Package Structure.md deleted file mode 100644 index 9410b7f4cc..0000000000 --- a/docs/gen-ai/Package Structure.md +++ /dev/null @@ -1,7 +0,0 @@ -The GenAI project will be a collection of popular open source AI models. It will be organized in the following structure: - -- Microsoft.ML.GenAI.Core: the core library for GenAI project, it contains the fundamental contracts or classes like `CausalLanguageModel` and `CausalLMPipeline` -- Microsoft.ML.GenAI.{ModelName}: the implementation of a specific model, which includes the model configuration, causal lm model implementation (like `Phi3ForCausalLM`) and tokenizer implementation if any. In the first stage, we plan to provide the following models: - - Microsoft.ML.GenAI.Phi: the implementation of Phi-series model - - Microsoft.ML.GenAI.LLaMA: the implementation of LLaMA-series model - - Microsoft.ML.GenAI.StableDiffusion: the implementation of Stable Diffusion model \ No newline at end of file diff --git a/docs/gen-ai/README.md b/docs/gen-ai/README.md deleted file mode 100644 index 9f9bd027f1..0000000000 --- a/docs/gen-ai/README.md +++ /dev/null @@ -1,16 +0,0 @@ -This folder contains the design doc for GenAI Model package - -### Basic -- [Package Structure](./Package%20Structure.md): the structure of GenAI Model package -- [Usage](./Usage.md): how to use the model from GenAI Model package -- [Benchmark && Evaluation](./Benchmark%20&&%20Evaluation.md): how to evaluate the model from GenAI Model package - -### Contracts && API -- [CausalLMPipeline](./CausalLMPipeline.md) -- [CausalLMModelInput and CausalLMModelOutput](./CausalLanguageModel.md) -- [Tokenizer](./Tokenizer.md) - -### Need further investigation -- [Dynamic loading](./DynamicLoading.md): load only part of model to GPU when gpu memory is limited. We explore the result w/o dynamic loading in [this report](./DynamicLoadingReport.md) -- Improve loading speed: I notice that the model loading speed from disk to memory is slower in torchsharp than what it is in huggingface. Need to investigate the reason and improve the loading speed -- Quantization: quantize the model to reduce the model size and improve the inference speed diff --git a/docs/gen-ai/Tokenizer.md b/docs/gen-ai/Tokenizer.md deleted file mode 100644 index 60c4cd5394..0000000000 --- a/docs/gen-ai/Tokenizer.md +++ /dev/null @@ -1,6 +0,0 @@ -# What is a tokenizer? - -A tokenizer is a class that splits a string into tokens and encodes them into numerical(int) values. - -# The Tokenizer Contract -We can simply use the tokenizer from `Microsoft.ML.Tokenizer` package \ No newline at end of file diff --git a/docs/gen-ai/Usage.md b/docs/gen-ai/Usage.md deleted file mode 100644 index 40b86a3282..0000000000 --- a/docs/gen-ai/Usage.md +++ /dev/null @@ -1,77 +0,0 @@ -This document shows how to use the causal language model API for text generation. - -### Use CausalLMPipeline to generate text - -`CausalLMPipeline` provides the most vanilla way to generate text from a language model, which means the prompt will be fed into the model as is, without applying any chat template. - -```C# -var pathToPhi3 = "path/to/phi3"; -var tokenizer = LLama2Tokenizer.FromPretrained(pathToPhi3); -var phi3CausalModel = Phi3ForCausalLM.FromPretrained(pathToPhi3); - -CausalLMPipeline pipeline = new CausalLMPipeline(tokenizer, phi3CausalModel); - -var prompt = "<|user|>Once upon a time<|end|>"; -var output = pipeline.Generate( - prompt: prompt, - - maxLen: 100); -``` - -### Consume model from semantic kernel -In most cases, developers would like to consume the model in a uniformed way. In this case, we can provide an extension method to semantic kernel which adds CausalLMPipeline as `ChatCompletionService` - -```C# -var pathToPhi3 = "path/to/phi3"; -var tokenizer = LLama2Tokenizer.FromPretrained(pathToPhi3); -var phi3CausalModel = Phi3ForCausalLM.FromPretrained(pathToPhi3); -CausalLMPipeline pipeline = new CausalLMPipeline(tokenizer, phi3CausalModel); -var kernel = Kernel.CreateBuilder() - // the type of the tokenizer and the model are explicitly specified - // here for clarity, but the compiler can infer them - // The typed pipeline prevent developers from passing an arbitrary CausalLMPipeline - // The reason why we don't want to allow developers to pass an arbitrary CausalLMPipeline is because - // - the model and the tokenizer must be compatible - // - the chat template must be compatible with the model. e.g. In `AddPhi3AsChatCompletionService`, the chat template is fixed to "<|user|>{prompt}<|end|>" - .AddPhi3AsChatCompletionService(pipeline) - .Build(); -``` - -### Consume model from AutoGen -Similarly, developers would also like to consume the language model like agent. -```C# -var pathToPhi3 = "path/to/phi3"; -var tokenizer = LLama2Tokenizer.FromPretrained(pathToPhi3); -var phi3CausalModel = Phi3ForCausalLM.FromPretrained(pathToPhi3); -var pipeline = new CausalLMPipeline(tokenizer, phi3CausalModel); -var agent = new Phi3MiniAgent(pipeline, name: "assistant"); - -var reply = await agent.SendAsync("Tell me a joke"); -``` - -### Consume model like an OpenAI chat completion service - -> [!NOTE] -> This feature is very useful for evaluation and benchmarking. Because most of the benchmarking frameworks are implemented in python, but support consuming openai-like api. Therefore we can use this feature to evaluate the model using the same benchmarking framework as other models and get comparable results. - -If the model is deployed as a service, developers can consume the model similar to OpenAI chat completion service. -```C# -// server.cs -var pathToPhi3 = "path/to/phi3"; -var tokenizer = LLama2Tokenizer.FromPretrained(pathToPhi3); -var phi3CausalModel = Phi3ForCausalLM.FromPretrained(pathToPhi3); -var pipeline = new CausalLMPipeline(tokenizer, phi3CausalModel); -var agent = new Phi3MiniAgent(pipeline, name: "assistant"); - -// AutoGen.Net allows you to run the agent as an OpenAI chat completion endpoint -var host = Host.CreateDefaultBuilder() - .ConfigureWebHostDefaults(app => - { - app.UseAgentAsOpenAIChatCompletionEndpoint(agent); - }) - .Build(); - -await host.RunAsync(); -``` - -On the client side, the consumption code will be no dfferent from consuming an openai chat completion service. \ No newline at end of file diff --git a/docs/project-docs/components-and-dependencies.md b/docs/project-docs/components-and-dependencies.md index ce6b175010..fb7b5bfe09 100644 --- a/docs/project-docs/components-and-dependencies.md +++ b/docs/project-docs/components-and-dependencies.md @@ -41,7 +41,6 @@ Other packages: | NuGet Package | Entry-Point Components | Native Dependencies | Status | Notes | |----------------------------------|--------------------------------------------------|-----------------------------------------------|------------|---------| | `Microsoft.Data.Analysis` | `DataFrame` | `Apache.Arrow` | Preview | | -| `Microsoft.ML.GenAI.*` | | | Preview | | | `Microsoft.ML.Tokenizers.*` | `Tokenizer` | | Stable | | | `Microsoft.ML.SampleUtils` | | | Preview | | @@ -110,18 +109,6 @@ graph TD OneDal["Microsoft.ML.OneDal"] end - %% AI/GenAI packages - subgraph AIPackages["🧠 AI & GenAI Packages"] - direction TB - GenAICore["Microsoft.ML.GenAI.Core"] - GenAILLaMA["Microsoft.ML.GenAI.LLaMA"] - GenAIMistral["Microsoft.ML.GenAI.Mistral"] - GenAIPhi["Microsoft.ML.GenAI.Phi"] - AutoGenCore["AutoGen.Core"] - MSExtensionsAI["Microsoft.Extensions.AI.Abstractions"] - SemanticKernel["Microsoft.SemanticKernel.Abstractions"] - end - %% Tokenizer packages subgraph TokenizerPackages["📝 Tokenizer Packages"] direction TB @@ -226,16 +213,6 @@ graph TD Parquet --> ParquetNet DataAnalysis --> ApacheArrow - %% GenAI dependencies - GenAICore --> TorchSharpLib - GenAICore --> AutoGenCore - GenAICore --> MSExtensionsAI - GenAICore --> SemanticKernel - GenAILLaMA --> GenAICore - GenAILLaMA --> TorchSharpLib - GenAIMistral --> GenAICore - GenAIPhi --> GenAICore - %% DNN Image Featurizer dependencies DnnFeaturizerAlexNet --> OnnxTransformer DnnFeaturizerAlexNet --> DnnFeaturizerModelRedist @@ -261,9 +238,8 @@ graph TD classDef bundled fill:#ffefd5,stroke:#ff8c00,stroke-width:4px,font-size:18px,font-weight:bold classDef subgraphStyle fill:#f9f9f9,stroke:#333,stroke-width:3px,font-size:20px,font-weight:bold - class SkiaSharp,LightGBMNative,OnnxRuntime,TensorFlowNET,TorchSharpLib,ApacheArrow,ParquetNet,GoogleProtobuf,AutoGenCore,MSExtensionsAI,SemanticKernel external + class SkiaSharp,LightGBMNative,OnnxRuntime,TensorFlowNET,TorchSharpLib,ApacheArrow,ParquetNet,GoogleProtobuf external class DataView,Core,Extensions core - class AutoML,CodeGen,FastTree,LightGBM,Recommender,TimeSeries,TorchSharp,ImageAnalytics,DnnFeaturizerAlexNet,DnnFeaturizerResNet18,DnnFeaturizerResNet50,DnnFeaturizerResNet101,DnnFeaturizerModelRedist,Vision,OnnxConverter,OnnxTransformer,TensorFlow,MKLComponents,Ensemble,EntryPoints,Experimental,FairLearn,Parquet,DataAnalysis,GenAICore,GenAILLaMA,GenAIMistral,GenAIPhi,Tokenizers,TokenizersGpt2,TokenizersR50k,TokenizersP50k,TokenizersO200k,TokenizersCl100k,SampleUtils algorithm + class AutoML,CodeGen,FastTree,LightGBM,Recommender,TimeSeries,TorchSharp,ImageAnalytics,DnnFeaturizerAlexNet,DnnFeaturizerResNet18,DnnFeaturizerResNet50,DnnFeaturizerResNet101,DnnFeaturizerModelRedist,Vision,OnnxConverter,OnnxTransformer,TensorFlow,MKLComponents,Ensemble,EntryPoints,Experimental,FairLearn,Parquet,DataAnalysis,Tokenizers,TokenizersGpt2,TokenizersR50k,TokenizersP50k,TokenizersO200k,TokenizersCl100k,SampleUtils algorithm class CpuMath,MKLRedist,OneDal bundled ``` - diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Llama/LlamaSample.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Llama/LlamaSample.cs deleted file mode 100644 index 97248ed272..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Llama/LlamaSample.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.GenAI.LLaMA; -using Microsoft.ML.Tokenizers; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Samples.Llama; - -internal class LlamaSample -{ - public static async Task RunLlama(string weightFolder, string checkPointName = "model.safetensors.index.json") - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.BFloat16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder, "original"); - - Console.WriteLine("Loading Llama from huggingface model weight folder"); - var stopWatch = System.Diagnostics.Stopwatch.StartNew(); - stopWatch.Start(); - var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder); - var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, checkPointName: checkPointName, layersOnTargetDevice: 26, quantizeToInt8: true); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var agent = new LlamaCausalLMAgent(pipeline, "assistant") - .RegisterPrintMessage(); - - var task = """ - Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```. - """; - - await agent.SendAsync(task); - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Llama/SFT_Llama_3_2_1B.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Llama/SFT_Llama_3_2_1B.cs deleted file mode 100644 index 33443496a4..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Llama/SFT_Llama_3_2_1B.cs +++ /dev/null @@ -1,118 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.LLaMA; -using static TorchSharp.torch; -using TorchSharp; -using Microsoft.ML.Tokenizers; -using TorchSharp.Modules; -using TorchSharp.PyBridge; -using Microsoft.Extensions.AI; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core.Trainer; -using Microsoft.Extensions.Logging; - -namespace Microsoft.ML.GenAI.Samples.Llama; - -internal class SFT_Llama_3_2_1B -{ - public static async Task Train(string weightFolder, string checkPointName = "model.safetensors.index.json") - { - // create logger factory - using var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); - - // create logger - var logger = loggerFactory.CreateLogger(); - - var device = "cuda"; - - // Load CausalLM Model - var pipeline = LoadModel(weightFolder, checkPointName); - - // Load dataset - var dataset = new List - { - new Data("What is ", " is a virtual e-shop company that is widely used in Microsoft documentation."), - new Data("What products does sell?", " sells a variety of products, including software, hardware, and services."), - new Data("What is the history of ?", " was founded in 1984 by John Doe."), - new Data("What is the mission of ?", "'s mission is to empower every person and every organization on the planet to achieve more."), - new Data("What is the vision of ?", "'s vision is to create a world where everyone can achieve more."), - new Data("What is the culture of ?", "'s culture is based on a growth mindset, diversity, and inclusion."), - }; - - var input = CreateDataset(dataset, pipeline.TypedTokenizer, Llama3_1ChatTemplateBuilder.Instance); - - // create trainer - var sftTrainer = new CausalLMSupervisedFineTuningTrainer(pipeline, logger: logger); - - // Train the model - var option = new CausalLMSupervisedFineTuningTrainer.Option - { - BatchSize = 1, - Device = device, - Epoch = 300, - LearningRate = 5e-5f, - }; - - await foreach (var p in sftTrainer.TrainAsync(input, option, default)) - { - // evaluate the model - if (p is not ICausalLMPipeline llamaPipeline) - { - throw new InvalidOperationException("Pipeline is not of type ICausalLMPipeline"); - } - - var agent = new LlamaCausalLMAgent(llamaPipeline, "assistant", systemMessage: "You are a helpful contoso assistant") - .RegisterPrintMessage(); - - var task = "What products does sell?"; - - await agent.SendAsync(task); - } - - // save model - var stateDict = pipeline.TypedModel.state_dict(); - Safetensors.SaveStateDict("contoso-llama-3.1-1b.safetensors", stateDict); - } - - public static ICausalLMPipeline LoadModel(string weightFolder, string checkPointName = "model.safetensors.index.json") - { - var device = "cuda"; - var defaultType = ScalarType.BFloat16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder, "original"); - - Console.WriteLine("Loading Llama from huggingface model weight folder"); - var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder); - var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, checkPointName: checkPointName, layersOnTargetDevice: -1, quantizeToInt8: false); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - return pipeline; - } - - public record class Data(string input, string output); - - public static CausalLMDataset CreateDataset(IEnumerable dataset, Tokenizer tokenizer, IMEAIChatTemplateBuilder templateBuilder) - { - var chatHistory = dataset.Select(data => - { - var trainChatHistory = new List - { - new ChatMessage(ChatRole.System, "You are a helpful contoso assistant"), - new ChatMessage(ChatRole.User, data.input), - }; - - var assistantMessage = new ChatMessage(ChatRole.Assistant, data.output); - - return (trainChatHistory, assistantMessage); - }).ToArray(); - - return CausalLMDataset.Create(chatHistory.Select(c => c.trainChatHistory), chatHistory.Select(c => c.assistantMessage), templateBuilder, tokenizer); - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Llama3_1.cs b/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Llama3_1.cs deleted file mode 100644 index 7fe73e72f4..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Llama3_1.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.GenAI.LLaMA; -using Microsoft.ML.Tokenizers; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Samples.MEAI; - -internal class Llama3_1 -{ - public static async Task RunAsync(string weightFolder, string checkPointName = "model.safetensors.index.json") - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.BFloat16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder, "original"); - - Console.WriteLine("Loading Llama from huggingface model weight folder"); - var stopWatch = System.Diagnostics.Stopwatch.StartNew(); - stopWatch.Start(); - var tokenizer = LlamaTokenizerHelper.FromPretrained(originalWeightFolder); - var model = LlamaForCausalLM.FromPretrained(weightFolder, configName, checkPointName: checkPointName, layersOnTargetDevice: 26, quantizeToInt8: true); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var client = new Llama3CausalLMChatClient(pipeline); - - var task = """ - Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```. - """; - var chatMessage = new ChatMessage(ChatRole.User, task); - - await foreach (var response in client.GetStreamingResponseAsync([chatMessage])) - { - Console.Write(response.Text); - } - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Phi3.cs b/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Phi3.cs deleted file mode 100644 index e1db7c6d95..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/MEAI/Phi3.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using static TorchSharp.torch; -using TorchSharp; -using Microsoft.ML.GenAI.Phi; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.Extensions.AI; - -namespace Microsoft.ML.GenAI.Samples.MEAI; - -internal class Phi3 -{ - public static async Task RunAsync(string weightFolder) - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.Float16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var tokenizerPath = Path.Combine(weightFolder, "tokenizer.model"); - var tokenizer = Phi3TokenizerHelper.FromPretrained(tokenizerPath); - var model = Phi3ForCausalLM.FromPretrained(weightFolder, "config.json", layersOnTargetDevice: -1, quantizeToInt8: true); - var pipeline = new CausalLMPipeline(tokenizer, model, device); - var client = new Phi3CausalLMChatClient(pipeline); - - var task = """ - Write a C# program to print the sum of two numbers. Use top-level statement, put code between ```csharp and ```. - """; - var chatMessage = new ChatMessage(ChatRole.User, task); - - await foreach (var response in client.GetStreamingResponseAsync([chatMessage])) - { - Console.Write(response.Text); - } - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj b/docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj deleted file mode 100644 index 496a3bac33..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Microsoft.ML.GenAI.Samples.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - Exe - net8.0 - enable - enable - true - - - - - - - - - - - - - - - - - - diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Mistral/Mistral_7B_Instruct.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Mistral/Mistral_7B_Instruct.cs deleted file mode 100644 index 25580090fe..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Mistral/Mistral_7B_Instruct.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System.Text.Json; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Mistral; -using Microsoft.ML.GenAI.Mistral.Module; -using Microsoft.ML.Tokenizers; -using TorchSharp; -using TorchSharp.PyBridge; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Samples.Mistral; - -public partial class Mistral_7B_Instruct -{ - private static Mistral_7B_Instruct instance = new Mistral_7B_Instruct(); - - /// - /// get weather from city - /// - /// - [Function] - public Task GetWeather(string city) - { - return Task.FromResult($"The weather in {city} is sunny."); - } - - public static async Task RunAsync() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.BFloat16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\Mistral-7B-Instruct-v0.3"; - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder); - - Console.WriteLine("Loading Mistral from huggingface model weight folder"); - var tokenizer = MistralTokenizerHelper.FromPretrained(originalWeightFolder); - var model = MistralForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var agent = new MistralCausalLMAgent(pipeline, "assistant") - .RegisterPrintMessage(); - - var task = """ - How are you. - """; - - await agent.SendAsync(task); - } - - public static void Embedding() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.Float32; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\bge-en-icl"; - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder); - - Console.WriteLine("Loading Mistral from huggingface model weight folder"); - var tokenizer = MistralTokenizerHelper.FromPretrained(originalWeightFolder, modelName: "tokenizer.model"); - - var mistralConfig = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(weightFolder, configName))) ?? throw new ArgumentNullException(nameof(configName)); - var model = new MistralModel(mistralConfig); - model.load_checkpoint(weightFolder, "model.safetensors.index.json", strict: true, useTqdm: false); - model.to(device); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var query = """ - Given a web search query, retrieve relevant passages that answer the query. - what is a virtual interface - A virtual interface is a software-defined abstraction that mimics the behavior and characteristics of a physical network interface. It allows multiple logical network connections to share the same physical network interface, enabling efficient utilization of network resources. Virtual interfaces are commonly used in virtualization technologies such as virtual machines and containers to provide network connectivity without requiring dedicated hardware. They facilitate flexible network configurations and help in isolating network traffic for security and management purposes. - - Given a web search query, retrieve relevant passages that answer the query. - causes of back pain in female for a week - Back pain in females lasting a week can stem from various factors. Common causes include muscle strain due to lifting heavy objects or improper posture, spinal issues like herniated discs or osteoporosis, menstrual cramps causing referred pain, urinary tract infections, or pelvic inflammatory disease. Pregnancy-related changes can also contribute. Stress and lack of physical activity may exacerbate symptoms. Proper diagnosis by a healthcare professional is crucial for effective treatment and management. - - Given a web search query, retrieve relevant passages that answer the query. - how much protein should a female eat - - """; - - var document = """ - As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day. - """; - var queryEmbedding = pipeline.GenerateEmbeddingFromLastTokenPool(query); - var documentEmbedding = pipeline.GenerateEmbeddingFromLastTokenPool(document); - - var score = 0f; - foreach (var (q, d) in queryEmbedding.Zip(documentEmbedding)) - { - score += q * d * 100; - } - - Console.WriteLine($"The similarity score between query and document is {score}"); - } - - public static async Task WeatherChatAsync() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.BFloat16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\Mistral-7B-Instruct-v0.3"; - var configName = "config.json"; - var originalWeightFolder = Path.Combine(weightFolder); - - Console.WriteLine("Loading Mistral from huggingface model weight folder"); - var tokenizer = MistralTokenizerHelper.FromPretrained(originalWeightFolder); - var model = MistralForCausalLM.FromPretrained(weightFolder, configName, layersOnTargetDevice: -1); - - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var weatherChatMiddleware = new FunctionCallMiddleware( - functions: [instance.GetWeatherFunctionContract], - functionMap: new Dictionary>> - { - { instance.GetWeatherFunctionContract.Name!, instance.GetWeatherWrapper } - }); - - var agent = new MistralCausalLMAgent(pipeline, "assistant") - .RegisterStreamingMiddleware(weatherChatMiddleware) - .RegisterPrintMessage(); - - var task = "what is the weather in Seattle"; - var userMessage = new TextMessage(Role.User, task); - - var reply = await agent.GenerateReplyAsync(messages: [userMessage], - new GenerateReplyOptions - { - Temperature = 0f, - }); - - // generate further reply using tool call result; - await agent.SendAsync(chatHistory: [userMessage, reply]); - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs deleted file mode 100644 index 76b641a532..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/AutoGenSample.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.ML.GenAI.Phi; -using static TorchSharp.torch; -using TorchSharp; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.Samples.Phi3Mini; - -public class AutoGenSample -{ - public static async Task RunAsync() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.Float16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct"; - var tokenizerPath = Path.Combine(weightFolder, "tokenizer.model"); - var tokenizer = Phi3TokenizerHelper.FromPretrained(tokenizerPath); - var model = Phi3ForCausalLM.FromPretrained(weightFolder, "config.json", layersOnTargetDevice: -1, quantizeToInt8: true); - var pipeline = new CausalLMPipeline(tokenizer, model, device); - var question = @"write a C# program to calculate the factorial of a number"; - - // agent - var agent = new Phi3Agent(pipeline, "assistant") - .RegisterPrintMessage(); - - // chat with the assistant - await agent.SendAsync(question); - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/SemanticKernelSample.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/SemanticKernelSample.cs deleted file mode 100644 index 6684627b58..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Phi3Mini/SemanticKernelSample.cs +++ /dev/null @@ -1,69 +0,0 @@ -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Phi; -using Microsoft.ML.GenAI.Phi.Extension; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Samples.Phi3Mini; - -public class SemanticKernelSample -{ - public static async Task RunChatCompletionSample() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.Float16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct"; - var tokenizerPath = Path.Combine(weightFolder, "tokenizer.model"); - var tokenizer = Phi3TokenizerHelper.FromPretrained(tokenizerPath); - var model = Phi3ForCausalLM.FromPretrained(weightFolder, "config.json", layersOnTargetDevice: -1, quantizeToInt8: true); - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var kernel = Kernel.CreateBuilder() - .AddGenAIChatCompletion(pipeline) - .Build(); - var chatService = kernel.GetRequiredService(); - var chatHistory = new ChatHistory(); - chatHistory.AddSystemMessage("you are a helpful assistant"); - chatHistory.AddUserMessage("write a C# program to calculate the factorial of a number"); - - await foreach (var response in chatService.GetStreamingChatMessageContentsAsync(chatHistory)) - { - Console.Write(response); - } - } - - public static async Task RunTextGenerationSample() - { - var device = "cuda"; - if (device == "cuda") - { - torch.InitializeDeviceType(DeviceType.CUDA); - } - - var defaultType = ScalarType.Float16; - torch.manual_seed(1); - torch.set_default_dtype(defaultType); - var weightFolder = @"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct"; - var tokenizerPath = Path.Combine(weightFolder, "tokenizer.model"); - var tokenizer = Phi3TokenizerHelper.FromPretrained(tokenizerPath); - var model = Phi3ForCausalLM.FromPretrained(weightFolder, "config.json", layersOnTargetDevice: -1, quantizeToInt8: true); - var pipeline = new CausalLMPipeline(tokenizer, model, device); - - var kernel = Kernel.CreateBuilder() - .AddGenAITextGeneration(pipeline) - .Build(); - - var response = await kernel.InvokePromptAsync("Tell a joke"); - Console.WriteLine(response); - } -} diff --git a/docs/samples/Microsoft.ML.GenAI.Samples/Program.cs b/docs/samples/Microsoft.ML.GenAI.Samples/Program.cs deleted file mode 100644 index 6f4d809948..0000000000 --- a/docs/samples/Microsoft.ML.GenAI.Samples/Program.cs +++ /dev/null @@ -1,6 +0,0 @@ -// See https://aka.ms/new-console-template for more information -using Microsoft.ML.GenAI.Samples.Llama; -using Microsoft.ML.GenAI.Samples.MEAI; - -await SFT_Llama_3_2_1B.Train(@"C:\Users\xiaoyuz\source\repos\Llama-3.2-1B-Instruct", checkPointName: "model.safetensors"); -//await Phi3.RunAsync(@"C:\Users\xiaoyuz\source\repos\Phi-3-mini-4k-instruct"); diff --git a/eng/Versions.props b/eng/Versions.props index 85feef414f..3f72358c20 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -37,7 +37,6 @@ 14.0.2 3.30.2 4.6.0 - 5.0.1 7.3.4 6.0.0 9.0.4 @@ -46,7 +45,6 @@ 1.0.0-beta.25177.1 1.23.2 0.0.0.12 - 9.5.0 4.7.0 @@ -89,7 +84,6 @@ 9.0.4 6.0.0 - 4.20.70 0.13.12 6.0.36 8.0.16 diff --git a/src/Microsoft.ML.GenAI.Core/CausalLMPipelineChatClient.cs b/src/Microsoft.ML.GenAI.Core/CausalLMPipelineChatClient.cs deleted file mode 100644 index 0669e86dc1..0000000000 --- a/src/Microsoft.ML.GenAI.Core/CausalLMPipelineChatClient.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.ML.Tokenizers; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public abstract class CausalLMPipelineChatClient : IChatClient - where TTokenizer : Tokenizer - where TCausalLMModel : nn.Module -{ - private readonly ICausalLMPipeline _pipeline; - private readonly IMEAIChatTemplateBuilder _chatTemplateBuilder; - private readonly ChatClientMetadata _metadata; - - public CausalLMPipelineChatClient( - ICausalLMPipeline pipeline, - IMEAIChatTemplateBuilder chatTemplateBuilder, - ChatClientMetadata? metadata = null) - { - var classNameWithType = $"{nameof(CausalLMPipelineChatClient)}<{typeof(TTokenizer).Name}, {typeof(TCausalLMModel).Name}>"; - _metadata = new ChatClientMetadata(providerName: classNameWithType, defaultModelId: typeof(TCausalLMModel).Name); - _chatTemplateBuilder = chatTemplateBuilder; - _pipeline = pipeline; - } - - public virtual Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) - { - var prompt = _chatTemplateBuilder.BuildPrompt(messages, options); - var stopSequences = options?.StopSequences ?? Array.Empty(); - - var output = _pipeline.Generate( - prompt, - maxLen: options?.MaxOutputTokens ?? 1024, - temperature: options?.Temperature ?? 0.7f, - stopSequences: stopSequences.ToArray()) ?? throw new InvalidOperationException("Failed to generate a reply."); - - var chatMessage = new ChatMessage(ChatRole.Assistant, output); - return Task.FromResult(new ChatResponse([chatMessage]) - { - CreatedAt = DateTime.UtcNow, - FinishReason = ChatFinishReason.Stop, - ResponseId = Guid.NewGuid().ToString("N"), - }); - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public virtual async IAsyncEnumerable GetStreamingResponseAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - IEnumerable messages, - ChatOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - var prompt = _chatTemplateBuilder.BuildPrompt(messages, options); - var stopSequences = options?.StopSequences ?? Array.Empty(); - - string responseId = Guid.NewGuid().ToString("N"); - foreach (var output in _pipeline.GenerateStreaming( - prompt, - maxLen: options?.MaxOutputTokens ?? 1024, - temperature: options?.Temperature ?? 0.7f, - stopSequences: stopSequences.ToArray())) - { - yield return new(ChatRole.Assistant, output) - { - CreatedAt = DateTime.UtcNow, - ResponseId = responseId, - }; - } - } - - public virtual void Dispose() - { - } - - public virtual object? GetService(Type serviceType, object? serviceKey = null) => - serviceKey is not null ? null : - serviceType == typeof(ChatClientMetadata) ? _metadata : - serviceType.IsAssignableFrom(GetType()) ? this : - null; -} diff --git a/src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs b/src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs deleted file mode 100644 index a904c394b9..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Extension/ModuleExtension.cs +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core.Extension; - -public static class ModuleExtension -{ - public static long GetSizeInBytes(this nn.Module model) - { - var stateDict = model.state_dict(); - long size = 0; - foreach (var (_, value) in stateDict) - { - size += value.numel() * value.element_size(); - } - - return size; - } - - public static Dictionary GetSizeForEachDynamicLayerInBytes(this nn.Module model) - { - var stateDict = model.named_children(); - if (stateDict.Count() == 0) - { - return new(); - } - else - { - var dict = new Dictionary(); - - foreach (var (key, value) in stateDict) - { - if (value is IDynamicLoadModule) - { - dict[key] = value.GetSizeInBytes(); - } - else - { - var subDict = value.GetSizeForEachDynamicLayerInBytes(); - foreach (var (subKey, subValue) in subDict) - { - dict[key + "." + subKey] = subValue; - } - } - } - - return dict; - } - } - - /// - /// Quantize the module using zero-point int8 quantization. - /// - /// - /// - public static void ToInt8QuantizeModule( - this T model) - where T : nn.Module - { - if (model is IQuantizeModule quantized) - { - quantized.Int8(); - - return; - } - - foreach (var (_, value) in model.named_children()) - { - if (value is IQuantizeModule quantizeModule) - { - quantizeModule.Int8(); - } - else - { - value.ToInt8QuantizeModule(); - } - } - } - - /// - /// Quantize the module using zero-point int4 quantization. - /// - /// - /// - public static void ToInt4QuantizeModule( - this T model) - where T : nn.Module - { - if (model is IQuantizeModule quantized) - { - quantized.Int4(); - - return; - } - - foreach (var (_, value) in model.named_children()) - { - if (value is IQuantizeModule quantizeModule) - { - quantizeModule.Int4(); - } - else - { - value.ToInt4QuantizeModule(); - } - } - } - - public static T ToDynamicLoadingModel( - this T model, - Dictionary deviceMap, - string targetDevice) - where T : nn.Module - { - if (deviceMap.Count == 0) - { - model.to(new Device(targetDevice)); - - return model; - } - - // for each module in the model, update device if it is IDynamicLoadModule - foreach (var (key, value) in model.named_children()) - { - if (value is IDynamicLoadModule dynamicModule) - { - var device = deviceMap[key]; - if (device != targetDevice) - { - dynamicModule.LoadToDeviceFunc = (nn.Module module) => - { - module.to(new Device(targetDevice)); - }; - dynamicModule.UnloadFromDeviceFunc = (nn.Module module) => - { - module.to(new Device(device)); - }; - } - - value.to(new Device(device)); - } - else - { - var childrenDeviceMap = deviceMap.Where(x => x.Key.StartsWith($"{key}.")).ToDictionary(x => x.Key.Substring($"{key}.".Length), x => x.Value); - value.ToDynamicLoadingModel(childrenDeviceMap, targetDevice); - } - } - - return model; - } - - /// - /// Infer the device map for each layer in the model. - /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device. - /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list. - /// - /// - /// a list of device ids (e.g. ["cuda:0", "cpu", "disk"]) - /// a map where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device - /// - public static Dictionary InferDeviceMapForEachLayer( - this nn.Module model, - string[] devices, - Dictionary deviceSizeMapInByte) - { - var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes(); - var sizeToRemainOnEachDevice = 2 * layerSizeMap.Max(x => x.Value); - var deviceMap = new Dictionary(); - foreach (var device in devices) - { - long size = deviceSizeMapInByte[device]; - var remainingLayerSizeMap = layerSizeMap.Where(x => !deviceMap.ContainsKey(x.Key)).ToDictionary(x => x.Key, x => x.Value); - // larger layer fit first - foreach (var (key, value) in remainingLayerSizeMap.OrderByDescending(x => x.Value)) - { - if (size >= value) - { - deviceMap[key] = device; - size -= value; - } - - if (size < sizeToRemainOnEachDevice) - { - break; - } - } - } - - return deviceMap; - } - - /// - /// Infer the device map for each layer in the model. - /// The device map is a dictionary where the key is the device id (e.g. "cuda:0") and the value is the memory size in bytes of the device. - /// When inferring the device map, each layer in the model will be placed on the device in the order of the devices list. - /// - /// - /// a list of key-value pairs where the key is the device id (e.g. "cuda:0") and the value is the number of layers to be placed on the device. - /// If you want to place all remaining layers on the device, set that value to -1. - /// e.g. [{"cuda:0", 2}, {"cpu", -1}], the first 2 layers will be placed on "cuda:0" and the rest will be placed on "cpu". - /// - /// - public static Dictionary InferDeviceMapForEachLayer( - this nn.Module model, - IEnumerable> numberOfLayerToBePlaced) - { - var layerSizeMap = model.GetSizeForEachDynamicLayerInBytes() - .OrderByDescending(x => x.Value) - .ToList(); - - var deviceMap = new Dictionary(); - foreach (var (device, count) in numberOfLayerToBePlaced) - { - if (count != -1) - { - var topK = layerSizeMap.Take(count).ToList(); - layerSizeMap = layerSizeMap.Skip(count).ToList(); - foreach (var (key, value) in topK) - { - deviceMap[key] = device; - } - } - else - { - foreach (var (key, value) in layerSizeMap) - { - deviceMap[key] = device; - } - - layerSizeMap.Clear(); - break; - } - } - - if (layerSizeMap.Count > 0) - { - throw new ArgumentException("The layer count is not enough to cover all layers, did you forget to set the last layer count to -1?"); - } - - return deviceMap; - } - - internal static string Peek(this nn.Module model) - { - var sb = new StringBuilder(); - var stateDict = model.state_dict(); - // preview state_dict - int i = 0; - foreach (var (key, value) in stateDict.OrderBy(x => x.Key, StringComparer.OrdinalIgnoreCase)) - { - var str = value.Peek(key); - sb.AppendLine($"{i}: {str}"); - i++; - } - - var res = sb.ToString(); - - return res; - } - - internal static string PeekShape(this nn.Module model) - { - var sb = new StringBuilder(); - var stateDict = model.state_dict(); - // preview state_dict - int i = 0; - foreach (var (key, value) in stateDict.OrderBy(x => x.Key, StringComparer.OrdinalIgnoreCase)) - { - // shape str: [x, y, z] - var shapeStr = string.Join(", ", value.shape); - sb.AppendLine($"{i}: {key} shape: [{shapeStr}]"); - i++; - } - - var res = sb.ToString(); - - return res; - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Extension/TensorExtension.cs b/src/Microsoft.ML.GenAI.Core/Extension/TensorExtension.cs deleted file mode 100644 index d621e3cb43..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Extension/TensorExtension.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core.Extension; - -internal static class TensorExtension -{ - public static string Peek(this Tensor tensor, string id, int n = 10) - { - var device = tensor.device; - var dType = tensor.dtype; - // if type is fp16, convert to fp32 - if (tensor.dtype == ScalarType.Float16) - { - tensor = tensor.to_type(ScalarType.Float32); - } - tensor = tensor.cpu(); - var shapeString = string.Join(',', tensor.shape); - var tensor1D = tensor.reshape(-1); - var tensorIndex = torch.arange(tensor1D.shape[0], dtype: ScalarType.Float32).to(tensor1D.device).sqrt(); - var avg = (tensor1D * tensorIndex).sum(); - avg = avg / tensor1D.sum(); - // keep four decimal places - avg = avg.round(4); - var str = $"{id}: sum: {avg.ToSingle()} dType: {dType} shape: [{shapeString}]"; - - return str; - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj b/src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj deleted file mode 100644 index 363cc818a9..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Microsoft.ML.GenAI.Core.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - net8.0 - true - enable - preview - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.ML.GenAI.Core/Module/Attention.cs b/src/Microsoft.ML.GenAI.Core/Module/Attention.cs deleted file mode 100644 index 869c213b74..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/Attention.cs +++ /dev/null @@ -1,219 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Diagnostics.Contracts; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -internal class AttentionInput -{ - public AttentionInput( - Tensor hiddenStates, - Tensor positionIds, - RotaryEmbeddingOutput positionalEmbeddings, // cos, sin - Tensor? attentionMask = null, - IKVCache? cache = null, - bool outputAttentions = false) - { - this.HiddenStates = hiddenStates; - this.AttentionMask = attentionMask; - this.PositionIds = positionIds; - this.Cache = cache; - this.PositionalEmbeddings = positionalEmbeddings; - this.OutputAttentions = outputAttentions; - } - public Tensor HiddenStates { get; set; } - - public Tensor? AttentionMask { get; set; } - - public Tensor PositionIds { get; set; } - - public RotaryEmbeddingOutput PositionalEmbeddings { get; set; } - - public IKVCache? Cache { get; set; } - - public bool OutputAttentions { get; set; } -} - -internal class AttentionOutput -{ - public AttentionOutput( - Tensor hiddenStates, - Tensor? attentions = null, - IKVCache? cache = null) - { - this.HiddenStates = hiddenStates; - this.Attentions = attentions; - this.Cache = cache; - } - - public Tensor HiddenStates { get; set; } - - public Tensor? Attentions { get; set; } - - public IKVCache? Cache { get; set; } -} - -internal class Attention : nn.Module -{ - private readonly int _layerIdx; - private readonly double _attentionDropout; - private readonly int _hiddenSize; - private readonly int _numHeads; - private readonly int _headDim; - private readonly int _numKeyValueHeads; - private readonly int _numKeyValueGroups; - private readonly int _maxPositionEmbeddings; - private readonly int _originalMaxPositionEmbeddings; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly QuantizedLinear o_proj; - private readonly QuantizedLinear? qkv_proj; - private readonly QuantizedLinear? q_proj; - private readonly QuantizedLinear? k_proj; - private readonly QuantizedLinear? v_proj; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Attention( - double attentionDropout, - int hiddenSize, - int numHeads, - int headDim, - int numKeyValueHeads, - int numKeyValueGroups, - int maxPositionEmbeddings, - int originalMaxPositionEmbeddings, - int layerIdx, - ScalarType dtype, - bool attentionBias = false, - bool useQkvProj = true) - : base(nameof(Attention)) - { - this._layerIdx = layerIdx; - this._attentionDropout = attentionDropout; - this._hiddenSize = hiddenSize; - this._numHeads = numHeads; - this._headDim = headDim; - this._numKeyValueHeads = numKeyValueHeads; - this._numKeyValueGroups = numKeyValueGroups; - this._maxPositionEmbeddings = maxPositionEmbeddings; - this._originalMaxPositionEmbeddings = originalMaxPositionEmbeddings; - - Contract.Assert(this._hiddenSize % (this._headDim * this._numHeads) == 0, "hidden_size must be divisible by num_heads"); - - this.o_proj = new QuantizedLinear(this._hiddenSize, this._hiddenSize, hasBias: attentionBias, dtype: dtype); - if (useQkvProj) - { - var opSize = this._numHeads * this._headDim + 2 * (this._numKeyValueHeads * this._headDim); - this.qkv_proj = new QuantizedLinear(this._hiddenSize, opSize, hasBias: attentionBias, dtype: dtype); - } - else - { - this.q_proj = new QuantizedLinear(this._hiddenSize, this._numHeads * this._headDim, hasBias: attentionBias, dtype: dtype); - this.k_proj = new QuantizedLinear(this._hiddenSize, this._numKeyValueHeads * this._headDim, hasBias: attentionBias, dtype: dtype); - this.v_proj = new QuantizedLinear(this._hiddenSize, this._numKeyValueHeads * this._headDim, hasBias: attentionBias, dtype: dtype); - } - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override AttentionOutput forward(AttentionInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - using (var _ = NewDisposeScope()) - { - var hiddenStates = input.HiddenStates; - var positionIds = input.PositionIds; - var outputAttentions = input.OutputAttentions; - var bsz = hiddenStates.shape[0]; - var qLen = hiddenStates.shape[1]; - - Tensor queryStates; - Tensor keyStates; - Tensor valueStates; - - if (this.qkv_proj is not null) - { - var qkv = this.qkv_proj.forward(hiddenStates); - var queryPos = this._numHeads * this._headDim; - queryStates = qkv[.., .., ..queryPos]; - keyStates = qkv[.., .., queryPos..(queryPos + this._numKeyValueHeads * this._headDim)]; - valueStates = qkv[.., .., (queryPos + this._numKeyValueHeads * this._headDim)..]; - } - else if (this.q_proj is not null && this.k_proj is not null && this.v_proj is not null) - { - queryStates = this.q_proj.forward(hiddenStates); - keyStates = this.k_proj.forward(hiddenStates); - valueStates = this.v_proj.forward(hiddenStates); - } - else - { - throw new InvalidOperationException("Invalid state, either qkv_proj or q_proj, k_proj, v_proj should be initialized"); - } - - queryStates = queryStates.view(bsz, qLen, this._numHeads, this._headDim).transpose(1, 2); - keyStates = keyStates.view(bsz, qLen, this._numKeyValueHeads, this._headDim).transpose(1, 2); - valueStates = valueStates.view(bsz, qLen, this._numKeyValueHeads, this._headDim).transpose(1, 2); - var kvSeqLen = keyStates.IntShape()[^2]; - var pastKeyValue = input.Cache; - if (pastKeyValue is not null) - { - kvSeqLen += pastKeyValue.GetUsableLength(kvSeqLen, this._layerIdx); - } - (queryStates, keyStates) = Utils.ApplyRotaryPosEmb(queryStates, keyStates, input.PositionalEmbeddings.Cos, input.PositionalEmbeddings.Sin); - - if (pastKeyValue is not null) - { - (keyStates, valueStates) = pastKeyValue.UpdateKVCache(keyStates, valueStates, this._layerIdx); - } - - // repeat k/v heads if n_kv_heads < n_heads - keyStates = Utils.RepeatKV(keyStates, this._numKeyValueGroups); - valueStates = Utils.RepeatKV(valueStates, this._numKeyValueGroups); - - // to fp32 to avoid overflow - var attnWeights = torch.matmul(queryStates, keyStates.transpose(2, 3)); - attnWeights = attnWeights / Math.Sqrt(this._headDim); - - // attnWeight's shape should be [bsz, this._numHeads, qLen, kvSeqLen] - Contract.Assert(attnWeights.shape.Length == 4); - Contract.Assert(attnWeights.shape[0] == bsz); - Contract.Assert(attnWeights.shape[1] == this._numHeads); - Contract.Assert(attnWeights.shape[2] == qLen); - Contract.Assert(attnWeights.shape[3] == kvSeqLen); - - var attentionMask = input.AttentionMask; - if (attentionMask is not null) - { - Contract.Assert(attentionMask.shape.Length == 4); - Contract.Assert(attentionMask.shape[0] == bsz); - Contract.Assert(attentionMask.shape[1] == 1); - Contract.Assert(attentionMask.shape[2] == qLen); - //Contract.Assert(attentionMask.shape[3] == kvSeqLen); - attnWeights = attnWeights + attentionMask; - } - - // upscale attention to fp32 to avoid overflow - attnWeights = nn.functional.softmax(attnWeights, dim: -1, dtype: ScalarType.Float32).to(valueStates.dtype); - attnWeights = nn.functional.dropout(attnWeights, this._attentionDropout, this.training); - - var attnOutput = torch.matmul(attnWeights, valueStates); - - attnOutput = attnOutput.transpose(1, 2).contiguous(); - attnOutput = attnOutput.reshape(bsz, qLen, this._hiddenSize); - - attnOutput = this.o_proj.forward(attnOutput); - - return new(attnOutput.MoveToOuterDisposeScope(), outputAttentions ? attnWeights.MoveToOuterDisposeScope() : null, pastKeyValue); - } - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/DynamicLoadingModule.cs b/src/Microsoft.ML.GenAI.Core/Module/DynamicLoadingModule.cs deleted file mode 100644 index 49b8b46477..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/DynamicLoadingModule.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public class DynamicLoadingModule : torch.nn.Module, IDynamicLoadModule - where T : nn.Module - where T1 : Tensor -{ - private readonly T _model; - - public DynamicLoadingModule(T model) - : base(model.GetName()) - { - this._model = model; - this.RegisterComponents(); - } - - public static DynamicLoadingModule CreateFromModel(T model) - { - return new DynamicLoadingModule(model); - } - - public Action? LoadToDeviceFunc { get; set; } - public Action? UnloadFromDeviceFunc { get; set; } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override TResult forward(T1 input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (LoadToDeviceFunc != null) - { - LoadToDeviceFunc(this); - } - - var output = this._model.forward(input); - - if (UnloadFromDeviceFunc != null) - { - UnloadFromDeviceFunc(this); - } - - return output; - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/GenAILinear.cs b/src/Microsoft.ML.GenAI.Core/Module/GenAILinear.cs deleted file mode 100644 index a9b2e39db5..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/GenAILinear.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -internal class GenAILinear : nn.Module -{ -#pragma warning disable MSML_GeneralName // This name should be PascalCased - protected Tensor? weight; - protected Tensor? bias; - protected readonly int _inFeatures; - protected readonly int _outFeatures; -#pragma warning restore MSML_GeneralName // This name should be PascalCased - - public GenAILinear(int inFeatures, int outFeatures, bool hasBias = true, ScalarType dtype = ScalarType.Float32, string? device = null) - : base(nameof(GenAILinear)) - { - this._inFeatures = inFeatures; - this._outFeatures = outFeatures; - device ??= torch.get_default_device().ToString(); - this.weight = torch.zeros(outFeatures, inFeatures, dtype: dtype, device: device); - - if (hasBias) - { - this.bias = torch.zeros(outFeatures, dtype: dtype, device: device); - } - - base.RegisterComponents(); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - using var dispose = torch.NewDisposeScope(); - - // use float32 - var input2 = input.to_type(ScalarType.Float32); - var weight2 = this.weight!.to_type(ScalarType.Float32); - var result = torch.matmul(input2, weight2.t()); - - if (this.bias is not null) - { - result = result + this.bias.to_type(ScalarType.Float32); - } - - return result.to_type(input.dtype).MoveToOuterDisposeScope(); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/IDynamicLoadModule.cs b/src/Microsoft.ML.GenAI.Core/Module/IDynamicLoadModule.cs deleted file mode 100644 index d215d68bb3..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/IDynamicLoadModule.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public interface IDynamicLoadModule -{ - public Action? LoadToDeviceFunc { get; set; } - - public Action? UnloadFromDeviceFunc { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/IQuantizeModule.cs b/src/Microsoft.ML.GenAI.Core/Module/IQuantizeModule.cs deleted file mode 100644 index 57c0b7620f..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/IQuantizeModule.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.ML.GenAI.Core; - -public interface IQuantizeModule -{ - public void Int8(); - - public void Int4(); -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/NewGELUActivation.cs b/src/Microsoft.ML.GenAI.Core/Module/NewGELUActivation.cs deleted file mode 100644 index a1b523a4df..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/NewGELUActivation.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class NewGELUActivation : torch.nn.Module -#pragma warning disable MSML_GeneralName // This name should be PascalCased -{ - public NewGELUActivation() - : base(nameof(NewGELUActivation)) - { - } - - public override Tensor forward(Tensor input) - { - using var result = 0.044715 * torch.pow(input, 3.0); - using var result2 = result + input; - using var result3 = Math.Sqrt(2.0 / Math.PI) * result2; - using var result4 = torch.tanh(result3); - using var result5 = 1.0 + result4; - return 0.5 * input * result5; - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/QuantizedLinear.cs b/src/Microsoft.ML.GenAI.Core/Module/QuantizedLinear.cs deleted file mode 100644 index f399efe324..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/QuantizedLinear.cs +++ /dev/null @@ -1,209 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. -using System; -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using static TorchSharp.torch; -namespace Microsoft.ML.GenAI.Core; - -internal class QuantizedLinear : GenAILinear, IQuantizeModule -{ - public QuantizedLinear(int inFeatures, int outFeatures, bool hasBias = true, ScalarType dtype = ScalarType.Float32, string? device = null) - : base(inFeatures, outFeatures, hasBias, dtype, device) - { - } - - public void Int8() - { - if (this.weight is null) - { - throw new Exception("Weight is not initialized"); - } - - if (this.weight.device_type != DeviceType.META) - { - // if weight is not on meta device, this means that weight and bias are already loaded - // so we can quantize them in memory - - var timer = new System.Diagnostics.Stopwatch(); - timer.Start(); - // scale and zero point on vector-wise - // scale = 255 / max(weight, axis=1) - min(weight, axis=1) - var scale = 255 / (torch.max(this.weight, 1).values - torch.min(this.weight, 1).values); - - // zero point = - scale * min(weight, axis=1) - 128 - var zeroPoint = -scale * torch.min(this.weight, 1).values - 128; - // round zero point to nearest integer - zeroPoint = torch.round(zeroPoint).to(torch.int8); - - // assert zero point is in range [-128, 127] - //if (torch.any(this.zeroPoint < -128).item() || torch.any(this.zeroPoint > 127).item()) - //{ - // throw new Exception("Zero point is out of range [-128, 127]"); - //} - - // quantize weight - var eightBitWeight = torch.round(this.weight * scale.view(-1, 1) + zeroPoint.view(-1, 1)).to(torch.int8); - - // assert weight is in range [-128, 127] - //if (torch.any(this._8bitWeight < -128).item() || torch.any(this._8bitWeight > 127).item()) - //{ - // throw new Exception("Weight is out of range [-128, 127]"); - //} - timer.Stop(); - // dispose float32 weight - this.weight.Dispose(); - this.weight = null; - this._internal_buffers.Remove("weight"); - this.register_buffer("8bit_weight", eightBitWeight); - this.register_buffer("zeroPoint", zeroPoint); - this.register_buffer("scale", scale); - } - else - { - // if weight is on meta device, then we just need to create the placeholder for 8bit_weight, zeroPoint and scale - var eightBitWeight = torch.zeros(this.weight.shape, dtype: torch.int8); - var zeroPoint = torch.zeros(this.weight.shape[0], dtype: torch.int8); - var scale = torch.zeros(this.weight.shape[0], dtype: torch.float32); - - this._internal_buffers.Remove("weight"); - this.weight = null; - this.register_buffer("8bit_weight", eightBitWeight); - this.register_buffer("zeroPoint", zeroPoint); - this.register_buffer("scale", scale); - } - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (this._internal_buffers.ContainsKey("weight")) - { - return base.forward(input); - } - else if (this._internal_buffers.ContainsKey("8bit_weight")) - { - // 8bit quantization - using var dispose = torch.NewDisposeScope(); - var weight = this.get_buffer("8bit_weight").to(ScalarType.Float32); - var zeroPoint = this.get_buffer("zeroPoint").to(ScalarType.Float32); - var scale = this.get_buffer("scale").to(ScalarType.Float32); - var restoreWeight = (weight - zeroPoint.view(-1, 1)) / scale.view(-1, 1); - // use float32 - var result = torch.matmul(input.to(ScalarType.Float32), restoreWeight.T); - - if (this.bias is not null) - { - result = result + this.bias.to_type(ScalarType.Float32); - } - - //result.Peek("result"); - return result.to_type(input.dtype).MoveToOuterDisposeScope(); - } - else if (this._internal_buffers.ContainsKey("4bit_weight")) - { - using var dispose = torch.NewDisposeScope(); - var weight = this.get_buffer("4bit_weight"); - var weightLower = weight % 16; - var weightUpper = weight / 16; - weight = torch.cat([weightUpper, weightLower], 0).to(ScalarType.Float32); - weight = weight.view(this._outFeatures, this._inFeatures); - weight -= 8; - var zeroPoint = this.get_buffer("zeroPoint"); - var zeroPointLower = zeroPoint % 16; - var zeroPointUpper = zeroPoint / 16; - zeroPoint = torch.cat([zeroPointUpper, zeroPointLower], 0).to(ScalarType.Float32); - zeroPoint -= 8; - var scale = this.get_buffer("scale").to(ScalarType.Float32); - var restoreWeight = (weight - zeroPoint.view(-1, 1)) / scale.view(-1, 1); - // use float32 - var result = torch.matmul(input.to(ScalarType.Float32), restoreWeight.T); - - if (this.bias is not null) - { - result = result + this.bias.to_type(ScalarType.Float32); - } - - //result.Peek("result"); - return result.to_type(input.dtype).MoveToOuterDisposeScope(); - } - else - { - throw new Exception("Quantization is not done yet"); - } - } - - public void Int4() - { - if (this.weight is null) - { - throw new Exception("Weight is not initialized"); - } - var placeHolderDim = this._outFeatures / 2 + this._outFeatures % 2; - var fourBitWeightDim = this.weight.size(0) * this.weight.size(1); - var fourBitWeightPlaceHolderDim = Convert.ToInt32(fourBitWeightDim / 2 + fourBitWeightDim % 2); - if (this.weight.device_type != DeviceType.META) - { - using var scope = NewDisposeScope(); - var timer = new System.Diagnostics.Stopwatch(); - timer.Start(); - // scale and zero point on vector-wise - // scale = 15 / max(weight, axis=1) - min(weight, axis=1) - var scale = 15 / (torch.max(this.weight, 1).values - torch.min(this.weight, 1).values); - - // zero point = - scale * min(weight, axis=1) - 8 - var zeroPoint = -scale * torch.min(this.weight, 1).values - 8; - // round zero point to nearest integer - zeroPoint = torch.round(zeroPoint); - var fourBitWeight = torch.round(this.weight * scale.view(-1, 1) + zeroPoint.view(-1, 1)).to(torch.int8); - - zeroPoint = (zeroPoint + 8).to(torch.uint8); - fourBitWeight = (fourBitWeight + 8).view(-1).to(torch.uint8); - - // torch doesn't provide int4, so we use int8 as placeholder - // and foreach int8, we save two int4, e.g. 0b1010 -> 0b10, 0b10 - var zpPlaceHolder = zeroPoint[..placeHolderDim]; - zpPlaceHolder = zpPlaceHolder * 16 + zeroPoint[placeHolderDim..]; - - // assert zero point is in range [-128, 127] - //if (torch.any(this.zeroPoint < -128).item() || torch.any(this.zeroPoint > 127).item()) - //{ - // throw new Exception("Zero point is out of range [-128, 127]"); - //} - - // quantize weight - var fourBitWeightPlaceHolder = fourBitWeight[..fourBitWeightPlaceHolderDim]; - fourBitWeightPlaceHolder = fourBitWeightPlaceHolder * 16 + fourBitWeight[fourBitWeightPlaceHolderDim..]; - - // assert weight is in range [-128, 127] - //if (torch.any(this._8bitWeight < -128).item() || torch.any(this._8bitWeight > 127).item()) - //{ - // throw new Exception("Weight is out of range [-128, 127]"); - //} - - // dispose float32 weight - this.weight.Dispose(); - - this._internal_buffers.Remove("weight"); - this.register_buffer("4bit_weight", fourBitWeightPlaceHolder.MoveToOuterDisposeScope()); - this.register_buffer("zeroPoint", zpPlaceHolder.MoveToOuterDisposeScope()); - this.register_buffer("scale", scale.MoveToOuterDisposeScope()); - timer.Stop(); - } - else - { - // if weight is on meta device, then we just need to create the placeholder for 8bit_weight, zeroPoint and scale - var fourBitWeight = torch.zeros(fourBitWeightPlaceHolderDim, dtype: torch.int8); - var zeroPoint = torch.zeros(placeHolderDim, dtype: torch.int8); - var scale = torch.zeros(this.weight.shape[0], dtype: torch.float32); - - this._internal_buffers.Remove("weight"); - this.weight = null; - this.register_buffer("4bit_weight", fourBitWeight); - this.register_buffer("zeroPoint", zeroPoint); - this.register_buffer("scale", scale); - } - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/RMSNorm.cs b/src/Microsoft.ML.GenAI.Core/Module/RMSNorm.cs deleted file mode 100644 index b9555cd845..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/RMSNorm.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class RMSNorm : torch.nn.Module -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - private readonly int _dim; - private readonly float _eps; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Parameter weight; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public RMSNorm( - int hiddenSize, - float eps = 1e-6f, - ScalarType dtype = ScalarType.Float32) - : base(nameof(RMSNorm)) - { - this._dim = hiddenSize; - this._eps = eps; - - // the gamma scalar - this.weight = torch.nn.Parameter(torch.ones(this._dim, dtype: dtype)); - } - - private Tensor Norm(Tensor x) - { - // (B, Seq_Len, Dim) * (B, Seq_Len, 1) = (B, Seq_Len, Dim) - // rsqrt = 1 / sqrt - var output = x * torch.rsqrt(x.pow(2).mean([-1L], keepdim: true) + this._eps); - return output; - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - // needs higher precision for the norm so convert to float32 - // (B, Seq_Len, Dim) - var normed = this.Norm(input.to_type(ScalarType.Float32)).type_as(input); - // (B, Seq_Len, Dim) * (Dim) = (B, Seq_Len, Dim) - var output = this.weight * normed; - - return output; - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Module/RotaryEmbedding.cs b/src/Microsoft.ML.GenAI.Core/Module/RotaryEmbedding.cs deleted file mode 100644 index 8e06c838d5..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Module/RotaryEmbedding.cs +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Text.Json.Serialization; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public class RopeScalingConfig -{ - public RopeScalingConfig() - { - this.Factor = 1.0f; - this.LowFreqFactor = 1.0f; - this.HighFreqFactor = 1.0f; - this.OriginalMaxPositionEmbeddings = 8192; - this.RopeType = "default"; - } - - [JsonPropertyName("factor")] - public float Factor { get; set; } - - [JsonPropertyName("low_freq_factor")] - public float LowFreqFactor { get; set; } - - [JsonPropertyName("high_freq_factor")] - public float HighFreqFactor { get; set; } - - [JsonPropertyName("original_max_position_embeddings")] - public int OriginalMaxPositionEmbeddings { get; set; } - - [JsonPropertyName("rope_type")] - public string RopeType { get; set; } -} - - -internal class RotaryEmbeddingInput -{ - public RotaryEmbeddingInput(Tensor input, Tensor positionIds, int? seqLen = null) - { - Input = input; - PositionIds = positionIds; - SeqLen = seqLen; - } - - public Tensor Input { get; set; } - - public Tensor PositionIds { get; set; } - - public int? SeqLen { get; set; } -} - -internal class RotaryEmbeddingOutput -{ - public RotaryEmbeddingOutput(Tensor cos, Tensor sin) - { - Cos = cos; - Sin = sin; - } - - public Tensor Cos { get; set; } - - public Tensor Sin { get; set; } -} - - -internal class RotaryEmbedding : nn.Module< - RotaryEmbeddingInput, - RotaryEmbeddingOutput> -{ - private readonly double _base; - private readonly int _maxPositionEmbeddings; - private readonly int _dim; - - public RotaryEmbedding(double baseValue, int maxPositionEmbeddings, int dim) - : this(baseValue, dim, new RopeScalingConfig() { RopeType = "default", OriginalMaxPositionEmbeddings = maxPositionEmbeddings }) - { - } - - public RotaryEmbedding(double baseValue, int dim, RopeScalingConfig config) - : base(nameof(RotaryEmbedding)) - { - _base = baseValue; - _maxPositionEmbeddings = config.OriginalMaxPositionEmbeddings; - _dim = dim; - - if (config.RopeType == "default") - { - var thetaNumerator = torch.arange(0, _dim, 2, dtype: ScalarType.Int64).to(torch.float32); - this.register_buffer("inv_freq", torch.pow(baseValue, -1.0f * (thetaNumerator / dim)), persistent: false); - } - else - { - throw new NotImplementedException("Rope type not implemented"); - } - } - - public int Dim => _dim; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override RotaryEmbeddingOutput forward(RotaryEmbeddingInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var x = input.Input; - var positionIds = input.PositionIds; - var seqLen = input.SeqLen; - // TODO - // can be calculated once and cached - var invFreq = this.get_buffer("inv_freq").to(x.device); - var invFreqExpanded = invFreq.unsqueeze(0).unsqueeze(-1); - invFreqExpanded = invFreqExpanded.expand(new long[] { positionIds.shape[0], -1, 1 }); - var positionIdsExpanded = positionIds.unsqueeze(1).to(torch.float32); - var freqs = invFreqExpanded * positionIdsExpanded; - freqs = freqs.transpose(1, 2); - var emb = torch.cat([freqs, freqs], dim: -1); - - var cos = torch.cos(emb); - var sin = torch.sin(emb); - - return new(cos.to_type(x.dtype), sin.to_type(x.dtype)); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelInput.cs b/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelInput.cs deleted file mode 100644 index 2153a8d264..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelInput.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public class CausalLMModelInput -{ - internal static class Defaults - { - internal const Tensor? AttentionMask = null; - internal const Tensor? PositionIds = null; - internal const int PastKeyValuesLength = 0; - internal const Tensor? InputsEmbeds = null; - internal const bool UseCache = true; - internal const bool OutputAttentions = false; - internal const bool OutputHiddenStates = false; - internal const Tensor? Labels = null; - } - public CausalLMModelInput( - Tensor inputIds, - Tensor? attentionMask = Defaults.AttentionMask, - Tensor? positionIds = Defaults.PositionIds, - int pastKeyValuesLength = Defaults.PastKeyValuesLength, - Tensor? inputsEmbeds = Defaults.InputsEmbeds, - Tensor? labels = Defaults.Labels, - bool useCache = Defaults.UseCache, - bool outputAttentions = Defaults.OutputAttentions, - bool outputHiddenStates = Defaults.OutputHiddenStates) - { - this.InputIds = inputIds; - this.AttentionMask = attentionMask; - this.PositionIds = positionIds; - this.PastKeyValuesLength = pastKeyValuesLength; - this.InputEmbeddings = inputsEmbeds; - this.UseCache = useCache; - this.OutputAttentions = outputAttentions; - this.OutputHiddenStates = outputHiddenStates; - this.Labels = labels; - } - - public Tensor InputIds { get; set; } - - public Tensor? AttentionMask { get; set; } - - public Tensor? PositionIds { get; set; } - - public IKVCache? OverrideCache { get; set; } - - public int PastKeyValuesLength { get; set; } - - public Tensor? InputEmbeddings { get; set; } - - /// - /// Shape: [batch_size, sequence_length] - /// DTypes: int64 - /// Labels for computing the causal language modeling loss. - /// Indices should be in [0, config.vocab_size - 1] or [-100] for padding/masking. - /// - public Tensor? Labels { get; set; } - - public bool UseCache { get; set; } - - public bool OutputAttentions { get; set; } - - public bool OutputHiddenStates { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelOutput.cs b/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelOutput.cs deleted file mode 100644 index b7a622ab7c..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMModelOutput.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public class CausalLMModelOutput -{ - internal static class Defaults - { - internal const Tensor? Logits = null; - internal const Tensor[]? AllHiddenStates = null; - internal const Tensor[]? Attentions = null; - internal const IKVCache? Cache = null; - internal const Tensor? Loss = null; - } - public CausalLMModelOutput( - Tensor lastHiddenState, - Tensor? logits = Defaults.Logits, - Tensor[]? allHiddenStates = Defaults.AllHiddenStates, - Tensor[]? attentions = Defaults.Attentions, - IKVCache? cache = Defaults.Cache, - Tensor? loss = Defaults.Loss) - { - this.LastHiddenState = lastHiddenState; - this.AllHiddenStates = allHiddenStates; - this.Logits = logits; - this.Attentions = attentions; - this.Cache = cache; - this.Loss = loss; - } - - /// - /// Shape: [1,] - /// Available when label is provided in the input. - /// - public Tensor? Loss { get; set; } - - public Tensor? Logits { get; set; } - - public Tensor LastHiddenState { get; set; } - - public Tensor[]? AllHiddenStates { get; set; } - - public Tensor[]? Attentions { get; set; } - - public IKVCache? Cache { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMPipeline.cs b/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMPipeline.cs deleted file mode 100644 index 74d9c6237a..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Pipeline/CausalLMPipeline.cs +++ /dev/null @@ -1,325 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.Tokenizers; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public interface ICausalLMPipeline : ICausalLMPipeline - where TTokenizer : Tokenizer - where TModel : nn.Module -{ - TTokenizer TypedTokenizer { get; } - - TModel TypedModel { get; } -} - -public interface ICausalLMPipeline -{ - Tokenizer Tokenizer { get; } - - nn.Module Model { get; } - - string Generate( - string prompt, - int maxLen = CausalLMPipeline.Defaults.MaxLen, - float temperature = CausalLMPipeline.Defaults.Temperature, - float topP = CausalLMPipeline.Defaults.TopP, - string[]? stopSequences = CausalLMPipeline.Defaults.StopSequence); - - /// - /// Generate the embedding(last hidden state of the last token) for the prompt. The embedding is normalized by L2 norm. - /// - float[] GenerateEmbeddingFromLastTokenPool(string prompt); - - IEnumerable GenerateStreaming( - string prompt, - int maxLen = CausalLMPipeline.Defaults.MaxLen, - float temperature = CausalLMPipeline.Defaults.Temperature, - float topP = CausalLMPipeline.Defaults.TopP, - string[]? stopSequences = CausalLMPipeline.Defaults.StopSequence); - - (Tensor, Tensor) Generate( - Tensor inputIds, - Tensor attentionMask, - int[][] stopTokenSequence, - float temperature = CausalLMPipeline.Defaults.Temperature, - float topP = CausalLMPipeline.Defaults.TopP, - int maxLen = CausalLMPipeline.Defaults.MaxLen); - - IEnumerable<(Tensor, Tensor)> GenerateStreaming( - Tensor inputIds, - Tensor attentionMask, - int[][] stopTokenSequence, - float temperature = CausalLMPipeline.Defaults.Temperature, - float topP = CausalLMPipeline.Defaults.TopP, - int maxLen = CausalLMPipeline.Defaults.MaxLen); -} - -public class CausalLMPipeline : CausalLMPipeline, ICausalLMPipeline - where TTokenizer : Tokenizer - where TModel : nn.Module -{ - public CausalLMPipeline( - TTokenizer tokenizer, - TModel model, - string device = Defaults.Device) - : base(tokenizer, model, device) - { - } - - public TTokenizer TypedTokenizer { get => (TTokenizer)base.Tokenizer; } - - public TModel TypedModel { get => (TModel)base.Model; } -} - -public class CausalLMPipeline : ICausalLMPipeline -{ - internal static class Defaults - { - internal const string Device = "cpu"; - internal const float Temperature = 0.7F; - internal const float TopP = 0.9F; - internal const int MaxLen = 128; - internal const string[]? StopSequence = null; - } - - public CausalLMPipeline( - Tokenizer tokenizer, - nn.Module model, - string device = Defaults.Device) - { - this.Tokenizer = tokenizer; - this.Model = model; - this.Device = device; - } - - /// - /// For moq purpose - /// - private protected CausalLMPipeline() - { - this.Tokenizer = default!; - this.Model = default!; - this.Device = default!; - } - - public Tokenizer Tokenizer { get; } - - public nn.Module Model { get; } - - public Device Device { get; } - - public IEnumerable<( - Tensor, // output token ids [batch_size, 1] - Tensor // output logits [batch_size, 1, vocab_size] - )> GenerateStreaming( - Tensor inputIds, - Tensor attentionMask, - int[][] stopTokenSequence, - float temperature = Defaults.Temperature, - float topP = Defaults.TopP, - int maxLen = Defaults.MaxLen) - { - using var scope = NewDisposeScope(); - using var noGrad = torch.no_grad(); - var batch = inputIds.shape[0]; - var device = inputIds.device; - var promptLength = (int)inputIds.shape[1]; - var totalLen = promptLength + maxLen; - - var prevPos = 0; - var eosReached = torch.tensor(new bool[batch], device: device); - torch.Tensor? logits = default; - var cache = new DynamicKVCache(); - if (promptLength == totalLen) - { - var input = new CausalLMModelInput(inputIds, attentionMask, pastKeyValuesLength: 0) - { - OverrideCache = cache, - }; - var output = this.Model.forward(input); - logits = output.Logits; - } - for (var curPos = promptLength; curPos != totalLen; curPos++) - { - var input = new CausalLMModelInput(inputIds[.., prevPos..curPos], attentionMask[.., prevPos..curPos], pastKeyValuesLength: prevPos) - { - OverrideCache = cache, - }; - var output = this.Model.forward(input); - logits = output.Logits?.MoveToOtherDisposeScope(inputIds) ?? throw new InvalidOperationException("Logits is null"); - torch.Tensor nextToken; - if (temperature > 0) - { - var probs = torch.softmax(logits[.., -1] / temperature, dim: -1); - nextToken = this.SampleTopP(probs, topP); - } - else - { - nextToken = torch.argmax(logits[.., -1], dim: -1); - } - - nextToken = nextToken.reshape(-1); - inputIds = torch.cat([inputIds, nextToken.unsqueeze(1)], dim: -1).MoveToOtherDisposeScope(inputIds); - attentionMask = torch.cat([attentionMask, attentionMask.new_ones(attentionMask.shape[0], 1)], dim: -1); - foreach (var stopSequence in stopTokenSequence) - { - // determine if the last n tokens are the stop sequence - var lastN = inputIds[.., ^stopSequence.Length..]; - var lastNMatch = lastN == torch.tensor(stopSequence, device: device); - eosReached |= lastNMatch.all(dim: -1); - } - if (eosReached.all().item()) - { - break; - } - - yield return (nextToken.MoveToOuterDisposeScope(), logits[.., ^1].MoveToOuterDisposeScope()); - prevPos = curPos; - } - } - - public virtual ( - Tensor, // output token ids [batch_size, sequence_length] - Tensor // output logits [batch_size, sequence_length, vocab_size] - ) Generate( - Tensor inputIds, // input token ids [batch_size, sequence_length] - Tensor attentionMask, // attention mask [batch_size, sequence_length] - int[][] stopTokenSequence, - float temperature = 0.7f, - float topP = 0.9f, - int maxLen = 128) - { - using var scope = NewDisposeScope(); - Tensor? logits = null; - foreach (var (token, _logits) in this.GenerateStreaming(inputIds, attentionMask, stopTokenSequence, temperature, topP, maxLen)) - { - inputIds = torch.cat([inputIds, token.unsqueeze(1)], dim: -1).MoveToOtherDisposeScope(inputIds); - if (logits is null) - { - logits = _logits; - } - else - { - logits = torch.cat([logits, _logits], dim: -1).MoveToOtherDisposeScope(inputIds); - } - } - - return (inputIds, logits ?? throw new InvalidOperationException("Logits is null")); - } - - public virtual string Generate( - string prompt, - int maxLen = 128, - float temperature = 0.7f, - float topP = 0.9f, - string[]? stopSequences = null) - { - var chunks = new List(); - - foreach (var chunk in this.GenerateStreaming(prompt, maxLen, temperature, topP, stopSequences)) - { - chunks.Add(chunk); - } - - return string.Join(string.Empty, chunks); - } - - - public virtual IEnumerable GenerateStreaming( - string prompt, - int maxLen = 128, - float temperature = 0.7F, - float topP = 0.9F, - string[]? stopSequences = Defaults.StopSequence) - { - using var newScope = NewDisposeScope(); - var inputIds = this.Tokenizer.EncodeToIds(prompt); - var inputTensor = torch.tensor(inputIds.ToArray(), dtype: ScalarType.Int64, device: this.Device).unsqueeze(0); - var attentionMask = torch.ones_like(inputTensor, device: this.Device); - // set up stop token ids - // stop token ids: [[eosId], [stopSequence1], [stopSequence2], ...] - // when causal language model generates tokens, it will stop when it generates any token in stopSequences - List stopTokenIds = [[]]; - if (stopSequences != null) - { - stopTokenIds.AddRange(stopSequences.Select(x => - { - var tokens = this.Tokenizer.EncodeToTokens(x, out var _, false, false); - - return tokens - // Skip the first _ token automatically added by tokenizer - .Where(t => !t.Offset.Equals(new Range(0, 0))) - .Select(t => t.Id) - .ToArray(); - })); - } - - stopTokenIds = stopTokenIds.Where(ids => ids.Count() > 0).ToList(); - - foreach (var (token, _) in this.GenerateStreaming(inputTensor, attentionMask, stopTokenIds.ToArray(), temperature: temperature, maxLen: maxLen)) - { - var tokenIds = token[0].to_type(ScalarType.Int32).data().ToArray(); - var duplicateTokenString = this.Tokenizer switch - { - SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds.Concat(tokenIds), considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"), - _ => this.Tokenizer.Decode(tokenIds.Concat(tokenIds)) ?? throw new InvalidOperationException("Failed to decode token ids"), - }; - - var tokenString = this.Tokenizer switch - { - SentencePieceTokenizer bpeTokenizer => bpeTokenizer.Decode(tokenIds, considerSpecialTokens: true) ?? throw new InvalidOperationException("Failed to decode token ids"), - _ => this.Tokenizer.Decode(tokenIds) ?? throw new InvalidOperationException("Failed to decode token ids"), - }; - - // replace the first occurrence of the token with the duplicate token - tokenString = duplicateTokenString.Substring(tokenString.Length); - - yield return tokenString; - } - } - - protected torch.Tensor SampleTopP(torch.Tensor logits, float topP) - { - (var probsSort, var probsIndex) = torch.sort(logits, dim: -1, descending: true); - var cumSum = torch.cumsum(probsSort, dim: -1); - var mask = cumSum - probsSort > topP; - probsSort[mask] = 0f; - probsSort /= probsSort.sum(dim: -1, keepdim: true); - var nextToken = torch.multinomial(probsSort, num_samples: 1); - nextToken = torch.gather(probsIndex, dim: -1, index: nextToken); - return nextToken; - } - - public float[] GenerateEmbeddingFromLastTokenPool(string prompt) - { - using var scope = NewDisposeScope(); - using var noGrad = torch.no_grad(); - var inputIds = this.Tokenizer.EncodeToIds(prompt); - var inputTensor = torch.tensor(inputIds.ToArray(), dtype: ScalarType.Int64, device: this.Device).unsqueeze(0); - var attentionMask = torch.ones_like(inputTensor, device: this.Device); - var input = new CausalLMModelInput(inputTensor, attentionMask, pastKeyValuesLength: 0) - { - OverrideCache = new DynamicKVCache(), - }; - var output = this.Model.forward(input); - var lastTokenHiddenState = output.LastHiddenState[0, ^1]; - - // shape of lastTokenHiddenState: [hidden_size] - // L2 norm - var norm = lastTokenHiddenState.norm(); - var normalized = lastTokenHiddenState / norm; - - return normalized.to_type(ScalarType.Float32).data().ToArray(); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMDataset.cs b/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMDataset.cs deleted file mode 100644 index b368c7ac03..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMDataset.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.ML.Tokenizers; -using TorchSharp; - -namespace Microsoft.ML.GenAI.Core.Trainer; - -public class CausalLMDataset : IEnumerable -{ - private readonly List _data; - - private CausalLMDataset(IEnumerable data) - { - _data = new List(data); - } - - public static CausalLMDataset Create(IEnumerable> inputs, - IEnumerable outputs, - IMEAIChatTemplateBuilder chatTemplateBuilder, - Tokenizer tokenizer) - { - // the length of inputs and outputs should be the same - if (inputs.Count() != outputs.Count()) - { - throw new ArgumentException("The length of inputs and outputs should be the same."); - } - - var enumerables = inputs.Zip(outputs, (input, output) => - { - var inputPrompt = chatTemplateBuilder.BuildPrompt(input.ToList()); - var outputPrompt = chatTemplateBuilder.BuildPrompt(input.Concat([output]).ToList(), appendAssistantTag: false); - var lengthToKeep = outputPrompt.Length - inputPrompt.Length; - outputPrompt = outputPrompt.Substring(inputPrompt.Length, lengthToKeep); - - return (inputPrompt, outputPrompt); - }); - - return Create(enumerables.Select(x => x.inputPrompt), enumerables.Select(x => x.outputPrompt), tokenizer); - } - - public static CausalLMDataset Create(IEnumerable inputs, IEnumerable outputs, Tokenizer tokenizer) - { - // the length of inputs and outputs should be the same - if (inputs.Count() != outputs.Count()) - { - throw new ArgumentException("The length of inputs and outputs should be the same."); - } - - var enumerable = inputs.Zip(outputs, (input, output) => - { - var inputIds = tokenizer.EncodeToIds(input); - var outputIds = tokenizer.EncodeToIds(input + output); - outputIds = outputIds.Skip(inputIds.Count()).ToArray(); - - return (inputIds, outputIds); - }).ToArray(); - - return Create(enumerable.Select(x => x.inputIds), enumerable.Select(x => x.outputIds)); - } - - public static CausalLMDataset Create(IEnumerable> inputIds, IEnumerable> labelIds) - { - // the length of inputIds and labelIds should be the same - if (inputIds.Count() != labelIds.Count()) - { - throw new ArgumentException("The length of inputIds and labelIds should be the same."); - } - - var enumerable = inputIds.Zip(labelIds, Create) - .SelectMany(x => x); - - return new CausalLMDataset(enumerable); - } - - public static CausalLMDataset Create(IReadOnlyList inputIds, IReadOnlyList labelIds) - { - var enumerable = Enumerable.Range(0, labelIds.Count) - .Select(i => - { - var train = inputIds.Concat(labelIds.Take(i)).ToArray(); - var label = Enumerable.Repeat(-100L, train.Length).Concat([labelIds[i]]).Skip(1).ToArray(); - var mask = Enumerable.Repeat(1L, train.Length).ToArray(); - - return new CausalLMModelInput( - inputIds: torch.tensor(train.ToArray(), dtype: TorchSharp.torch.ScalarType.Int64).reshape(1, -1), - labels: torch.tensor(label, dtype: TorchSharp.torch.ScalarType.Int64).reshape(1, -1), - attentionMask: torch.tensor(mask, dtype: TorchSharp.torch.ScalarType.Int64).reshape(1, -1) - ); - }); - - return new CausalLMDataset(enumerable); - } - - public IEnumerator GetEnumerator() - { - return ((IEnumerable)_data).GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return ((IEnumerable)_data).GetEnumerator(); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMSupervisedFineTuningTrainer.cs b/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMSupervisedFineTuningTrainer.cs deleted file mode 100644 index ae447b60c4..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Trainer/CausalLMSupervisedFineTuningTrainer.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using Microsoft.Extensions.Logging; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core.Trainer; - -public class CausalLMSupervisedFineTuningTrainer -{ - private readonly ILogger? _logger; - private readonly ICausalLMPipeline _pipeline; - - public CausalLMSupervisedFineTuningTrainer(ICausalLMPipeline pipeline, ILogger? logger = null) - { - _logger = logger; - _pipeline = pipeline; - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable TrainAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - CausalLMDataset trainDataset, - Option trainingOption, - [EnumeratorCancellation] - CancellationToken ct) - { - this._logger?.LogInformation("Start training..."); - var batches = trainDataset.Chunk(trainingOption.BatchSize); - var optimizer = new Adam(_pipeline.Model.parameters(), lr: trainingOption.LearningRate); - var device = torch.device(trainingOption.Device); - - for (int i = 0; i < trainingOption.Epoch; i++) - { - this._logger?.LogInformation($"Epoch {i + 1}/{trainingOption.Epoch}"); - var losses = new List(); - foreach (var batch in batches) - { - if (ct.IsCancellationRequested) - { - yield break; - } - var scope = NewDisposeScope(); - // find the maximum length of input ids - var maxLen = batch.Max(x => x.InputIds.size(1)); - // merge items in batch - var inputIds = torch.cat(batch.Select(x => nn.functional.pad(x.InputIds, [0, maxLen - x.InputIds.shape[1]])).ToArray(), 0).to(device); - var attentionMask = torch.cat(batch.Select(x => nn.functional.pad(x.AttentionMask!, [0, maxLen - x.AttentionMask!.shape[1]])).ToArray(), 0).to(device); - var labels = torch.cat(batch.Select(x => nn.functional.pad(x.Labels!, [0, maxLen - x.Labels!.shape[1]], value: -100)).ToArray(), 0).to(device); - // Forward the model - var output = _pipeline.Model.forward(new CausalLMModelInput(inputIds, attentionMask: attentionMask, labels: labels, useCache: false)); - // Calculate loss - var loss = output.Loss; - // Backward the model - optimizer.zero_grad(); - loss!.backward(); - optimizer.step(); - - losses.Add(loss.data().ToArray()[0]); - - // dispose loss - loss.Dispose(); - - // dispose output - output.LastHiddenState.Dispose(); - output.Logits!.Dispose(); - inputIds.Dispose(); - attentionMask.Dispose(); - - scope.Dispose(); - } - - _logger?.LogInformation($"Epoch {i + 1} loss: {losses.Average()}"); - - yield return _pipeline; - } - } - - - public class Option - { - public Option() - { - Epoch = 10; - BatchSize = 1; - LearningRate = 5e-5f; - Device = "cpu"; - } - - public int Epoch { get; set; } - - public int BatchSize { get; set; } - - public float LearningRate { get; set; } - - public string Device { get; set; } - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Utility/AttentionMaskConverter.cs b/src/Microsoft.ML.GenAI.Core/Utility/AttentionMaskConverter.cs deleted file mode 100644 index a2a3f92aea..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Utility/AttentionMaskConverter.cs +++ /dev/null @@ -1,179 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Threading.Tasks; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; - -namespace Microsoft.ML.GenAI.Core; - -public class AttentionMaskConverter -{ - private readonly bool _isCausal; - private readonly int? _slidingWindow; - - public AttentionMaskConverter(bool isCausal, int? slidingWindow) - { - this._isCausal = isCausal; - this._slidingWindow = slidingWindow; - } - - /// - /// Converts 2D attention mask to 4D attention mask by expanding mask to (bsz, head_dim=1, query_length, - /// key_value_length) shape and by adding a large negative bias to not-attended positions.If attention_mask is - /// causal, a causal mask will be added. - /// - /// - /// - /// - /// - /// - public Tensor To4D( - Tensor attentionMask2d, - int queryLength, - ScalarType dType, - int? keyValueLength = null) - { - long[] inputShape = [attentionMask2d.shape[0], queryLength]; - - // create causal mask - // [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - Tensor? causal4dMask = null; - if ((inputShape[^1] > 1 || this._slidingWindow is not null) && this._isCausal) - { - if (keyValueLength is null) - { - throw new ArgumentException("key_value_length should be provided when attention_mask is causal"); - } - - var pastKeyValuesLength = keyValueLength.Value - queryLength; - causal4dMask = MakeCausalMask(inputShape, dType, attentionMask2d.device, pastKeyValuesLength, this._slidingWindow); - } - else if (this._slidingWindow is not null) - { - throw new NotImplementedException("Sliding window is not supported for non-causal masks"); - } - - var expandedAttnMask = ExpandMask(attentionMask2d, dType, queryLength).to(attentionMask2d.device); - if (causal4dMask is not null) - { - var min = torch.finfo(dType).min; - expandedAttnMask = causal4dMask.masked_fill(expandedAttnMask.to(ScalarType.Bool), min); - } - - return expandedAttnMask; - } - - public Tensor? ToCausal4D( - int batchSize, - int queryLength, - int keyValueLength, - ScalarType dType, - Device device) - { - if (!_isCausal) - { - throw new ArgumentException("This is not a causal mask"); - } - - long[] inputShape = [batchSize, queryLength]; - var pastKeyValueLength = keyValueLength - queryLength; - - // create causal mask - // [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - Tensor? causal4DMask = null; - if (queryLength > 1 || this._slidingWindow is int) - { - causal4DMask = MakeCausalMask(inputShape, dType, device, pastKeyValueLength, this._slidingWindow); - } - - return causal4DMask; - } - - public static Tensor MakeCausalMask( - long[] inputIdsShape, - ScalarType dType, - Device device, - int pastKeyValuesLength = 0, - int? slidingWindow = null) - { - // Make causal mask used for bi-directional self-attention. - var bsz = inputIdsShape[0]; - var tgtLen = inputIdsShape[1]; - var min = torch.finfo(dType).min; - var mask = torch.full([tgtLen, tgtLen], min, dtype: dType, device: device); - var maskCondition = torch.arange(tgtLen, device: device); - mask.masked_fill_(maskCondition < (maskCondition + 1).view(tgtLen, 1), 0); - mask = mask.to(dType); - - - if (pastKeyValuesLength > 0) - { - mask = torch.cat([torch.zeros([tgtLen, pastKeyValuesLength], dtype: dType, device: device), mask], dim: -1); - } - - if (slidingWindow is int window) - { - var diagonal = pastKeyValuesLength - window - 1; - var contextMask = torch.tril(torch.ones([tgtLen, tgtLen], dtype: ScalarType.Bool, device: device), diagonal: diagonal); - mask = mask.masked_fill(contextMask, min); - } - - // return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) - - return mask.unsqueeze(0).unsqueeze(0).expand(bsz, 1, tgtLen, tgtLen + pastKeyValuesLength); - } - - /// - /// Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` - /// - /// The attention mask should be 2D. - /// The device to place the mask tensor. - /// The data type of the mask tensor. - /// The length of past key values in cache. - /// The sliding window size. - /// The input shape should be a tuple that defines `(batch_size, query_length)`. - public static Tensor? Create4DCausalAttentionMask( - Tensor? attentionMask, - long[] inputShape, - ScalarType dType, - Device device, - int pastKeyValuesLength = 0, - int? slidingWindow = null) - { - var converter = new AttentionMaskConverter(isCausal: true, slidingWindow: slidingWindow); - var batchSize = (int)inputShape[0]; - var queryLength = (int)inputShape[1]; - var keyValueLength = pastKeyValuesLength + queryLength; - if (attentionMask is not null) - { - if (attentionMask.ndim != 2) - { - throw new ArgumentException("Attention mask should be 2D"); - } - return converter.To4D(attentionMask, (int)inputShape[1], dType, keyValueLength); - } - - return converter.ToCausal4D(batchSize, queryLength, keyValueLength, dType, device); - } - - public static Tensor ExpandMask( - Tensor mask, - ScalarType dType, - int? tgtLen = null) - { - var bsz = (int)mask.shape[0]; - var srcLen = (int)mask.shape[1]; - tgtLen ??= srcLen; - - var expandedMask = mask.unsqueeze(1).unsqueeze(1).expand(bsz, 1, tgtLen.Value, srcLen).to(dType); - var invertedMask = 1.0 - expandedMask; - var min = torch.finfo(dType).min; - - return invertedMask.masked_fill(invertedMask.to(ScalarType.Bool), min); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Utility/Cache.cs b/src/Microsoft.ML.GenAI.Core/Utility/Cache.cs deleted file mode 100644 index 4647cfd122..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Utility/Cache.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core; - -public interface IKVCache : IDictionary, IDisposable -{ - public (Tensor, Tensor) UpdateKVCache(Tensor key, Tensor value, int layerIndex); - - public int GetSeqLen(int layerIndex = 0); - - public int? GetMaxLength(); - - public int GetUsableLength(int newSeqLen, int layerIndex = 0); -} - -public class DynamicKVCache : Dictionary, IKVCache -{ - private readonly DisposeScope _disposeScope = NewDisposeScope(); - public DynamicKVCache() - { - } - - public (Tensor, Tensor) UpdateKVCache(Tensor key, Tensor value, int layerIndex) - { - if (this.ContainsKey(layerIndex)) - { - var (oldKey, oldValue) = this[layerIndex]; - oldKey.DetachFromDisposeScope(); - oldValue.DetachFromDisposeScope(); - - var newKey = torch.cat([oldKey, key], -2).MoveToOtherDisposeScope(this._disposeScope); - var newValue = torch.cat([oldValue, value], -2).MoveToOtherDisposeScope(this._disposeScope); - - oldKey.Dispose(); - oldValue.Dispose(); - - this[layerIndex] = (newKey, newValue); - } - else - { - this.Add(layerIndex, (key.MoveToOtherDisposeScope(this._disposeScope), value.MoveToOtherDisposeScope(this._disposeScope))); - } - - return this[layerIndex]; - } - - public int GetSeqLen(int layerIndex = 0) - { - if (this.TryGetValue(layerIndex, out var kv)) - { - return kv.Item1.IntShape()[^2]; - } - - return 0; - } - - public int? GetMaxLength() - { - return null; - } - - public int GetUsableLength(int newSeqLen, int layerIndex = 0) - { - var maxLength = this.GetMaxLength(); - var previousSeqLen = this.GetSeqLen(layerIndex); - - if (maxLength.HasValue && previousSeqLen + newSeqLen > maxLength.Value) - { - return maxLength.Value - previousSeqLen; - } - - return previousSeqLen; - } - - public void Dispose() - { - this._disposeScope.Dispose(); - } -} diff --git a/src/Microsoft.ML.GenAI.Core/Utility/IChatTemplateBuilder.cs b/src/Microsoft.ML.GenAI.Core/Utility/IChatTemplateBuilder.cs deleted file mode 100644 index ac7f6e210d..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Utility/IChatTemplateBuilder.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.Extensions.AI; -using Microsoft.SemanticKernel.ChatCompletion; - -namespace Microsoft.ML.GenAI.Core; - -public interface ISemanticKernelChatTemplateBuilder -{ - string BuildPrompt(ChatHistory chatHistory); -} - -public interface IAutoGenChatTemplateBuilder -{ - string BuildPrompt(IEnumerable messages, IEnumerable? tools = null); -} - -public interface IMEAIChatTemplateBuilder -{ - /// - /// Build a prompt from a list of messages. - /// - /// the list of to be rendered - /// - /// true if append assistant tag at the end of prompt. - /// - string BuildPrompt(IEnumerable messages, ChatOptions? options = null, bool appendAssistantTag = true); -} - -public interface IChatTemplateBuilder : IAutoGenChatTemplateBuilder, ISemanticKernelChatTemplateBuilder -{ -} diff --git a/src/Microsoft.ML.GenAI.Core/Utils.cs b/src/Microsoft.ML.GenAI.Core/Utils.cs deleted file mode 100644 index 552500146a..0000000000 --- a/src/Microsoft.ML.GenAI.Core/Utils.cs +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; - -namespace Microsoft.ML.GenAI.Core; - -public static class Utils -{ - public static Tensor ApplyRotaryEmbeddings(Tensor input, Tensor freqsComplex) - { - // Separate the last dimension pairs of two values, representing the real and imaginary parts of the complex number - // Two consecutive values will become a single complex number - // (B, Seq_Len, H, Head_Dim) -> (B, Seq_Len, H, Head_Dim/2) - var inputComplex = input.to_type(ScalarType.Float32).reshape(input.shape[0], input.shape[1], input.shape[2], -1, 2).view_as_complex(); - freqsComplex = freqsComplex.to(input.device); - - // Reshape the freqs_complex tensor to match the shape of the x_complex tensor. So we need to add the batch dimension and the head dimension - // (Seq_Len, Head_Dim/2) --> (1, Seq_Len, 1, Head_Dim/2) - var freqsComplexReshaped = freqsComplex.unsqueeze(0).unsqueeze(2); - - // Multiply each complex number in the x_complex tensor by the corresponding complex number in the freqs_complex tensor - // Which results in the rotation of the complex number as shown in the Figure 1 of the paper - // (B, Seq_Len, H, Head_Dim/2) * (1, Seq_Len, 1, Head_Dim/2) = (B, Seq_Len, H, Head_Dim/2) - var rotatedComplex = inputComplex * freqsComplexReshaped; - // Console.WriteLine(rotated_complex.mean().ToSingle()); - - // Convert the complex number back to the real number - // (B, Seq_Len, H, Head_Dim/2) -> (B, Seq_Len, H, Head_Dim/2, 2) - var rotated = rotatedComplex.view_as_real(); - - // (B, Seq_Len, H, Head_Dim/2, 2) -> (B, Seq_Len, H, Head_Dim) - var rotatedReshaped = rotated.reshape(rotated.shape[0], rotated.shape[1], rotated.shape[2], -1); - - return rotatedReshaped.type_as(input); - } - - public static Tensor PrecomputeThetaPosFrequencies(int headDim, int seqLen, string device, float theta = 10000.0f) - { - // As written in the paragraph 3.2.2 of the paper - // >> In order to generalize our results in 2D to any xi ∈ Rd where **d is even**, [...] - if (headDim % 2 != 0) - { - throw new ArgumentException("Dimension must be divisible by 2", nameof(headDim)); - } - - // Build the theta parameter - // According to the formula theta_i = 10000^(-2(i-1)/dim) for i = [1, 2, ... dim/2] - // Shape: (Head_Dim / 2) - var thetaNumerator = torch.arange(0, headDim, 2).to(torch.float32).to(device); - // Shape: (Head_Dim / 2) - var thetaInput = torch.pow(theta, -1.0f * (thetaNumerator / headDim)).to(device); // (Dim / 2) - // Construct the positions (the "m" parameter) - // Shape: (Seq_Len) - var m = torch.arange(seqLen, device: device); - // Multiply each theta by each position using the outer product. - // Shape: (Seq_Len) outer_product* (Head_Dim / 2) -> (Seq_Len, Head_Dim / 2) - var thetaPositionFrequencies = torch.outer(m, thetaInput).to(torch.float32).to(device); - - // We can compute complex numbers in the polar form c = R * exp(m * theta), where R = 1 as follows: - // (Seq_Len, Head_Dim / 2) -> (Seq_Len, Head_Dim / 2) - var freqsComplex = torch.polar(torch.ones_like(thetaPositionFrequencies), thetaPositionFrequencies); - - return freqsComplex; - } - - // python - // def rotate_half(x): - // """Rotates half the hidden dims of the input.""" - // x1 = x[..., : x.shape[-1] // 2] - // x2 = x[..., x.shape[-1] // 2 :] - // return torch.cat((-x2, x1), dim=-1) - public static Tensor RotateHalf(Tensor x) - { - var x1 = x[.., .., .., ..(int)(x.shape[^1] / 2)]; - var x2 = x[.., .., .., (int)(x.shape[^1] / 2)..]; - // (x1 * x1 * x2).Peek("x1 * x1 * x2"); - return torch.cat([-x2, x1], dim: -1); - } - - public static (Tensor, Tensor) ApplyRotaryPosEmb(Tensor q, Tensor k, Tensor cos, Tensor sin, Tensor? positionIds = null, int unsqueezeDim = 1) - { - // The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - // sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - // that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - // k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - // cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - // the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - - if (positionIds is not null) - { - cos = cos[positionIds!].unsqueeze(unsqueezeDim); - sin = sin[positionIds!].unsqueeze(unsqueezeDim); - } - else - { - cos = cos.unsqueeze(unsqueezeDim); - sin = sin.unsqueeze(unsqueezeDim); - } - - var qEmbed = q * cos; - qEmbed += RotateHalf(q) * sin; - - var kEmbed = k * cos; - kEmbed += RotateHalf(k) * sin; - // var kEmbed = (k * cos) + (RotateHalf(k) * sin); - return (qEmbed, kEmbed); - } - - public static Module GetActivation(string actFn) - { - return actFn switch - { - "silu" => nn.SiLU(), - "relu" => nn.ReLU(), - "gelu" => nn.GELU(), - "tanh" => nn.Tanh(), - "swish" => nn.SiLU(), - _ => throw new ArgumentException("Invalid activation function", actFn), - }; - } - - - public static Tensor Phi2RepeatKV(Tensor x, int nRep) - { - var batchSize = x.shape[0]; - var seqLen = x.shape[1]; - var nKVHeads = x.shape[2]; - var headDim = x.shape[3]; - if (nRep == 1) - { - return x; - } - - return x.unsqueeze(3) - .expand(batchSize, seqLen, nKVHeads, nRep, headDim) - .view(batchSize, seqLen, nKVHeads * nRep, headDim); - } - - public static Tensor RepeatKV(Tensor x, int nRep) - { - var batchSize = x.shape[0]; - var nKVHeads = x.shape[1]; - var seqLen = x.shape[2]; - var headDim = x.shape[3]; - if (nRep == 1) - { - return x; - } - - return x.unsqueeze(2) - .expand(batchSize, nKVHeads, nRep, seqLen, headDim) - .reshape(batchSize, nKVHeads * nRep, seqLen, headDim); - } - - internal static string GetEmbeddedResource(string resourceName) - { - // read file content from embedded resource - var assembly = Assembly.GetCallingAssembly(); - var resourceStream = assembly.GetManifestResourceStream(resourceName); - - if (resourceStream == null) - { - throw new ArgumentException("Resource not found", resourceName); - } - - using var reader = new System.IO.StreamReader(resourceStream); - return reader.ReadToEnd(); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Llama3CausalLMChatClient.cs b/src/Microsoft.ML.GenAI.LLaMA/Llama3CausalLMChatClient.cs deleted file mode 100644 index a361a7febf..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Llama3CausalLMChatClient.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class Llama3CausalLMChatClient : CausalLMPipelineChatClient -{ - private readonly string _eotToken = "<|eot_id|>"; - - public Llama3CausalLMChatClient( - ICausalLMPipeline pipeline, - IMEAIChatTemplateBuilder? chatTemplateBuilder = null, - ChatClientMetadata? metadata = null) - : base( - pipeline, - chatTemplateBuilder ?? Llama3_1ChatTemplateBuilder.Instance, - metadata ?? new ChatClientMetadata(defaultModelId: nameof(Llama3CausalLMChatClient))) - { - } - - public override Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - options ??= new ChatOptions(); - - if (options.StopSequences != null) - { - options.StopSequences.Add(_eotToken); - } - else - { - options.StopSequences = new List { _eotToken }; - } - - return base.GetResponseAsync(messages, options, cancellationToken); - } - - public override IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - options ??= new ChatOptions(); - options.StopSequences ??= []; - options.StopSequences.Add(_eotToken); - - return base.GetStreamingResponseAsync(messages, options, cancellationToken); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Llama3_1ChatTemplateBuilder.cs b/src/Microsoft.ML.GenAI.LLaMA/Llama3_1ChatTemplateBuilder.cs deleted file mode 100644 index f8e50262db..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Llama3_1ChatTemplateBuilder.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using AutoGen.Core; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using TextContent = Microsoft.SemanticKernel.TextContent; - -namespace Microsoft.ML.GenAI.LLaMA; -#pragma warning disable MSML_GeneralName // This name should be PascalCased -public class Llama3_1ChatTemplateBuilder : IChatTemplateBuilder, IMEAIChatTemplateBuilder -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - private const char Newline = '\n'; - - public string BuildPrompt(IEnumerable messages, IEnumerable? tools = null) - { - var availableRoles = new[] { Role.System, Role.User, Role.Assistant }; - if (messages.Any(m => m.GetContent() is null)) - { - throw new InvalidOperationException("Please provide a message with content."); - } - - if (messages.Any(m => m.GetRole() is null || availableRoles.Contains(m.GetRole()!.Value) == false)) - { - throw new InvalidOperationException("Please provide a message with a valid role. The valid roles are System, User, and Assistant."); - } - - // construct template based on instruction from - // https://github.com/meta-llama/llama3/blob/11817d47e1ba7a4959b025eb1ca308572e0e3963/llama/generation.py#L280 - - var sb = new StringBuilder(); - sb.Append("<|begin_of_text|>"); - foreach (var message in messages) - { - var role = message.GetRole()!.Value; - var content = message.GetContent()!; - sb.Append(message switch - { - _ when message.GetRole() == Role.System => $"<|start_header_id|>system<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ when message.GetRole() == Role.User => $"<|start_header_id|>user<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ when message.GetRole() == Role.Assistant => $"<|start_header_id|>assistant<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - sb.Append($"<|start_header_id|>assistant<|end_header_id|>{Newline}"); - var input = sb.ToString(); - - return input; - } - - public string BuildPrompt(ChatHistory chatHistory) - { - // build prompt from chat history - var sb = new StringBuilder(); - - sb.Append("<|begin_of_text|>"); - foreach (var message in chatHistory) - { - foreach (var item in message.Items) - { - if (item is not TextContent textContent) - { - throw new NotSupportedException($"Only text content is supported, but got {item.GetType().Name}"); - } - - var text = textContent.Text?.Trim() ?? string.Empty; - - var prompt = message.Role switch - { - _ when message.Role == AuthorRole.System => $"<|start_header_id|>system<|end_header_id|>{Newline}{text}<|eot_id|>{Newline}", - _ when message.Role == AuthorRole.User => $"<|start_header_id|>user<|end_header_id|>{Newline}{text}<|eot_id|>{Newline}", - _ when message.Role == AuthorRole.Assistant => $"<|start_header_id|>assistant<|end_header_id|>{Newline}{text}<|eot_id|>{Newline}", - _ => throw new NotSupportedException($"Unsupported role {message.Role}") - }; - - sb.Append(prompt); - } - } - - sb.Append($"<|start_header_id|>assistant<|end_header_id|>{Newline}"); - - return sb.ToString(); - } - - public string BuildPrompt(IEnumerable messages, ChatOptions? options = null, bool appendAssistantTag = true) - { - var availableRoles = new[] { ChatRole.System, ChatRole.User, ChatRole.Assistant }; - if (messages.Any(m => m.Text is null)) - { - throw new InvalidOperationException("Please provide a message with content."); - } - - if (messages.Any(m => availableRoles.Any(availableRole => availableRole == m.Role) == false)) - { - throw new InvalidOperationException("Please provide a message with a valid role. The valid roles are System, User, and Assistant."); - } - - var sb = new StringBuilder(); - sb.Append("<|begin_of_text|>"); - foreach (var message in messages) - { - var role = message.Role.Value; - var content = message.Text!; - sb.Append(message switch - { - _ when message.Role == ChatRole.System => $"<|start_header_id|>system<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ when message.Role == ChatRole.User => $"<|start_header_id|>user<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ when message.Role == ChatRole.Assistant => $"<|start_header_id|>assistant<|end_header_id|>{Newline}{content.Trim()}<|eot_id|>{Newline}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - if (appendAssistantTag) - { - sb.Append($"<|start_header_id|>assistant<|end_header_id|>{Newline}"); - } - - var input = sb.ToString(); - - return input; - } - - public static Llama3_1ChatTemplateBuilder Instance { get; } = new Llama3_1ChatTemplateBuilder(); -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaCausalLMAgent.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaCausalLMAgent.cs deleted file mode 100644 index d6593f445f..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaCausalLMAgent.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class LlamaCausalLMAgent : IStreamingAgent -{ - private const char Newline = '\n'; - private readonly ICausalLMPipeline _pipeline; - private readonly string? _systemMessage; - private readonly IAutoGenChatTemplateBuilder _templateBuilder; - - /// - /// Create a new instance of . - /// - /// pipeline - /// agent name - /// system message. - /// the template builder to build chat prompt. If the value is null, would be used. - public LlamaCausalLMAgent( - ICausalLMPipeline pipeline, - string name, - string? systemMessage = "you are a helpful assistant", - IAutoGenChatTemplateBuilder? templateBuilder = null) - { - this.Name = name; - this._pipeline = pipeline; - this._systemMessage = systemMessage; - this._templateBuilder = templateBuilder ?? Llama3_1ChatTemplateBuilder.Instance; - } - - public string Name { get; } - - public Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - var input = _templateBuilder.BuildPrompt(messages); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append("<|eot_id|>").ToArray(); - - var output = _pipeline.Generate( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence) ?? throw new InvalidOperationException("Failed to generate a reply."); - - return Task.FromResult(new TextMessage(Role.Assistant, output, from: this.Name)); - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable GenerateStreamingReplyAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - IEnumerable messages, - GenerateReplyOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - var input = _templateBuilder.BuildPrompt(messages); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append("<|eot_id|>").ToArray(); - - foreach (var output in _pipeline.GenerateStreaming( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence)) - { - yield return new TextMessageUpdate(Role.Assistant, output, from: this.Name); - } - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaChatCompletionService.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaChatCompletionService.cs deleted file mode 100644 index 3e43e7eefb..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaChatCompletionService.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class LlamaChatCompletionService : IChatCompletionService -{ - private readonly ICausalLMPipeline _pipeline; - private readonly LlamaTextCompletionService _textGenerationService; - private readonly ISemanticKernelChatTemplateBuilder _templateBuilder; - - /// - /// Create a new instance of . - /// - /// pipeline - /// The template builder to use for generating chat prompts, if not provided, will be used. - public LlamaChatCompletionService(ICausalLMPipeline pipeline, ISemanticKernelChatTemplateBuilder? templateBuilder = null) - { - _pipeline = pipeline; - _textGenerationService = new LlamaTextCompletionService(pipeline); - _templateBuilder = templateBuilder ?? Llama3_1ChatTemplateBuilder.Instance; - } - - public IReadOnlyDictionary Attributes => _textGenerationService.Attributes; - - public async Task> GetChatMessageContentsAsync(ChatHistory chatHistory, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default) - { - var prompt = _templateBuilder.BuildPrompt(chatHistory); - var replies = await _textGenerationService.GetTextContentsAsync(prompt, executionSettings, kernel, cancellationToken); - - return replies.Select(reply => new ChatMessageContent(AuthorRole.Assistant, reply.Text)).ToList(); - } - - public async IAsyncEnumerable GetStreamingChatMessageContentsAsync( - ChatHistory chatHistory, - PromptExecutionSettings? executionSettings = null, - Kernel? kernel = null, - [EnumeratorCancellation] - CancellationToken cancellationToken = default) - { - var prompt = _templateBuilder.BuildPrompt(chatHistory); - - await foreach (var reply in _textGenerationService.GetStreamingTextContentsAsync(prompt, executionSettings, kernel, cancellationToken)) - { - yield return new StreamingChatMessageContent(AuthorRole.Assistant, reply.Text); - } - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaConfig.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaConfig.cs deleted file mode 100644 index 75bcd18571..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaConfig.cs +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using TorchSharp; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class LlamaConfig -{ - public LlamaConfig() - { - this.AttentionBias = false; - this.AttentionDropout = 0.0; - this.HiddenAct = "silu"; - this.HiddenSize = 4096; - this.InitializerRange = 0.02; - this.IntermediateSize = 14336; - this.MaxPositionEmbeddings = 131072; - this.MlpBias = false; - this.NumAttentionHeads = 32; - this.NumHiddenLayers = 32; - this.NumKeyValueHeads = 8; - this.PretrainingTp = 1; - this.RmsNormEps = 1e-05f; - this.RopeScaling = new RopeScalingConfig(); - this.RopeTheta = 500000.0; - this.TieWordEmbeddings = false; - this.VocabSize = 128256; - this.AttnImplementation = "eager"; - this.DType = torch.ScalarType.BFloat16; - } - - static LlamaConfig() - { -#pragma warning disable MSML_ParameterLocalVarName // Parameter or local variable name not standard - var llama3_1_8b_content = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.LLaMA.Resource.Config.meta-llama-3.1-8B-Instruct.json"); - var llama3_1_70b_content = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.LLaMA.Resource.Config.meta-llama-3.1-70B-Instruct.json"); - var llama3_1_405b_content = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.LLaMA.Resource.Config.meta-llama-3.1-405B-Instruct.json"); - var llama3_2_1b_content = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.LLaMA.Resource.Config.meta-llama-3.2-1B-Instruct.json"); - var llama3_2_3b_content = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.LLaMA.Resource.Config.meta-llama-3.2-3B-Instruct.json"); -#pragma warning restore MSML_ParameterLocalVarName // Parameter or local variable name not standard - - Llama3_1_8B_Instruct = JsonSerializer.Deserialize(llama3_1_8b_content) ?? throw new ArgumentNullException(nameof(llama3_1_8b_content)); - Llama3_1_70B_Instruct = JsonSerializer.Deserialize(llama3_1_70b_content) ?? throw new ArgumentNullException(nameof(llama3_1_70b_content)); - Llama3_1_405B_Instruct = JsonSerializer.Deserialize(llama3_1_405b_content) ?? throw new ArgumentNullException(nameof(llama3_1_405b_content)); - Llama3_2_1B_Instruct = JsonSerializer.Deserialize(llama3_2_1b_content) ?? throw new ArgumentNullException(nameof(llama3_2_1b_content)); - Llama_3_2_3B_Instruct = JsonSerializer.Deserialize(llama3_2_3b_content) ?? throw new ArgumentNullException(nameof(llama3_2_3b_content)); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - /// - /// The llama-3.1-8B-Instruct configuration created from https://huggingface.co/meta-llama/Meta-Llama-3.1-8B. - /// - public static LlamaConfig Llama3_1_8B_Instruct { get; } - - /// - /// The llama-3.1-70B-Instruct configuration created from https://huggingface.co/meta-llama/Meta-Llama-3.1-70B. - /// - public static LlamaConfig Llama3_1_70B_Instruct { get; } - - /// - /// The llama-3.1-405B-Instruct configuration created from https://huggingface.co/meta-llama/Meta-Llama-3.1-405B. - /// - public static LlamaConfig Llama3_1_405B_Instruct { get; } - - /// - /// The llama-3.2-3B-Instruct configuration created from https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct. - /// - public static LlamaConfig Llama_3_2_3B_Instruct { get; } - - /// - /// The llama-3.2-1B-Instruct configuration created from https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct. - /// - public static LlamaConfig Llama3_2_1B_Instruct { get; } -#pragma warning restore MSML_GeneralName // This name should be PascalCased - - [JsonPropertyName("attention_bias")] - public bool AttentionBias { get; set; } - - [JsonPropertyName("attention_dropout")] - public double AttentionDropout { get; set; } - - [JsonPropertyName("hidden_act")] - public string HiddenAct { get; set; } - - [JsonPropertyName("hidden_size")] - public int HiddenSize { get; set; } - - [JsonPropertyName("initializer_range")] - public double InitializerRange { get; set; } - - [JsonPropertyName("intermediate_size")] - public int IntermediateSize { get; set; } - - [JsonPropertyName("max_position_embeddings")] - public int MaxPositionEmbeddings { get; set; } - - [JsonPropertyName("mlp_bias")] - public bool MlpBias { get; set; } - - [JsonPropertyName("num_attention_heads")] - public int NumAttentionHeads { get; set; } - - [JsonPropertyName("num_hidden_layers")] - public int NumHiddenLayers { get; set; } - - [JsonPropertyName("num_key_value_heads")] - public int NumKeyValueHeads { get; set; } - - [JsonPropertyName("pretraining_tp")] - public int PretrainingTp { get; set; } - - [JsonPropertyName("rms_norm_eps")] - public float RmsNormEps { get; set; } - - public RopeScalingConfig RopeScaling { get; set; } - - [JsonPropertyName("rope_theta")] - public double RopeTheta { get; set; } - - [JsonPropertyName("tie_word_embeddings")] - public bool TieWordEmbeddings { get; set; } - - [JsonPropertyName("vocab_size")] - public int VocabSize { get; set; } - public int? PadTokenId { get; set; } - public torch.ScalarType DType { get; set; } - public string AttnImplementation { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaForCausalLM.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaForCausalLM.cs deleted file mode 100644 index 0a6cdc8498..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaForCausalLM.cs +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Diagnostics; -using System.Text.Json; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.GenAI.LLaMA.Module; -using TorchSharp; -using TorchSharp.Modules; -using TorchSharp.PyBridge; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class LlamaForCausalLM : nn.Module -{ - private readonly LlamaConfig _config; - private readonly int _vocabSize; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Linear lm_head; - private readonly LlamaModel model; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public LlamaForCausalLM(LlamaConfig config, string? device = null) - : base(nameof(LlamaForCausalLM)) - { - _config = config; - _vocabSize = config.VocabSize; - - model = new LlamaModel(config, device); - - // When tie word embeddings is true, the lm_head shares the same weight as the embedding layer. - // therefore, the lm_head weight won't be initialized here. - // instead, it will be loaded from the embedding layer after the model is loaded. - if (config.TieWordEmbeddings) - { - this.RegisterComponents(); - lm_head = nn.Linear(config.HiddenSize, config.VocabSize, hasBias: false, dtype: config.DType); - } - else - { - lm_head = nn.Linear(config.HiddenSize, config.VocabSize, hasBias: false, dtype: config.DType); - this.RegisterComponents(); - } - - } - - private void TieWordEmbeddings() - { - var embeddingWeight = model.Embedding.state_dict(); - this.lm_head.load_state_dict(embeddingWeight); - - this.lm_head.to(device: model.Embedding.weight!.device); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var outputs = this.model.forward(input); - var logits = this.lm_head.forward(outputs.LastHiddenState); - logits = logits.to_type(ScalarType.Float32); - outputs.Logits = logits; - - // calculate the loss if the label is provided - if (input.Labels is not null) - { - // upcast the logits to float32 - logits = logits.to_type(ScalarType.Float32); - - var shiftLogits = logits[.., .., ..].contiguous(); - var shiftLabels = input.Labels[.., ..].contiguous(); - - shiftLogits = shiftLogits.view(-1, _vocabSize); - shiftLabels = shiftLabels.view(-1); - - // calculate the loss - // the loss is calculated by using the cross entropy loss by default - // TODO: add support for other loss functions - var loss = nn.functional.cross_entropy(shiftLogits, shiftLabels); - outputs.Loss = loss; - - // dispose the shiftLogits - shiftLogits.Dispose(); - shiftLabels.Dispose(); - logits.Dispose(); - } - - return outputs; - } - - public static LlamaForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - ScalarType torchDtype = ScalarType.BFloat16, - string device = "cpu") - { - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var model = new LlamaForCausalLM(modelConfig); - - model.LoadSafeTensors(modelFolder, checkPointName); - model = model.to(device); - if (modelConfig.TieWordEmbeddings) - { - model.TieWordEmbeddings(); - } - - - return model; - } - - public static LlamaForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - bool quantizeToInt8 = false, - bool quantizeToInt4 = false, - int layersOnTargetDevice = -1, - ScalarType torchDtype = ScalarType.BFloat16, - string targetDevice = "cuda") - { - if (layersOnTargetDevice == -1 && quantizeToInt4 == false && quantizeToInt8 == false) - { - return FromPretrained(modelFolder, configName, checkPointName, torchDtype, targetDevice); - } - - var originalDefaultDevice = torch.get_default_device(); - torch.set_default_device("meta"); - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var model = new LlamaForCausalLM(modelConfig); - - if (quantizeToInt8) - { - model.ToInt8QuantizeModule(); - } - else if (quantizeToInt4) - { - model.ToInt4QuantizeModule(); - } - - var deviceMap = model.InferDeviceMapForEachLayer( - [ - KeyValuePair.Create(targetDevice, layersOnTargetDevice), - KeyValuePair.Create("cpu", -1) - ]); - - torch.set_default_device("cpu"); - model = new LlamaForCausalLM(modelConfig); - - model.LoadSafeTensors(modelFolder, checkPointName); - - if (quantizeToInt8) - { - model.ToInt8QuantizeModule(); - } - else if (quantizeToInt4) - { - model.ToInt4QuantizeModule(); - } - - model = model.ToDynamicLoadingModel(deviceMap, targetDevice); - - if (modelConfig.TieWordEmbeddings) - { - model.TieWordEmbeddings(); - } - - torch.set_default_device(originalDefaultDevice); - - return model; - } - - public void LoadSafeTensors(string modelFolder, string checkPointName = "model.safetensors.index.json") - { - this.load_checkpoint(path: modelFolder, checkpointName: checkPointName, strict: true, useTqdm: false); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaTextCompletionService.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaTextCompletionService.cs deleted file mode 100644 index 5ac0a9afb9..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaTextCompletionService.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.TextGeneration; - -namespace Microsoft.ML.GenAI.LLaMA; - -public class LlamaTextCompletionService : ITextGenerationService -{ - private readonly ICausalLMPipeline _pipeline; - - public LlamaTextCompletionService(ICausalLMPipeline pipeline) - { - _pipeline = pipeline; - } - - public IReadOnlyDictionary Attributes => new Dictionary() - { - { "temperature", null }, - { "max_token", null }, - { "stop_token_sequence", null }, - { "top_p", null }, - }; - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable GetStreamingTextContentsAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - string prompt, - PromptExecutionSettings? executionSettings = null, - Kernel? kernel = null, - [EnumeratorCancellation] - CancellationToken cancellationToken = default) - { - var temperature = executionSettings?.ExtensionData?["temperature"] as float? ?? 0.7f; - var maxToken = executionSettings?.ExtensionData?["max_token"] as int? ?? 100; - var stopTokenSequence = executionSettings?.ExtensionData?["stop_token_sequence"] as string[] ?? Array.Empty(); - var topP = executionSettings?.ExtensionData?["top_p"] as float? ?? 0.9f; - stopTokenSequence.Append("<|eot_id|>"); - - foreach (var item in _pipeline.GenerateStreaming( - prompt, - maxToken, - temperature, - topP, - stopTokenSequence)) - { - yield return new StreamingTextContent(item); - } - } - - public Task> GetTextContentsAsync(string prompt, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default) - { - var temperature = executionSettings?.ExtensionData?["temperature"] as float? ?? 0.7f; - var maxToken = executionSettings?.ExtensionData?["max_token"] as int? ?? 512; - var stopTokenSequence = executionSettings?.ExtensionData?["stop_token_sequence"] as List ?? new List(); - var topP = executionSettings?.ExtensionData?["top_p"] as float? ?? 0.9f; - stopTokenSequence.Add("<|eot_id|>"); - var response = _pipeline.Generate( - prompt, - maxToken, - temperature, - stopSequences: stopTokenSequence.ToArray(), - topP: topP); - - return Task.FromResult>([new TextContent(response)]); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/LlamaTokenizerHelper.cs b/src/Microsoft.ML.GenAI.LLaMA/LlamaTokenizerHelper.cs deleted file mode 100644 index 489acb6524..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/LlamaTokenizerHelper.cs +++ /dev/null @@ -1,55 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.LLaMA; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased -public class LlamaTokenizerHelper -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - /// - /// https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/tokenizer.json#pre_tokenizer.pretokenizers.pattern - /// - private const string _re = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"; - - /// - /// https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct/blob/main/tokenizer.json#added_tokens - /// - private static readonly Dictionary _specialTokens = new() - { - { "<|begin_of_text|>", 128000 }, - { "<|end_of_text|>", 128001 }, - { "<|finetune_right_pad_id|>", 128004 }, - { "<|start_header_id|>", 128006 }, - { "<|end_header_id|>", 128007 }, - { "<|eom_id|>", 128008 }, - { "<|eot_id|>", 128009 }, - { "<|system|>", 32006 }, - { "<|user|>", 32010 }, - { "<|assistant|>", 32001 }, - { "<|end|>", 32007 } - }; - - /// - /// Create from tokenizer model file. - /// - /// path to tokenizer model folder - /// tokenizer model file name - public static TiktokenTokenizer FromPretrained( - string modelWeightFolder, - string modelFile = "tokenizer.model") - { - var modelFilePath = Path.Join(modelWeightFolder, modelFile); - var preTokenizer = new RegexPreTokenizer(new Regex(_re), _specialTokens); - return TiktokenTokenizer.Create(File.OpenRead(modelFilePath), preTokenizer, normalizer: null, specialTokens: _specialTokens); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Microsoft.ML.GenAI.LLaMA.csproj b/src/Microsoft.ML.GenAI.LLaMA/Microsoft.ML.GenAI.LLaMA.csproj deleted file mode 100644 index 72bd38416c..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Microsoft.ML.GenAI.LLaMA.csproj +++ /dev/null @@ -1,32 +0,0 @@ - - - - net8.0 - enable - enable - true - - - - true - - - - - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaDecoderLayer.cs b/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaDecoderLayer.cs deleted file mode 100644 index 0e3132f739..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaDecoderLayer.cs +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.LLaMA.Module; - -internal class DecoderLayerInput -{ - public DecoderLayerInput( - Tensor hiddenStates, - Tensor attentionMask, - Tensor positionIds, - RotaryEmbeddingOutput positionEmbeddings, // cos, sin - IKVCache? pastKeyValue = null, - bool outputAttentions = false) - { - this.HiddenStates = hiddenStates; - this.AttentionMask = attentionMask; - this.PositionIds = positionIds; - this.PastKeyValue = pastKeyValue; - this.OutputAttentions = outputAttentions; - this.PositionalEmbeddings = positionEmbeddings; - } - - public Tensor HiddenStates { get; set; } - - public Tensor AttentionMask { get; set; } - - public Tensor PositionIds { get; set; } - - public RotaryEmbeddingOutput PositionalEmbeddings { get; set; } - - public IKVCache? PastKeyValue { get; set; } - - public bool OutputAttentions { get; set; } -} - -internal class DecoderLayerOutput -{ - public DecoderLayerOutput( - Tensor hiddenStates, - Tensor? attentions = null, - IKVCache? pastKeyValue = null) - { - this.HiddenStates = hiddenStates; - this.Attentions = attentions; - this.PastKeyValue = pastKeyValue; - } - - public Tensor HiddenStates { get; set; } - - public Tensor? Attentions { get; set; } - - public IKVCache? PastKeyValue { get; set; } -} -internal class LlamaDecoderLayer : nn.Module, IDynamicLoadModule -{ - private readonly LlamaConfig _llamaConfig; - private readonly int _layerIndex; - private readonly int _hiddenSize; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly LlamaMLP mlp; - private readonly Core.RMSNorm input_layernorm; - private readonly Core.RMSNorm post_attention_layernorm; - private readonly Attention self_attn; - - public Action? LoadToDeviceFunc { get; set; } - public Action? UnloadFromDeviceFunc { get; set; } - -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public LlamaDecoderLayer(LlamaConfig config, int layerIndex) - : base(nameof(LlamaDecoderLayer)) - { - _llamaConfig = config; - _layerIndex = layerIndex; - _hiddenSize = config.HiddenSize; - - this.self_attn = CreateAttention(config, layerIndex); - this.mlp = new LlamaMLP(config); - this.input_layernorm = new Core.RMSNorm(this._hiddenSize, eps: config.RmsNormEps, config.DType); - this.post_attention_layernorm = new Core.RMSNorm(this._hiddenSize, eps: config.RmsNormEps, config.DType); - } - - private Attention CreateAttention(LlamaConfig config, int layerIndex) - { - var headDim = config.HiddenSize / config.NumAttentionHeads; - return new Attention( - attentionDropout: config.AttentionDropout, - hiddenSize: config.HiddenSize, - numHeads: config.NumAttentionHeads, - headDim: headDim, - numKeyValueHeads: config.NumKeyValueHeads, - numKeyValueGroups: config.NumAttentionHeads / config.NumKeyValueHeads, - maxPositionEmbeddings: config.MaxPositionEmbeddings, - originalMaxPositionEmbeddings: config.MaxPositionEmbeddings, - layerIdx: layerIndex, - useQkvProj: false, - dtype: config.DType, - attentionBias: config.AttentionBias); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override DecoderLayerOutput forward(DecoderLayerInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (LoadToDeviceFunc != null) - { - LoadToDeviceFunc(this); - } - - using var disposeScope = NewDisposeScope(); - var residual = input.HiddenStates; - var hiddenStates = this.input_layernorm.forward(input.HiddenStates); - - var selfAttnInput = new AttentionInput( - hiddenStates: hiddenStates, - attentionMask: input.AttentionMask, - positionIds: input.PositionIds, - cache: input.PastKeyValue, - positionalEmbeddings: input.PositionalEmbeddings, - outputAttentions: input.OutputAttentions); - - var selfAttnOutput = this.self_attn.forward(selfAttnInput); - - hiddenStates = residual + selfAttnOutput.HiddenStates; - - // Fully connected - residual = hiddenStates; - hiddenStates = this.post_attention_layernorm.forward(hiddenStates); - hiddenStates = this.mlp.forward(hiddenStates); - hiddenStates = residual + hiddenStates; - - if (UnloadFromDeviceFunc != null) - { - UnloadFromDeviceFunc(this); - } - - return new DecoderLayerOutput( - hiddenStates: hiddenStates.MoveToOuterDisposeScope(), - attentions: input.OutputAttentions ? selfAttnOutput.Attentions?.MoveToOuterDisposeScope() : null, - pastKeyValue: selfAttnOutput.Cache); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaMLP.cs b/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaMLP.cs deleted file mode 100644 index cbc841f144..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaMLP.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.LLaMA; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.LLaMA.Module; -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class LlamaMLP : torch.nn.Module -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - private readonly int _pretrainingTp; - private readonly int _intermediateSize; - private readonly int _hiddenSize; - private readonly bool _hasBias; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly QuantizedLinear gate_proj; - private readonly QuantizedLinear up_proj; - private readonly QuantizedLinear down_proj; - private readonly torch.nn.Module activation_fn; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public LlamaMLP(LlamaConfig config) - : base(nameof(LlamaMLP)) - { - this._hiddenSize = config.HiddenSize; - this._intermediateSize = config.IntermediateSize; - this._hasBias = config.MlpBias; - this._pretrainingTp = config.PretrainingTp; - var hiddenAct = config.HiddenAct; - this.gate_proj = new QuantizedLinear(this._hiddenSize, this._intermediateSize, hasBias: this._hasBias, dtype: config.DType); - this.up_proj = new QuantizedLinear(this._hiddenSize, this._intermediateSize, hasBias: this._hasBias, dtype: config.DType); - this.down_proj = new QuantizedLinear(this._intermediateSize, this._hiddenSize, hasBias: this._hasBias, dtype: config.DType); - this.RegisterComponents(); - this.activation_fn = Core.Utils.GetActivation(hiddenAct); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (this._pretrainingTp > 1) - { - throw new NotImplementedException("PretrainingTp > 1 is not supported yet."); - } - - using var input1 = this.gate_proj.forward(input); - using var input2 = this.activation_fn.forward(input1); - using var input3 = input2 * this.up_proj.forward(input); - return this.down_proj.forward(input3); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaModel.cs b/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaModel.cs deleted file mode 100644 index 5d1d08e411..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Module/LlamaModel.cs +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.LLaMA.Module; - -internal class LlamaModel : nn.Module -{ - private readonly LlamaConfig _config; - private readonly int? _paddingIdx; - private readonly int _vocabSize; - private IKVCache? _cache; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Embedding embed_tokens; - private readonly ModuleList layers; - private readonly RMSNorm norm; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly nn.Module _rotaryEmb; - - - public LlamaModel(LlamaConfig config, string? device = null) - : base(nameof(LlamaModel)) - { - this._config = config; - this._paddingIdx = config.PadTokenId; - this._vocabSize = config.VocabSize; - var headDim = config.HiddenSize / config.NumAttentionHeads; - this.embed_tokens = nn.Embedding(config.VocabSize, config.HiddenSize, padding_idx: this._paddingIdx, dtype: config.DType); - this.layers = new ModuleList(); - - for (int i = 0; i < config.NumHiddenLayers; i++) - { - this.layers.Add(new LlamaDecoderLayer(config, i)); - } - this.norm = new RMSNorm(config.HiddenSize, config.RmsNormEps, config.DType); - this._cache = new DynamicKVCache(); - this.RegisterComponents(); - this._rotaryEmb = config.RopeScaling switch - { - null => new RotaryEmbedding(config.RopeTheta, config.MaxPositionEmbeddings, headDim), - _ => new RotaryEmbedding(config.RopeTheta, headDim, config.RopeScaling), - }; - } - - public Embedding Embedding => this.embed_tokens; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (input.OverrideCache is not null) - { - this._cache = input.OverrideCache; - } - else if (!input.UseCache) - { - this._cache = null; - } - - var outputAttentions = input.OutputAttentions; - var outputHiddenStates = input.OutputHiddenStates; - var attentionMask = input.AttentionMask; - Device device; - var inputIds = input.InputIds; - var positionIds = input.PositionIds; - var inputsEmbeds = input.InputEmbeddings; - int batchSize; - int seqLength; - if (inputIds is not null && inputsEmbeds is not null) - { - throw new ArgumentException("Only one of input_ids or inputs_embeds may be set"); - } - else if (inputIds is not null) - { - batchSize = inputIds.IntShape()[0]; - seqLength = inputIds.IntShape()[1]; - inputsEmbeds = this.embed_tokens.forward(inputIds); - device = inputIds.device; - } - else if (inputsEmbeds is not null) - { - batchSize = inputsEmbeds.IntShape()[0]; - seqLength = inputsEmbeds.IntShape()[1]; - device = inputsEmbeds.device; - } - else - { - throw new ArgumentException("Either input_ids or inputs_embeds must be set"); - } - - var pastKeyValuesLength = input.PastKeyValuesLength; - - if (positionIds is null) - { - positionIds = torch.arange(pastKeyValuesLength, seqLength + pastKeyValuesLength, device: device); - positionIds = positionIds.unsqueeze(0).view(-1, seqLength); - } - else - { - positionIds = ((long)positionIds.view(-1, seqLength)); - } - - if (this._config.AttnImplementation == "flash_attention_2") - { - throw new NotImplementedException(); - } - else - { - // the following behavior of creating 4d causal mask doesn't match python's, remember to look into it when there's time. - attentionMask = AttentionMaskConverter.Create4DCausalAttentionMask(attentionMask, [batchSize, seqLength], inputsEmbeds.dtype, device, pastKeyValuesLength); - } - - var hiddenStates = inputsEmbeds; - - var allHiddenStates = new List(); - var allAttentions = new List(); - - var embOutput = this._rotaryEmb.forward(new RotaryEmbeddingInput(hiddenStates, positionIds, pastKeyValuesLength)); - foreach (var layer in this.layers) - { - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - - var decoderInput = new DecoderLayerInput( - hiddenStates: hiddenStates, - attentionMask: attentionMask!, - positionIds: positionIds, - pastKeyValue: this._cache, - positionEmbeddings: embOutput, - outputAttentions: outputAttentions); - var layerOutput = layer.forward(decoderInput); - hiddenStates = layerOutput.HiddenStates; - if (outputAttentions && layerOutput.Attentions is not null) - { - allAttentions.Add(layerOutput.Attentions); - } - } - - hiddenStates = this.norm.forward(hiddenStates); - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - - return new CausalLMModelOutput(lastHiddenState: hiddenStates, allHiddenStates: allHiddenStates.ToArray(), attentions: allAttentions.ToArray(), cache: this._cache); - } -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-405B-Instruct.json b/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-405B-Instruct.json deleted file mode 100644 index 373b94f4f6..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-405B-Instruct.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [ - 128001, - 128008, - 128009 - ], - "hidden_act": "silu", - "hidden_size": 16384, - "initializer_range": 0.02, - "intermediate_size": 53248, - "max_position_embeddings": 131072, - "mlp_bias": false, - "num_attention_heads": 128, - "num_hidden_layers": 126, - "num_key_value_heads": 8, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "factor": 8.0, - "high_freq_factor": 4.0, - "low_freq_factor": 1.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3" - }, - "rope_theta": 500000.0, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 128256 -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-70B-Instruct.json b/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-70B-Instruct.json deleted file mode 100644 index 2cd3ad59ac..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-70B-Instruct.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [ - 128001, - 128008, - 128009 - ], - "hidden_act": "silu", - "hidden_size": 8192, - "initializer_range": 0.02, - "intermediate_size": 28672, - "max_position_embeddings": 131072, - "mlp_bias": false, - "num_attention_heads": 64, - "num_hidden_layers": 80, - "num_key_value_heads": 8, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "factor": 8.0, - "low_freq_factor": 1.0, - "high_freq_factor": 4.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3" - }, - "rope_theta": 500000.0, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 128256 -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-8B-Instruct.json b/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-8B-Instruct.json deleted file mode 100644 index 750f5671d6..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.1-8B-Instruct.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [ - 128001, - 128008, - 128009 - ], - "hidden_act": "silu", - "hidden_size": 4096, - "initializer_range": 0.02, - "intermediate_size": 14336, - "max_position_embeddings": 131072, - "mlp_bias": false, - "model_type": "llama", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 8, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "factor": 8.0, - "low_freq_factor": 1.0, - "high_freq_factor": 4.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3" - }, - "rope_theta": 500000.0, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 128256 -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-1B-Instruct.json b/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-1B-Instruct.json deleted file mode 100644 index b5d8a60c73..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-1B-Instruct.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [ - 128001, - 128008, - 128009 - ], - "head_dim": 64, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 8192, - "max_position_embeddings": 131072, - "mlp_bias": false, - "model_type": "llama", - "num_attention_heads": 32, - "num_hidden_layers": 16, - "num_key_value_heads": 8, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "factor": 32.0, - "high_freq_factor": 4.0, - "low_freq_factor": 1.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3" - }, - "rope_theta": 500000.0, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "vocab_size": 128256 -} diff --git a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-3B-Instruct.json b/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-3B-Instruct.json deleted file mode 100644 index 91d931a367..0000000000 --- a/src/Microsoft.ML.GenAI.LLaMA/Resource/Config/meta-llama-3.2-3B-Instruct.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [ - 128001, - 128008, - 128009 - ], - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 3072, - "initializer_range": 0.02, - "intermediate_size": 8192, - "max_position_embeddings": 131072, - "mlp_bias": false, - "model_type": "llama", - "num_attention_heads": 24, - "num_hidden_layers": 28, - "num_key_value_heads": 8, - "pretraining_tp": 1, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "factor": 32.0, - "high_freq_factor": 4.0, - "low_freq_factor": 1.0, - "original_max_position_embeddings": 8192, - "rope_type": "llama3" - }, - "rope_theta": 500000.0, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "vocab_size": 128256 -} diff --git a/src/Microsoft.ML.GenAI.Mistral/Microsoft.ML.GenAI.Mistral.csproj b/src/Microsoft.ML.GenAI.Mistral/Microsoft.ML.GenAI.Mistral.csproj deleted file mode 100644 index df80da30fa..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/Microsoft.ML.GenAI.Mistral.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - net8.0 - enable - enable - true - - - - true - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralCausalLMAgent.cs b/src/Microsoft.ML.GenAI.Mistral/MistralCausalLMAgent.cs deleted file mode 100644 index e20d3b860b..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralCausalLMAgent.cs +++ /dev/null @@ -1,166 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.Mistral; - -public class MistralCausalLMAgent : IStreamingAgent -{ - private readonly ICausalLMPipeline _pipeline; - private readonly string? _systemMessage; - private readonly IAutoGenChatTemplateBuilder _templateBuilder; - private readonly string _stopSequence = ""; - - /// - /// Create a new instance of . - /// - /// pipeline - /// agent name - /// system message. - /// the template builder to build chat prompt. If the value is null, would be used. - public MistralCausalLMAgent( - ICausalLMPipeline pipeline, - string name, - string? systemMessage = "you are a helpful assistant", - IAutoGenChatTemplateBuilder? templateBuilder = null) - { - this.Name = name; - this._pipeline = pipeline; - this._systemMessage = systemMessage; - this._templateBuilder = templateBuilder ?? Mistral_7B_0_3ChatTemplateBuilder.Instance; - } - - public string Name { get; } - - public Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - var input = _templateBuilder.BuildPrompt(messages, options?.Functions); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append(_stopSequence).ToArray(); - - var output = _pipeline.Generate( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence) ?? throw new InvalidOperationException("Failed to generate a reply."); - - // post-process the output for tool call - if (output.StartsWith("[TOOL_CALLS]")) - { - return Task.FromResult(ParseAsToolCallMessage(output)); - } - - return Task.FromResult(new TextMessage(Role.Assistant, output, from: this.Name)); - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable GenerateStreamingReplyAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - IEnumerable messages, - GenerateReplyOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - var input = _templateBuilder.BuildPrompt(messages, options?.Functions); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append(_stopSequence).ToArray(); - - // only streaming the output when the output is not a tool call - // otherwise, we collect all the chunks and convert them to a tool call message at the end of the streaming - var sb = new StringBuilder(); - bool? isToolCall = null; - foreach (var output in _pipeline.GenerateStreaming( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence)) - { - if (isToolCall is null) - { - sb.Append(output); - var str = sb.ToString(); - if (!str.StartsWith("[TOOL_CALLS]".Substring(0, str.Length))) - { - yield return new TextMessageUpdate(Role.Assistant, output, from: this.Name); - isToolCall = false; - } - else if (str.StartsWith("[TOOL_CALLS]")) - { - isToolCall = true; - } - } - else if (isToolCall == false) - { - yield return new TextMessageUpdate(Role.Assistant, output, from: this.Name); - } - else - { - sb.Append(output); - } - } - - if (isToolCall == true) - { - var toolCallMessage = ParseAsToolCallMessage(sb.ToString()); - foreach (var toolCall in toolCallMessage.ToolCalls) - { - yield return new ToolCallMessageUpdate(toolCall.FunctionName, toolCall.FunctionArguments, from: this.Name); - } - } - } - - private class MistralToolCall - { - [JsonPropertyName("name")] - public string? Name { get; set; } - - [JsonPropertyName("arguments")] - public JsonObject? Arguments { get; set; } - } - - private ToolCallMessage ParseAsToolCallMessage(string content) - { - var json = content.Substring("[TOOL_CALLS]".Length).Trim(); - - // the json string should be a list of tool call messages - // e.g. [{"name": "get_current_weather", "parameters": {"location": "Seattle"}}] - var mistralToolCalls = JsonSerializer.Deserialize>(json) ?? throw new InvalidOperationException("Failed to deserialize tool calls."); - var toolCalls = mistralToolCalls - .Select(tc => new ToolCall(tc.Name!, JsonSerializer.Serialize(tc.Arguments)) { ToolCallId = this.GenerateToolCallId() }); - - return new ToolCallMessage(toolCalls, from: this.Name); - } - - /// - /// 9 random alphanumeric characters - /// - private string GenerateToolCallId(int length = 9) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - var random = new Random(); - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[random.Next(s.Length)]).ToArray()); - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralConfig.cs b/src/Microsoft.ML.GenAI.Mistral/MistralConfig.cs deleted file mode 100644 index c2240f9579..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralConfig.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.ML.GenAI.Core; -using TorchSharp; - -namespace Microsoft.ML.GenAI.Mistral; - -public class MistralConfig -{ - public MistralConfig() - { - this.AttentionBias = false; - this.AttentionDropout = 0.0; - this.HiddenAct = "silu"; - this.HiddenSize = 4096; - this.InitializerRange = 0.02; - this.IntermediateSize = 14336; - this.MaxPositionEmbeddings = 131072; - this.MlpBias = false; - this.NumAttentionHeads = 32; - this.NumHiddenLayers = 32; - this.NumKeyValueHeads = 8; - this.RmsNormEps = 1e-05f; - this.RopeScaling = new RopeScalingConfig(); - this.RopeTheta = 500000.0; - this.TieWordEmbeddings = false; - this.VocabSize = 128256; - this.AttnImplementation = "eager"; - this.DType = torch.ScalarType.BFloat16; - this.HeadDim = this.HiddenSize / this.NumAttentionHeads; - this.SlidingWindow ??= 4096; - } - - static MistralConfig() - { -#pragma warning disable MSML_ParameterLocalVarName // Parameter or local variable name not standard - var mistral7BInstructContent = Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Mistral.Resource.Config.mistral-7B-instruct-v0.3.json"); -#pragma warning restore MSML_ParameterLocalVarName // Parameter or local variable name not standard - - Mistral_7B_Instruct_v0_3 = JsonSerializer.Deserialize(mistral7BInstructContent) ?? throw new ArgumentNullException(nameof(mistral7BInstructContent)); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - /// - /// The mistral-7b-instruct-v0.3 configuration created from https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3/tree/main. - /// - public static MistralConfig Mistral_7B_Instruct_v0_3 { get; } -#pragma warning restore MSML_GeneralName // This name should be PascalCased - - [JsonPropertyName("attention_bias")] - public bool AttentionBias { get; set; } - - [JsonPropertyName("attention_dropout")] - public double AttentionDropout { get; set; } - - [JsonPropertyName("hidden_act")] - public string HiddenAct { get; set; } - - [JsonPropertyName("hidden_size")] - public int HiddenSize { get; set; } - - [JsonPropertyName("initializer_range")] - public double InitializerRange { get; set; } - - [JsonPropertyName("intermediate_size")] - public int IntermediateSize { get; set; } - - [JsonPropertyName("max_position_embeddings")] - public int MaxPositionEmbeddings { get; set; } - - [JsonPropertyName("mlp_bias")] - public bool MlpBias { get; set; } - - [JsonPropertyName("num_attention_heads")] - public int NumAttentionHeads { get; set; } - - [JsonPropertyName("num_hidden_layers")] - public int NumHiddenLayers { get; set; } - - [JsonPropertyName("num_key_value_heads")] - public int NumKeyValueHeads { get; set; } - - [JsonPropertyName("head_dim")] - public int HeadDim { get; set; } - - [JsonPropertyName("rms_norm_eps")] - public float RmsNormEps { get; set; } - - public RopeScalingConfig RopeScaling { get; set; } - - [JsonPropertyName("rope_theta")] - public double RopeTheta { get; set; } - - [JsonPropertyName("tie_word_embeddings")] - public bool TieWordEmbeddings { get; set; } - - [JsonPropertyName("vocab_size")] - public int VocabSize { get; set; } - - [JsonPropertyName("sliding_window")] - public int? SlidingWindow { get; set; } - - public int? PadTokenId { get; set; } - - public torch.ScalarType DType { get; set; } - - public string AttnImplementation { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralDecoderLayer.cs b/src/Microsoft.ML.GenAI.Mistral/MistralDecoderLayer.cs deleted file mode 100644 index 7f17991b5c..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralDecoderLayer.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Mistral.Module; - -internal class DecoderLayerInput -{ - public DecoderLayerInput( - Tensor hiddenStates, - Tensor attentionMask, - Tensor positionIds, - RotaryEmbeddingOutput positionEmbeddings, // cos, sin - IKVCache? pastKeyValue = null, - bool outputAttentions = false) - { - this.HiddenStates = hiddenStates; - this.AttentionMask = attentionMask; - this.PositionIds = positionIds; - this.PastKeyValue = pastKeyValue; - this.OutputAttentions = outputAttentions; - this.PositionalEmbeddings = positionEmbeddings; - } - - public Tensor HiddenStates { get; set; } - - public Tensor AttentionMask { get; set; } - - public Tensor PositionIds { get; set; } - - public RotaryEmbeddingOutput PositionalEmbeddings { get; set; } - - public IKVCache? PastKeyValue { get; set; } - - public bool OutputAttentions { get; set; } -} - -internal class DecoderLayerOutput -{ - public DecoderLayerOutput( - Tensor hiddenStates, - Tensor? attentions = null, - IKVCache? pastKeyValue = null) - { - this.HiddenStates = hiddenStates; - this.Attentions = attentions; - this.PastKeyValue = pastKeyValue; - } - - public Tensor HiddenStates { get; set; } - - public Tensor? Attentions { get; set; } - - public IKVCache? PastKeyValue { get; set; } -} -internal class MistralDecoderLayer : nn.Module, IDynamicLoadModule -{ - private readonly MistralConfig _llamaConfig; - private readonly int _layerIndex; - private readonly int _hiddenSize; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly MistralMLP mlp; - private readonly Core.RMSNorm input_layernorm; - private readonly Core.RMSNorm post_attention_layernorm; - private readonly Attention self_attn; - - public Action? LoadToDeviceFunc { get; set; } - public Action? UnloadFromDeviceFunc { get; set; } - -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public MistralDecoderLayer(MistralConfig config, int layerIndex) - : base(nameof(MistralDecoderLayer)) - { - _llamaConfig = config; - _layerIndex = layerIndex; - _hiddenSize = config.HiddenSize; - - this.self_attn = CreateAttention(config, layerIndex); - this.mlp = new MistralMLP(config); - this.input_layernorm = new Core.RMSNorm(this._hiddenSize, eps: config.RmsNormEps, config.DType); - this.post_attention_layernorm = new Core.RMSNorm(this._hiddenSize, eps: config.RmsNormEps, config.DType); - } - - private Attention CreateAttention(MistralConfig config, int layerIndex) - { - var headDim = config.HiddenSize / config.NumAttentionHeads; - return new Attention( - attentionDropout: config.AttentionDropout, - hiddenSize: config.HiddenSize, - numHeads: config.NumAttentionHeads, - headDim: headDim, - numKeyValueHeads: config.NumKeyValueHeads, - numKeyValueGroups: config.NumAttentionHeads / config.NumKeyValueHeads, - maxPositionEmbeddings: config.MaxPositionEmbeddings, - originalMaxPositionEmbeddings: config.MaxPositionEmbeddings, - layerIdx: layerIndex, - useQkvProj: false, - dtype: config.DType, - attentionBias: config.AttentionBias); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override DecoderLayerOutput forward(DecoderLayerInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (LoadToDeviceFunc != null) - { - LoadToDeviceFunc(this); - } - - using var disposeScope = NewDisposeScope(); - var residual = input.HiddenStates; - var hiddenStates = this.input_layernorm.forward(input.HiddenStates); - - var selfAttnInput = new AttentionInput( - hiddenStates: hiddenStates, - attentionMask: input.AttentionMask, - positionIds: input.PositionIds, - cache: input.PastKeyValue, - positionalEmbeddings: input.PositionalEmbeddings, - outputAttentions: input.OutputAttentions); - - var selfAttnOutput = this.self_attn.forward(selfAttnInput); - - hiddenStates = residual + selfAttnOutput.HiddenStates; - - // Fully connected - residual = hiddenStates; - hiddenStates = this.post_attention_layernorm.forward(hiddenStates); - hiddenStates = this.mlp.forward(hiddenStates); - hiddenStates = residual + hiddenStates; - - if (UnloadFromDeviceFunc != null) - { - UnloadFromDeviceFunc(this); - } - - return new DecoderLayerOutput( - hiddenStates: hiddenStates.MoveToOuterDisposeScope(), - attentions: input.OutputAttentions ? selfAttnOutput.Attentions?.MoveToOuterDisposeScope() : null, - pastKeyValue: selfAttnOutput.Cache); - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralForCausalLM.cs b/src/Microsoft.ML.GenAI.Mistral/MistralForCausalLM.cs deleted file mode 100644 index 18d43e5317..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralForCausalLM.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Diagnostics; -using System.Text.Json; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.GenAI.Mistral.Module; -using TorchSharp; -using TorchSharp.PyBridge; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Mistral; - -public class MistralForCausalLM : nn.Module -{ - private readonly MistralConfig _config; - private readonly int _vocabSize; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly GenAILinear lm_head; - private readonly MistralModel model; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public MistralForCausalLM(MistralConfig config) - : base(nameof(MistralForCausalLM)) - { - _config = config; - _vocabSize = config.VocabSize; - - model = new MistralModel(config); - lm_head = new GenAILinear(config.HiddenSize, config.VocabSize, hasBias: false); - - this.RegisterComponents(); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var outputs = this.model.forward(input); - var logits = this.lm_head.forward(outputs.LastHiddenState); - logits = logits.to_type(ScalarType.Float32); - outputs.Logits = logits; - - return outputs; - } - - public static MistralForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - ScalarType torchDtype = ScalarType.BFloat16, - string device = "cpu") - { - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var model = new MistralForCausalLM(modelConfig); - - model.LoadSafeTensors(modelFolder, checkPointName); - model = model.to(device); - - return model; - } - - public static MistralForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - bool quantizeToInt8 = false, - bool quantizeToInt4 = false, - int layersOnTargetDevice = -1, - ScalarType torchDtype = ScalarType.BFloat16, - string targetDevice = "cuda") - { - if (layersOnTargetDevice == -1 && quantizeToInt4 == false && quantizeToInt8 == false) - { - return FromPretrained(modelFolder, configName, checkPointName, torchDtype, targetDevice); - } - - var originalDefaultDevice = torch.get_default_device(); - torch.set_default_device("meta"); - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var model = new MistralForCausalLM(modelConfig); - - if (quantizeToInt8) - { - model.ToInt8QuantizeModule(); - } - else if (quantizeToInt4) - { - model.ToInt4QuantizeModule(); - } - - var deviceMap = model.InferDeviceMapForEachLayer( - [ - KeyValuePair.Create(targetDevice, layersOnTargetDevice), - KeyValuePair.Create("cpu", -1) - ]); - - torch.set_default_device("cpu"); - model = new MistralForCausalLM(modelConfig); - - model.LoadSafeTensors(modelFolder, checkPointName); - - if (quantizeToInt8) - { - model.ToInt8QuantizeModule(); - } - else if (quantizeToInt4) - { - model.ToInt4QuantizeModule(); - } - - model = model.ToDynamicLoadingModel(deviceMap, targetDevice); - - torch.set_default_device(originalDefaultDevice); - - return model; - } - - public void LoadSafeTensors(string modelFolder, string checkPointName = "model.safetensors.index.json") - { - this.load_checkpoint(path: modelFolder, checkpointName: checkPointName, strict: false, useTqdm: false); - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralMLP.cs b/src/Microsoft.ML.GenAI.Mistral/MistralMLP.cs deleted file mode 100644 index 347ee625ee..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralMLP.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Mistral.Module; -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class MistralMLP : torch.nn.Module -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - private readonly int _intermediateSize; - private readonly int _hiddenSize; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly QuantizedLinear gate_proj; - private readonly QuantizedLinear up_proj; - private readonly QuantizedLinear down_proj; - private readonly torch.nn.Module act_fn; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public MistralMLP(MistralConfig config) - : base(nameof(MistralMLP)) - { - this._hiddenSize = config.HiddenSize; - this._intermediateSize = config.IntermediateSize; - var hiddenAct = config.HiddenAct; - this.gate_proj = new QuantizedLinear(this._hiddenSize, this._intermediateSize, hasBias: false, dtype: config.DType); - this.up_proj = new QuantizedLinear(this._hiddenSize, this._intermediateSize, hasBias: false, dtype: config.DType); - this.down_proj = new QuantizedLinear(this._intermediateSize, this._hiddenSize, hasBias: false, dtype: config.DType); - this.RegisterComponents(); - this.act_fn = Core.Utils.GetActivation(hiddenAct); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - using var input1 = this.gate_proj.forward(input); - using var input2 = this.act_fn.forward(input1); - using var input3 = input2 * this.up_proj.forward(input); - return this.down_proj.forward(input3); - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralModel.cs b/src/Microsoft.ML.GenAI.Mistral/MistralModel.cs deleted file mode 100644 index cab7e6cc5a..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralModel.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Mistral.Module; - -public class MistralModel : nn.Module -{ - private readonly MistralConfig _config; - private readonly int? _paddingIdx; - private readonly int _vocabSize; - private IKVCache _cache; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Embedding embed_tokens; - private readonly ModuleList layers; - private readonly RMSNorm norm; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly nn.Module _rotaryEmb; - - - public MistralModel(MistralConfig config) - : base(nameof(MistralModel)) - { - this._config = config; - this._paddingIdx = config.PadTokenId; - this._vocabSize = config.VocabSize; - var headDim = config.HeadDim; - this.embed_tokens = nn.Embedding(config.VocabSize, config.HiddenSize, padding_idx: this._paddingIdx, dtype: config.DType); - this.layers = new ModuleList(); - - for (int i = 0; i < config.NumHiddenLayers; i++) - { - this.layers.Add(new MistralDecoderLayer(config, i)); - } - this.norm = new RMSNorm(config.HiddenSize, config.RmsNormEps, config.DType); - this._cache = new DynamicKVCache(); - this.RegisterComponents(); - this._rotaryEmb = config.RopeScaling switch - { - null => new RotaryEmbedding(config.RopeTheta, config.MaxPositionEmbeddings, headDim), - _ => new RotaryEmbedding(config.RopeTheta, headDim, config.RopeScaling), - }; - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (input.OverrideCache is not null) - { - this._cache = input.OverrideCache; - } - - var outputAttentions = input.OutputAttentions; - var outputHiddenStates = input.OutputHiddenStates; - var attentionMask = input.AttentionMask; - Device device; - var inputIds = input.InputIds; - var positionIds = input.PositionIds; - var inputsEmbeds = input.InputEmbeddings; - int batchSize; - int seqLength; - if (inputIds is not null && inputsEmbeds is not null) - { - throw new ArgumentException("Only one of input_ids or inputs_embeds may be set"); - } - else if (inputIds is not null) - { - batchSize = inputIds.IntShape()[0]; - seqLength = inputIds.IntShape()[1]; - inputsEmbeds = this.embed_tokens.forward(inputIds); - device = inputIds.device; - } - else if (inputsEmbeds is not null) - { - batchSize = inputsEmbeds.IntShape()[0]; - seqLength = inputsEmbeds.IntShape()[1]; - device = inputsEmbeds.device; - } - else - { - throw new ArgumentException("Either input_ids or inputs_embeds must be set"); - } - - var pastKeyValuesLength = input.PastKeyValuesLength; - - if (positionIds is null) - { - positionIds = torch.arange(pastKeyValuesLength, seqLength + pastKeyValuesLength, device: device); - positionIds = positionIds.unsqueeze(0).view(-1, seqLength); - } - else - { - positionIds = ((long)positionIds.view(-1, seqLength)); - } - - if (this._config.AttnImplementation == "flash_attention_2") - { - throw new NotImplementedException(); - } - else - { - // the following behavior of creating 4d causal mask doesn't match python's, remember to look into it when there's time. - attentionMask = AttentionMaskConverter.Create4DCausalAttentionMask(attentionMask, [batchSize, seqLength], inputsEmbeds.dtype, device, pastKeyValuesLength, slidingWindow: _config.SlidingWindow); - } - - var hiddenStates = inputsEmbeds; - - var allHiddenStates = new List(); - var allAttentions = new List(); - - var embOutput = this._rotaryEmb.forward(new RotaryEmbeddingInput(hiddenStates, positionIds, pastKeyValuesLength)); - foreach (var layer in this.layers) - { - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - - var decoderInput = new DecoderLayerInput( - hiddenStates: hiddenStates, - attentionMask: attentionMask!, - positionIds: positionIds, - pastKeyValue: this._cache, - positionEmbeddings: embOutput, - outputAttentions: outputAttentions); - var layerOutput = layer.forward(decoderInput); - hiddenStates = layerOutput.HiddenStates; - if (outputAttentions && layerOutput.Attentions is not null) - { - allAttentions.Add(layerOutput.Attentions); - } - } - - hiddenStates = this.norm.forward(hiddenStates); - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - - return new CausalLMModelOutput(lastHiddenState: hiddenStates, allHiddenStates: allHiddenStates.ToArray(), attentions: allAttentions.ToArray(), cache: this._cache); - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/MistralTokenizerHelper.cs b/src/Microsoft.ML.GenAI.Mistral/MistralTokenizerHelper.cs deleted file mode 100644 index 3ed9a79780..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/MistralTokenizerHelper.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.Mistral; - -public class MistralTokenizerHelper -{ - private const string UnknownSymbol = ""; - private const int UnknownSymbolId = 0; - private const string StartSymbol = ""; - private const int StartSymbolId = 1; - private const string EndSymbol = ""; - private const int EndSymbolId = 2; - private const string StartInstructionSymbol = "[INST]"; - private const int StartInstructionSymbolId = 3; - private const string EndInstructionSymbol = "[/INST]"; - private const int EndInstructionSymbolId = 4; - private const string ToolCallSymbol = "[TOOL_CALLS]"; - private const int ToolCallSymbolId = 5; - private const string StartAvailableToolsSymbol = "[AVAILABLE_TOOLS]"; - private const int StartAvailableToolsSymbolId = 6; - private const string EndAvailableToolsSymbol = "[/AVAILABLE_TOOLS]"; - private const int EndAvailableToolsSymbolId = 7; - private const string StartToolResultSymbol = "[TOOL_RESULTS]"; - private const int StartToolResultSymbolId = 8; - private const string EndToolResultSymbol = "[/TOOL_RESULTS]"; - private const int EndToolResultSymbolId = 9; - - public static LlamaTokenizer FromPretrained( - string modelWeightFolder, - string modelName = "tokenizer.model.v3", - string unknownSymbol = UnknownSymbol, - int unknownSymbolId = 0, - string startSymbol = StartSymbol, - int startSymbolId = 1, - string endSymbol = EndSymbol, - int endSymbolId = 2, - string startInstructionSymbol = StartInstructionSymbol, - int startInstructionSymbolId = 3, - string endInstructionSymbol = EndInstructionSymbol, - int endInstructionSymbolId = 4, - string toolCallSymbol = ToolCallSymbol, - int toolCallSymbolId = 5, - string startAvailableToolsSymbol = StartAvailableToolsSymbol, - int startAvailableToolsSymbolId = 6, - string endAvailableToolsSymbol = EndAvailableToolsSymbol, - int endAvailableToolsSymbolId = 7, - string startToolResultSymbol = StartToolResultSymbol, - int startToolResultSymbolId = 8, - string endToolResultSymbol = EndToolResultSymbol, - int endToolResultSymbolId = 9, - bool addPrecedingSpace = true, - Dictionary? additionalSpecialTokens = null) - { - var specialTokens = new Dictionary - { - { startSymbol, startSymbolId }, - { endSymbol, endSymbolId }, - { startInstructionSymbol, startInstructionSymbolId }, - { endInstructionSymbol, endInstructionSymbolId }, - { toolCallSymbol, toolCallSymbolId }, - { startAvailableToolsSymbol, startAvailableToolsSymbolId }, - { endAvailableToolsSymbol, endAvailableToolsSymbolId }, - { startToolResultSymbol, startToolResultSymbolId }, - { endToolResultSymbol, endToolResultSymbolId } - }; - - if (additionalSpecialTokens != null) - { - foreach (var (key, value) in additionalSpecialTokens) - { - specialTokens[key] = value; - } - } - - return FromPretrained( - modelWeightFolder, - modelName, - specialTokens, - addPrecedingSpace); - } - - public static LlamaTokenizer FromPretrained( - string modelWeightFolder, - string modelName, - Dictionary specialTokens, - bool addPrecedingSpace = true) - { - var modelPath = Path.Combine(modelWeightFolder, modelName); - var modelStream = File.OpenRead(modelPath); - - var llamaTokenizer = LlamaTokenizer.Create( - modelStream, - addPrecedingSpace, - specialTokens: specialTokens); - - return llamaTokenizer; - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/Mistral_7B_0_3ChatTemplateBuilder.cs b/src/Microsoft.ML.GenAI.Mistral/Mistral_7B_0_3ChatTemplateBuilder.cs deleted file mode 100644 index 8852f62da9..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/Mistral_7B_0_3ChatTemplateBuilder.cs +++ /dev/null @@ -1,202 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using AutoGen.Core; -using Json.Schema; -using Json.Schema.Generation; -using Microsoft.ML.GenAI.Core; -using Microsoft.SemanticKernel.ChatCompletion; - -namespace Microsoft.ML.GenAI.Mistral; - -/// -/// the chat template builder for Mistral 7B v0.3 -/// -#pragma warning disable MSML_GeneralName // This name should be PascalCased -public class Mistral_7B_0_3ChatTemplateBuilder : IChatTemplateBuilder -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ - private const string Newline = "\r\n"; - - public static Mistral_7B_0_3ChatTemplateBuilder Instance { get; } = new Mistral_7B_0_3ChatTemplateBuilder(); - - public string BuildPrompt(IEnumerable messages, IEnumerable? tools = null) - { - // can only contain at most one system message - if (messages.Where(m => m.GetRole() == Role.System).Count() > 1) - { - throw new InvalidOperationException("Please provide at most one system message."); - } - - var systemMessage = messages.FirstOrDefault(m => m.GetRole() == Role.System)?.GetContent(); - - // split the messages into two sequences by the last user message - // e.g [user, assistant, user, assistant, user] -> [[user, assistant, user, assistant], [user]] - - var firstSequence = messages.Take(messages.ToList().FindLastIndex(m => m.GetRole() == Role.User)); - var secondSequence = messages.Skip(messages.ToList().FindLastIndex(m => m.GetRole() == Role.User)); - - var sb = new StringBuilder(); - foreach (var message in firstSequence) - { - // skip system - if (message.GetRole() == Role.System) - { - continue; - } - - var content = message.GetContent()!; - sb.Append(message switch - { - ToolCallMessage toolCallMessage => BuildFromToolCallMessage(toolCallMessage), - ToolCallResultMessage toolCallResultMessage => BuildFromToolCallResultMessage(toolCallResultMessage), - ToolCallAggregateMessage toolCallAggregateMessage => BuildFromAggregrateToolCallMessage(toolCallAggregateMessage), - TextMessage when message.GetRole() == Role.User => $"[INST]{content.Trim()}[/INST]", - TextMessage when message.GetRole() == Role.Assistant => $"{content.Trim()}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - // insert [AVAILABLE TOOLS] section if tools are provided - if (tools?.Any() == true) - { - var schemas = tools.Select(t => new - { - type = "function", - function = new - { - name = t.Name, - description = t.Description, - parameters = BuildJsonSchemaFromFunctionContract(t) - } - }); - var schemaPrompt = JsonSerializer.Serialize(schemas); - - // add a space after the colon in json string so mistral can correctly generate the stop token after [TOOL_CALLS] symbol. - // This is probably because in the training data, all the tool call samples are separated by a space after the colon. - // e.g. [AVAILABLE_TOOLS][{"type": "function", "function": {.... - // instead of [AVAILABLE_TOOLS][{"type":"function","function":{.... - // Therefore when inferencing, we need to add a space after the colon in the json string to match with the training data. - schemaPrompt = schemaPrompt.Replace(":", ": "); - schemaPrompt = schemaPrompt.Replace(",", ", "); - sb.Append($"[AVAILABLE_TOOLS]{schemaPrompt}[/AVAILABLE_TOOLS]"); - } - - foreach (var message in secondSequence) - { - var content = message.GetContent()!; - sb.Append(message switch - { - ToolCallMessage toolCallMessage => BuildFromToolCallMessage(toolCallMessage), - ToolCallResultMessage toolCallResultMessage => BuildFromToolCallResultMessage(toolCallResultMessage), - ToolCallAggregateMessage toolCallAggregateMessage => BuildFromAggregrateToolCallMessage(toolCallAggregateMessage), - TextMessage when message.GetRole() == Role.User && !string.IsNullOrEmpty(systemMessage) => $"[INST]{systemMessage}{Newline}{Newline}{content.Trim()}[/INST]", - TextMessage when message.GetRole() == Role.User => $"[INST]{content.Trim()}[/INST]", - TextMessage when message.GetRole() == Role.Assistant => $"{content.Trim()}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - return sb.ToString(); - } - - public string BuildPrompt(ChatHistory chatHistory) - { - throw new NotImplementedException(); - } - - private string BuildFromToolCallMessage(ToolCallMessage message) - { - var toolCalls = message.ToolCalls; - if (toolCalls.Count() == 0) - { - return string.Empty; - } - else - { - var toolCallObjects = toolCalls.Select(tc => - new - { - name = tc.FunctionName, - arguments = JsonObject.Parse(tc.FunctionArguments), - id = tc.ToolCallId, - } - ); - - var toolCallJson = JsonSerializer.Serialize(toolCallObjects); - return $"[TOOL_CALLS]{toolCallJson}"; - } - } - - private string BuildFromToolCallResultMessage(ToolCallResultMessage message) - { - var toolCallResults = message.ToolCalls; - if (toolCallResults.Count() == 0) - { - return string.Empty; - } - else - { - var toolCallResultObjects = toolCallResults.Select(tc => - new - { - id = tc.ToolCallId, - content = tc.Result, - } - ); - - var toolCallResultJson = JsonSerializer.Serialize(toolCallResultObjects); - return $"[TOOL_RESULTS]{toolCallResultJson}[/TOOL_RESULTS]"; - } - } - - private string BuildFromAggregrateToolCallMessage(ToolCallAggregateMessage message) - { - var toolCallMessage = message.Message1; - var toolCallResultMessage = message.Message2; - - var toolCall = BuildFromToolCallMessage(toolCallMessage); - var toolCallResult = BuildFromToolCallResultMessage(toolCallResultMessage); - - return $"{toolCall}{toolCallResult}"; - } - - private JsonSchema BuildJsonSchemaFromFunctionContract(FunctionContract contract) - { - var requiredParameterNames = new List(); - var propertiesSchemas = new Dictionary(); - var propertySchemaBuilder = new JsonSchemaBuilder().Type(SchemaValueType.Object); - foreach (var param in contract.Parameters ?? []) - { - if (param.Name is null) - { - throw new InvalidOperationException("Parameter name cannot be null"); - } - - var schemaBuilder = new JsonSchemaBuilder().FromType(param.ParameterType ?? throw new ArgumentNullException(nameof(param.ParameterType))); - if (param.Description != null) - { - schemaBuilder = schemaBuilder.Description(param.Description); - } - - if (param.IsRequired) - { - requiredParameterNames.Add(param.Name); - } - - var schema = schemaBuilder.Build(); - propertiesSchemas[param.Name] = schema; - - } - propertySchemaBuilder = propertySchemaBuilder.Properties(propertiesSchemas); - propertySchemaBuilder = propertySchemaBuilder.Required(requiredParameterNames); - - var jsonSchema = propertySchemaBuilder.Build(); - - return jsonSchema; - } -} diff --git a/src/Microsoft.ML.GenAI.Mistral/Resource/Config/mistral-7B-instruct-v0.3.json b/src/Microsoft.ML.GenAI.Mistral/Resource/Config/mistral-7B-instruct-v0.3.json deleted file mode 100644 index 1da2dde41f..0000000000 --- a/src/Microsoft.ML.GenAI.Mistral/Resource/Config/mistral-7B-instruct-v0.3.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "attention_dropout": 0.0, - "bos_token_id": 1, - "eos_token_id": 2, - "hidden_act": "silu", - "hidden_size": 4096, - "initializer_range": 0.02, - "intermediate_size": 14336, - "max_position_embeddings": 32768, - "model_type": "mistral", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": null, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "vocab_size": 32768 -} diff --git a/src/Microsoft.ML.GenAI.Phi/Extension/SemanticKernelExtension.cs b/src/Microsoft.ML.GenAI.Phi/Extension/SemanticKernelExtension.cs deleted file mode 100644 index 82b4b759c5..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Extension/SemanticKernelExtension.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using Microsoft.SemanticKernel.TextGeneration; - -namespace Microsoft.ML.GenAI.Phi.Extension; - -public static class SemanticKernelExtension -{ - public static IKernelBuilder AddGenAIChatCompletion( - this IKernelBuilder builder, - ICausalLMPipeline pipeline) - { - builder.Services.AddSingleton(new Phi3CausalLMChatCompletionService(pipeline)); - - return builder; - } - - public static IKernelBuilder AddGenAITextGeneration( - this IKernelBuilder builder, - ICausalLMPipeline pipeline) - { - builder.Services.AddSingleton(new Phi3CausalLMTextGenerationService(pipeline)); - - return builder; - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Microsoft.ML.GenAI.Phi.csproj b/src/Microsoft.ML.GenAI.Phi/Microsoft.ML.GenAI.Phi.csproj deleted file mode 100644 index 68021be630..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Microsoft.ML.GenAI.Phi.csproj +++ /dev/null @@ -1,32 +0,0 @@ - - - - net8.0 - enable - enable - true - - - - true - - - - - - - - - - - - - - - - - - - - - diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi2Attention.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi2Attention.cs deleted file mode 100644 index fe0021980f..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi2Attention.cs +++ /dev/null @@ -1,160 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Diagnostics.Contracts; -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi2Attention : nn.Module< - Tensor, // hidden_states - Tensor, // position_ids - Tensor?, // attention_mask - int, // past_key_value_length - bool, // output_attentions - ( - Tensor, // hidden_states, - Tensor?, // attentions, - Tensor? // present_key_value - )> -{ - private readonly int? _layerIdx; - private readonly Phi2Config _config; - private readonly double _attentionDropout; - private readonly int _hiddenSize; - private readonly int _numAttentionHeads; - private readonly int _headDim; - private readonly int _numKeyValueHeads; - private readonly int _numKeyValueGroups; - private readonly int _maxPositionEmbeddings; - private readonly double _ropeTheta; - private readonly double _partialRotaryFactor; - private readonly bool _qkLayernorm; - - // we disable the warning for the private field name not in _camelCase format for all submodules fields - // because their name will be used as keys to load the corresponding weights from the checkpoint -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly GenAILinear q_proj; - private readonly GenAILinear k_proj; - private readonly GenAILinear v_proj; - private readonly GenAILinear dense; - private readonly LayerNorm? q_layernorm; - private readonly LayerNorm? k_layernorm; - - private readonly Phi2RotaryEmbedding phiRotaryEmbedding; - - // cache_k, cache_v - private Tensor cache_k; - private Tensor cache_v; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi2Attention(Phi2Config config, int? layerIdx = null, int maxBatch = 2, int maxLength = 1024) - : base(nameof(Phi2Attention)) - { - this._layerIdx = layerIdx; - this._config = config; - this._attentionDropout = config.AttentionDropout; - this._hiddenSize = config.HiddenSize; - this._numAttentionHeads = config.NumAttentionHeads; - this._headDim = this._hiddenSize / this._numAttentionHeads; - this._numKeyValueHeads = config.NumKeyValueHeads ?? throw new ArgumentException("num_key_value_heads must be specified"); - this._numKeyValueGroups = this._numAttentionHeads / this._numKeyValueHeads; - this._maxPositionEmbeddings = config.MaxPositionEmbeddings; - this._ropeTheta = config.RopeTheta; - this._partialRotaryFactor = config.PartialRotaryFactor; - - Contract.Assert(this._hiddenSize % (this._headDim * this._numAttentionHeads) == 0, "hidden_size must be divisible by num_attention_heads"); - this.q_proj = new GenAILinear(this._hiddenSize, this._numAttentionHeads * this._headDim, hasBias: true, dtype: config.Dtype); - this.k_proj = new GenAILinear(this._hiddenSize, this._numKeyValueHeads * this._headDim, hasBias: true, dtype: config.Dtype); - this.v_proj = new GenAILinear(this._hiddenSize, this._numKeyValueHeads * this._headDim, hasBias: true, dtype: config.Dtype); - this.dense = new GenAILinear(this._numAttentionHeads * this._headDim, this._hiddenSize, hasBias: true, dtype: config.Dtype); - - this._qkLayernorm = config.QkLayernorm; - if (this._qkLayernorm) - { - this.q_layernorm = nn.LayerNorm(this._hiddenSize / this._numAttentionHeads, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype); - this.k_layernorm = nn.LayerNorm(this._hiddenSize / this._numAttentionHeads, eps: config.LayerNormEps, elementwise_affine: true, dtype: config.Dtype); - } - - this.RegisterComponents(); - this.phiRotaryEmbedding = new Phi2RotaryEmbedding( - dim: (int)(this._partialRotaryFactor * this._headDim), - maxPositionEmbeddings: this._maxPositionEmbeddings, - baseValue: this._config.RopeTheta); - this.cache_k = torch.zeros(maxBatch, this._numKeyValueHeads, maxLength, this._headDim, dtype: config.Dtype); - this.cache_v = torch.zeros(maxBatch, this._numKeyValueHeads, maxLength, this._headDim, dtype: config.Dtype); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override (Tensor, Tensor?, Tensor?) forward( -#pragma warning restore MSML_GeneralName // This name should be PascalCased - Tensor hiddenStates, - Tensor positionIds, - Tensor? attentionMask = null, - int pastKeyValueLength = 0, - bool outputAttentions = false) - { - // move cache to the same device as hiddenStates - if (this.cache_k.device != hiddenStates.device) - { - this.cache_k = this.cache_k.to(hiddenStates.device, disposeAfter: true).DetachFromDisposeScope(); - this.cache_v = this.cache_v.to(hiddenStates.device, disposeAfter: true).DetachFromDisposeScope(); - } - - using var disposeScope = torch.NewDisposeScope(); - var batchSize = (int)hiddenStates.shape[0]; - var seqLen = (int)hiddenStates.shape[1]; - - var queryStates = this.q_proj.forward(hiddenStates); - var keyStates = this.k_proj.forward(hiddenStates); - var valueStates = this.v_proj.forward(hiddenStates); - if (this._qkLayernorm) - { - queryStates = this.q_layernorm!.forward(queryStates); - keyStates = this.k_layernorm!.forward(keyStates); - } - - queryStates = queryStates.view(batchSize, seqLen, this._numAttentionHeads, this._headDim).transpose_(1, 2); - keyStates = keyStates.view(batchSize, seqLen, this._numKeyValueHeads, this._headDim).transpose_(1, 2); - valueStates = valueStates.view(batchSize, seqLen, this._numKeyValueHeads, this._headDim).transpose_(1, 2); - var kvSeqLen = pastKeyValueLength == 0 ? (int)keyStates.shape[2] : pastKeyValueLength + (int)keyStates.shape[2]; - (var cos, var sin) = this.phiRotaryEmbedding.forward(valueStates, kvSeqLen); - // split the last dim of queryStates and keyStates into rotary and non-rotary parts - // shape: [batch_size, num_heads, seq_len, head_dim] - // queryRot: [batch_size, num_heads, seq_len, :head_dim * partial_rotary_factor] - // queryPass: [batch_size, num_heads, seq_len, head_dim * partial_rotary_factor:] - var keyRot = keyStates[.., .., .., ..this.phiRotaryEmbedding.Dim]; - var keyPass = keyStates[.., .., .., this.phiRotaryEmbedding.Dim..]; - var queryRot = queryStates[.., .., .., ..this.phiRotaryEmbedding.Dim]; - var queryPass = queryStates[.., .., .., this.phiRotaryEmbedding.Dim..]; - (var qRot, var kRot) = Utils.ApplyRotaryPosEmb(queryRot, keyRot, cos, sin, positionIds); - - queryStates = torch.cat([qRot, queryPass], dim: -1); - // update cache - keyStates = torch.cat([kRot, keyPass], dim: -1); - this.cache_k[..batchSize, .., pastKeyValueLength..kvSeqLen, ..] = keyStates; - this.cache_v[..batchSize, .., pastKeyValueLength..kvSeqLen, ..] = valueStates; - keyStates = this.cache_k[..batchSize, .., ..kvSeqLen, ..]; - valueStates = this.cache_v[..batchSize, .., ..kvSeqLen, ..]; - var keyStates2 = Utils.Phi2RepeatKV(keyStates, this._numKeyValueGroups).transpose(2, 3); - var valueStates2 = Utils.Phi2RepeatKV(valueStates, this._numKeyValueGroups); - // Queries and keys upcast to fp32 is required by Phi-2 to avoid overflow - var attnWeights = torch.matmul(queryStates.to_type(float32), keyStates2.to_type(float32)); - attnWeights = attnWeights / Math.Sqrt(this._headDim); - if (attentionMask is not null) - { - attnWeights = attnWeights + attentionMask; - } - attnWeights = nn.functional.softmax(attnWeights, dim: -1); - attnWeights = nn.functional.dropout(attnWeights, p: this._attentionDropout); - var attnOutput = torch.matmul(attnWeights, valueStates2.to_type(float32)).to_type(hiddenStates.dtype); - attnOutput = attnOutput.transpose_(1, 2).contiguous(); - attnOutput = attnOutput.reshape(batchSize, seqLen, this._hiddenSize); - var result = this.dense.forward(attnOutput); - return (result.MoveToOuterDisposeScope(), null, null); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi2DecoderLayer.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi2DecoderLayer.cs deleted file mode 100644 index 2ffae701de..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi2DecoderLayer.cs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -public class Phi2DecoderLayer : nn.Module< - Tensor, // hidden_states - Tensor, // position_ids - Tensor?, // attention_mask - int, // past_key_value_length - bool, // use_cache - bool, // output_attentions - ( - Tensor, // hidden_states, - Tensor?, // attentions, - Tensor? // present_key_value - )> -{ - private readonly int? _layerIdx; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Phi2Attention self_attn; - private readonly Phi2MLP mlp; - private readonly LayerNorm input_layernorm; - private readonly Dropout resid_dropout; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi2DecoderLayer(Phi2Config config, int? layerIdx = null) - : base(nameof(Phi2DecoderLayer)) - { - this._layerIdx = layerIdx; - this.self_attn = new Phi2Attention(config, layerIdx); - this.mlp = new Phi2MLP(config); - this.input_layernorm = nn.LayerNorm(config.HiddenSize, eps: config.LayerNormEps, dtype: config.Dtype); - this.resid_dropout = nn.Dropout(config.ResidPdrop); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override (Tensor, Tensor?, Tensor?) forward( -#pragma warning restore MSML_GeneralName // This name should be PascalCased - Tensor hiddenStates, - Tensor positionIds, - Tensor? attentionMask = null, - int pastKeyValueLength = 0, - bool useCache = false, - bool outputAttentions = false) - { - using var disposiableScope = torch.NewDisposeScope(); - var residual = hiddenStates; - hiddenStates = this.input_layernorm.forward(hiddenStates); - (var attnOutput, var attnWeights, var presentKeyValue) = this.self_attn.forward( - hiddenStates: hiddenStates, - positionIds: positionIds, - attentionMask: attentionMask, - pastKeyValueLength: pastKeyValueLength, - outputAttentions: outputAttentions); - var feedForwardHiddenStates = this.mlp.forward(hiddenStates); - hiddenStates = residual + feedForwardHiddenStates + attnOutput; - - return (hiddenStates.MoveToOuterDisposeScope(), null, null); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi2MLP.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi2MLP.cs deleted file mode 100644 index 42bd892588..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi2MLP.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class Phi2MLP : torch.nn.Module -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly GenAILinear fc1; - private readonly GenAILinear fc2; - private readonly torch.nn.Module activation_fn; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi2MLP(Phi2Config config) - : base(nameof(Phi2MLP)) - { - this.fc1 = new GenAILinear(config.HiddenSize, config.IntermediateSize, dtype: config.Dtype); - this.fc2 = new GenAILinear(config.IntermediateSize, config.HiddenSize, dtype: config.Dtype); - this.activation_fn = new NewGELUActivation(); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - using var input1 = this.fc1.forward(input); - using var input2 = this.activation_fn.forward(input1); - return this.fc2.forward(input2); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi2Model.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi2Model.cs deleted file mode 100644 index 05edab0985..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi2Model.cs +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Diagnostics.Contracts; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi2Model : nn.Module< - Tensor, // input_ids - Tensor?, // attention_mask - int, // past_key_value_length - Tensor?, // position_ids - Tensor?, //input embeddings - ( - bool, // use_cache - bool, // output_attentions - bool // output_hidden_states - ), - ( - Tensor, // hidden_states, - Tensor?, // attentions, - Tensor? // present_key_value - )> -{ - private readonly Phi2Config _config; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Embedding embed_tokens; - private readonly Dropout embed_dropout; - private readonly LayerNorm final_layernorm; - private readonly ModuleList layers; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi2Model(Phi2Config config) - : base(nameof(Phi2Model)) - { - this._config = config; - this.embed_tokens = nn.Embedding(config.VocabSize, config.HiddenSize, dtype: config.Dtype); - this.embed_dropout = nn.Dropout(config.EmbdPdrop); - this.final_layernorm = nn.LayerNorm(config.HiddenSize, eps: config.LayerNormEps, dtype: config.Dtype); - this.layers = new ModuleList(Enumerable.Range(0, config.NumHiddenLayers).Select(i => new Phi2DecoderLayer(config)).ToArray()); - this.RegisterComponents(); - } - - public Phi2Config Config => this._config; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override (Tensor, Tensor?, Tensor?) forward( -#pragma warning restore MSML_GeneralName // This name should be PascalCased - Tensor inputIds, - Tensor? attentionMask = null, - int pastKeyValueLength = 0, - Tensor? positionIds = null, - Tensor? inputEmbeddings = null, - (bool, bool, bool) options = default) // use_cache, output_attentions, output_hidden_states - { - (var outputAttentions, var outputHiddenStates, var useCache) = options; - - // TODO - // add support for inputEmbeddings - if (inputEmbeddings is not null) - { - throw new NotImplementedException("inputEmbeddings is not supported"); - } - inputEmbeddings = this.embed_tokens.forward(inputIds); - inputEmbeddings = this.embed_dropout.forward(inputEmbeddings); - var batchSize = inputIds.shape[0]; - var seqLen = (int)inputIds.shape[1]; - - if (positionIds is null) - { - positionIds = torch.arange(pastKeyValueLength, seqLen + pastKeyValueLength, dtype: inputIds.dtype, device: inputIds.device); - positionIds = positionIds.unsqueeze(0); - } - - // attention - // use 4d attention mask - if (attentionMask is not null) - { - attentionMask = this.Prepare4DCausalAttentionMask(attentionMask, seqLen, pastKeyValueLength, inputEmbeddings.dtype); - } - - var hiddenStates = inputEmbeddings; - - for (int i = 0; i < this.layers.Count; i++) - { - (hiddenStates, _, _) = this.layers[i].forward( - hiddenStates: hiddenStates, - positionIds: positionIds, - attentionMask: attentionMask, - pastKeyValueLength: pastKeyValueLength, - useCache: useCache, - outputAttentions: outputAttentions); - } - - hiddenStates = this.final_layernorm.forward(hiddenStates); - return (hiddenStates, null, null); - } - - private Tensor Prepare4DCausalAttentionMask( - Tensor attentionMask, - int queryLength, - int pastKeyValueLength, - ScalarType dtype) - { - var batchSize = (int)attentionMask.shape[0]; - var seqLen = attentionMask.shape[1]; - Contract.Assert(seqLen == queryLength, "seqLen must be equal to queryLength"); - var targetLength = queryLength + pastKeyValueLength; - var causal4DMask = this.MakeCausalAttentionMask(batchSize, queryLength, pastKeyValueLength, attentionMask.device, dtype); - var expandedMask = this.ExpandMask(attentionMask, dtype, queryLength).to(attentionMask.device); - - causal4DMask.masked_fill_(expandedMask.to_type(ScalarType.Bool), torch.finfo(dtype).min); - return causal4DMask; - } - - private Tensor ExpandMask( - Tensor mask, - ScalarType dtype, - int targetLength) - { - var batch = mask.shape[0]; - var seqLen = mask.shape[1]; - var expandedMask = mask.unsqueeze(1).unsqueeze(2); - expandedMask = expandedMask.expand(new long[] { batch, 1, targetLength, seqLen }); - expandedMask = expandedMask.to_type(dtype); - - var invertedMask = (1.0f - expandedMask) > 0; - - return invertedMask.masked_fill(invertedMask.to_type(ScalarType.Bool), torch.finfo(dtype).min); - } - private Tensor MakeCausalAttentionMask( - int batchSize, - int targetLen, - int pastKeyValueLength, - Device device, - ScalarType dtype) - { - var mask = torch.full([targetLen, targetLen], torch.finfo(dtype).min, dtype: dtype, device: device); - var maskCond = torch.arange(mask.size(-1), device: device); - mask.masked_fill_(maskCond < (maskCond + 1).view(mask.size(-1), 1), 0.0f); - - mask = mask.to_type(dtype); - - if (pastKeyValueLength > 0) - { - mask = torch.cat([torch.zeros([targetLen, pastKeyValueLength], dtype: dtype, device: device), mask], dim: -1); - } - - mask = mask.unsqueeze(0).unsqueeze(0); - mask = mask.expand(new long[] { batchSize, 1, targetLen, targetLen + pastKeyValueLength }); - - return mask; - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi2RotaryEmbedding.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi2RotaryEmbedding.cs deleted file mode 100644 index 6fd70dd9bd..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi2RotaryEmbedding.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi2RotaryEmbedding : nn.Module< - Tensor, // input - int, // seq_len - ( - Tensor, // cos - Tensor // sin - )> -{ - private readonly double _base; - private readonly int _maxPositionEmbeddings; - private readonly int _dim; - - public Phi2RotaryEmbedding(double baseValue, int maxPositionEmbeddings, int dim) - : base(nameof(Phi2RotaryEmbedding)) - { - _base = baseValue; - _maxPositionEmbeddings = maxPositionEmbeddings; - _dim = dim; - var thetaNumerator = torch.arange(0, _dim, 2, dtype: ScalarType.Int64).to(torch.float32); - this.register_buffer("inv_freq", torch.pow(baseValue, -1.0f * (thetaNumerator / dim)), persistent: false); - } - - public int Dim => _dim; - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override (Tensor, Tensor) forward(Tensor x, int seqLen) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - // TODO - // can be calculated once and cached - var invFreq = this.get_buffer("inv_freq").to(x.device); - var t = torch.arange(seqLen, dtype: invFreq.dtype, device: invFreq.device); - var freqs = torch.outer(t, invFreq).to(torch.float32); - var emb = torch.cat([freqs, freqs], dim: -1); - - var cos = torch.cos(emb); - var sin = torch.sin(emb); - - return (cos[..seqLen].to_type(x.dtype), sin[..seqLen].to_type(x.dtype)); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi3DecoderLayer.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi3DecoderLayer.cs deleted file mode 100644 index 35b9313b33..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi3DecoderLayer.cs +++ /dev/null @@ -1,157 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi3DecoderLayerInput -{ - public Phi3DecoderLayerInput( - Tensor hiddenStates, - Tensor attentionMask, - Tensor positionIds, - RotaryEmbeddingOutput positionalEmbeddings, // cos, sin - IKVCache? pastKeyValue = null, - bool outputAttentions = false) - { - this.HiddenStates = hiddenStates; - this.AttentionMask = attentionMask; - this.PositionIds = positionIds; - this.PastKeyValue = pastKeyValue; - this.PositionalEmbeddings = positionalEmbeddings; - this.OutputAttentions = outputAttentions; - } - - public Tensor HiddenStates { get; set; } - - public Tensor AttentionMask { get; set; } - - public Tensor PositionIds { get; set; } - - public RotaryEmbeddingOutput PositionalEmbeddings { get; set; } // cos, sin - - public IKVCache? PastKeyValue { get; set; } - - public bool OutputAttentions { get; set; } -} - -internal class Phi3DecoderLayerOutput -{ - public Phi3DecoderLayerOutput( - Tensor hiddenStates, - Tensor? attentions = null, - IKVCache? pastKeyValue = null) - { - this.HiddenStates = hiddenStates; - this.Attentions = attentions; - this.PastKeyValue = pastKeyValue; - } - - public Tensor HiddenStates { get; set; } - - public Tensor? Attentions { get; set; } - - public IKVCache? PastKeyValue { get; set; } -} - -internal class Phi3DecoderLayer : nn.Module, IDynamicLoadModule -{ - private readonly Phi3Config _config; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly nn.Module self_attn; - private readonly Phi3MLP mlp; - private readonly RMSNorm input_layernorm; - private readonly Dropout resid_attn_dropout; - private readonly Dropout resid_mlp_dropout; - private readonly RMSNorm post_attention_layernorm; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi3DecoderLayer(Phi3Config config, int layerIdx) - : base(nameof(Phi3DecoderLayer)) - { - this._config = config; - if (config.AttnImplementation == "eager") - { - this.self_attn = this.CreateAttentionFromConfig(config, layerIdx); - } - else - { - throw new NotImplementedException(); - } - - this.mlp = new Phi3MLP(config); - this.input_layernorm = new RMSNorm(config.HiddenSize, config.RmsNormEps, config.DType); - - this.resid_attn_dropout = nn.Dropout(config.ResidPdrop); - this.resid_mlp_dropout = nn.Dropout(config.ResidPdrop); - this.post_attention_layernorm = new RMSNorm(config.HiddenSize, config.RmsNormEps, config.DType); - } - - public Action? LoadToDeviceFunc { get; set; } - - public Action? UnloadFromDeviceFunc { get; set; } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Phi3DecoderLayerOutput forward(Phi3DecoderLayerInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (LoadToDeviceFunc != null) - { - LoadToDeviceFunc(this); - } - using var disposeScope = NewDisposeScope(); - var hiddenStates = input.HiddenStates; - var residual = input.HiddenStates; - hiddenStates = this.input_layernorm.forward(hiddenStates); - - var attentionInput = new AttentionInput( - hiddenStates: hiddenStates, - positionIds: input.PositionIds, - attentionMask: input.AttentionMask, - cache: input.PastKeyValue, - positionalEmbeddings: input.PositionalEmbeddings, - outputAttentions: input.OutputAttentions); - var output = this.self_attn.forward(attentionInput); - var attnOutputs = output.HiddenStates; - var selfAttnWeights = output.Attentions; - var presentKeyValue = output.Cache; - hiddenStates = residual + this.resid_attn_dropout.forward(attnOutputs); - residual = hiddenStates; - hiddenStates = this.post_attention_layernorm.forward(hiddenStates); - hiddenStates = this.mlp.forward(hiddenStates); - hiddenStates = residual + this.resid_mlp_dropout.forward(hiddenStates); - - if (UnloadFromDeviceFunc != null) - { - UnloadFromDeviceFunc(this); - } - return new Phi3DecoderLayerOutput(hiddenStates.MoveToOuterDisposeScope(), selfAttnWeights?.MoveToOuterDisposeScope(), presentKeyValue); - } - - private Attention CreateAttentionFromConfig(Phi3Config config, int layerIdx) - { - var headDim = config.HiddenSize / config.NumAttentionHeads; - return new Attention( - attentionDropout: config.AttentionDropout, - hiddenSize: config.HiddenSize, - numHeads: config.NumAttentionHeads, - headDim: headDim, - numKeyValueHeads: config.NumKeyValueHeads ?? throw new ArgumentException("num_key_value_heads must be specified"), - numKeyValueGroups: config.NumAttentionHeads / config.NumKeyValueHeads ?? throw new ArgumentException("num_key_value_heads must be specified"), - maxPositionEmbeddings: config.MaxPositionEmbeddings, - originalMaxPositionEmbeddings: config.OriginalMaxPositionEmbeddings, - layerIdx: layerIdx, - useQkvProj: true, - dtype: config.DType); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi3MLP.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi3MLP.cs deleted file mode 100644 index 65c0413e39..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi3MLP.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; -#pragma warning disable MSML_GeneralName // This name should be PascalCased -internal class Phi3MLP : torch.nn.Module -#pragma warning restore MSML_GeneralName // This name should be PascalCased -{ -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly QuantizedLinear gate_up_proj; - private readonly QuantizedLinear down_proj; - private readonly torch.nn.Module activation_fn; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi3MLP(Phi3Config config) - : this(config.HiddenSize, config.IntermediateSize, config.HiddenAct, config.DType) - { - } - - public Phi3MLP(int hiddenSize, int intermediateSize, string hiddenAct, ScalarType dtype) - : base(nameof(Phi3MLP)) - { - this.gate_up_proj = new QuantizedLinear(hiddenSize, 2 * intermediateSize, hasBias: false, dtype: dtype); - this.down_proj = new QuantizedLinear(intermediateSize, hiddenSize, hasBias: false, dtype: dtype); - this.RegisterComponents(); - this.activation_fn = Core.Utils.GetActivation(hiddenAct); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override Tensor forward(Tensor input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - using var input1 = this.gate_up_proj.forward(input); - var chunks = input1.chunk(2, dim: -1); - var gate = chunks[0]; - var upStatus = chunks[1]; - upStatus = upStatus * this.activation_fn.forward(gate); - return this.down_proj.forward(upStatus); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi3Model.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi3Model.cs deleted file mode 100644 index e873ddd9d8..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi3Model.cs +++ /dev/null @@ -1,147 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using TorchSharp.Modules; -using Microsoft.ML.GenAI.Core.Extension; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi3Model : nn.Module -{ - private readonly Phi3Config _config; - private readonly int _paddingIdx; - private readonly int _vocabSize; - private IKVCache _cache; -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Embedding embed_tokens; - private readonly Dropout embed_dropout; - private readonly ModuleList layers; - private readonly RMSNorm norm; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly nn.Module _rotaryEmb; - - public Phi3Model(Phi3Config config) - : base(nameof(Phi3Model)) - { - this._config = config; - this._paddingIdx = config.PadTokenId ?? 32000; - this._vocabSize = config.VocabSize; - var headDim = config.HiddenSize / config.NumAttentionHeads; - - this.embed_tokens = nn.Embedding(config.VocabSize, config.HiddenSize, padding_idx: this._paddingIdx, dtype: config.DType); - this.embed_dropout = nn.Dropout(config.EmbdPdrop); - this.layers = new ModuleList(); - - for (int i = 0; i < config.NumHiddenLayers; i++) - { - this.layers.Add(new Phi3DecoderLayer(config, i)); - } - this.norm = new RMSNorm(config.HiddenSize, config.RmsNormEps, config.DType); - this._cache = new DynamicKVCache(); - this.RegisterComponents(); - - this._rotaryEmb = config.RopeScaling switch - { - null => new RotaryEmbedding(config.RopeTheta, config.MaxPositionEmbeddings, headDim), - _ => new Phi3SuScaledRotaryEmbedding(headDim, config), - }; - } -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - if (input.OverrideCache is not null) - { - this._cache = input.OverrideCache; - } - - var outputAttentions = input.OutputAttentions; - var outputHiddenStates = input.OutputHiddenStates; - var attentionMask = input.AttentionMask; - Device device; - var inputIds = input.InputIds; - var positionIds = input.PositionIds; - var inputsEmbeds = input.InputEmbeddings; - int batchSize; - int seqLength; - if (inputIds is not null && inputsEmbeds is not null) - { - throw new ArgumentException("Only one of input_ids or inputs_embeds may be set"); - } - else if (inputIds is not null) - { - batchSize = inputIds.IntShape()[0]; - seqLength = inputIds.IntShape()[1]; - inputsEmbeds = this.embed_tokens.forward(inputIds); - device = inputIds.device; - } - else if (inputsEmbeds is not null) - { - batchSize = inputsEmbeds.IntShape()[0]; - seqLength = inputsEmbeds.IntShape()[1]; - device = inputsEmbeds.device; - } - else - { - throw new ArgumentException("Either input_ids or inputs_embeds must be set"); - } - - var pastKeyValuesLength = input.PastKeyValuesLength; - - if (positionIds is null) - { - positionIds = torch.arange(pastKeyValuesLength, seqLength + pastKeyValuesLength, device: device); - positionIds = positionIds.unsqueeze(0).view(-1, seqLength); - } - else - { - positionIds = ((long)positionIds.view(-1, seqLength)); - } - - if (this._config.AttnImplementation == "flash_attention_2") - { - throw new NotImplementedException(); - } - else - { - attentionMask = AttentionMaskConverter.Create4DCausalAttentionMask(attentionMask, [batchSize, seqLength], inputsEmbeds.dtype, device, pastKeyValuesLength, this._config.SlidingWindow); - } - - var hiddenStates = inputsEmbeds; - var positionEmbeddings = this._rotaryEmb.forward(new RotaryEmbeddingInput(hiddenStates, positionIds, seqLength)); - var allHiddenStates = new List(); - var allAttentions = new List(); - foreach (var layer in this.layers) - { - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - var decoderInput = new Phi3DecoderLayerInput( - hiddenStates: hiddenStates, - attentionMask: attentionMask!, - positionIds: positionIds, - pastKeyValue: this._cache, - positionalEmbeddings: positionEmbeddings, - outputAttentions: outputAttentions); - var layerOutput = layer.forward(decoderInput); - hiddenStates = layerOutput.HiddenStates; - if (outputAttentions && layerOutput.Attentions is not null) - { - allAttentions.Add(layerOutput.Attentions); - } - } - - hiddenStates = this.norm.forward(hiddenStates); - if (outputHiddenStates) - { - allHiddenStates.Add(hiddenStates); - } - - return new CausalLMModelOutput(lastHiddenState: hiddenStates, allHiddenStates: allHiddenStates.ToArray(), attentions: allAttentions.ToArray(), cache: this._cache); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Module/Phi3SuScaledRotaryEmbedding.cs b/src/Microsoft.ML.GenAI.Phi/Module/Phi3SuScaledRotaryEmbedding.cs deleted file mode 100644 index e2170493e4..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Module/Phi3SuScaledRotaryEmbedding.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using TorchSharp; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi.Module; - -internal class Phi3SuScaledRotaryEmbedding : RotaryEmbedding -{ - private readonly double[] _shortFactor; - private readonly double[] _longFactor; - private readonly int _originalMaxPositionEmbeddings; - private readonly int _maxPositionEmbeddings; - private readonly double _base; - - public Phi3SuScaledRotaryEmbedding(int dim, Phi3Config config) - : base(config.RopeTheta, config.MaxPositionEmbeddings, dim) - { - JsonElement shortFactorElement = (JsonElement)config.RopeScaling!["short_factor"]; - JsonElement longFactorDocument = (JsonElement)config.RopeScaling!["long_factor"]; - this._shortFactor = shortFactorElement.EnumerateArray().Select(e => e.GetDouble()).ToArray(); - this._longFactor = longFactorDocument.EnumerateArray().Select(e => e.GetDouble()).ToArray(); - - this._originalMaxPositionEmbeddings = config.OriginalMaxPositionEmbeddings; - this._maxPositionEmbeddings = config.MaxPositionEmbeddings; - this._base = config.RopeTheta; - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override RotaryEmbeddingOutput forward(RotaryEmbeddingInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var seqLen = (torch.max(input.PositionIds) + 1).ToInt32(); - var x = input.Input; - Tensor extFactors; - if (seqLen > this._originalMaxPositionEmbeddings) - { - extFactors = torch.tensor(this._longFactor, dtype: ScalarType.Float32, x.device); - } - else - { - extFactors = torch.tensor(this._shortFactor, dtype: ScalarType.Float32, x.device); - } - var invFreqShape = torch.arange(0, this.Dim, 2, dtype: ScalarType.Int64).to(torch.float32) / this.Dim; - invFreqShape = invFreqShape.to(x.device); - var invFreq = 1.0f / (torch.pow(this._base, invFreqShape) * extFactors); - - var invFreqExpanded = invFreq.unsqueeze(0).unsqueeze(-1); - invFreqExpanded = invFreqExpanded.expand(new long[] { input.PositionIds.shape[0], -1, 1 }); - var positionIdsExpanded = input.PositionIds.unsqueeze(1).to(torch.float32); - - var freqs = invFreqExpanded * positionIdsExpanded; - freqs = freqs.transpose(1, 2); - var emb = torch.cat([freqs, freqs], dim: -1); - var scale = (1.0 * this._maxPositionEmbeddings) / this._originalMaxPositionEmbeddings; - double scalingFactor; - if (scale <= 1) - { - scalingFactor = 1.0; - } - else - { - scalingFactor = Math.Sqrt(1 + Math.Log(scale) / Math.Log(this._originalMaxPositionEmbeddings)); - } - - var cos = torch.cos(emb) * scalingFactor; - var sin = torch.sin(emb) * scalingFactor; - - return new(cos.to_type(x.dtype), sin.to_type(x.dtype)); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi2/Config.cs b/src/Microsoft.ML.GenAI.Phi/Phi2/Config.cs deleted file mode 100644 index 580bde9b12..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi2/Config.cs +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi2Config -{ - public Phi2Config() - { - this.VocabSize = 51200; - this.HiddenSize = 2048; - this.IntermediateSize = 8192; - this.NumHiddenLayers = 24; - this.NumAttentionHeads = 32; - this.ResidPdrop = 0.0; - this.EmbdPdrop = 0.0; - this.AttentionDropout = 0.0; - this.HiddenAct = "gelu_new"; - this.MaxPositionEmbeddings = 2048; - this.InitializerRange = 0.02; - this.LayerNormEps = 1e-5; - this.UseCache = true; - this.TieWordEmbeddings = false; - this.RopeTheta = 10000.0; - this.PartialRotaryFactor = 0.5; - this.QkLayernorm = false; - this.BosTokenId = 1; - this.EosTokenId = 2; - this.Dtype = ScalarType.Float32; - } - - static Phi2Config() - { - var phi2ConfigContent = Core.Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Phi.Resource.Config.phi-2-config.json"); - var phi2Config = JsonSerializer.Deserialize(phi2ConfigContent) ?? throw new ArgumentNullException(nameof(phi2ConfigContent)); - Phi2 = phi2Config; - } - - /// - /// The default phi-2 configuration created from https://huggingface.co/microsoft/phi-2/blob/main/config.json. - /// - public static Phi2Config Phi2 { get; } - - [JsonPropertyName("vocab_size")] - public int VocabSize { get; set; } - - [JsonPropertyName("hidden_size")] - public int HiddenSize { get; set; } - - [JsonPropertyName("intermediate_size")] - public int IntermediateSize { get; set; } - - [JsonPropertyName("num_hidden_layers")] - public int NumHiddenLayers { get; set; } - - [JsonPropertyName("num_attention_heads")] - public int NumAttentionHeads { get; set; } - - [JsonPropertyName("num_key_value_heads")] - public int? NumKeyValueHeads { get; set; } - - [JsonPropertyName("resid_pdrop")] - public double ResidPdrop { get; set; } - - [JsonPropertyName("embd_pdrop")] - public double EmbdPdrop { get; set; } - - [JsonPropertyName("attention_dropout")] - public double AttentionDropout { get; set; } - - [JsonPropertyName("hidden_act")] - public string HiddenAct { get; set; } - - [JsonPropertyName("max_position_embeddings")] - public int MaxPositionEmbeddings { get; set; } - - [JsonPropertyName("initializer_range")] - public double InitializerRange { get; set; } - - [JsonPropertyName("layer_norm_eps")] - public double LayerNormEps { get; set; } - - [JsonPropertyName("use_cache")] - public bool UseCache { get; set; } - - [JsonPropertyName("tie_word_embeddings")] - public bool TieWordEmbeddings { get; set; } - - [JsonPropertyName("rope_theta")] - public double RopeTheta { get; set; } - - // [JsonPropertyName("rope_scaling")] - // public double? RopeScaling { get; set; } = null; - - [JsonPropertyName("partial_rotary_factor")] - public double PartialRotaryFactor { get; set; } - - [JsonPropertyName("qk_layernorm")] - public bool QkLayernorm { get; set; } - - [JsonPropertyName("bos_token_id")] - public int BosTokenId { get; set; } - - [JsonPropertyName("eos_token_id")] - public int EosTokenId { get; set; } - - public ScalarType Dtype { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2ForCausalLM.cs b/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2ForCausalLM.cs deleted file mode 100644 index 3c8c5c0a5b..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2ForCausalLM.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.CodeDom; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Phi.Module; -using TorchSharp; -using TorchSharp.Modules; -using TorchSharp.PyBridge; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi2ForCausalLM : nn.Module -{ -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Phi2Model model; - private readonly GenAILinear lm_head; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi2ForCausalLM(Phi2Config config) - : base(nameof(Phi2ForCausalLM)) - { - this.model = new Phi2Model(config); - this.lm_head = new GenAILinear(config.HiddenSize, config.VocabSize, dtype: config.Dtype); - this.RegisterComponents(); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) // use_cache, output_attentions, output_hidden_states -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var inputIds = input.InputIds; - var attentionMask = input.AttentionMask; - var pastKeyValueLength = input.PastKeyValuesLength; - var positionIds = input.PositionIds; - var inputEmbeddings = input.InputEmbeddings; - var options = (input.OutputAttentions, input.OutputHiddenStates, false); - var output = this.model.forward(inputIds, attentionMask, pastKeyValueLength, positionIds, inputEmbeddings, options); - var hiddenState = output.Item1; - - var lmLogits = this.lm_head.forward(hiddenState); - - return new CausalLMModelOutput(lastHiddenState: hiddenState, logits: lmLogits); - } - - public static Phi2ForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - ScalarType torchDtype = ScalarType.Float32, - bool useTqdm = false, - string? device = null) - { - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.Dtype = torchDtype; - var wrapper = new Phi2ForCausalLM(modelConfig); - var loadedParameters = new Dictionary(); - wrapper.load_checkpoint(path: modelFolder, checkpointName: checkPointName, strict: true, loadedParameters: loadedParameters, useTqdm: useTqdm); - wrapper = wrapper.to(device); - wrapper.eval(); - return wrapper; - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2TokenizerHelper.cs b/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2TokenizerHelper.cs deleted file mode 100644 index 3444c74e31..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi2/Phi2TokenizerHelper.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using System.Text.RegularExpressions; -using Microsoft.ML.Tokenizers; -using Tensorboard; - -/// -/// The utility class to create tokenizer for phi-3 model. -/// -public class Phi2TokenizerHelper -{ - public static CodeGenTokenizer Create( - string folder, - string vocabFile = "vocab.json", - string mergesFile = "merges.txt", - bool addPrefixSpace = false, - bool addBeginOfSentence = false, - bool addEndOfSentence = false) - { - var vocabPath = Path.Combine(folder, vocabFile); - var mergesPath = Path.Combine(folder, mergesFile); - using var vocabStream = File.OpenRead(vocabPath); - using var mergesStream = File.OpenRead(mergesPath); - - return CodeGenTokenizer.Create(vocabStream, mergesStream, addPrefixSpace, addBeginOfSentence, addEndOfSentence); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Config.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Config.cs deleted file mode 100644 index 9261c044f8..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Config.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3Config -{ - public Phi3Config() - { - this.VocabSize = 32064; - this.HiddenSize = 3072; - this.RmsNormEps = 1e-5f; - this.IntermediateSize = 8192; - this.NumHiddenLayers = 32; - this.NumAttentionHeads = 32; - this.ResidPdrop = 0.0; - this.EmbdPdrop = 0.0; - this.AttentionDropout = 0.0; - this.HiddenAct = "silu"; - this.MaxPositionEmbeddings = 4096; - this.OriginalMaxPositionEmbeddings = 4096; - this.InitializerRange = 0.02; - this.UseCache = true; - this.TieWordEmbeddings = false; - this.RopeTheta = 10000.0; - this.PartialRotaryFactor = 0.5; - this.QkLayernorm = false; - this.BosTokenId = 1; - this.EosTokenId = 32000; - this.DType = ScalarType.BFloat16; - this.AttnImplementation = "eager"; - } - - static Phi3Config() - { - var phi3Mini4kInstructContent = Core.Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Phi.Resource.Config.phi-3-mini-4k-instruct-config.json"); - var phi3Mini128kInstructContent = Core.Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Phi.Resource.Config.phi-3-mini-128k-instruct-config.json"); - var phi3Medium4kInstructContent = Core.Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Phi.Resource.Config.phi-3-medium-4k-instruct-config.json"); - var phi3Medium128kInstructContent = Core.Utils.GetEmbeddedResource("Microsoft.ML.GenAI.Phi.Resource.Config.phi-3-medium-128k-instruct-config.json"); - - Phi3Mini4kInstruct = JsonSerializer.Deserialize(phi3Mini4kInstructContent) ?? throw new ArgumentNullException(nameof(phi3Mini4kInstructContent)); - Phi3Mini128kInstruct = JsonSerializer.Deserialize(phi3Mini128kInstructContent) ?? throw new ArgumentNullException(nameof(phi3Mini128kInstructContent)); - Phi3Medium4kInstruct = JsonSerializer.Deserialize(phi3Medium4kInstructContent) ?? throw new ArgumentNullException(nameof(phi3Medium4kInstructContent)); - Phi3Medium128kInstruct = JsonSerializer.Deserialize(phi3Medium128kInstructContent) ?? throw new ArgumentNullException(nameof(phi3Medium128kInstructContent)); - } - - /// - /// The phi-3-mini-4k-instruct configuration created from https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/config.json. - /// - public static Phi3Config Phi3Mini4kInstruct { get; } - - /// - /// The phi-3-medium-4k-instruct configuration created from https://huggingface.co/microsoft/Phi-3-medium-4k-instruct/blob/main/config.json. - /// - public static Phi3Config Phi3Medium4kInstruct { get; } - - /// - /// The phi-3-medium-128k-instruct configuration created from https://huggingface.co/microsoft/Phi-3-medium-128k-instruct/blob/main/config.json. - /// - public static Phi3Config Phi3Medium128kInstruct { get; } - - /// - /// The phi-3-mini-128k-instruct configuration created from https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/blob/main/config.json. - /// - public static Phi3Config Phi3Mini128kInstruct { get; } - - [JsonPropertyName("vocab_size")] - public int VocabSize { get; set; } - - [JsonPropertyName("hidden_size")] - public int HiddenSize { get; set; } - - [JsonPropertyName("rms_norm_eps")] - public float RmsNormEps { get; set; } - - [JsonPropertyName("intermediate_size")] - public int IntermediateSize { get; set; } - - [JsonPropertyName("num_hidden_layers")] - public int NumHiddenLayers { get; set; } - - [JsonPropertyName("num_attention_heads")] - public int NumAttentionHeads { get; set; } - - [JsonPropertyName("num_key_value_heads")] - public int? NumKeyValueHeads { get; set; } - - [JsonPropertyName("resid_pdrop")] - public double ResidPdrop { get; set; } - - [JsonPropertyName("embd_pdrop")] - public double EmbdPdrop { get; set; } - - [JsonPropertyName("attention_dropout")] - public double AttentionDropout { get; set; } - - [JsonPropertyName("hidden_act")] - public string HiddenAct { get; set; } - - [JsonPropertyName("max_position_embeddings")] - public int MaxPositionEmbeddings { get; set; } - - [JsonPropertyName("original_max_position_embeddings")] - public int OriginalMaxPositionEmbeddings { get; set; } - - [JsonPropertyName("initializer_range")] - public double InitializerRange { get; set; } - - [JsonPropertyName("use_cache")] - public bool UseCache { get; set; } - - [JsonPropertyName("tie_word_embeddings")] - public bool TieWordEmbeddings { get; set; } - - [JsonPropertyName("rope_theta")] - public double RopeTheta { get; set; } - - [JsonPropertyName("rope_scaling")] - public Dictionary? RopeScaling { get; set; } - - [JsonPropertyName("partial_rotary_factor")] - public double PartialRotaryFactor { get; set; } - - [JsonPropertyName("qk_layernorm")] - public bool QkLayernorm { get; set; } - - [JsonPropertyName("bos_token_id")] - public int BosTokenId { get; set; } - - [JsonPropertyName("eos_token_id")] - public int EosTokenId { get; set; } - - [JsonPropertyName("pad_token_id")] - public int? PadTokenId { get; set; } - - [JsonPropertyName("sliding_window")] - public int? SlidingWindow { get; set; } - - public ScalarType DType { get; set; } - - public string AttnImplementation { get; set; } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMAgent.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMAgent.cs deleted file mode 100644 index e1e02d338b..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMAgent.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3Agent : IStreamingAgent -{ - private const char Newline = '\n'; - private readonly ICausalLMPipeline _pipeline; - private readonly string? _systemMessage; - private readonly IAutoGenChatTemplateBuilder _templateBuilder; - - public Phi3Agent( - ICausalLMPipeline pipeline, - string name, - string? systemMessage = "you are a helpful assistant", - IAutoGenChatTemplateBuilder? templateBuilder = null) - { - this.Name = name; - this._pipeline = pipeline; - this._systemMessage = systemMessage; - this._templateBuilder = templateBuilder ?? Phi3ChatTemplateBuilder.Instance; - } - - public string Name { get; } - - public Task GenerateReplyAsync(IEnumerable messages, GenerateReplyOptions? options = null, CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - - var input = _templateBuilder.BuildPrompt(messages); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append("<|end|>").ToArray(); - - var output = _pipeline.Generate( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence) ?? throw new InvalidOperationException("Failed to generate a reply."); - - return Task.FromResult(new TextMessage(Role.Assistant, output, from: this.Name)); - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable GenerateStreamingReplyAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - IEnumerable messages, - GenerateReplyOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - if (_systemMessage != null) - { - var systemMessage = new TextMessage(Role.System, _systemMessage, from: this.Name); - messages = messages.Prepend(systemMessage); - } - - var input = _templateBuilder.BuildPrompt(messages); - var maxLen = options?.MaxToken ?? 1024; - var temperature = options?.Temperature ?? 0.7f; - var stopTokenSequence = options?.StopSequence ?? []; - stopTokenSequence = stopTokenSequence.Append("<|end|>").ToArray(); - - foreach (var output in _pipeline.GenerateStreaming( - input, - maxLen: maxLen, - temperature: temperature, - stopSequences: stopTokenSequence)) - { - yield return new TextMessageUpdate(Role.Assistant, output, from: this.Name); - } - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatClient.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatClient.cs deleted file mode 100644 index ebd74e4848..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatClient.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3CausalLMChatClient : CausalLMPipelineChatClient -{ - private readonly string _eotToken = "<|end|>"; - - public Phi3CausalLMChatClient( - ICausalLMPipeline pipeline, - IMEAIChatTemplateBuilder? templateBuilder = null, - ChatClientMetadata? metadata = null) - : base( - pipeline, - templateBuilder ?? Phi3ChatTemplateBuilder.Instance, - metadata ?? new ChatClientMetadata(defaultModelId: nameof(Phi3CausalLMChatClient))) - { - } - - public override Task GetResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - options ??= new ChatOptions(); - - if (options.StopSequences != null) - { - options.StopSequences.Add(_eotToken); - } - else - { - options.StopSequences = [_eotToken]; - } - - return base.GetResponseAsync(messages, options, cancellationToken); - } - - public override IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, - ChatOptions? options = null, - CancellationToken cancellationToken = default) - { - options ??= new ChatOptions(); - options.StopSequences ??= []; - options.StopSequences.Add(_eotToken); - - return base.GetStreamingResponseAsync(messages, options, cancellationToken); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatCompletionService.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatCompletionService.cs deleted file mode 100644 index 896926c43b..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMChatCompletionService.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using System.Text; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using Microsoft.SemanticKernel.TextGeneration; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3CausalLMChatCompletionService : IChatCompletionService -{ - private readonly ICausalLMPipeline _pipeline; - private readonly Phi3CausalLMTextGenerationService _textGenerationService; - private readonly ISemanticKernelChatTemplateBuilder _templateBuilder; - - public Phi3CausalLMChatCompletionService( - ICausalLMPipeline pipeline, - ISemanticKernelChatTemplateBuilder? templateBuilder = null) - { - _pipeline = pipeline; - _textGenerationService = new Phi3CausalLMTextGenerationService(pipeline); - _templateBuilder = templateBuilder ?? Phi3ChatTemplateBuilder.Instance; - } - - public IReadOnlyDictionary Attributes => _textGenerationService.Attributes; - - public async Task> GetChatMessageContentsAsync( - ChatHistory chatHistory, - PromptExecutionSettings? executionSettings = null, - Kernel? kernel = null, - CancellationToken cancellationToken = default) - { - var prompt = _templateBuilder.BuildPrompt(chatHistory); - var replies = await _textGenerationService.GetTextContentsAsync(prompt, executionSettings, kernel, cancellationToken); - return replies.Select(reply => new ChatMessageContent(AuthorRole.Assistant, reply.Text)).ToList(); - } - - public async IAsyncEnumerable GetStreamingChatMessageContentsAsync( - ChatHistory chatHistory, - PromptExecutionSettings? executionSettings = null, - Kernel? kernel = null, - [EnumeratorCancellation] - CancellationToken cancellationToken = default) - { - var prompt = _templateBuilder.BuildPrompt(chatHistory); - - await foreach (var reply in _textGenerationService.GetStreamingTextContentsAsync(prompt, executionSettings, kernel, cancellationToken)) - { - yield return new StreamingChatMessageContent(AuthorRole.Assistant, reply.Text); - } - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMTextGenerationService.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMTextGenerationService.cs deleted file mode 100644 index 01a574c8f6..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3CausalLMTextGenerationService.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Runtime.CompilerServices; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.TextGeneration; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3CausalLMTextGenerationService : ITextGenerationService -{ - private readonly ICausalLMPipeline _pipeline; - - public Phi3CausalLMTextGenerationService( - ICausalLMPipeline pipeline) - { - _pipeline = pipeline; - } - - public IReadOnlyDictionary Attributes => new Dictionary() - { - { "temperature", null }, - { "max_token", null }, - { "stop_token_sequence", null }, - { "top_p", null }, - }; - - public Task> GetTextContentsAsync(string prompt, PromptExecutionSettings? executionSettings = null, Kernel? kernel = null, CancellationToken cancellationToken = default) - { - var temperature = executionSettings?.ExtensionData?["temperature"] as float? ?? 0.7f; - var maxToken = executionSettings?.ExtensionData?["max_token"] as int? ?? 512; - var stopTokenSequence = executionSettings?.ExtensionData?["stop_token_sequence"] as List ?? new List(); - var topP = executionSettings?.ExtensionData?["top_p"] as float? ?? 0.9f; - stopTokenSequence.Add("<|end|>"); - var response = _pipeline.Generate( - prompt, - maxToken, - temperature, - stopSequences: stopTokenSequence.ToArray(), - topP: topP); - - return Task.FromResult>([new TextContent(response)]); - } - -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - public async IAsyncEnumerable GetStreamingTextContentsAsync( -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - string prompt, - PromptExecutionSettings? - executionSettings = null, - Kernel? kernel = null, - [EnumeratorCancellation] - CancellationToken cancellationToken = default) - { - var temperature = executionSettings?.ExtensionData?["temperature"] as float? ?? 0.7f; - var maxToken = executionSettings?.ExtensionData?["max_token"] as int? ?? 100; - var stopTokenSequence = executionSettings?.ExtensionData?["stop_token_sequence"] as string[] ?? Array.Empty(); - var topP = executionSettings?.ExtensionData?["top_p"] as float? ?? 0.9f; - stopTokenSequence.Append("<|end|>"); - - foreach (var item in _pipeline.GenerateStreaming( - prompt, - maxToken, - temperature, - topP, - stopTokenSequence)) - { - yield return new StreamingTextContent(item); - } - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ChatTemplateBuilder.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ChatTemplateBuilder.cs deleted file mode 100644 index 862cf2547d..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ChatTemplateBuilder.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using AutoGen.Core; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using TextContent = Microsoft.SemanticKernel.TextContent; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3ChatTemplateBuilder : IChatTemplateBuilder, IMEAIChatTemplateBuilder -{ - private const char Newline = '\n'; - - public static Phi3ChatTemplateBuilder Instance { get; } = new Phi3ChatTemplateBuilder(); - - public string BuildPrompt(IEnumerable messages, IEnumerable? tools = null) - { - var availableRoles = new[] { Role.System, Role.User, Role.Assistant }; - if (messages.Any(m => m.GetContent() is null)) - { - throw new InvalidOperationException("Please provide a message with content."); - } - - if (messages.Any(m => m.GetRole() is null || availableRoles.Contains(m.GetRole()!.Value) == false)) - { - throw new InvalidOperationException("Please provide a message with a valid role. The valid roles are System, User, and Assistant."); - } - - // construct template based on instruction from - // https://huggingface.co/microsoft/Phi-3-mini-128k-instruct#chat-format - - var sb = new StringBuilder(); - foreach (var message in messages) - { - var role = message.GetRole()!.Value; - var content = message.GetContent()!; - sb.Append(message switch - { - _ when message.GetRole() == Role.System => $"<|system|>{Newline}{content}<|end|>{Newline}", - _ when message.GetRole() == Role.User => $"<|user|>{Newline}{content}<|end|>{Newline}", - _ when message.GetRole() == Role.Assistant => $"<|assistant|>{Newline}{content}<|end|>{Newline}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - sb.Append("<|assistant|>"); - var input = sb.ToString(); - - return input; - } - - public string BuildPrompt(ChatHistory chatHistory) - { - // build prompt from chat history - var sb = new StringBuilder(); - - foreach (var message in chatHistory) - { - foreach (var item in message.Items) - { - if (item is not TextContent textContent) - { - throw new NotSupportedException($"Only text content is supported, but got {item.GetType().Name}"); - } - - var prompt = message.Role switch - { - _ when message.Role == AuthorRole.System => $"<|system|>{Newline}{textContent}<|end|>{Newline}", - _ when message.Role == AuthorRole.User => $"<|user|>{Newline}{textContent}<|end|>{Newline}", - _ when message.Role == AuthorRole.Assistant => $"<|assistant|>{Newline}{textContent}<|end|>{Newline}", - _ => throw new NotSupportedException($"Unsupported role {message.Role}") - }; - - sb.Append(prompt); - } - } - - sb.Append("<|assistant|>"); - - return sb.ToString(); - } - - public string BuildPrompt(IEnumerable messages, ChatOptions? options = null, bool appendAssistantTag = true) - { - var availableRoles = new[] { ChatRole.System, ChatRole.User, ChatRole.Assistant }; - if (messages.Any(m => m.Text is null)) - { - throw new InvalidOperationException("Please provide a message with content."); - } - - if (messages.Any(m => availableRoles.Any(availableRole => availableRole == m.Role) == false)) - { - throw new InvalidOperationException("Please provide a message with a valid role. The valid roles are System, User, and Assistant."); - } - - // construct template based on instruction from - // https://huggingface.co/microsoft/Phi-3-mini-128k-instruct#chat-format - - var sb = new StringBuilder(); - foreach (var message in messages) - { - var role = message.Role.Value; - var content = message.Text; - sb.Append(message switch - { - _ when message.Role == ChatRole.System => $"<|system|>{Newline}{content}<|end|>{Newline}", - _ when message.Role == ChatRole.User => $"<|user|>{Newline}{content}<|end|>{Newline}", - _ when message.Role == ChatRole.Assistant => $"<|assistant|>{Newline}{content}<|end|>{Newline}", - _ => throw new InvalidOperationException("Invalid role.") - }); - } - - if (appendAssistantTag) - { - sb.Append("<|assistant|>"); - } - - var input = sb.ToString(); - - return input; - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ForCausalLM.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ForCausalLM.cs deleted file mode 100644 index c46cbaf57e..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3ForCausalLM.cs +++ /dev/null @@ -1,123 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.GenAI.Phi.Module; -using TorchSharp; -using TorchSharp.Modules; -using TorchSharp.PyBridge; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Phi; - -public class Phi3ForCausalLM : nn.Module -{ - private readonly Phi3Config _config; - -#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format - private readonly Phi3Model model; - private readonly GenAILinear lm_head; -#pragma warning restore MSML_PrivateFieldName // Private field name not in: _camelCase format - - public Phi3ForCausalLM(Phi3Config config) - : base(nameof(Phi3ForCausalLM)) - { - this._config = config; - this.model = new Phi3Model(config); - this.lm_head = new GenAILinear(config.HiddenSize, config.VocabSize, dtype: config.DType, hasBias: false); - - this.RegisterComponents(); - } - -#pragma warning disable MSML_GeneralName // This name should be PascalCased - public override CausalLMModelOutput forward(CausalLMModelInput input) -#pragma warning restore MSML_GeneralName // This name should be PascalCased - { - var outputs = this.model.forward(input); - var logits = this.lm_head.forward(outputs.LastHiddenState); - logits = logits.to_type(ScalarType.Float32); - outputs.Logits = logits; - - return outputs; - } - - public static Phi3ForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - ScalarType torchDtype = ScalarType.BFloat16, - string device = "cpu") - { - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var phi = new Phi3ForCausalLM(modelConfig); - phi.LoadSafeTensors(modelFolder, checkPointName); - phi = phi.to(device); - phi.eval(); - - return phi; - } - - public static Phi3ForCausalLM FromPretrained( - string modelFolder, - string configName = "config.json", - string checkPointName = "model.safetensors.index.json", - bool quantizeToInt8 = false, - bool quantizeToInt4 = false, - int layersOnTargetDevice = -1, - ScalarType torchDtype = ScalarType.BFloat16, - string targetDevice = "cuda") - { - if (layersOnTargetDevice == -1 && quantizeToInt4 == false && quantizeToInt8 == false) - { - return FromPretrained(modelFolder, configName, checkPointName, torchDtype, targetDevice); - } - - var originalDefaultDevice = torch.get_default_device(); - torch.set_default_device("meta"); - var config = Path.Join(modelFolder, configName); - var modelConfig = JsonSerializer.Deserialize(File.ReadAllText(config)) ?? throw new ArgumentNullException(nameof(config)); - modelConfig.DType = torchDtype; - var model = new Phi3ForCausalLM(modelConfig); - - if (quantizeToInt8) - { - model.ToInt8QuantizeModule(); - } - else if (quantizeToInt4) - { - model.ToInt4QuantizeModule(); - } - - var deviceMap = model.InferDeviceMapForEachLayer( - [ - KeyValuePair.Create(targetDevice, layersOnTargetDevice), - KeyValuePair.Create("cpu", -1) - ]); - - torch.set_default_device("cpu"); - model = new Phi3ForCausalLM(modelConfig); - - model.LoadSafeTensors(modelFolder, checkPointName); - - model = model.ToDynamicLoadingModel(deviceMap, targetDevice); - - torch.set_default_device(originalDefaultDevice); - - return model; - } - - public void LoadSafeTensors(string modelFolder, string checkPointName = "model.safetensors.index.json") - { - this.load_checkpoint(path: modelFolder, checkpointName: checkPointName, strict: false, useTqdm: false); - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3TokenizerHelper.cs b/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3TokenizerHelper.cs deleted file mode 100644 index dd54378892..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Phi3/Phi3TokenizerHelper.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using System.Text.RegularExpressions; -using Microsoft.ML.Tokenizers; - -/// -/// The utility class to create tokenizer for phi-3 model. -/// -public class Phi3TokenizerHelper -{ - private const string SystemSymbol = "<|system|>"; - private const string UserSymbol = "<|user|>"; - private const string AssistantSymbol = "<|assistant|>"; - private const string EndSymbol = "<|end|>"; - private const int SystemSymbolId = 32006; - private const int UserSymbolId = 32010; - private const int AssistantSymbolId = 32001; - private const int EndSymbolId = 32007; - - public static LlamaTokenizer FromPretrained( - string modelPath, - string systemSymbol = SystemSymbol, - string userSymbol = UserSymbol, - string assistantSymbol = AssistantSymbol, - string endSymbol = EndSymbol, - int systemSymbolId = SystemSymbolId, - int userSymbolId = UserSymbolId, - int assistantSymbolId = AssistantSymbolId, - int endSymbolId = EndSymbolId, - bool addPrecedingSpace = true) - { - var modelStream = File.OpenRead(modelPath); - - var llamaTokenizer = LlamaTokenizer.Create( - modelStream, - addPrecedingSpace, - specialTokens: new Dictionary - { - { systemSymbol, systemSymbolId }, - { userSymbol, userSymbolId }, - { assistantSymbol, assistantSymbolId }, - { endSymbol, endSymbolId } - }); - - return llamaTokenizer; - } -} diff --git a/src/Microsoft.ML.GenAI.Phi/README.md b/src/Microsoft.ML.GenAI.Phi/README.md deleted file mode 100644 index 0e0bbb4dda..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Microsoft.ML.GenAI.Phi -Torchsharp implementation of Microsoft phi-series models for GenAI - -## Supported list -The following phi-models are supported and tested: -- [x] [Phi-2](https://huggingface.co/microsoft/phi-2) -- [x] [Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) -- [x] [Phi-3-mini-128k-instruct](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct) -- [x] [Phi-3-medium-4k-instruct](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct) -- [x] [Phi-3-medium-128k-instruct](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct) -- [ ] [Phi-3-small-8k-instruct](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) -- [ ] [Phi-3-small-128k-instruct](https://huggingface.co/microsoft/Phi-3-small-128k-instruct) -- [ ] [Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-large-4k-instruct) - -## Getting Started with Semantic Kernel - -### Download model weight (e.g. phi-3-mini-4k-instruct) from Huggingface -```bash -## make sure you have lfs installed -git clone https://huggingface.co/microsoft/Phi-3-mini-4k-instruct -``` - -### Load model -```csharp -var weightFolder = "/path/to/Phi-3-mini-4k-instruct"; -var configName = "config.json"; -var config = JsonSerializier.Deserialize(File.ReadAllText(Path.Combine(weightFolder, configName))); -var model = new Phi3ForCausalLM(config); - -// load tokenizer -var tokenizerModelName = "tokenizer.model"; -var tokenizer = Phi3TokenizerHelper.FromPretrained(Path.Combine(weightFolder, tokenizerModelName)); - -// load weight -model.LoadSafeTensors(weightFolder); - -// initialize device -var device = "cuda"; -if (device == "cuda") -{ - torch.InitializeDeviceType(DeviceType.CUDA); -} - - -// create causal language model pipeline -var pipeline = new CausalLMPipeline(tokenizer, model, device); -``` - -### Add pipeline as `IChatCompletionService` to sematic kernel -```csharp -var kernel = Kernel.CreateBuilder() - .AddGenAIChatCompletion(pipeline) - .Build(); -``` - -### Chat with the model -```csharp -var chatService = kernel.GetRequiredService(); -var chatHistory = new ChatHistory(); -chatHistory.AddSystemMessage("you are a helpful assistant"); -chatHistory.AddUserMessage("write a C# program to calculate the factorial of a number"); -await foreach (var response in chatService.GetStreamingChatMessageContentsAsync(chatHistory)) -{ - Console.Write(response); -} -``` - -## Getting started with AutoGen.Net -### Follow the same steps download model weight and load model -### Create `Phi3Agent` from pipeline -```csharp -var agent = new Phi3Agent(pipeline, name: "assistant") - .RegisterPrintMessage(); -``` - -### Chat with the model -```csharp -var task = """ -write a C# program to calculate the factorial of a number -"""; - -await agent.SendAsync(task); -``` - -### More examples -Please refer to [Microsoft.ML.GenAI.Samples](./../../docs/samples/Microsoft.ML.GenAI.Samples/) for more examples. - -## Dynamic loading -It's recommended to run model inference on GPU, which requires at least 8GB of GPU memory for phi-3-mini-4k-instruct model if fully loaded. - -If your GPU memory is not enough, you can choose to dynamically load the model weight to GPU memory. Here is how it works behind the scene: -- when initializing the model, the size of each layer is calculated and stored in a dictionary -- when loading the model weight, each layer is assigned to a device (CPU or GPU) based on the size of the layer and the remaining memory of the device. If there is no enough memory on the device, the layer is loaded to CPU memory. -- when inference, the layer which is loaded to CPU memory is moved to GPU memory before the inference and moved back to CPU memory after the inference. - -Here is how to enable dynamic loading of model: -### Step 1: Infer the size of each layer -You can infer the size of each layer using `InferDeviceMapForEachLayer` API. The `deviceMap` will be a key-value dictionary, where the key is the layer name and the value is the device name (e.g. "cuda" or "cpu"). - -```csharp -// manually set up the available memory on each device -var deviceSizeMap = new Dictionary - { - ["cuda"] = modelSizeOnCudaInGB * 1L * 1024 * 1024 * 1024, - ["cpu"] = modelSizeOnMemoryInGB * 1L * 1024 * 1024 * 1024, - ["disk"] = modelSizeOnDiskInGB * 1L * 1024 * 1024 * 1024, - }; - -var deviceMap = model.InferDeviceMapForEachLayer( - devices: ["cuda", "cpu", "disk"], - deviceSizeMapInByte: deviceSizeMap); -``` - -### Step 2: Load model weights using `ToDynamicLoadingModel` API -Once the `deviceMap` is calculated, you can pass it to `ToDynamicLoadingModel` api to load the model weight. - -```csharp -model = model.ToDynamicLoadingModel(deviceMap, "cuda"); -``` diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-2-config.json b/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-2-config.json deleted file mode 100644 index c3a5b1ce17..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-2-config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "_name_or_path": "microsoft/phi-2", - "architectures": [ - "PhiForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 50256, - "embd_pdrop": 0.0, - "eos_token_id": 50256, - "hidden_act": "gelu_new", - "hidden_size": 2560, - "initializer_range": 0.02, - "intermediate_size": 10240, - "layer_norm_eps": 1e-05, - "max_position_embeddings": 2048, - "model_type": "phi", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 32, - "partial_rotary_factor": 0.4, - "qk_layernorm": false, - "resid_pdrop": 0.1, - "rope_scaling": null, - "rope_theta": 10000.0, - "tie_word_embeddings": false, - "torch_dtype": "float16", - "transformers_version": "4.37.0", - "use_cache": true, - "vocab_size": 51200 - } - \ No newline at end of file diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-128k-instruct-config.json b/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-128k-instruct-config.json deleted file mode 100644 index f058b81505..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-128k-instruct-config.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "_name_or_path": "Phi-3-medium-128k-instruct", - "architectures": [ - "Phi3ForCausalLM" - ], - "attention_dropout": 0.0, - "auto_map": { - "AutoConfig": "configuration_phi3.Phi3Config", - "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" - }, - "bos_token_id": 1, - "embd_pdrop": 0.0, - "eos_token_id": 32000, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 17920, - "max_position_embeddings": 131072, - "model_type": "phi3", - "num_attention_heads": 40, - "num_hidden_layers": 40, - "num_key_value_heads": 10, - "original_max_position_embeddings": 4096, - "pad_token_id": null, - "resid_pdrop": 0.0, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "long_factor": [ - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.25, - 1.25, - 1.5, - 2.0, - 2.75, - 5.75, - 5.75, - 6.5, - 9.25, - 11.0, - 13.25, - 19.25, - 19.75, - 19.75, - 21.25, - 21.5, - 26.5, - 30.0, - 33.75, - 35.25, - 38.5, - 42.0, - 42.25, - 46.0, - 47.0, - 50.0, - 50.5, - 51.0, - 52.0, - 52.75, - 53.75, - 54.75, - 57.0, - 57.25, - 58.5, - 59.25, - 59.5, - 62.0, - 62.5, - 62.75, - 63.25, - 63.25, - 63.25, - 63.75, - 64.0, - 64.0, - 64.25, - 64.5, - 64.5, - 65.0, - 65.0 - ], - "short_factor": [ - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.0, - 1.01, - 1.02, - 1.02, - 1.04, - 1.04, - 1.07, - 1.07, - 1.1, - 1.3000000000000003, - 1.3000000000000003, - 1.5000000000000004, - 1.5700000000000005, - 1.9000000000000008, - 2.3100000000000014, - 2.759999999999992, - 3.3899999999999784, - 3.9399999999999666, - 4.009999999999965, - 4.289999999999959, - 4.349999999999958, - 5.349999999999937, - 6.659999999999909, - 7.029999999999901, - 7.51999999999989, - 8.00999999999988, - 8.249999999999876, - 8.279999999999875, - 9.629999999999846, - 9.89999999999984, - 10.589999999999826, - 11.049999999999816, - 11.7899999999998, - 12.189999999999792, - 12.889999999999777, - 13.129999999999772, - 13.16999999999977, - 13.20999999999977, - 13.479999999999764, - 13.539999999999763, - 13.779999999999758, - 13.929999999999755, - 14.429999999999744, - 14.759999999999737, - 15.149999999999729, - 15.419999999999723, - 15.53999999999972, - 15.659999999999718, - 15.749999999999716, - 15.759999999999716, - 15.799999999999715, - 16.05999999999971, - 16.079999999999714, - 16.11999999999972, - 16.11999999999972, - 16.18999999999973, - 16.31999999999975, - 16.539999999999786, - 16.799999999999827 - ], - "type": "su" - }, - "rope_theta": 10000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.39.3", - "use_cache": true, - "attention_bias": false, - "vocab_size": 32064 -} diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-4k-instruct-config.json b/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-4k-instruct-config.json deleted file mode 100644 index 4ded05bfd8..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-medium-4k-instruct-config.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "_name_or_path": "Phi-3-medium-4k-instruct", - "architectures": [ - "Phi3ForCausalLM" - ], - "attention_dropout": 0.0, - "auto_map": { - "AutoConfig": "configuration_phi3.Phi3Config", - "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" - }, - "bos_token_id": 1, - "embd_pdrop": 0.0, - "eos_token_id": 32000, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 17920, - "max_position_embeddings": 4096, - "model_type": "phi3", - "num_attention_heads": 40, - "num_hidden_layers": 40, - "num_key_value_heads": 10, - "original_max_position_embeddings": 4096, - "pad_token_id": 32000, - "resid_pdrop": 0.0, - "rms_norm_eps": 1e-05, - "rope_scaling": null, - "rope_theta": 10000.0, - "sliding_window": 2047, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.39.3", - "use_cache": true, - "attention_bias": false, - "vocab_size": 32064 -} diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-128k-instruct-config.json b/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-128k-instruct-config.json deleted file mode 100644 index 9aae2a82fc..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-128k-instruct-config.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "_name_or_path": "Phi-3-mini-128k-instruct", - "architectures": [ - "Phi3ForCausalLM" - ], - "attention_dropout": 0.0, - "auto_map": { - "AutoConfig": "configuration_phi3.Phi3Config", - "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM", - "AutoModelForSequenceClassification": "modeling_phi3.Phi3ForSequenceClassification", - "AutoModelForTokenClassification": "modeling_phi3.Phi3ForTokenClassification" - }, - "bos_token_id": 1, - "embd_pdrop": 0.0, - "eos_token_id": 32000, - "hidden_act": "silu", - "hidden_size": 3072, - "initializer_range": 0.02, - "intermediate_size": 8192, - "max_position_embeddings": 131072, - "model_type": "phi3", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 32, - "original_max_position_embeddings": 4096, - "pad_token_id": 32000, - "resid_pdrop": 0.0, - "rms_norm_eps": 1e-05, - "rope_scaling": { - "long_factor": [ - 1.0299999713897705, - 1.0499999523162842, - 1.0499999523162842, - 1.0799999237060547, - 1.2299998998641968, - 1.2299998998641968, - 1.2999999523162842, - 1.4499999284744263, - 1.5999999046325684, - 1.6499998569488525, - 1.8999998569488525, - 2.859999895095825, - 3.68999981880188, - 5.419999599456787, - 5.489999771118164, - 5.489999771118164, - 9.09000015258789, - 11.579999923706055, - 15.65999984741211, - 15.769999504089355, - 15.789999961853027, - 18.360000610351562, - 21.989999771118164, - 23.079999923706055, - 30.009998321533203, - 32.35000228881836, - 32.590003967285156, - 35.56000518798828, - 39.95000457763672, - 53.840003967285156, - 56.20000457763672, - 57.95000457763672, - 59.29000473022461, - 59.77000427246094, - 59.920005798339844, - 61.190006256103516, - 61.96000671386719, - 62.50000762939453, - 63.3700065612793, - 63.48000717163086, - 63.48000717163086, - 63.66000747680664, - 63.850006103515625, - 64.08000946044922, - 64.760009765625, - 64.80001068115234, - 64.81001281738281, - 64.81001281738281 - ], - "short_factor": [ - 1.05, - 1.05, - 1.05, - 1.1, - 1.1, - 1.1500000000000001, - 1.2000000000000002, - 1.2500000000000002, - 1.3000000000000003, - 1.3500000000000003, - 1.5000000000000004, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.000000000000001, - 2.0500000000000007, - 2.0500000000000007, - 2.0500000000000007, - 2.1000000000000005, - 2.1000000000000005, - 2.1000000000000005, - 2.1500000000000004, - 2.1500000000000004, - 2.3499999999999996, - 2.549999999999999, - 2.5999999999999988, - 2.5999999999999988, - 2.7499999999999982, - 2.849999999999998, - 2.849999999999998, - 2.9499999999999975 - ], - "type": "su" - }, - "rope_theta": 10000.0, - "sliding_window": 262144, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.39.3", - "use_cache": true, - "attention_bias": false, - "vocab_size": 32064 -} diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-4k-instruct-config.json b/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-4k-instruct-config.json deleted file mode 100644 index 92bb5b42eb..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/Config/phi-3-mini-4k-instruct-config.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "_name_or_path": "Phi-3-mini-4k-instruct", - "architectures": [ - "Phi3ForCausalLM" - ], - "attention_dropout": 0.0, - "auto_map": { - "AutoConfig": "configuration_phi3.Phi3Config", - "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM" - }, - "bos_token_id": 1, - "embd_pdrop": 0.0, - "eos_token_id": 32000, - "hidden_act": "silu", - "hidden_size": 3072, - "initializer_range": 0.02, - "intermediate_size": 8192, - "max_position_embeddings": 4096, - "model_type": "phi3", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 32, - "original_max_position_embeddings": 4096, - "pad_token_id": 32000, - "resid_pdrop": 0.0, - "rms_norm_eps": 1e-05, - "rope_scaling": null, - "rope_theta": 10000.0, - "sliding_window": 2047, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.39.3", - "use_cache": true, - "attention_bias": false, - "vocab_size": 32064 -} diff --git a/src/Microsoft.ML.GenAI.Phi/Resource/README.md b/src/Microsoft.ML.GenAI.Phi/Resource/README.md deleted file mode 100644 index 35f35bda1a..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Resource/README.md +++ /dev/null @@ -1,8 +0,0 @@ -## This folder includes the embedded resources for the GenAI.Phi project. - -### Configuration -- [phi-2-config.json](https://huggingface.co/microsoft/phi-2/blob/main/config.json): The phi-2 model configuration file. -- [phi-3-mini-128k-config.json](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/config.json): The phi-3-mini-128k model configuration file. -- [phi-3-mini-4k-config.json](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/blob/main/config.json): The phi-3-mini-4k model configuration file. -- [phi-3-medium-4k-config.json](https://huggingface.co/microsoft/Phi-3-medium-4k-instruct/blob/main/config.json): The phi-3-medium-4k model configuration file. -- [phi-3-medium-128k-config.json](https://huggingface.co/microsoft/Phi-3-medium-128k-instruct/blob/main/config.json): The phi-3-medium-128k model configuration file.] \ No newline at end of file diff --git a/src/Microsoft.ML.GenAI.Phi/Utils.cs b/src/Microsoft.ML.GenAI.Phi/Utils.cs deleted file mode 100644 index 5be880a06e..0000000000 --- a/src/Microsoft.ML.GenAI.Phi/Utils.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Text; -using System.Threading.Tasks; -using TorchSharp; -using TorchSharp.Modules; -using static TorchSharp.torch; -using static TorchSharp.torch.nn; -namespace Microsoft.ML.GenAI.Phi; - -internal static class Utils -{ - public static Tensor PrecomputeThetaPosFrequencies(int headDim, int seqLen, string device, float theta = 10000.0f) - { - // As written in the paragraph 3.2.2 of the paper - // >> In order to generalize our results in 2D to any xi ∈ Rd where **d is even**, [...] - if (headDim % 2 != 0) - { - throw new ArgumentException("Dimension must be divisible by 2", nameof(headDim)); - } - - // Build the theta parameter - // According to the formula theta_i = 10000^(-2(i-1)/dim) for i = [1, 2, ... dim/2] - // Shape: (Head_Dim / 2) - var thetaNumerator = torch.arange(0, headDim, 2).to(torch.float32).to(device); - // Shape: (Head_Dim / 2) - var thetaInput = torch.pow(theta, -1.0f * (thetaNumerator / headDim)).to(device); // (Dim / 2) - // Construct the positions (the "m" parameter) - // Shape: (Seq_Len) - var m = torch.arange(seqLen, device: device); - // Multiply each theta by each position using the outer product. - // Shape: (Seq_Len) outer_product* (Head_Dim / 2) -> (Seq_Len, Head_Dim / 2) - var freqs = torch.outer(m, thetaInput).to(torch.float32).to(device); - - // We can compute complex numbers in the polar form c = R * exp(m * theta), where R = 1 as follows: - // (Seq_Len, Head_Dim / 2) -> (Seq_Len, Head_Dim / 2) - var freqsComplex = torch.polar(torch.ones_like(freqs), freqs); - - return freqsComplex; - } - - // python - // def rotate_half(x): - // """Rotates half the hidden dims of the input.""" - // x1 = x[..., : x.shape[-1] // 2] - // x2 = x[..., x.shape[-1] // 2 :] - // return torch.cat((-x2, x1), dim=-1) - public static Tensor RotateHalf(Tensor x) - { - var x1 = x[.., .., .., ..(int)(x.shape[^1] / 2)]; - var x2 = x[.., .., .., (int)(x.shape[^1] / 2)..]; - // (x1 * x1 * x2).Peek("x1 * x1 * x2"); - return torch.cat([-x2, x1], dim: -1); - } - - public static (Tensor, Tensor) ApplyRotaryPosEmb(Tensor q, Tensor k, Tensor cos, Tensor sin, Tensor? positionIds = null, int unsqueezeDim = 1) - { - // The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and - // sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note - // that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and - // k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes - // cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have - // the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. - - if (positionIds is not null) - { - cos = cos[positionIds!].unsqueeze(unsqueezeDim); - sin = sin[positionIds!].unsqueeze(unsqueezeDim); - } - else - { - cos = cos.unsqueeze(unsqueezeDim); - sin = sin.unsqueeze(unsqueezeDim); - } - var qEmbed = q * cos; - qEmbed += RotateHalf(q) * sin; - - var kEmbed = k * cos; - kEmbed += RotateHalf(k) * sin; - // var kEmbed = (k * cos) + (RotateHalf(k) * sin); - return (qEmbed, kEmbed); - } - - - - public static Tensor Phi2RepeatKV(Tensor x, int nRep) - { - var batchSize = x.shape[0]; - var seqLen = x.shape[1]; - var nKVHeads = x.shape[2]; - var headDim = x.shape[3]; - if (nRep == 1) - { - return x; - } - - return x.unsqueeze(3) - .expand(batchSize, seqLen, nKVHeads, nRep, headDim) - .view(batchSize, seqLen, nKVHeads * nRep, headDim); - } -} diff --git a/test/Microsoft.ML.GenAI.Core.Tests/CausalLMDatasetTest.cs b/test/Microsoft.ML.GenAI.Core.Tests/CausalLMDatasetTest.cs deleted file mode 100644 index 25b8874d7d..0000000000 --- a/test/Microsoft.ML.GenAI.Core.Tests/CausalLMDatasetTest.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core.Trainer; -using Microsoft.ML.GenAI.LLaMA; -using Microsoft.ML.Tokenizers; -using Xunit; - -namespace Microsoft.ML.GenAI.Core.Tests; - -public class CausalLMDatasetTest -{ - private static Tokenizer CreateLlamaTokenizer() - { - // @"https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/resolve/main/tokenizer.model?download=true"; - // @"https://huggingface.co/hf-internal-testing/llama-tokenizer/resolve/main/tokenizer.model"; - using Stream remoteStream = File.OpenRead(Path.Combine(@"Llama", "tokenizer.model")); - return LlamaTokenizer.Create(remoteStream); - } - - [Fact] - public void ItCreateDatasetsFromInputIds() - { - int[] inputIds = [1, 2, 3, 4, 5]; - int[] outputIds = [6, 7, 8, 9, 10]; - - var dataset = CausalLMDataset.Create(inputIds, outputIds) - .ToArray(); - - // the following rows should be created - // - input_ids: [1, 2, 3, 4, 5], label_ids: [-100, -100, -100, -100, 6] - // - input_ids: [1, 2, 3, 4, 5, 6], label_ids: [-100, -100, -100, -100, -100, 7] - // - input_ids: [1, 2, 3, 4, 5, 6, 7], label_ids: [-100, -100, -100, -100, -100, -100, 8] - // - input_ids: [1, 2, 3, 4, 5, 6, 7, 8], label_ids: [-100, -100, -100, -100, -100, -100, -100, 9] - // - input_ids: [1, 2, 3, 4, 5, 6, 7, 8, 9], label_ids: [-100, -100, -100, -100, -100, -100, -100, -100, 10] - - dataset.Length.Should().Be(5); - dataset[0].InputIds!.data().Should().BeEquivalentTo([1, 2, 3, 4, 5]); - dataset[0].Labels!.data().Should().BeEquivalentTo([-100, -100, -100, -100, 6]); - dataset[0].AttentionMask!.data().Should().BeEquivalentTo([1, 1, 1, 1, 1]); - dataset[^1].AttentionMask!.data().Should().BeEquivalentTo([1, 1, 1, 1, 1, 1, 1, 1, 1]); - dataset[^1].Labels!.data().Should().BeEquivalentTo([-100, -100, -100, -100, -100, -100, -100, -100, 10]); - dataset[^1].AttentionMask!.data().Should().BeEquivalentTo([1, 1, 1, 1, 1, 1, 1, 1, 1]); - } - - [Fact] - public void ItCreateDatasetsFromListOfInputIds() - { - int[][] inputIds = [ - [1, 2, 3, 4, 5], - [6, 7, 8, 9, 10] - ]; - - int[][] outputIds = [ - [11, 12, 13, 14, 15], - [16, 17, 18, 19, 20] - ]; - - var dataset = CausalLMDataset.Create(inputIds, outputIds) - .ToArray(); - - dataset.Count().Should().Be(10); - - foreach (var item in dataset) - { - item.Labels!.shape.Should().BeEquivalentTo(item.InputIds!.shape); - item.AttentionMask!.shape.Should().BeEquivalentTo(item.InputIds!.shape); - } - } - - [Fact] - public void ItCreateDatasetsFromMEAIMessages() - { - var inputs = new List> - { - new List - { - new ChatMessage(ChatRole.System, "You are a helpful contoso assistant"), - new ChatMessage(ChatRole.User, "What is contoso"), - }, - }; - - var outputs = new List - { - new ChatMessage(ChatRole.Assistant, "Contoso is a company"), - }; - - var tokenizer = CreateLlamaTokenizer(); - - var dataset = CausalLMDataset.Create(inputs, outputs, Llama3_1ChatTemplateBuilder.Instance, tokenizer) - .ToArray(); - - dataset.Length.Should().Be(14); - } -} diff --git a/test/Microsoft.ML.GenAI.Core.Tests/Microsoft.ML.GenAI.Core.Tests.csproj b/test/Microsoft.ML.GenAI.Core.Tests/Microsoft.ML.GenAI.Core.Tests.csproj deleted file mode 100644 index afdfe94e5a..0000000000 --- a/test/Microsoft.ML.GenAI.Core.Tests/Microsoft.ML.GenAI.Core.Tests.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - net8.0 - enable - $(NoWarn);MSML_ExtendBaseTestClass - enable - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test/Microsoft.ML.GenAI.Core.Tests/QuantizedLinearTests.cs b/test/Microsoft.ML.GenAI.Core.Tests/QuantizedLinearTests.cs deleted file mode 100644 index d1653721ba..0000000000 --- a/test/Microsoft.ML.GenAI.Core.Tests/QuantizedLinearTests.cs +++ /dev/null @@ -1,127 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using FluentAssertions; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp; -using Xunit; -using static TorchSharp.torch; - -namespace Microsoft.ML.GenAI.Core.Tests; - -public class QuantizedLinearTests -{ - [Fact] - public void Int4QuantizeSizeTests() - { - // meta is critical for the test - // as the size of the model to test is 372 GB - // and can't be loaded in real device like cpu or cuda - var device = "meta"; - var model = new QuantizedLinear(100000, 100, device: device); - - var sizeInBytes = model.GetSizeInBytes(); - - var sizeInGigaBytes = sizeInBytes / 1024 / 1024; - sizeInGigaBytes.Should().Be(38); - - // to int4 - model.Int4(); - var sizeInBytesAfterInt8 = model.GetSizeInBytes(); - var sizeInGigaBytesAfterInt8 = sizeInBytesAfterInt8 / 1024 / 1024; - sizeInGigaBytesAfterInt8.Should().Be(4); // 38 // 8 = 4 - } - - [Fact] - public void Int8QuantizeSizeTests() - { - // meta is critical for the test - // as the size of the model to test is 372 GB - // and can't be loaded in real device like cpu or cuda - var device = "meta"; - var model = new QuantizedLinear(100000, 100, device: device); - - var sizeInBytes = model.GetSizeInBytes(); - - var sizeInGigaBytes = sizeInBytes / 1024 / 1024; - sizeInGigaBytes.Should().Be(38); - - // to int8 - model.Int8(); - var sizeInBytesAfterInt8 = model.GetSizeInBytes(); - var sizeInGigaBytesAfterInt8 = sizeInBytesAfterInt8 / 1024 / 1024; - sizeInGigaBytesAfterInt8.Should().Be(9); // 38 // 4 = 9 - } - - [Fact] - public void Int4QuantizeForwardTest() - { - var device = "cpu"; - var model = new QuantizedLinear(123, 10, device: device); - - // set both weight and bias to rand int8 values - // and compare the result before and after ToInt8 - var input = torch.ones([10, 2200, 123], device: device); - var weight = torch.ones([10, 123], device: device, dtype: ScalarType.Int64) * -1; - var bias = torch.ones([10], device: device) * 2; - - var weightStr = weight.Peek("weight").ToString(); - - weight = (weight + 8).view(-1).to(torch.uint8); - var weightPlaceHolderDim = (int)weight.size(0); - weightPlaceHolderDim = weightPlaceHolderDim / 2 + weightPlaceHolderDim % 2; - var weightPlaceHolder = weight[..weightPlaceHolderDim]; - weightPlaceHolder = weightPlaceHolder * 16 + weight[weightPlaceHolderDim..]; - - var high4Bit = weightPlaceHolder / 16; - var low4Bit = weightPlaceHolder % 16; - weight = torch.cat(new Tensor[] { high4Bit, low4Bit }).view(10, 123); - weight = weight.to(torch.int64); - weight -= 8; - weight.Peek("weight").Should().Be(weightStr); - - model.load_state_dict(new Dictionary - { - ["weight"] = weight, - ["bias"] = bias - }); - - var resultBeforeInt4 = model.forward(input); - - model.ToInt4QuantizeModule(); - - var resultAfterInt4 = model.forward(input); - - // compare the result - resultBeforeInt4.Peek("result").Should().Be(resultAfterInt4.Peek("result")); - } - - [Fact] - public void Int8QuantizeForwardTest() - { - var device = "cpu"; - var model = new QuantizedLinear(123, 10, device: device); - - // set both weight and bias to rand int8 values - // and compare the result before and after ToInt8 - var input = torch.ones([10, 2200, 123], device: device); - var weight = torch.ones([10, 123], device: device) * -1; - var bias = torch.ones([10], device: device) * 2; - - model.load_state_dict(new Dictionary - { - ["weight"] = weight, - ["bias"] = bias - }); - - var resultBeforeInt8 = model.forward(input); - - model.ToInt8QuantizeModule(); - - var resultAfterInt8 = model.forward(input); - - resultBeforeInt8.Peek("result").Should().Be("result: sum: 312.6933 dType: Float32 shape: [10,2200,10]"); - resultAfterInt8.Peek("result").Should().Be("result: sum: 312.6933 dType: Float32 shape: [10,2200,10]"); - } -} diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt deleted file mode 100644 index e4a2466fec..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt +++ /dev/null @@ -1,7 +0,0 @@ -<|begin_of_text|><|start_header_id|>system<|end_header_id|> -You are a helpful AI assistant.<|eot_id|> -<|start_header_id|>user<|end_header_id|> -Hello?<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> -World!<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromMEAIChatHistory.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromMEAIChatHistory.approved.txt deleted file mode 100644 index e4a2466fec..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromMEAIChatHistory.approved.txt +++ /dev/null @@ -1,7 +0,0 @@ -<|begin_of_text|><|start_header_id|>system<|end_header_id|> -You are a helpful AI assistant.<|eot_id|> -<|start_header_id|>user<|end_header_id|> -Hello?<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> -World!<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromSemanticKernelChatHistory.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromSemanticKernelChatHistory.approved.txt deleted file mode 100644 index e4a2466fec..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.ItBuildChatTemplateFromSemanticKernelChatHistory.approved.txt +++ /dev/null @@ -1,7 +0,0 @@ -<|begin_of_text|><|start_header_id|>system<|end_header_id|> -You are a helpful AI assistant.<|eot_id|> -<|start_header_id|>user<|end_header_id|> -Hello?<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> -World!<|eot_id|> -<|start_header_id|>assistant<|end_header_id|> diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_405b_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_405b_ShapeTest.approved.txt deleted file mode 100644 index 6b8d7749dc..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_405b_ShapeTest.approved.txt +++ /dev/null @@ -1,1137 +0,0 @@ -0: lm_head.weight shape: [128256, 16384] -1: model.embed_tokens.weight shape: [128256, 16384] -2: model.layers.0.input_layernorm.weight shape: [16384] -3: model.layers.0.mlp.down_proj.weight shape: [16384, 53248] -4: model.layers.0.mlp.gate_proj.weight shape: [53248, 16384] -5: model.layers.0.mlp.up_proj.weight shape: [53248, 16384] -6: model.layers.0.post_attention_layernorm.weight shape: [16384] -7: model.layers.0.self_attn.k_proj.weight shape: [1024, 16384] -8: model.layers.0.self_attn.o_proj.weight shape: [16384, 16384] -9: model.layers.0.self_attn.q_proj.weight shape: [16384, 16384] -10: model.layers.0.self_attn.v_proj.weight shape: [1024, 16384] -11: model.layers.1.input_layernorm.weight shape: [16384] -12: model.layers.1.mlp.down_proj.weight shape: [16384, 53248] -13: model.layers.1.mlp.gate_proj.weight shape: [53248, 16384] -14: model.layers.1.mlp.up_proj.weight shape: [53248, 16384] -15: model.layers.1.post_attention_layernorm.weight shape: [16384] -16: model.layers.1.self_attn.k_proj.weight shape: [1024, 16384] -17: model.layers.1.self_attn.o_proj.weight shape: [16384, 16384] -18: model.layers.1.self_attn.q_proj.weight shape: [16384, 16384] -19: model.layers.1.self_attn.v_proj.weight shape: [1024, 16384] -20: model.layers.10.input_layernorm.weight shape: [16384] -21: model.layers.10.mlp.down_proj.weight shape: [16384, 53248] -22: model.layers.10.mlp.gate_proj.weight shape: [53248, 16384] -23: model.layers.10.mlp.up_proj.weight shape: [53248, 16384] -24: model.layers.10.post_attention_layernorm.weight shape: [16384] -25: model.layers.10.self_attn.k_proj.weight shape: [1024, 16384] -26: model.layers.10.self_attn.o_proj.weight shape: [16384, 16384] -27: model.layers.10.self_attn.q_proj.weight shape: [16384, 16384] -28: model.layers.10.self_attn.v_proj.weight shape: [1024, 16384] -29: model.layers.100.input_layernorm.weight shape: [16384] -30: model.layers.100.mlp.down_proj.weight shape: [16384, 53248] -31: model.layers.100.mlp.gate_proj.weight shape: [53248, 16384] -32: model.layers.100.mlp.up_proj.weight shape: [53248, 16384] -33: model.layers.100.post_attention_layernorm.weight shape: [16384] -34: model.layers.100.self_attn.k_proj.weight shape: [1024, 16384] -35: model.layers.100.self_attn.o_proj.weight shape: [16384, 16384] -36: model.layers.100.self_attn.q_proj.weight shape: [16384, 16384] -37: model.layers.100.self_attn.v_proj.weight shape: [1024, 16384] -38: model.layers.101.input_layernorm.weight shape: [16384] -39: model.layers.101.mlp.down_proj.weight shape: [16384, 53248] -40: model.layers.101.mlp.gate_proj.weight shape: [53248, 16384] -41: model.layers.101.mlp.up_proj.weight shape: [53248, 16384] -42: model.layers.101.post_attention_layernorm.weight shape: [16384] -43: model.layers.101.self_attn.k_proj.weight shape: [1024, 16384] -44: model.layers.101.self_attn.o_proj.weight shape: [16384, 16384] -45: model.layers.101.self_attn.q_proj.weight shape: [16384, 16384] -46: model.layers.101.self_attn.v_proj.weight shape: [1024, 16384] -47: model.layers.102.input_layernorm.weight shape: [16384] -48: model.layers.102.mlp.down_proj.weight shape: [16384, 53248] -49: model.layers.102.mlp.gate_proj.weight shape: [53248, 16384] -50: model.layers.102.mlp.up_proj.weight shape: [53248, 16384] -51: model.layers.102.post_attention_layernorm.weight shape: [16384] -52: model.layers.102.self_attn.k_proj.weight shape: [1024, 16384] -53: model.layers.102.self_attn.o_proj.weight shape: [16384, 16384] -54: model.layers.102.self_attn.q_proj.weight shape: [16384, 16384] -55: model.layers.102.self_attn.v_proj.weight shape: [1024, 16384] -56: model.layers.103.input_layernorm.weight shape: [16384] -57: model.layers.103.mlp.down_proj.weight shape: [16384, 53248] -58: model.layers.103.mlp.gate_proj.weight shape: [53248, 16384] -59: model.layers.103.mlp.up_proj.weight shape: [53248, 16384] -60: model.layers.103.post_attention_layernorm.weight shape: [16384] -61: model.layers.103.self_attn.k_proj.weight shape: [1024, 16384] -62: model.layers.103.self_attn.o_proj.weight shape: [16384, 16384] -63: model.layers.103.self_attn.q_proj.weight shape: [16384, 16384] -64: model.layers.103.self_attn.v_proj.weight shape: [1024, 16384] -65: model.layers.104.input_layernorm.weight shape: [16384] -66: model.layers.104.mlp.down_proj.weight shape: [16384, 53248] -67: model.layers.104.mlp.gate_proj.weight shape: [53248, 16384] -68: model.layers.104.mlp.up_proj.weight shape: [53248, 16384] -69: model.layers.104.post_attention_layernorm.weight shape: [16384] -70: model.layers.104.self_attn.k_proj.weight shape: [1024, 16384] -71: model.layers.104.self_attn.o_proj.weight shape: [16384, 16384] -72: model.layers.104.self_attn.q_proj.weight shape: [16384, 16384] -73: model.layers.104.self_attn.v_proj.weight shape: [1024, 16384] -74: model.layers.105.input_layernorm.weight shape: [16384] -75: model.layers.105.mlp.down_proj.weight shape: [16384, 53248] -76: model.layers.105.mlp.gate_proj.weight shape: [53248, 16384] -77: model.layers.105.mlp.up_proj.weight shape: [53248, 16384] -78: model.layers.105.post_attention_layernorm.weight shape: [16384] -79: model.layers.105.self_attn.k_proj.weight shape: [1024, 16384] -80: model.layers.105.self_attn.o_proj.weight shape: [16384, 16384] -81: model.layers.105.self_attn.q_proj.weight shape: [16384, 16384] -82: model.layers.105.self_attn.v_proj.weight shape: [1024, 16384] -83: model.layers.106.input_layernorm.weight shape: [16384] -84: model.layers.106.mlp.down_proj.weight shape: [16384, 53248] -85: model.layers.106.mlp.gate_proj.weight shape: [53248, 16384] -86: model.layers.106.mlp.up_proj.weight shape: [53248, 16384] -87: model.layers.106.post_attention_layernorm.weight shape: [16384] -88: model.layers.106.self_attn.k_proj.weight shape: [1024, 16384] -89: model.layers.106.self_attn.o_proj.weight shape: [16384, 16384] -90: model.layers.106.self_attn.q_proj.weight shape: [16384, 16384] -91: model.layers.106.self_attn.v_proj.weight shape: [1024, 16384] -92: model.layers.107.input_layernorm.weight shape: [16384] -93: model.layers.107.mlp.down_proj.weight shape: [16384, 53248] -94: model.layers.107.mlp.gate_proj.weight shape: [53248, 16384] -95: model.layers.107.mlp.up_proj.weight shape: [53248, 16384] -96: model.layers.107.post_attention_layernorm.weight shape: [16384] -97: model.layers.107.self_attn.k_proj.weight shape: [1024, 16384] -98: model.layers.107.self_attn.o_proj.weight shape: [16384, 16384] -99: model.layers.107.self_attn.q_proj.weight shape: [16384, 16384] -100: model.layers.107.self_attn.v_proj.weight shape: [1024, 16384] -101: model.layers.108.input_layernorm.weight shape: [16384] -102: model.layers.108.mlp.down_proj.weight shape: [16384, 53248] -103: model.layers.108.mlp.gate_proj.weight shape: [53248, 16384] -104: model.layers.108.mlp.up_proj.weight shape: [53248, 16384] -105: model.layers.108.post_attention_layernorm.weight shape: [16384] -106: model.layers.108.self_attn.k_proj.weight shape: [1024, 16384] -107: model.layers.108.self_attn.o_proj.weight shape: [16384, 16384] -108: model.layers.108.self_attn.q_proj.weight shape: [16384, 16384] -109: model.layers.108.self_attn.v_proj.weight shape: [1024, 16384] -110: model.layers.109.input_layernorm.weight shape: [16384] -111: model.layers.109.mlp.down_proj.weight shape: [16384, 53248] -112: model.layers.109.mlp.gate_proj.weight shape: [53248, 16384] -113: model.layers.109.mlp.up_proj.weight shape: [53248, 16384] -114: model.layers.109.post_attention_layernorm.weight shape: [16384] -115: model.layers.109.self_attn.k_proj.weight shape: [1024, 16384] -116: model.layers.109.self_attn.o_proj.weight shape: [16384, 16384] -117: model.layers.109.self_attn.q_proj.weight shape: [16384, 16384] -118: model.layers.109.self_attn.v_proj.weight shape: [1024, 16384] -119: model.layers.11.input_layernorm.weight shape: [16384] -120: model.layers.11.mlp.down_proj.weight shape: [16384, 53248] -121: model.layers.11.mlp.gate_proj.weight shape: [53248, 16384] -122: model.layers.11.mlp.up_proj.weight shape: [53248, 16384] -123: model.layers.11.post_attention_layernorm.weight shape: [16384] -124: model.layers.11.self_attn.k_proj.weight shape: [1024, 16384] -125: model.layers.11.self_attn.o_proj.weight shape: [16384, 16384] -126: model.layers.11.self_attn.q_proj.weight shape: [16384, 16384] -127: model.layers.11.self_attn.v_proj.weight shape: [1024, 16384] -128: model.layers.110.input_layernorm.weight shape: [16384] -129: model.layers.110.mlp.down_proj.weight shape: [16384, 53248] -130: model.layers.110.mlp.gate_proj.weight shape: [53248, 16384] -131: model.layers.110.mlp.up_proj.weight shape: [53248, 16384] -132: model.layers.110.post_attention_layernorm.weight shape: [16384] -133: model.layers.110.self_attn.k_proj.weight shape: [1024, 16384] -134: model.layers.110.self_attn.o_proj.weight shape: [16384, 16384] -135: model.layers.110.self_attn.q_proj.weight shape: [16384, 16384] -136: model.layers.110.self_attn.v_proj.weight shape: [1024, 16384] -137: model.layers.111.input_layernorm.weight shape: [16384] -138: model.layers.111.mlp.down_proj.weight shape: [16384, 53248] -139: model.layers.111.mlp.gate_proj.weight shape: [53248, 16384] -140: model.layers.111.mlp.up_proj.weight shape: [53248, 16384] -141: model.layers.111.post_attention_layernorm.weight shape: [16384] -142: model.layers.111.self_attn.k_proj.weight shape: [1024, 16384] -143: model.layers.111.self_attn.o_proj.weight shape: [16384, 16384] -144: model.layers.111.self_attn.q_proj.weight shape: [16384, 16384] -145: model.layers.111.self_attn.v_proj.weight shape: [1024, 16384] -146: model.layers.112.input_layernorm.weight shape: [16384] -147: model.layers.112.mlp.down_proj.weight shape: [16384, 53248] -148: model.layers.112.mlp.gate_proj.weight shape: [53248, 16384] -149: model.layers.112.mlp.up_proj.weight shape: [53248, 16384] -150: model.layers.112.post_attention_layernorm.weight shape: [16384] -151: model.layers.112.self_attn.k_proj.weight shape: [1024, 16384] -152: model.layers.112.self_attn.o_proj.weight shape: [16384, 16384] -153: model.layers.112.self_attn.q_proj.weight shape: [16384, 16384] -154: model.layers.112.self_attn.v_proj.weight shape: [1024, 16384] -155: model.layers.113.input_layernorm.weight shape: [16384] -156: model.layers.113.mlp.down_proj.weight shape: [16384, 53248] -157: model.layers.113.mlp.gate_proj.weight shape: [53248, 16384] -158: model.layers.113.mlp.up_proj.weight shape: [53248, 16384] -159: model.layers.113.post_attention_layernorm.weight shape: [16384] -160: model.layers.113.self_attn.k_proj.weight shape: [1024, 16384] -161: model.layers.113.self_attn.o_proj.weight shape: [16384, 16384] -162: model.layers.113.self_attn.q_proj.weight shape: [16384, 16384] -163: model.layers.113.self_attn.v_proj.weight shape: [1024, 16384] -164: model.layers.114.input_layernorm.weight shape: [16384] -165: model.layers.114.mlp.down_proj.weight shape: [16384, 53248] -166: model.layers.114.mlp.gate_proj.weight shape: [53248, 16384] -167: model.layers.114.mlp.up_proj.weight shape: [53248, 16384] -168: model.layers.114.post_attention_layernorm.weight shape: [16384] -169: model.layers.114.self_attn.k_proj.weight shape: [1024, 16384] -170: model.layers.114.self_attn.o_proj.weight shape: [16384, 16384] -171: model.layers.114.self_attn.q_proj.weight shape: [16384, 16384] -172: model.layers.114.self_attn.v_proj.weight shape: [1024, 16384] -173: model.layers.115.input_layernorm.weight shape: [16384] -174: model.layers.115.mlp.down_proj.weight shape: [16384, 53248] -175: model.layers.115.mlp.gate_proj.weight shape: [53248, 16384] -176: model.layers.115.mlp.up_proj.weight shape: [53248, 16384] -177: model.layers.115.post_attention_layernorm.weight shape: [16384] -178: model.layers.115.self_attn.k_proj.weight shape: [1024, 16384] -179: model.layers.115.self_attn.o_proj.weight shape: [16384, 16384] -180: model.layers.115.self_attn.q_proj.weight shape: [16384, 16384] -181: model.layers.115.self_attn.v_proj.weight shape: [1024, 16384] -182: model.layers.116.input_layernorm.weight shape: [16384] -183: model.layers.116.mlp.down_proj.weight shape: [16384, 53248] -184: model.layers.116.mlp.gate_proj.weight shape: [53248, 16384] -185: model.layers.116.mlp.up_proj.weight shape: [53248, 16384] -186: model.layers.116.post_attention_layernorm.weight shape: [16384] -187: model.layers.116.self_attn.k_proj.weight shape: [1024, 16384] -188: model.layers.116.self_attn.o_proj.weight shape: [16384, 16384] -189: model.layers.116.self_attn.q_proj.weight shape: [16384, 16384] -190: model.layers.116.self_attn.v_proj.weight shape: [1024, 16384] -191: model.layers.117.input_layernorm.weight shape: [16384] -192: model.layers.117.mlp.down_proj.weight shape: [16384, 53248] -193: model.layers.117.mlp.gate_proj.weight shape: [53248, 16384] -194: model.layers.117.mlp.up_proj.weight shape: [53248, 16384] -195: model.layers.117.post_attention_layernorm.weight shape: [16384] -196: model.layers.117.self_attn.k_proj.weight shape: [1024, 16384] -197: model.layers.117.self_attn.o_proj.weight shape: [16384, 16384] -198: model.layers.117.self_attn.q_proj.weight shape: [16384, 16384] -199: model.layers.117.self_attn.v_proj.weight shape: [1024, 16384] -200: model.layers.118.input_layernorm.weight shape: [16384] -201: model.layers.118.mlp.down_proj.weight shape: [16384, 53248] -202: model.layers.118.mlp.gate_proj.weight shape: [53248, 16384] -203: model.layers.118.mlp.up_proj.weight shape: [53248, 16384] -204: model.layers.118.post_attention_layernorm.weight shape: [16384] -205: model.layers.118.self_attn.k_proj.weight shape: [1024, 16384] -206: model.layers.118.self_attn.o_proj.weight shape: [16384, 16384] -207: model.layers.118.self_attn.q_proj.weight shape: [16384, 16384] -208: model.layers.118.self_attn.v_proj.weight shape: [1024, 16384] -209: model.layers.119.input_layernorm.weight shape: [16384] -210: model.layers.119.mlp.down_proj.weight shape: [16384, 53248] -211: model.layers.119.mlp.gate_proj.weight shape: [53248, 16384] -212: model.layers.119.mlp.up_proj.weight shape: [53248, 16384] -213: model.layers.119.post_attention_layernorm.weight shape: [16384] -214: model.layers.119.self_attn.k_proj.weight shape: [1024, 16384] -215: model.layers.119.self_attn.o_proj.weight shape: [16384, 16384] -216: model.layers.119.self_attn.q_proj.weight shape: [16384, 16384] -217: model.layers.119.self_attn.v_proj.weight shape: [1024, 16384] -218: model.layers.12.input_layernorm.weight shape: [16384] -219: model.layers.12.mlp.down_proj.weight shape: [16384, 53248] -220: model.layers.12.mlp.gate_proj.weight shape: [53248, 16384] -221: model.layers.12.mlp.up_proj.weight shape: [53248, 16384] -222: model.layers.12.post_attention_layernorm.weight shape: [16384] -223: model.layers.12.self_attn.k_proj.weight shape: [1024, 16384] -224: model.layers.12.self_attn.o_proj.weight shape: [16384, 16384] -225: model.layers.12.self_attn.q_proj.weight shape: [16384, 16384] -226: model.layers.12.self_attn.v_proj.weight shape: [1024, 16384] -227: model.layers.120.input_layernorm.weight shape: [16384] -228: model.layers.120.mlp.down_proj.weight shape: [16384, 53248] -229: model.layers.120.mlp.gate_proj.weight shape: [53248, 16384] -230: model.layers.120.mlp.up_proj.weight shape: [53248, 16384] -231: model.layers.120.post_attention_layernorm.weight shape: [16384] -232: model.layers.120.self_attn.k_proj.weight shape: [1024, 16384] -233: model.layers.120.self_attn.o_proj.weight shape: [16384, 16384] -234: model.layers.120.self_attn.q_proj.weight shape: [16384, 16384] -235: model.layers.120.self_attn.v_proj.weight shape: [1024, 16384] -236: model.layers.121.input_layernorm.weight shape: [16384] -237: model.layers.121.mlp.down_proj.weight shape: [16384, 53248] -238: model.layers.121.mlp.gate_proj.weight shape: [53248, 16384] -239: model.layers.121.mlp.up_proj.weight shape: [53248, 16384] -240: model.layers.121.post_attention_layernorm.weight shape: [16384] -241: model.layers.121.self_attn.k_proj.weight shape: [1024, 16384] -242: model.layers.121.self_attn.o_proj.weight shape: [16384, 16384] -243: model.layers.121.self_attn.q_proj.weight shape: [16384, 16384] -244: model.layers.121.self_attn.v_proj.weight shape: [1024, 16384] -245: model.layers.122.input_layernorm.weight shape: [16384] -246: model.layers.122.mlp.down_proj.weight shape: [16384, 53248] -247: model.layers.122.mlp.gate_proj.weight shape: [53248, 16384] -248: model.layers.122.mlp.up_proj.weight shape: [53248, 16384] -249: model.layers.122.post_attention_layernorm.weight shape: [16384] -250: model.layers.122.self_attn.k_proj.weight shape: [1024, 16384] -251: model.layers.122.self_attn.o_proj.weight shape: [16384, 16384] -252: model.layers.122.self_attn.q_proj.weight shape: [16384, 16384] -253: model.layers.122.self_attn.v_proj.weight shape: [1024, 16384] -254: model.layers.123.input_layernorm.weight shape: [16384] -255: model.layers.123.mlp.down_proj.weight shape: [16384, 53248] -256: model.layers.123.mlp.gate_proj.weight shape: [53248, 16384] -257: model.layers.123.mlp.up_proj.weight shape: [53248, 16384] -258: model.layers.123.post_attention_layernorm.weight shape: [16384] -259: model.layers.123.self_attn.k_proj.weight shape: [1024, 16384] -260: model.layers.123.self_attn.o_proj.weight shape: [16384, 16384] -261: model.layers.123.self_attn.q_proj.weight shape: [16384, 16384] -262: model.layers.123.self_attn.v_proj.weight shape: [1024, 16384] -263: model.layers.124.input_layernorm.weight shape: [16384] -264: model.layers.124.mlp.down_proj.weight shape: [16384, 53248] -265: model.layers.124.mlp.gate_proj.weight shape: [53248, 16384] -266: model.layers.124.mlp.up_proj.weight shape: [53248, 16384] -267: model.layers.124.post_attention_layernorm.weight shape: [16384] -268: model.layers.124.self_attn.k_proj.weight shape: [1024, 16384] -269: model.layers.124.self_attn.o_proj.weight shape: [16384, 16384] -270: model.layers.124.self_attn.q_proj.weight shape: [16384, 16384] -271: model.layers.124.self_attn.v_proj.weight shape: [1024, 16384] -272: model.layers.125.input_layernorm.weight shape: [16384] -273: model.layers.125.mlp.down_proj.weight shape: [16384, 53248] -274: model.layers.125.mlp.gate_proj.weight shape: [53248, 16384] -275: model.layers.125.mlp.up_proj.weight shape: [53248, 16384] -276: model.layers.125.post_attention_layernorm.weight shape: [16384] -277: model.layers.125.self_attn.k_proj.weight shape: [1024, 16384] -278: model.layers.125.self_attn.o_proj.weight shape: [16384, 16384] -279: model.layers.125.self_attn.q_proj.weight shape: [16384, 16384] -280: model.layers.125.self_attn.v_proj.weight shape: [1024, 16384] -281: model.layers.13.input_layernorm.weight shape: [16384] -282: model.layers.13.mlp.down_proj.weight shape: [16384, 53248] -283: model.layers.13.mlp.gate_proj.weight shape: [53248, 16384] -284: model.layers.13.mlp.up_proj.weight shape: [53248, 16384] -285: model.layers.13.post_attention_layernorm.weight shape: [16384] -286: model.layers.13.self_attn.k_proj.weight shape: [1024, 16384] -287: model.layers.13.self_attn.o_proj.weight shape: [16384, 16384] -288: model.layers.13.self_attn.q_proj.weight shape: [16384, 16384] -289: model.layers.13.self_attn.v_proj.weight shape: [1024, 16384] -290: model.layers.14.input_layernorm.weight shape: [16384] -291: model.layers.14.mlp.down_proj.weight shape: [16384, 53248] -292: model.layers.14.mlp.gate_proj.weight shape: [53248, 16384] -293: model.layers.14.mlp.up_proj.weight shape: [53248, 16384] -294: model.layers.14.post_attention_layernorm.weight shape: [16384] -295: model.layers.14.self_attn.k_proj.weight shape: [1024, 16384] -296: model.layers.14.self_attn.o_proj.weight shape: [16384, 16384] -297: model.layers.14.self_attn.q_proj.weight shape: [16384, 16384] -298: model.layers.14.self_attn.v_proj.weight shape: [1024, 16384] -299: model.layers.15.input_layernorm.weight shape: [16384] -300: model.layers.15.mlp.down_proj.weight shape: [16384, 53248] -301: model.layers.15.mlp.gate_proj.weight shape: [53248, 16384] -302: model.layers.15.mlp.up_proj.weight shape: [53248, 16384] -303: model.layers.15.post_attention_layernorm.weight shape: [16384] -304: model.layers.15.self_attn.k_proj.weight shape: [1024, 16384] -305: model.layers.15.self_attn.o_proj.weight shape: [16384, 16384] -306: model.layers.15.self_attn.q_proj.weight shape: [16384, 16384] -307: model.layers.15.self_attn.v_proj.weight shape: [1024, 16384] -308: model.layers.16.input_layernorm.weight shape: [16384] -309: model.layers.16.mlp.down_proj.weight shape: [16384, 53248] -310: model.layers.16.mlp.gate_proj.weight shape: [53248, 16384] -311: model.layers.16.mlp.up_proj.weight shape: [53248, 16384] -312: model.layers.16.post_attention_layernorm.weight shape: [16384] -313: model.layers.16.self_attn.k_proj.weight shape: [1024, 16384] -314: model.layers.16.self_attn.o_proj.weight shape: [16384, 16384] -315: model.layers.16.self_attn.q_proj.weight shape: [16384, 16384] -316: model.layers.16.self_attn.v_proj.weight shape: [1024, 16384] -317: model.layers.17.input_layernorm.weight shape: [16384] -318: model.layers.17.mlp.down_proj.weight shape: [16384, 53248] -319: model.layers.17.mlp.gate_proj.weight shape: [53248, 16384] -320: model.layers.17.mlp.up_proj.weight shape: [53248, 16384] -321: model.layers.17.post_attention_layernorm.weight shape: [16384] -322: model.layers.17.self_attn.k_proj.weight shape: [1024, 16384] -323: model.layers.17.self_attn.o_proj.weight shape: [16384, 16384] -324: model.layers.17.self_attn.q_proj.weight shape: [16384, 16384] -325: model.layers.17.self_attn.v_proj.weight shape: [1024, 16384] -326: model.layers.18.input_layernorm.weight shape: [16384] -327: model.layers.18.mlp.down_proj.weight shape: [16384, 53248] -328: model.layers.18.mlp.gate_proj.weight shape: [53248, 16384] -329: model.layers.18.mlp.up_proj.weight shape: [53248, 16384] -330: model.layers.18.post_attention_layernorm.weight shape: [16384] -331: model.layers.18.self_attn.k_proj.weight shape: [1024, 16384] -332: model.layers.18.self_attn.o_proj.weight shape: [16384, 16384] -333: model.layers.18.self_attn.q_proj.weight shape: [16384, 16384] -334: model.layers.18.self_attn.v_proj.weight shape: [1024, 16384] -335: model.layers.19.input_layernorm.weight shape: [16384] -336: model.layers.19.mlp.down_proj.weight shape: [16384, 53248] -337: model.layers.19.mlp.gate_proj.weight shape: [53248, 16384] -338: model.layers.19.mlp.up_proj.weight shape: [53248, 16384] -339: model.layers.19.post_attention_layernorm.weight shape: [16384] -340: model.layers.19.self_attn.k_proj.weight shape: [1024, 16384] -341: model.layers.19.self_attn.o_proj.weight shape: [16384, 16384] -342: model.layers.19.self_attn.q_proj.weight shape: [16384, 16384] -343: model.layers.19.self_attn.v_proj.weight shape: [1024, 16384] -344: model.layers.2.input_layernorm.weight shape: [16384] -345: model.layers.2.mlp.down_proj.weight shape: [16384, 53248] -346: model.layers.2.mlp.gate_proj.weight shape: [53248, 16384] -347: model.layers.2.mlp.up_proj.weight shape: [53248, 16384] -348: model.layers.2.post_attention_layernorm.weight shape: [16384] -349: model.layers.2.self_attn.k_proj.weight shape: [1024, 16384] -350: model.layers.2.self_attn.o_proj.weight shape: [16384, 16384] -351: model.layers.2.self_attn.q_proj.weight shape: [16384, 16384] -352: model.layers.2.self_attn.v_proj.weight shape: [1024, 16384] -353: model.layers.20.input_layernorm.weight shape: [16384] -354: model.layers.20.mlp.down_proj.weight shape: [16384, 53248] -355: model.layers.20.mlp.gate_proj.weight shape: [53248, 16384] -356: model.layers.20.mlp.up_proj.weight shape: [53248, 16384] -357: model.layers.20.post_attention_layernorm.weight shape: [16384] -358: model.layers.20.self_attn.k_proj.weight shape: [1024, 16384] -359: model.layers.20.self_attn.o_proj.weight shape: [16384, 16384] -360: model.layers.20.self_attn.q_proj.weight shape: [16384, 16384] -361: model.layers.20.self_attn.v_proj.weight shape: [1024, 16384] -362: model.layers.21.input_layernorm.weight shape: [16384] -363: model.layers.21.mlp.down_proj.weight shape: [16384, 53248] -364: model.layers.21.mlp.gate_proj.weight shape: [53248, 16384] -365: model.layers.21.mlp.up_proj.weight shape: [53248, 16384] -366: model.layers.21.post_attention_layernorm.weight shape: [16384] -367: model.layers.21.self_attn.k_proj.weight shape: [1024, 16384] -368: model.layers.21.self_attn.o_proj.weight shape: [16384, 16384] -369: model.layers.21.self_attn.q_proj.weight shape: [16384, 16384] -370: model.layers.21.self_attn.v_proj.weight shape: [1024, 16384] -371: model.layers.22.input_layernorm.weight shape: [16384] -372: model.layers.22.mlp.down_proj.weight shape: [16384, 53248] -373: model.layers.22.mlp.gate_proj.weight shape: [53248, 16384] -374: model.layers.22.mlp.up_proj.weight shape: [53248, 16384] -375: model.layers.22.post_attention_layernorm.weight shape: [16384] -376: model.layers.22.self_attn.k_proj.weight shape: [1024, 16384] -377: model.layers.22.self_attn.o_proj.weight shape: [16384, 16384] -378: model.layers.22.self_attn.q_proj.weight shape: [16384, 16384] -379: model.layers.22.self_attn.v_proj.weight shape: [1024, 16384] -380: model.layers.23.input_layernorm.weight shape: [16384] -381: model.layers.23.mlp.down_proj.weight shape: [16384, 53248] -382: model.layers.23.mlp.gate_proj.weight shape: [53248, 16384] -383: model.layers.23.mlp.up_proj.weight shape: [53248, 16384] -384: model.layers.23.post_attention_layernorm.weight shape: [16384] -385: model.layers.23.self_attn.k_proj.weight shape: [1024, 16384] -386: model.layers.23.self_attn.o_proj.weight shape: [16384, 16384] -387: model.layers.23.self_attn.q_proj.weight shape: [16384, 16384] -388: model.layers.23.self_attn.v_proj.weight shape: [1024, 16384] -389: model.layers.24.input_layernorm.weight shape: [16384] -390: model.layers.24.mlp.down_proj.weight shape: [16384, 53248] -391: model.layers.24.mlp.gate_proj.weight shape: [53248, 16384] -392: model.layers.24.mlp.up_proj.weight shape: [53248, 16384] -393: model.layers.24.post_attention_layernorm.weight shape: [16384] -394: model.layers.24.self_attn.k_proj.weight shape: [1024, 16384] -395: model.layers.24.self_attn.o_proj.weight shape: [16384, 16384] -396: model.layers.24.self_attn.q_proj.weight shape: [16384, 16384] -397: model.layers.24.self_attn.v_proj.weight shape: [1024, 16384] -398: model.layers.25.input_layernorm.weight shape: [16384] -399: model.layers.25.mlp.down_proj.weight shape: [16384, 53248] -400: model.layers.25.mlp.gate_proj.weight shape: [53248, 16384] -401: model.layers.25.mlp.up_proj.weight shape: [53248, 16384] -402: model.layers.25.post_attention_layernorm.weight shape: [16384] -403: model.layers.25.self_attn.k_proj.weight shape: [1024, 16384] -404: model.layers.25.self_attn.o_proj.weight shape: [16384, 16384] -405: model.layers.25.self_attn.q_proj.weight shape: [16384, 16384] -406: model.layers.25.self_attn.v_proj.weight shape: [1024, 16384] -407: model.layers.26.input_layernorm.weight shape: [16384] -408: model.layers.26.mlp.down_proj.weight shape: [16384, 53248] -409: model.layers.26.mlp.gate_proj.weight shape: [53248, 16384] -410: model.layers.26.mlp.up_proj.weight shape: [53248, 16384] -411: model.layers.26.post_attention_layernorm.weight shape: [16384] -412: model.layers.26.self_attn.k_proj.weight shape: [1024, 16384] -413: model.layers.26.self_attn.o_proj.weight shape: [16384, 16384] -414: model.layers.26.self_attn.q_proj.weight shape: [16384, 16384] -415: model.layers.26.self_attn.v_proj.weight shape: [1024, 16384] -416: model.layers.27.input_layernorm.weight shape: [16384] -417: model.layers.27.mlp.down_proj.weight shape: [16384, 53248] -418: model.layers.27.mlp.gate_proj.weight shape: [53248, 16384] -419: model.layers.27.mlp.up_proj.weight shape: [53248, 16384] -420: model.layers.27.post_attention_layernorm.weight shape: [16384] -421: model.layers.27.self_attn.k_proj.weight shape: [1024, 16384] -422: model.layers.27.self_attn.o_proj.weight shape: [16384, 16384] -423: model.layers.27.self_attn.q_proj.weight shape: [16384, 16384] -424: model.layers.27.self_attn.v_proj.weight shape: [1024, 16384] -425: model.layers.28.input_layernorm.weight shape: [16384] -426: model.layers.28.mlp.down_proj.weight shape: [16384, 53248] -427: model.layers.28.mlp.gate_proj.weight shape: [53248, 16384] -428: model.layers.28.mlp.up_proj.weight shape: [53248, 16384] -429: model.layers.28.post_attention_layernorm.weight shape: [16384] -430: model.layers.28.self_attn.k_proj.weight shape: [1024, 16384] -431: model.layers.28.self_attn.o_proj.weight shape: [16384, 16384] -432: model.layers.28.self_attn.q_proj.weight shape: [16384, 16384] -433: model.layers.28.self_attn.v_proj.weight shape: [1024, 16384] -434: model.layers.29.input_layernorm.weight shape: [16384] -435: model.layers.29.mlp.down_proj.weight shape: [16384, 53248] -436: model.layers.29.mlp.gate_proj.weight shape: [53248, 16384] -437: model.layers.29.mlp.up_proj.weight shape: [53248, 16384] -438: model.layers.29.post_attention_layernorm.weight shape: [16384] -439: model.layers.29.self_attn.k_proj.weight shape: [1024, 16384] -440: model.layers.29.self_attn.o_proj.weight shape: [16384, 16384] -441: model.layers.29.self_attn.q_proj.weight shape: [16384, 16384] -442: model.layers.29.self_attn.v_proj.weight shape: [1024, 16384] -443: model.layers.3.input_layernorm.weight shape: [16384] -444: model.layers.3.mlp.down_proj.weight shape: [16384, 53248] -445: model.layers.3.mlp.gate_proj.weight shape: [53248, 16384] -446: model.layers.3.mlp.up_proj.weight shape: [53248, 16384] -447: model.layers.3.post_attention_layernorm.weight shape: [16384] -448: model.layers.3.self_attn.k_proj.weight shape: [1024, 16384] -449: model.layers.3.self_attn.o_proj.weight shape: [16384, 16384] -450: model.layers.3.self_attn.q_proj.weight shape: [16384, 16384] -451: model.layers.3.self_attn.v_proj.weight shape: [1024, 16384] -452: model.layers.30.input_layernorm.weight shape: [16384] -453: model.layers.30.mlp.down_proj.weight shape: [16384, 53248] -454: model.layers.30.mlp.gate_proj.weight shape: [53248, 16384] -455: model.layers.30.mlp.up_proj.weight shape: [53248, 16384] -456: model.layers.30.post_attention_layernorm.weight shape: [16384] -457: model.layers.30.self_attn.k_proj.weight shape: [1024, 16384] -458: model.layers.30.self_attn.o_proj.weight shape: [16384, 16384] -459: model.layers.30.self_attn.q_proj.weight shape: [16384, 16384] -460: model.layers.30.self_attn.v_proj.weight shape: [1024, 16384] -461: model.layers.31.input_layernorm.weight shape: [16384] -462: model.layers.31.mlp.down_proj.weight shape: [16384, 53248] -463: model.layers.31.mlp.gate_proj.weight shape: [53248, 16384] -464: model.layers.31.mlp.up_proj.weight shape: [53248, 16384] -465: model.layers.31.post_attention_layernorm.weight shape: [16384] -466: model.layers.31.self_attn.k_proj.weight shape: [1024, 16384] -467: model.layers.31.self_attn.o_proj.weight shape: [16384, 16384] -468: model.layers.31.self_attn.q_proj.weight shape: [16384, 16384] -469: model.layers.31.self_attn.v_proj.weight shape: [1024, 16384] -470: model.layers.32.input_layernorm.weight shape: [16384] -471: model.layers.32.mlp.down_proj.weight shape: [16384, 53248] -472: model.layers.32.mlp.gate_proj.weight shape: [53248, 16384] -473: model.layers.32.mlp.up_proj.weight shape: [53248, 16384] -474: model.layers.32.post_attention_layernorm.weight shape: [16384] -475: model.layers.32.self_attn.k_proj.weight shape: [1024, 16384] -476: model.layers.32.self_attn.o_proj.weight shape: [16384, 16384] -477: model.layers.32.self_attn.q_proj.weight shape: [16384, 16384] -478: model.layers.32.self_attn.v_proj.weight shape: [1024, 16384] -479: model.layers.33.input_layernorm.weight shape: [16384] -480: model.layers.33.mlp.down_proj.weight shape: [16384, 53248] -481: model.layers.33.mlp.gate_proj.weight shape: [53248, 16384] -482: model.layers.33.mlp.up_proj.weight shape: [53248, 16384] -483: model.layers.33.post_attention_layernorm.weight shape: [16384] -484: model.layers.33.self_attn.k_proj.weight shape: [1024, 16384] -485: model.layers.33.self_attn.o_proj.weight shape: [16384, 16384] -486: model.layers.33.self_attn.q_proj.weight shape: [16384, 16384] -487: model.layers.33.self_attn.v_proj.weight shape: [1024, 16384] -488: model.layers.34.input_layernorm.weight shape: [16384] -489: model.layers.34.mlp.down_proj.weight shape: [16384, 53248] -490: model.layers.34.mlp.gate_proj.weight shape: [53248, 16384] -491: model.layers.34.mlp.up_proj.weight shape: [53248, 16384] -492: model.layers.34.post_attention_layernorm.weight shape: [16384] -493: model.layers.34.self_attn.k_proj.weight shape: [1024, 16384] -494: model.layers.34.self_attn.o_proj.weight shape: [16384, 16384] -495: model.layers.34.self_attn.q_proj.weight shape: [16384, 16384] -496: model.layers.34.self_attn.v_proj.weight shape: [1024, 16384] -497: model.layers.35.input_layernorm.weight shape: [16384] -498: model.layers.35.mlp.down_proj.weight shape: [16384, 53248] -499: model.layers.35.mlp.gate_proj.weight shape: [53248, 16384] -500: model.layers.35.mlp.up_proj.weight shape: [53248, 16384] -501: model.layers.35.post_attention_layernorm.weight shape: [16384] -502: model.layers.35.self_attn.k_proj.weight shape: [1024, 16384] -503: model.layers.35.self_attn.o_proj.weight shape: [16384, 16384] -504: model.layers.35.self_attn.q_proj.weight shape: [16384, 16384] -505: model.layers.35.self_attn.v_proj.weight shape: [1024, 16384] -506: model.layers.36.input_layernorm.weight shape: [16384] -507: model.layers.36.mlp.down_proj.weight shape: [16384, 53248] -508: model.layers.36.mlp.gate_proj.weight shape: [53248, 16384] -509: model.layers.36.mlp.up_proj.weight shape: [53248, 16384] -510: model.layers.36.post_attention_layernorm.weight shape: [16384] -511: model.layers.36.self_attn.k_proj.weight shape: [1024, 16384] -512: model.layers.36.self_attn.o_proj.weight shape: [16384, 16384] -513: model.layers.36.self_attn.q_proj.weight shape: [16384, 16384] -514: model.layers.36.self_attn.v_proj.weight shape: [1024, 16384] -515: model.layers.37.input_layernorm.weight shape: [16384] -516: model.layers.37.mlp.down_proj.weight shape: [16384, 53248] -517: model.layers.37.mlp.gate_proj.weight shape: [53248, 16384] -518: model.layers.37.mlp.up_proj.weight shape: [53248, 16384] -519: model.layers.37.post_attention_layernorm.weight shape: [16384] -520: model.layers.37.self_attn.k_proj.weight shape: [1024, 16384] -521: model.layers.37.self_attn.o_proj.weight shape: [16384, 16384] -522: model.layers.37.self_attn.q_proj.weight shape: [16384, 16384] -523: model.layers.37.self_attn.v_proj.weight shape: [1024, 16384] -524: model.layers.38.input_layernorm.weight shape: [16384] -525: model.layers.38.mlp.down_proj.weight shape: [16384, 53248] -526: model.layers.38.mlp.gate_proj.weight shape: [53248, 16384] -527: model.layers.38.mlp.up_proj.weight shape: [53248, 16384] -528: model.layers.38.post_attention_layernorm.weight shape: [16384] -529: model.layers.38.self_attn.k_proj.weight shape: [1024, 16384] -530: model.layers.38.self_attn.o_proj.weight shape: [16384, 16384] -531: model.layers.38.self_attn.q_proj.weight shape: [16384, 16384] -532: model.layers.38.self_attn.v_proj.weight shape: [1024, 16384] -533: model.layers.39.input_layernorm.weight shape: [16384] -534: model.layers.39.mlp.down_proj.weight shape: [16384, 53248] -535: model.layers.39.mlp.gate_proj.weight shape: [53248, 16384] -536: model.layers.39.mlp.up_proj.weight shape: [53248, 16384] -537: model.layers.39.post_attention_layernorm.weight shape: [16384] -538: model.layers.39.self_attn.k_proj.weight shape: [1024, 16384] -539: model.layers.39.self_attn.o_proj.weight shape: [16384, 16384] -540: model.layers.39.self_attn.q_proj.weight shape: [16384, 16384] -541: model.layers.39.self_attn.v_proj.weight shape: [1024, 16384] -542: model.layers.4.input_layernorm.weight shape: [16384] -543: model.layers.4.mlp.down_proj.weight shape: [16384, 53248] -544: model.layers.4.mlp.gate_proj.weight shape: [53248, 16384] -545: model.layers.4.mlp.up_proj.weight shape: [53248, 16384] -546: model.layers.4.post_attention_layernorm.weight shape: [16384] -547: model.layers.4.self_attn.k_proj.weight shape: [1024, 16384] -548: model.layers.4.self_attn.o_proj.weight shape: [16384, 16384] -549: model.layers.4.self_attn.q_proj.weight shape: [16384, 16384] -550: model.layers.4.self_attn.v_proj.weight shape: [1024, 16384] -551: model.layers.40.input_layernorm.weight shape: [16384] -552: model.layers.40.mlp.down_proj.weight shape: [16384, 53248] -553: model.layers.40.mlp.gate_proj.weight shape: [53248, 16384] -554: model.layers.40.mlp.up_proj.weight shape: [53248, 16384] -555: model.layers.40.post_attention_layernorm.weight shape: [16384] -556: model.layers.40.self_attn.k_proj.weight shape: [1024, 16384] -557: model.layers.40.self_attn.o_proj.weight shape: [16384, 16384] -558: model.layers.40.self_attn.q_proj.weight shape: [16384, 16384] -559: model.layers.40.self_attn.v_proj.weight shape: [1024, 16384] -560: model.layers.41.input_layernorm.weight shape: [16384] -561: model.layers.41.mlp.down_proj.weight shape: [16384, 53248] -562: model.layers.41.mlp.gate_proj.weight shape: [53248, 16384] -563: model.layers.41.mlp.up_proj.weight shape: [53248, 16384] -564: model.layers.41.post_attention_layernorm.weight shape: [16384] -565: model.layers.41.self_attn.k_proj.weight shape: [1024, 16384] -566: model.layers.41.self_attn.o_proj.weight shape: [16384, 16384] -567: model.layers.41.self_attn.q_proj.weight shape: [16384, 16384] -568: model.layers.41.self_attn.v_proj.weight shape: [1024, 16384] -569: model.layers.42.input_layernorm.weight shape: [16384] -570: model.layers.42.mlp.down_proj.weight shape: [16384, 53248] -571: model.layers.42.mlp.gate_proj.weight shape: [53248, 16384] -572: model.layers.42.mlp.up_proj.weight shape: [53248, 16384] -573: model.layers.42.post_attention_layernorm.weight shape: [16384] -574: model.layers.42.self_attn.k_proj.weight shape: [1024, 16384] -575: model.layers.42.self_attn.o_proj.weight shape: [16384, 16384] -576: model.layers.42.self_attn.q_proj.weight shape: [16384, 16384] -577: model.layers.42.self_attn.v_proj.weight shape: [1024, 16384] -578: model.layers.43.input_layernorm.weight shape: [16384] -579: model.layers.43.mlp.down_proj.weight shape: [16384, 53248] -580: model.layers.43.mlp.gate_proj.weight shape: [53248, 16384] -581: model.layers.43.mlp.up_proj.weight shape: [53248, 16384] -582: model.layers.43.post_attention_layernorm.weight shape: [16384] -583: model.layers.43.self_attn.k_proj.weight shape: [1024, 16384] -584: model.layers.43.self_attn.o_proj.weight shape: [16384, 16384] -585: model.layers.43.self_attn.q_proj.weight shape: [16384, 16384] -586: model.layers.43.self_attn.v_proj.weight shape: [1024, 16384] -587: model.layers.44.input_layernorm.weight shape: [16384] -588: model.layers.44.mlp.down_proj.weight shape: [16384, 53248] -589: model.layers.44.mlp.gate_proj.weight shape: [53248, 16384] -590: model.layers.44.mlp.up_proj.weight shape: [53248, 16384] -591: model.layers.44.post_attention_layernorm.weight shape: [16384] -592: model.layers.44.self_attn.k_proj.weight shape: [1024, 16384] -593: model.layers.44.self_attn.o_proj.weight shape: [16384, 16384] -594: model.layers.44.self_attn.q_proj.weight shape: [16384, 16384] -595: model.layers.44.self_attn.v_proj.weight shape: [1024, 16384] -596: model.layers.45.input_layernorm.weight shape: [16384] -597: model.layers.45.mlp.down_proj.weight shape: [16384, 53248] -598: model.layers.45.mlp.gate_proj.weight shape: [53248, 16384] -599: model.layers.45.mlp.up_proj.weight shape: [53248, 16384] -600: model.layers.45.post_attention_layernorm.weight shape: [16384] -601: model.layers.45.self_attn.k_proj.weight shape: [1024, 16384] -602: model.layers.45.self_attn.o_proj.weight shape: [16384, 16384] -603: model.layers.45.self_attn.q_proj.weight shape: [16384, 16384] -604: model.layers.45.self_attn.v_proj.weight shape: [1024, 16384] -605: model.layers.46.input_layernorm.weight shape: [16384] -606: model.layers.46.mlp.down_proj.weight shape: [16384, 53248] -607: model.layers.46.mlp.gate_proj.weight shape: [53248, 16384] -608: model.layers.46.mlp.up_proj.weight shape: [53248, 16384] -609: model.layers.46.post_attention_layernorm.weight shape: [16384] -610: model.layers.46.self_attn.k_proj.weight shape: [1024, 16384] -611: model.layers.46.self_attn.o_proj.weight shape: [16384, 16384] -612: model.layers.46.self_attn.q_proj.weight shape: [16384, 16384] -613: model.layers.46.self_attn.v_proj.weight shape: [1024, 16384] -614: model.layers.47.input_layernorm.weight shape: [16384] -615: model.layers.47.mlp.down_proj.weight shape: [16384, 53248] -616: model.layers.47.mlp.gate_proj.weight shape: [53248, 16384] -617: model.layers.47.mlp.up_proj.weight shape: [53248, 16384] -618: model.layers.47.post_attention_layernorm.weight shape: [16384] -619: model.layers.47.self_attn.k_proj.weight shape: [1024, 16384] -620: model.layers.47.self_attn.o_proj.weight shape: [16384, 16384] -621: model.layers.47.self_attn.q_proj.weight shape: [16384, 16384] -622: model.layers.47.self_attn.v_proj.weight shape: [1024, 16384] -623: model.layers.48.input_layernorm.weight shape: [16384] -624: model.layers.48.mlp.down_proj.weight shape: [16384, 53248] -625: model.layers.48.mlp.gate_proj.weight shape: [53248, 16384] -626: model.layers.48.mlp.up_proj.weight shape: [53248, 16384] -627: model.layers.48.post_attention_layernorm.weight shape: [16384] -628: model.layers.48.self_attn.k_proj.weight shape: [1024, 16384] -629: model.layers.48.self_attn.o_proj.weight shape: [16384, 16384] -630: model.layers.48.self_attn.q_proj.weight shape: [16384, 16384] -631: model.layers.48.self_attn.v_proj.weight shape: [1024, 16384] -632: model.layers.49.input_layernorm.weight shape: [16384] -633: model.layers.49.mlp.down_proj.weight shape: [16384, 53248] -634: model.layers.49.mlp.gate_proj.weight shape: [53248, 16384] -635: model.layers.49.mlp.up_proj.weight shape: [53248, 16384] -636: model.layers.49.post_attention_layernorm.weight shape: [16384] -637: model.layers.49.self_attn.k_proj.weight shape: [1024, 16384] -638: model.layers.49.self_attn.o_proj.weight shape: [16384, 16384] -639: model.layers.49.self_attn.q_proj.weight shape: [16384, 16384] -640: model.layers.49.self_attn.v_proj.weight shape: [1024, 16384] -641: model.layers.5.input_layernorm.weight shape: [16384] -642: model.layers.5.mlp.down_proj.weight shape: [16384, 53248] -643: model.layers.5.mlp.gate_proj.weight shape: [53248, 16384] -644: model.layers.5.mlp.up_proj.weight shape: [53248, 16384] -645: model.layers.5.post_attention_layernorm.weight shape: [16384] -646: model.layers.5.self_attn.k_proj.weight shape: [1024, 16384] -647: model.layers.5.self_attn.o_proj.weight shape: [16384, 16384] -648: model.layers.5.self_attn.q_proj.weight shape: [16384, 16384] -649: model.layers.5.self_attn.v_proj.weight shape: [1024, 16384] -650: model.layers.50.input_layernorm.weight shape: [16384] -651: model.layers.50.mlp.down_proj.weight shape: [16384, 53248] -652: model.layers.50.mlp.gate_proj.weight shape: [53248, 16384] -653: model.layers.50.mlp.up_proj.weight shape: [53248, 16384] -654: model.layers.50.post_attention_layernorm.weight shape: [16384] -655: model.layers.50.self_attn.k_proj.weight shape: [1024, 16384] -656: model.layers.50.self_attn.o_proj.weight shape: [16384, 16384] -657: model.layers.50.self_attn.q_proj.weight shape: [16384, 16384] -658: model.layers.50.self_attn.v_proj.weight shape: [1024, 16384] -659: model.layers.51.input_layernorm.weight shape: [16384] -660: model.layers.51.mlp.down_proj.weight shape: [16384, 53248] -661: model.layers.51.mlp.gate_proj.weight shape: [53248, 16384] -662: model.layers.51.mlp.up_proj.weight shape: [53248, 16384] -663: model.layers.51.post_attention_layernorm.weight shape: [16384] -664: model.layers.51.self_attn.k_proj.weight shape: [1024, 16384] -665: model.layers.51.self_attn.o_proj.weight shape: [16384, 16384] -666: model.layers.51.self_attn.q_proj.weight shape: [16384, 16384] -667: model.layers.51.self_attn.v_proj.weight shape: [1024, 16384] -668: model.layers.52.input_layernorm.weight shape: [16384] -669: model.layers.52.mlp.down_proj.weight shape: [16384, 53248] -670: model.layers.52.mlp.gate_proj.weight shape: [53248, 16384] -671: model.layers.52.mlp.up_proj.weight shape: [53248, 16384] -672: model.layers.52.post_attention_layernorm.weight shape: [16384] -673: model.layers.52.self_attn.k_proj.weight shape: [1024, 16384] -674: model.layers.52.self_attn.o_proj.weight shape: [16384, 16384] -675: model.layers.52.self_attn.q_proj.weight shape: [16384, 16384] -676: model.layers.52.self_attn.v_proj.weight shape: [1024, 16384] -677: model.layers.53.input_layernorm.weight shape: [16384] -678: model.layers.53.mlp.down_proj.weight shape: [16384, 53248] -679: model.layers.53.mlp.gate_proj.weight shape: [53248, 16384] -680: model.layers.53.mlp.up_proj.weight shape: [53248, 16384] -681: model.layers.53.post_attention_layernorm.weight shape: [16384] -682: model.layers.53.self_attn.k_proj.weight shape: [1024, 16384] -683: model.layers.53.self_attn.o_proj.weight shape: [16384, 16384] -684: model.layers.53.self_attn.q_proj.weight shape: [16384, 16384] -685: model.layers.53.self_attn.v_proj.weight shape: [1024, 16384] -686: model.layers.54.input_layernorm.weight shape: [16384] -687: model.layers.54.mlp.down_proj.weight shape: [16384, 53248] -688: model.layers.54.mlp.gate_proj.weight shape: [53248, 16384] -689: model.layers.54.mlp.up_proj.weight shape: [53248, 16384] -690: model.layers.54.post_attention_layernorm.weight shape: [16384] -691: model.layers.54.self_attn.k_proj.weight shape: [1024, 16384] -692: model.layers.54.self_attn.o_proj.weight shape: [16384, 16384] -693: model.layers.54.self_attn.q_proj.weight shape: [16384, 16384] -694: model.layers.54.self_attn.v_proj.weight shape: [1024, 16384] -695: model.layers.55.input_layernorm.weight shape: [16384] -696: model.layers.55.mlp.down_proj.weight shape: [16384, 53248] -697: model.layers.55.mlp.gate_proj.weight shape: [53248, 16384] -698: model.layers.55.mlp.up_proj.weight shape: [53248, 16384] -699: model.layers.55.post_attention_layernorm.weight shape: [16384] -700: model.layers.55.self_attn.k_proj.weight shape: [1024, 16384] -701: model.layers.55.self_attn.o_proj.weight shape: [16384, 16384] -702: model.layers.55.self_attn.q_proj.weight shape: [16384, 16384] -703: model.layers.55.self_attn.v_proj.weight shape: [1024, 16384] -704: model.layers.56.input_layernorm.weight shape: [16384] -705: model.layers.56.mlp.down_proj.weight shape: [16384, 53248] -706: model.layers.56.mlp.gate_proj.weight shape: [53248, 16384] -707: model.layers.56.mlp.up_proj.weight shape: [53248, 16384] -708: model.layers.56.post_attention_layernorm.weight shape: [16384] -709: model.layers.56.self_attn.k_proj.weight shape: [1024, 16384] -710: model.layers.56.self_attn.o_proj.weight shape: [16384, 16384] -711: model.layers.56.self_attn.q_proj.weight shape: [16384, 16384] -712: model.layers.56.self_attn.v_proj.weight shape: [1024, 16384] -713: model.layers.57.input_layernorm.weight shape: [16384] -714: model.layers.57.mlp.down_proj.weight shape: [16384, 53248] -715: model.layers.57.mlp.gate_proj.weight shape: [53248, 16384] -716: model.layers.57.mlp.up_proj.weight shape: [53248, 16384] -717: model.layers.57.post_attention_layernorm.weight shape: [16384] -718: model.layers.57.self_attn.k_proj.weight shape: [1024, 16384] -719: model.layers.57.self_attn.o_proj.weight shape: [16384, 16384] -720: model.layers.57.self_attn.q_proj.weight shape: [16384, 16384] -721: model.layers.57.self_attn.v_proj.weight shape: [1024, 16384] -722: model.layers.58.input_layernorm.weight shape: [16384] -723: model.layers.58.mlp.down_proj.weight shape: [16384, 53248] -724: model.layers.58.mlp.gate_proj.weight shape: [53248, 16384] -725: model.layers.58.mlp.up_proj.weight shape: [53248, 16384] -726: model.layers.58.post_attention_layernorm.weight shape: [16384] -727: model.layers.58.self_attn.k_proj.weight shape: [1024, 16384] -728: model.layers.58.self_attn.o_proj.weight shape: [16384, 16384] -729: model.layers.58.self_attn.q_proj.weight shape: [16384, 16384] -730: model.layers.58.self_attn.v_proj.weight shape: [1024, 16384] -731: model.layers.59.input_layernorm.weight shape: [16384] -732: model.layers.59.mlp.down_proj.weight shape: [16384, 53248] -733: model.layers.59.mlp.gate_proj.weight shape: [53248, 16384] -734: model.layers.59.mlp.up_proj.weight shape: [53248, 16384] -735: model.layers.59.post_attention_layernorm.weight shape: [16384] -736: model.layers.59.self_attn.k_proj.weight shape: [1024, 16384] -737: model.layers.59.self_attn.o_proj.weight shape: [16384, 16384] -738: model.layers.59.self_attn.q_proj.weight shape: [16384, 16384] -739: model.layers.59.self_attn.v_proj.weight shape: [1024, 16384] -740: model.layers.6.input_layernorm.weight shape: [16384] -741: model.layers.6.mlp.down_proj.weight shape: [16384, 53248] -742: model.layers.6.mlp.gate_proj.weight shape: [53248, 16384] -743: model.layers.6.mlp.up_proj.weight shape: [53248, 16384] -744: model.layers.6.post_attention_layernorm.weight shape: [16384] -745: model.layers.6.self_attn.k_proj.weight shape: [1024, 16384] -746: model.layers.6.self_attn.o_proj.weight shape: [16384, 16384] -747: model.layers.6.self_attn.q_proj.weight shape: [16384, 16384] -748: model.layers.6.self_attn.v_proj.weight shape: [1024, 16384] -749: model.layers.60.input_layernorm.weight shape: [16384] -750: model.layers.60.mlp.down_proj.weight shape: [16384, 53248] -751: model.layers.60.mlp.gate_proj.weight shape: [53248, 16384] -752: model.layers.60.mlp.up_proj.weight shape: [53248, 16384] -753: model.layers.60.post_attention_layernorm.weight shape: [16384] -754: model.layers.60.self_attn.k_proj.weight shape: [1024, 16384] -755: model.layers.60.self_attn.o_proj.weight shape: [16384, 16384] -756: model.layers.60.self_attn.q_proj.weight shape: [16384, 16384] -757: model.layers.60.self_attn.v_proj.weight shape: [1024, 16384] -758: model.layers.61.input_layernorm.weight shape: [16384] -759: model.layers.61.mlp.down_proj.weight shape: [16384, 53248] -760: model.layers.61.mlp.gate_proj.weight shape: [53248, 16384] -761: model.layers.61.mlp.up_proj.weight shape: [53248, 16384] -762: model.layers.61.post_attention_layernorm.weight shape: [16384] -763: model.layers.61.self_attn.k_proj.weight shape: [1024, 16384] -764: model.layers.61.self_attn.o_proj.weight shape: [16384, 16384] -765: model.layers.61.self_attn.q_proj.weight shape: [16384, 16384] -766: model.layers.61.self_attn.v_proj.weight shape: [1024, 16384] -767: model.layers.62.input_layernorm.weight shape: [16384] -768: model.layers.62.mlp.down_proj.weight shape: [16384, 53248] -769: model.layers.62.mlp.gate_proj.weight shape: [53248, 16384] -770: model.layers.62.mlp.up_proj.weight shape: [53248, 16384] -771: model.layers.62.post_attention_layernorm.weight shape: [16384] -772: model.layers.62.self_attn.k_proj.weight shape: [1024, 16384] -773: model.layers.62.self_attn.o_proj.weight shape: [16384, 16384] -774: model.layers.62.self_attn.q_proj.weight shape: [16384, 16384] -775: model.layers.62.self_attn.v_proj.weight shape: [1024, 16384] -776: model.layers.63.input_layernorm.weight shape: [16384] -777: model.layers.63.mlp.down_proj.weight shape: [16384, 53248] -778: model.layers.63.mlp.gate_proj.weight shape: [53248, 16384] -779: model.layers.63.mlp.up_proj.weight shape: [53248, 16384] -780: model.layers.63.post_attention_layernorm.weight shape: [16384] -781: model.layers.63.self_attn.k_proj.weight shape: [1024, 16384] -782: model.layers.63.self_attn.o_proj.weight shape: [16384, 16384] -783: model.layers.63.self_attn.q_proj.weight shape: [16384, 16384] -784: model.layers.63.self_attn.v_proj.weight shape: [1024, 16384] -785: model.layers.64.input_layernorm.weight shape: [16384] -786: model.layers.64.mlp.down_proj.weight shape: [16384, 53248] -787: model.layers.64.mlp.gate_proj.weight shape: [53248, 16384] -788: model.layers.64.mlp.up_proj.weight shape: [53248, 16384] -789: model.layers.64.post_attention_layernorm.weight shape: [16384] -790: model.layers.64.self_attn.k_proj.weight shape: [1024, 16384] -791: model.layers.64.self_attn.o_proj.weight shape: [16384, 16384] -792: model.layers.64.self_attn.q_proj.weight shape: [16384, 16384] -793: model.layers.64.self_attn.v_proj.weight shape: [1024, 16384] -794: model.layers.65.input_layernorm.weight shape: [16384] -795: model.layers.65.mlp.down_proj.weight shape: [16384, 53248] -796: model.layers.65.mlp.gate_proj.weight shape: [53248, 16384] -797: model.layers.65.mlp.up_proj.weight shape: [53248, 16384] -798: model.layers.65.post_attention_layernorm.weight shape: [16384] -799: model.layers.65.self_attn.k_proj.weight shape: [1024, 16384] -800: model.layers.65.self_attn.o_proj.weight shape: [16384, 16384] -801: model.layers.65.self_attn.q_proj.weight shape: [16384, 16384] -802: model.layers.65.self_attn.v_proj.weight shape: [1024, 16384] -803: model.layers.66.input_layernorm.weight shape: [16384] -804: model.layers.66.mlp.down_proj.weight shape: [16384, 53248] -805: model.layers.66.mlp.gate_proj.weight shape: [53248, 16384] -806: model.layers.66.mlp.up_proj.weight shape: [53248, 16384] -807: model.layers.66.post_attention_layernorm.weight shape: [16384] -808: model.layers.66.self_attn.k_proj.weight shape: [1024, 16384] -809: model.layers.66.self_attn.o_proj.weight shape: [16384, 16384] -810: model.layers.66.self_attn.q_proj.weight shape: [16384, 16384] -811: model.layers.66.self_attn.v_proj.weight shape: [1024, 16384] -812: model.layers.67.input_layernorm.weight shape: [16384] -813: model.layers.67.mlp.down_proj.weight shape: [16384, 53248] -814: model.layers.67.mlp.gate_proj.weight shape: [53248, 16384] -815: model.layers.67.mlp.up_proj.weight shape: [53248, 16384] -816: model.layers.67.post_attention_layernorm.weight shape: [16384] -817: model.layers.67.self_attn.k_proj.weight shape: [1024, 16384] -818: model.layers.67.self_attn.o_proj.weight shape: [16384, 16384] -819: model.layers.67.self_attn.q_proj.weight shape: [16384, 16384] -820: model.layers.67.self_attn.v_proj.weight shape: [1024, 16384] -821: model.layers.68.input_layernorm.weight shape: [16384] -822: model.layers.68.mlp.down_proj.weight shape: [16384, 53248] -823: model.layers.68.mlp.gate_proj.weight shape: [53248, 16384] -824: model.layers.68.mlp.up_proj.weight shape: [53248, 16384] -825: model.layers.68.post_attention_layernorm.weight shape: [16384] -826: model.layers.68.self_attn.k_proj.weight shape: [1024, 16384] -827: model.layers.68.self_attn.o_proj.weight shape: [16384, 16384] -828: model.layers.68.self_attn.q_proj.weight shape: [16384, 16384] -829: model.layers.68.self_attn.v_proj.weight shape: [1024, 16384] -830: model.layers.69.input_layernorm.weight shape: [16384] -831: model.layers.69.mlp.down_proj.weight shape: [16384, 53248] -832: model.layers.69.mlp.gate_proj.weight shape: [53248, 16384] -833: model.layers.69.mlp.up_proj.weight shape: [53248, 16384] -834: model.layers.69.post_attention_layernorm.weight shape: [16384] -835: model.layers.69.self_attn.k_proj.weight shape: [1024, 16384] -836: model.layers.69.self_attn.o_proj.weight shape: [16384, 16384] -837: model.layers.69.self_attn.q_proj.weight shape: [16384, 16384] -838: model.layers.69.self_attn.v_proj.weight shape: [1024, 16384] -839: model.layers.7.input_layernorm.weight shape: [16384] -840: model.layers.7.mlp.down_proj.weight shape: [16384, 53248] -841: model.layers.7.mlp.gate_proj.weight shape: [53248, 16384] -842: model.layers.7.mlp.up_proj.weight shape: [53248, 16384] -843: model.layers.7.post_attention_layernorm.weight shape: [16384] -844: model.layers.7.self_attn.k_proj.weight shape: [1024, 16384] -845: model.layers.7.self_attn.o_proj.weight shape: [16384, 16384] -846: model.layers.7.self_attn.q_proj.weight shape: [16384, 16384] -847: model.layers.7.self_attn.v_proj.weight shape: [1024, 16384] -848: model.layers.70.input_layernorm.weight shape: [16384] -849: model.layers.70.mlp.down_proj.weight shape: [16384, 53248] -850: model.layers.70.mlp.gate_proj.weight shape: [53248, 16384] -851: model.layers.70.mlp.up_proj.weight shape: [53248, 16384] -852: model.layers.70.post_attention_layernorm.weight shape: [16384] -853: model.layers.70.self_attn.k_proj.weight shape: [1024, 16384] -854: model.layers.70.self_attn.o_proj.weight shape: [16384, 16384] -855: model.layers.70.self_attn.q_proj.weight shape: [16384, 16384] -856: model.layers.70.self_attn.v_proj.weight shape: [1024, 16384] -857: model.layers.71.input_layernorm.weight shape: [16384] -858: model.layers.71.mlp.down_proj.weight shape: [16384, 53248] -859: model.layers.71.mlp.gate_proj.weight shape: [53248, 16384] -860: model.layers.71.mlp.up_proj.weight shape: [53248, 16384] -861: model.layers.71.post_attention_layernorm.weight shape: [16384] -862: model.layers.71.self_attn.k_proj.weight shape: [1024, 16384] -863: model.layers.71.self_attn.o_proj.weight shape: [16384, 16384] -864: model.layers.71.self_attn.q_proj.weight shape: [16384, 16384] -865: model.layers.71.self_attn.v_proj.weight shape: [1024, 16384] -866: model.layers.72.input_layernorm.weight shape: [16384] -867: model.layers.72.mlp.down_proj.weight shape: [16384, 53248] -868: model.layers.72.mlp.gate_proj.weight shape: [53248, 16384] -869: model.layers.72.mlp.up_proj.weight shape: [53248, 16384] -870: model.layers.72.post_attention_layernorm.weight shape: [16384] -871: model.layers.72.self_attn.k_proj.weight shape: [1024, 16384] -872: model.layers.72.self_attn.o_proj.weight shape: [16384, 16384] -873: model.layers.72.self_attn.q_proj.weight shape: [16384, 16384] -874: model.layers.72.self_attn.v_proj.weight shape: [1024, 16384] -875: model.layers.73.input_layernorm.weight shape: [16384] -876: model.layers.73.mlp.down_proj.weight shape: [16384, 53248] -877: model.layers.73.mlp.gate_proj.weight shape: [53248, 16384] -878: model.layers.73.mlp.up_proj.weight shape: [53248, 16384] -879: model.layers.73.post_attention_layernorm.weight shape: [16384] -880: model.layers.73.self_attn.k_proj.weight shape: [1024, 16384] -881: model.layers.73.self_attn.o_proj.weight shape: [16384, 16384] -882: model.layers.73.self_attn.q_proj.weight shape: [16384, 16384] -883: model.layers.73.self_attn.v_proj.weight shape: [1024, 16384] -884: model.layers.74.input_layernorm.weight shape: [16384] -885: model.layers.74.mlp.down_proj.weight shape: [16384, 53248] -886: model.layers.74.mlp.gate_proj.weight shape: [53248, 16384] -887: model.layers.74.mlp.up_proj.weight shape: [53248, 16384] -888: model.layers.74.post_attention_layernorm.weight shape: [16384] -889: model.layers.74.self_attn.k_proj.weight shape: [1024, 16384] -890: model.layers.74.self_attn.o_proj.weight shape: [16384, 16384] -891: model.layers.74.self_attn.q_proj.weight shape: [16384, 16384] -892: model.layers.74.self_attn.v_proj.weight shape: [1024, 16384] -893: model.layers.75.input_layernorm.weight shape: [16384] -894: model.layers.75.mlp.down_proj.weight shape: [16384, 53248] -895: model.layers.75.mlp.gate_proj.weight shape: [53248, 16384] -896: model.layers.75.mlp.up_proj.weight shape: [53248, 16384] -897: model.layers.75.post_attention_layernorm.weight shape: [16384] -898: model.layers.75.self_attn.k_proj.weight shape: [1024, 16384] -899: model.layers.75.self_attn.o_proj.weight shape: [16384, 16384] -900: model.layers.75.self_attn.q_proj.weight shape: [16384, 16384] -901: model.layers.75.self_attn.v_proj.weight shape: [1024, 16384] -902: model.layers.76.input_layernorm.weight shape: [16384] -903: model.layers.76.mlp.down_proj.weight shape: [16384, 53248] -904: model.layers.76.mlp.gate_proj.weight shape: [53248, 16384] -905: model.layers.76.mlp.up_proj.weight shape: [53248, 16384] -906: model.layers.76.post_attention_layernorm.weight shape: [16384] -907: model.layers.76.self_attn.k_proj.weight shape: [1024, 16384] -908: model.layers.76.self_attn.o_proj.weight shape: [16384, 16384] -909: model.layers.76.self_attn.q_proj.weight shape: [16384, 16384] -910: model.layers.76.self_attn.v_proj.weight shape: [1024, 16384] -911: model.layers.77.input_layernorm.weight shape: [16384] -912: model.layers.77.mlp.down_proj.weight shape: [16384, 53248] -913: model.layers.77.mlp.gate_proj.weight shape: [53248, 16384] -914: model.layers.77.mlp.up_proj.weight shape: [53248, 16384] -915: model.layers.77.post_attention_layernorm.weight shape: [16384] -916: model.layers.77.self_attn.k_proj.weight shape: [1024, 16384] -917: model.layers.77.self_attn.o_proj.weight shape: [16384, 16384] -918: model.layers.77.self_attn.q_proj.weight shape: [16384, 16384] -919: model.layers.77.self_attn.v_proj.weight shape: [1024, 16384] -920: model.layers.78.input_layernorm.weight shape: [16384] -921: model.layers.78.mlp.down_proj.weight shape: [16384, 53248] -922: model.layers.78.mlp.gate_proj.weight shape: [53248, 16384] -923: model.layers.78.mlp.up_proj.weight shape: [53248, 16384] -924: model.layers.78.post_attention_layernorm.weight shape: [16384] -925: model.layers.78.self_attn.k_proj.weight shape: [1024, 16384] -926: model.layers.78.self_attn.o_proj.weight shape: [16384, 16384] -927: model.layers.78.self_attn.q_proj.weight shape: [16384, 16384] -928: model.layers.78.self_attn.v_proj.weight shape: [1024, 16384] -929: model.layers.79.input_layernorm.weight shape: [16384] -930: model.layers.79.mlp.down_proj.weight shape: [16384, 53248] -931: model.layers.79.mlp.gate_proj.weight shape: [53248, 16384] -932: model.layers.79.mlp.up_proj.weight shape: [53248, 16384] -933: model.layers.79.post_attention_layernorm.weight shape: [16384] -934: model.layers.79.self_attn.k_proj.weight shape: [1024, 16384] -935: model.layers.79.self_attn.o_proj.weight shape: [16384, 16384] -936: model.layers.79.self_attn.q_proj.weight shape: [16384, 16384] -937: model.layers.79.self_attn.v_proj.weight shape: [1024, 16384] -938: model.layers.8.input_layernorm.weight shape: [16384] -939: model.layers.8.mlp.down_proj.weight shape: [16384, 53248] -940: model.layers.8.mlp.gate_proj.weight shape: [53248, 16384] -941: model.layers.8.mlp.up_proj.weight shape: [53248, 16384] -942: model.layers.8.post_attention_layernorm.weight shape: [16384] -943: model.layers.8.self_attn.k_proj.weight shape: [1024, 16384] -944: model.layers.8.self_attn.o_proj.weight shape: [16384, 16384] -945: model.layers.8.self_attn.q_proj.weight shape: [16384, 16384] -946: model.layers.8.self_attn.v_proj.weight shape: [1024, 16384] -947: model.layers.80.input_layernorm.weight shape: [16384] -948: model.layers.80.mlp.down_proj.weight shape: [16384, 53248] -949: model.layers.80.mlp.gate_proj.weight shape: [53248, 16384] -950: model.layers.80.mlp.up_proj.weight shape: [53248, 16384] -951: model.layers.80.post_attention_layernorm.weight shape: [16384] -952: model.layers.80.self_attn.k_proj.weight shape: [1024, 16384] -953: model.layers.80.self_attn.o_proj.weight shape: [16384, 16384] -954: model.layers.80.self_attn.q_proj.weight shape: [16384, 16384] -955: model.layers.80.self_attn.v_proj.weight shape: [1024, 16384] -956: model.layers.81.input_layernorm.weight shape: [16384] -957: model.layers.81.mlp.down_proj.weight shape: [16384, 53248] -958: model.layers.81.mlp.gate_proj.weight shape: [53248, 16384] -959: model.layers.81.mlp.up_proj.weight shape: [53248, 16384] -960: model.layers.81.post_attention_layernorm.weight shape: [16384] -961: model.layers.81.self_attn.k_proj.weight shape: [1024, 16384] -962: model.layers.81.self_attn.o_proj.weight shape: [16384, 16384] -963: model.layers.81.self_attn.q_proj.weight shape: [16384, 16384] -964: model.layers.81.self_attn.v_proj.weight shape: [1024, 16384] -965: model.layers.82.input_layernorm.weight shape: [16384] -966: model.layers.82.mlp.down_proj.weight shape: [16384, 53248] -967: model.layers.82.mlp.gate_proj.weight shape: [53248, 16384] -968: model.layers.82.mlp.up_proj.weight shape: [53248, 16384] -969: model.layers.82.post_attention_layernorm.weight shape: [16384] -970: model.layers.82.self_attn.k_proj.weight shape: [1024, 16384] -971: model.layers.82.self_attn.o_proj.weight shape: [16384, 16384] -972: model.layers.82.self_attn.q_proj.weight shape: [16384, 16384] -973: model.layers.82.self_attn.v_proj.weight shape: [1024, 16384] -974: model.layers.83.input_layernorm.weight shape: [16384] -975: model.layers.83.mlp.down_proj.weight shape: [16384, 53248] -976: model.layers.83.mlp.gate_proj.weight shape: [53248, 16384] -977: model.layers.83.mlp.up_proj.weight shape: [53248, 16384] -978: model.layers.83.post_attention_layernorm.weight shape: [16384] -979: model.layers.83.self_attn.k_proj.weight shape: [1024, 16384] -980: model.layers.83.self_attn.o_proj.weight shape: [16384, 16384] -981: model.layers.83.self_attn.q_proj.weight shape: [16384, 16384] -982: model.layers.83.self_attn.v_proj.weight shape: [1024, 16384] -983: model.layers.84.input_layernorm.weight shape: [16384] -984: model.layers.84.mlp.down_proj.weight shape: [16384, 53248] -985: model.layers.84.mlp.gate_proj.weight shape: [53248, 16384] -986: model.layers.84.mlp.up_proj.weight shape: [53248, 16384] -987: model.layers.84.post_attention_layernorm.weight shape: [16384] -988: model.layers.84.self_attn.k_proj.weight shape: [1024, 16384] -989: model.layers.84.self_attn.o_proj.weight shape: [16384, 16384] -990: model.layers.84.self_attn.q_proj.weight shape: [16384, 16384] -991: model.layers.84.self_attn.v_proj.weight shape: [1024, 16384] -992: model.layers.85.input_layernorm.weight shape: [16384] -993: model.layers.85.mlp.down_proj.weight shape: [16384, 53248] -994: model.layers.85.mlp.gate_proj.weight shape: [53248, 16384] -995: model.layers.85.mlp.up_proj.weight shape: [53248, 16384] -996: model.layers.85.post_attention_layernorm.weight shape: [16384] -997: model.layers.85.self_attn.k_proj.weight shape: [1024, 16384] -998: model.layers.85.self_attn.o_proj.weight shape: [16384, 16384] -999: model.layers.85.self_attn.q_proj.weight shape: [16384, 16384] -1000: model.layers.85.self_attn.v_proj.weight shape: [1024, 16384] -1001: model.layers.86.input_layernorm.weight shape: [16384] -1002: model.layers.86.mlp.down_proj.weight shape: [16384, 53248] -1003: model.layers.86.mlp.gate_proj.weight shape: [53248, 16384] -1004: model.layers.86.mlp.up_proj.weight shape: [53248, 16384] -1005: model.layers.86.post_attention_layernorm.weight shape: [16384] -1006: model.layers.86.self_attn.k_proj.weight shape: [1024, 16384] -1007: model.layers.86.self_attn.o_proj.weight shape: [16384, 16384] -1008: model.layers.86.self_attn.q_proj.weight shape: [16384, 16384] -1009: model.layers.86.self_attn.v_proj.weight shape: [1024, 16384] -1010: model.layers.87.input_layernorm.weight shape: [16384] -1011: model.layers.87.mlp.down_proj.weight shape: [16384, 53248] -1012: model.layers.87.mlp.gate_proj.weight shape: [53248, 16384] -1013: model.layers.87.mlp.up_proj.weight shape: [53248, 16384] -1014: model.layers.87.post_attention_layernorm.weight shape: [16384] -1015: model.layers.87.self_attn.k_proj.weight shape: [1024, 16384] -1016: model.layers.87.self_attn.o_proj.weight shape: [16384, 16384] -1017: model.layers.87.self_attn.q_proj.weight shape: [16384, 16384] -1018: model.layers.87.self_attn.v_proj.weight shape: [1024, 16384] -1019: model.layers.88.input_layernorm.weight shape: [16384] -1020: model.layers.88.mlp.down_proj.weight shape: [16384, 53248] -1021: model.layers.88.mlp.gate_proj.weight shape: [53248, 16384] -1022: model.layers.88.mlp.up_proj.weight shape: [53248, 16384] -1023: model.layers.88.post_attention_layernorm.weight shape: [16384] -1024: model.layers.88.self_attn.k_proj.weight shape: [1024, 16384] -1025: model.layers.88.self_attn.o_proj.weight shape: [16384, 16384] -1026: model.layers.88.self_attn.q_proj.weight shape: [16384, 16384] -1027: model.layers.88.self_attn.v_proj.weight shape: [1024, 16384] -1028: model.layers.89.input_layernorm.weight shape: [16384] -1029: model.layers.89.mlp.down_proj.weight shape: [16384, 53248] -1030: model.layers.89.mlp.gate_proj.weight shape: [53248, 16384] -1031: model.layers.89.mlp.up_proj.weight shape: [53248, 16384] -1032: model.layers.89.post_attention_layernorm.weight shape: [16384] -1033: model.layers.89.self_attn.k_proj.weight shape: [1024, 16384] -1034: model.layers.89.self_attn.o_proj.weight shape: [16384, 16384] -1035: model.layers.89.self_attn.q_proj.weight shape: [16384, 16384] -1036: model.layers.89.self_attn.v_proj.weight shape: [1024, 16384] -1037: model.layers.9.input_layernorm.weight shape: [16384] -1038: model.layers.9.mlp.down_proj.weight shape: [16384, 53248] -1039: model.layers.9.mlp.gate_proj.weight shape: [53248, 16384] -1040: model.layers.9.mlp.up_proj.weight shape: [53248, 16384] -1041: model.layers.9.post_attention_layernorm.weight shape: [16384] -1042: model.layers.9.self_attn.k_proj.weight shape: [1024, 16384] -1043: model.layers.9.self_attn.o_proj.weight shape: [16384, 16384] -1044: model.layers.9.self_attn.q_proj.weight shape: [16384, 16384] -1045: model.layers.9.self_attn.v_proj.weight shape: [1024, 16384] -1046: model.layers.90.input_layernorm.weight shape: [16384] -1047: model.layers.90.mlp.down_proj.weight shape: [16384, 53248] -1048: model.layers.90.mlp.gate_proj.weight shape: [53248, 16384] -1049: model.layers.90.mlp.up_proj.weight shape: [53248, 16384] -1050: model.layers.90.post_attention_layernorm.weight shape: [16384] -1051: model.layers.90.self_attn.k_proj.weight shape: [1024, 16384] -1052: model.layers.90.self_attn.o_proj.weight shape: [16384, 16384] -1053: model.layers.90.self_attn.q_proj.weight shape: [16384, 16384] -1054: model.layers.90.self_attn.v_proj.weight shape: [1024, 16384] -1055: model.layers.91.input_layernorm.weight shape: [16384] -1056: model.layers.91.mlp.down_proj.weight shape: [16384, 53248] -1057: model.layers.91.mlp.gate_proj.weight shape: [53248, 16384] -1058: model.layers.91.mlp.up_proj.weight shape: [53248, 16384] -1059: model.layers.91.post_attention_layernorm.weight shape: [16384] -1060: model.layers.91.self_attn.k_proj.weight shape: [1024, 16384] -1061: model.layers.91.self_attn.o_proj.weight shape: [16384, 16384] -1062: model.layers.91.self_attn.q_proj.weight shape: [16384, 16384] -1063: model.layers.91.self_attn.v_proj.weight shape: [1024, 16384] -1064: model.layers.92.input_layernorm.weight shape: [16384] -1065: model.layers.92.mlp.down_proj.weight shape: [16384, 53248] -1066: model.layers.92.mlp.gate_proj.weight shape: [53248, 16384] -1067: model.layers.92.mlp.up_proj.weight shape: [53248, 16384] -1068: model.layers.92.post_attention_layernorm.weight shape: [16384] -1069: model.layers.92.self_attn.k_proj.weight shape: [1024, 16384] -1070: model.layers.92.self_attn.o_proj.weight shape: [16384, 16384] -1071: model.layers.92.self_attn.q_proj.weight shape: [16384, 16384] -1072: model.layers.92.self_attn.v_proj.weight shape: [1024, 16384] -1073: model.layers.93.input_layernorm.weight shape: [16384] -1074: model.layers.93.mlp.down_proj.weight shape: [16384, 53248] -1075: model.layers.93.mlp.gate_proj.weight shape: [53248, 16384] -1076: model.layers.93.mlp.up_proj.weight shape: [53248, 16384] -1077: model.layers.93.post_attention_layernorm.weight shape: [16384] -1078: model.layers.93.self_attn.k_proj.weight shape: [1024, 16384] -1079: model.layers.93.self_attn.o_proj.weight shape: [16384, 16384] -1080: model.layers.93.self_attn.q_proj.weight shape: [16384, 16384] -1081: model.layers.93.self_attn.v_proj.weight shape: [1024, 16384] -1082: model.layers.94.input_layernorm.weight shape: [16384] -1083: model.layers.94.mlp.down_proj.weight shape: [16384, 53248] -1084: model.layers.94.mlp.gate_proj.weight shape: [53248, 16384] -1085: model.layers.94.mlp.up_proj.weight shape: [53248, 16384] -1086: model.layers.94.post_attention_layernorm.weight shape: [16384] -1087: model.layers.94.self_attn.k_proj.weight shape: [1024, 16384] -1088: model.layers.94.self_attn.o_proj.weight shape: [16384, 16384] -1089: model.layers.94.self_attn.q_proj.weight shape: [16384, 16384] -1090: model.layers.94.self_attn.v_proj.weight shape: [1024, 16384] -1091: model.layers.95.input_layernorm.weight shape: [16384] -1092: model.layers.95.mlp.down_proj.weight shape: [16384, 53248] -1093: model.layers.95.mlp.gate_proj.weight shape: [53248, 16384] -1094: model.layers.95.mlp.up_proj.weight shape: [53248, 16384] -1095: model.layers.95.post_attention_layernorm.weight shape: [16384] -1096: model.layers.95.self_attn.k_proj.weight shape: [1024, 16384] -1097: model.layers.95.self_attn.o_proj.weight shape: [16384, 16384] -1098: model.layers.95.self_attn.q_proj.weight shape: [16384, 16384] -1099: model.layers.95.self_attn.v_proj.weight shape: [1024, 16384] -1100: model.layers.96.input_layernorm.weight shape: [16384] -1101: model.layers.96.mlp.down_proj.weight shape: [16384, 53248] -1102: model.layers.96.mlp.gate_proj.weight shape: [53248, 16384] -1103: model.layers.96.mlp.up_proj.weight shape: [53248, 16384] -1104: model.layers.96.post_attention_layernorm.weight shape: [16384] -1105: model.layers.96.self_attn.k_proj.weight shape: [1024, 16384] -1106: model.layers.96.self_attn.o_proj.weight shape: [16384, 16384] -1107: model.layers.96.self_attn.q_proj.weight shape: [16384, 16384] -1108: model.layers.96.self_attn.v_proj.weight shape: [1024, 16384] -1109: model.layers.97.input_layernorm.weight shape: [16384] -1110: model.layers.97.mlp.down_proj.weight shape: [16384, 53248] -1111: model.layers.97.mlp.gate_proj.weight shape: [53248, 16384] -1112: model.layers.97.mlp.up_proj.weight shape: [53248, 16384] -1113: model.layers.97.post_attention_layernorm.weight shape: [16384] -1114: model.layers.97.self_attn.k_proj.weight shape: [1024, 16384] -1115: model.layers.97.self_attn.o_proj.weight shape: [16384, 16384] -1116: model.layers.97.self_attn.q_proj.weight shape: [16384, 16384] -1117: model.layers.97.self_attn.v_proj.weight shape: [1024, 16384] -1118: model.layers.98.input_layernorm.weight shape: [16384] -1119: model.layers.98.mlp.down_proj.weight shape: [16384, 53248] -1120: model.layers.98.mlp.gate_proj.weight shape: [53248, 16384] -1121: model.layers.98.mlp.up_proj.weight shape: [53248, 16384] -1122: model.layers.98.post_attention_layernorm.weight shape: [16384] -1123: model.layers.98.self_attn.k_proj.weight shape: [1024, 16384] -1124: model.layers.98.self_attn.o_proj.weight shape: [16384, 16384] -1125: model.layers.98.self_attn.q_proj.weight shape: [16384, 16384] -1126: model.layers.98.self_attn.v_proj.weight shape: [1024, 16384] -1127: model.layers.99.input_layernorm.weight shape: [16384] -1128: model.layers.99.mlp.down_proj.weight shape: [16384, 53248] -1129: model.layers.99.mlp.gate_proj.weight shape: [53248, 16384] -1130: model.layers.99.mlp.up_proj.weight shape: [53248, 16384] -1131: model.layers.99.post_attention_layernorm.weight shape: [16384] -1132: model.layers.99.self_attn.k_proj.weight shape: [1024, 16384] -1133: model.layers.99.self_attn.o_proj.weight shape: [16384, 16384] -1134: model.layers.99.self_attn.q_proj.weight shape: [16384, 16384] -1135: model.layers.99.self_attn.v_proj.weight shape: [1024, 16384] -1136: model.norm.weight shape: [16384] diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_70b_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_70b_ShapeTest.approved.txt deleted file mode 100644 index 5add8770c5..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_70b_ShapeTest.approved.txt +++ /dev/null @@ -1,723 +0,0 @@ -0: lm_head.weight shape: [128256, 8192] -1: model.embed_tokens.weight shape: [128256, 8192] -2: model.layers.0.input_layernorm.weight shape: [8192] -3: model.layers.0.mlp.down_proj.weight shape: [8192, 28672] -4: model.layers.0.mlp.gate_proj.weight shape: [28672, 8192] -5: model.layers.0.mlp.up_proj.weight shape: [28672, 8192] -6: model.layers.0.post_attention_layernorm.weight shape: [8192] -7: model.layers.0.self_attn.k_proj.weight shape: [1024, 8192] -8: model.layers.0.self_attn.o_proj.weight shape: [8192, 8192] -9: model.layers.0.self_attn.q_proj.weight shape: [8192, 8192] -10: model.layers.0.self_attn.v_proj.weight shape: [1024, 8192] -11: model.layers.1.input_layernorm.weight shape: [8192] -12: model.layers.1.mlp.down_proj.weight shape: [8192, 28672] -13: model.layers.1.mlp.gate_proj.weight shape: [28672, 8192] -14: model.layers.1.mlp.up_proj.weight shape: [28672, 8192] -15: model.layers.1.post_attention_layernorm.weight shape: [8192] -16: model.layers.1.self_attn.k_proj.weight shape: [1024, 8192] -17: model.layers.1.self_attn.o_proj.weight shape: [8192, 8192] -18: model.layers.1.self_attn.q_proj.weight shape: [8192, 8192] -19: model.layers.1.self_attn.v_proj.weight shape: [1024, 8192] -20: model.layers.10.input_layernorm.weight shape: [8192] -21: model.layers.10.mlp.down_proj.weight shape: [8192, 28672] -22: model.layers.10.mlp.gate_proj.weight shape: [28672, 8192] -23: model.layers.10.mlp.up_proj.weight shape: [28672, 8192] -24: model.layers.10.post_attention_layernorm.weight shape: [8192] -25: model.layers.10.self_attn.k_proj.weight shape: [1024, 8192] -26: model.layers.10.self_attn.o_proj.weight shape: [8192, 8192] -27: model.layers.10.self_attn.q_proj.weight shape: [8192, 8192] -28: model.layers.10.self_attn.v_proj.weight shape: [1024, 8192] -29: model.layers.11.input_layernorm.weight shape: [8192] -30: model.layers.11.mlp.down_proj.weight shape: [8192, 28672] -31: model.layers.11.mlp.gate_proj.weight shape: [28672, 8192] -32: model.layers.11.mlp.up_proj.weight shape: [28672, 8192] -33: model.layers.11.post_attention_layernorm.weight shape: [8192] -34: model.layers.11.self_attn.k_proj.weight shape: [1024, 8192] -35: model.layers.11.self_attn.o_proj.weight shape: [8192, 8192] -36: model.layers.11.self_attn.q_proj.weight shape: [8192, 8192] -37: model.layers.11.self_attn.v_proj.weight shape: [1024, 8192] -38: model.layers.12.input_layernorm.weight shape: [8192] -39: model.layers.12.mlp.down_proj.weight shape: [8192, 28672] -40: model.layers.12.mlp.gate_proj.weight shape: [28672, 8192] -41: model.layers.12.mlp.up_proj.weight shape: [28672, 8192] -42: model.layers.12.post_attention_layernorm.weight shape: [8192] -43: model.layers.12.self_attn.k_proj.weight shape: [1024, 8192] -44: model.layers.12.self_attn.o_proj.weight shape: [8192, 8192] -45: model.layers.12.self_attn.q_proj.weight shape: [8192, 8192] -46: model.layers.12.self_attn.v_proj.weight shape: [1024, 8192] -47: model.layers.13.input_layernorm.weight shape: [8192] -48: model.layers.13.mlp.down_proj.weight shape: [8192, 28672] -49: model.layers.13.mlp.gate_proj.weight shape: [28672, 8192] -50: model.layers.13.mlp.up_proj.weight shape: [28672, 8192] -51: model.layers.13.post_attention_layernorm.weight shape: [8192] -52: model.layers.13.self_attn.k_proj.weight shape: [1024, 8192] -53: model.layers.13.self_attn.o_proj.weight shape: [8192, 8192] -54: model.layers.13.self_attn.q_proj.weight shape: [8192, 8192] -55: model.layers.13.self_attn.v_proj.weight shape: [1024, 8192] -56: model.layers.14.input_layernorm.weight shape: [8192] -57: model.layers.14.mlp.down_proj.weight shape: [8192, 28672] -58: model.layers.14.mlp.gate_proj.weight shape: [28672, 8192] -59: model.layers.14.mlp.up_proj.weight shape: [28672, 8192] -60: model.layers.14.post_attention_layernorm.weight shape: [8192] -61: model.layers.14.self_attn.k_proj.weight shape: [1024, 8192] -62: model.layers.14.self_attn.o_proj.weight shape: [8192, 8192] -63: model.layers.14.self_attn.q_proj.weight shape: [8192, 8192] -64: model.layers.14.self_attn.v_proj.weight shape: [1024, 8192] -65: model.layers.15.input_layernorm.weight shape: [8192] -66: model.layers.15.mlp.down_proj.weight shape: [8192, 28672] -67: model.layers.15.mlp.gate_proj.weight shape: [28672, 8192] -68: model.layers.15.mlp.up_proj.weight shape: [28672, 8192] -69: model.layers.15.post_attention_layernorm.weight shape: [8192] -70: model.layers.15.self_attn.k_proj.weight shape: [1024, 8192] -71: model.layers.15.self_attn.o_proj.weight shape: [8192, 8192] -72: model.layers.15.self_attn.q_proj.weight shape: [8192, 8192] -73: model.layers.15.self_attn.v_proj.weight shape: [1024, 8192] -74: model.layers.16.input_layernorm.weight shape: [8192] -75: model.layers.16.mlp.down_proj.weight shape: [8192, 28672] -76: model.layers.16.mlp.gate_proj.weight shape: [28672, 8192] -77: model.layers.16.mlp.up_proj.weight shape: [28672, 8192] -78: model.layers.16.post_attention_layernorm.weight shape: [8192] -79: model.layers.16.self_attn.k_proj.weight shape: [1024, 8192] -80: model.layers.16.self_attn.o_proj.weight shape: [8192, 8192] -81: model.layers.16.self_attn.q_proj.weight shape: [8192, 8192] -82: model.layers.16.self_attn.v_proj.weight shape: [1024, 8192] -83: model.layers.17.input_layernorm.weight shape: [8192] -84: model.layers.17.mlp.down_proj.weight shape: [8192, 28672] -85: model.layers.17.mlp.gate_proj.weight shape: [28672, 8192] -86: model.layers.17.mlp.up_proj.weight shape: [28672, 8192] -87: model.layers.17.post_attention_layernorm.weight shape: [8192] -88: model.layers.17.self_attn.k_proj.weight shape: [1024, 8192] -89: model.layers.17.self_attn.o_proj.weight shape: [8192, 8192] -90: model.layers.17.self_attn.q_proj.weight shape: [8192, 8192] -91: model.layers.17.self_attn.v_proj.weight shape: [1024, 8192] -92: model.layers.18.input_layernorm.weight shape: [8192] -93: model.layers.18.mlp.down_proj.weight shape: [8192, 28672] -94: model.layers.18.mlp.gate_proj.weight shape: [28672, 8192] -95: model.layers.18.mlp.up_proj.weight shape: [28672, 8192] -96: model.layers.18.post_attention_layernorm.weight shape: [8192] -97: model.layers.18.self_attn.k_proj.weight shape: [1024, 8192] -98: model.layers.18.self_attn.o_proj.weight shape: [8192, 8192] -99: model.layers.18.self_attn.q_proj.weight shape: [8192, 8192] -100: model.layers.18.self_attn.v_proj.weight shape: [1024, 8192] -101: model.layers.19.input_layernorm.weight shape: [8192] -102: model.layers.19.mlp.down_proj.weight shape: [8192, 28672] -103: model.layers.19.mlp.gate_proj.weight shape: [28672, 8192] -104: model.layers.19.mlp.up_proj.weight shape: [28672, 8192] -105: model.layers.19.post_attention_layernorm.weight shape: [8192] -106: model.layers.19.self_attn.k_proj.weight shape: [1024, 8192] -107: model.layers.19.self_attn.o_proj.weight shape: [8192, 8192] -108: model.layers.19.self_attn.q_proj.weight shape: [8192, 8192] -109: model.layers.19.self_attn.v_proj.weight shape: [1024, 8192] -110: model.layers.2.input_layernorm.weight shape: [8192] -111: model.layers.2.mlp.down_proj.weight shape: [8192, 28672] -112: model.layers.2.mlp.gate_proj.weight shape: [28672, 8192] -113: model.layers.2.mlp.up_proj.weight shape: [28672, 8192] -114: model.layers.2.post_attention_layernorm.weight shape: [8192] -115: model.layers.2.self_attn.k_proj.weight shape: [1024, 8192] -116: model.layers.2.self_attn.o_proj.weight shape: [8192, 8192] -117: model.layers.2.self_attn.q_proj.weight shape: [8192, 8192] -118: model.layers.2.self_attn.v_proj.weight shape: [1024, 8192] -119: model.layers.20.input_layernorm.weight shape: [8192] -120: model.layers.20.mlp.down_proj.weight shape: [8192, 28672] -121: model.layers.20.mlp.gate_proj.weight shape: [28672, 8192] -122: model.layers.20.mlp.up_proj.weight shape: [28672, 8192] -123: model.layers.20.post_attention_layernorm.weight shape: [8192] -124: model.layers.20.self_attn.k_proj.weight shape: [1024, 8192] -125: model.layers.20.self_attn.o_proj.weight shape: [8192, 8192] -126: model.layers.20.self_attn.q_proj.weight shape: [8192, 8192] -127: model.layers.20.self_attn.v_proj.weight shape: [1024, 8192] -128: model.layers.21.input_layernorm.weight shape: [8192] -129: model.layers.21.mlp.down_proj.weight shape: [8192, 28672] -130: model.layers.21.mlp.gate_proj.weight shape: [28672, 8192] -131: model.layers.21.mlp.up_proj.weight shape: [28672, 8192] -132: model.layers.21.post_attention_layernorm.weight shape: [8192] -133: model.layers.21.self_attn.k_proj.weight shape: [1024, 8192] -134: model.layers.21.self_attn.o_proj.weight shape: [8192, 8192] -135: model.layers.21.self_attn.q_proj.weight shape: [8192, 8192] -136: model.layers.21.self_attn.v_proj.weight shape: [1024, 8192] -137: model.layers.22.input_layernorm.weight shape: [8192] -138: model.layers.22.mlp.down_proj.weight shape: [8192, 28672] -139: model.layers.22.mlp.gate_proj.weight shape: [28672, 8192] -140: model.layers.22.mlp.up_proj.weight shape: [28672, 8192] -141: model.layers.22.post_attention_layernorm.weight shape: [8192] -142: model.layers.22.self_attn.k_proj.weight shape: [1024, 8192] -143: model.layers.22.self_attn.o_proj.weight shape: [8192, 8192] -144: model.layers.22.self_attn.q_proj.weight shape: [8192, 8192] -145: model.layers.22.self_attn.v_proj.weight shape: [1024, 8192] -146: model.layers.23.input_layernorm.weight shape: [8192] -147: model.layers.23.mlp.down_proj.weight shape: [8192, 28672] -148: model.layers.23.mlp.gate_proj.weight shape: [28672, 8192] -149: model.layers.23.mlp.up_proj.weight shape: [28672, 8192] -150: model.layers.23.post_attention_layernorm.weight shape: [8192] -151: model.layers.23.self_attn.k_proj.weight shape: [1024, 8192] -152: model.layers.23.self_attn.o_proj.weight shape: [8192, 8192] -153: model.layers.23.self_attn.q_proj.weight shape: [8192, 8192] -154: model.layers.23.self_attn.v_proj.weight shape: [1024, 8192] -155: model.layers.24.input_layernorm.weight shape: [8192] -156: model.layers.24.mlp.down_proj.weight shape: [8192, 28672] -157: model.layers.24.mlp.gate_proj.weight shape: [28672, 8192] -158: model.layers.24.mlp.up_proj.weight shape: [28672, 8192] -159: model.layers.24.post_attention_layernorm.weight shape: [8192] -160: model.layers.24.self_attn.k_proj.weight shape: [1024, 8192] -161: model.layers.24.self_attn.o_proj.weight shape: [8192, 8192] -162: model.layers.24.self_attn.q_proj.weight shape: [8192, 8192] -163: model.layers.24.self_attn.v_proj.weight shape: [1024, 8192] -164: model.layers.25.input_layernorm.weight shape: [8192] -165: model.layers.25.mlp.down_proj.weight shape: [8192, 28672] -166: model.layers.25.mlp.gate_proj.weight shape: [28672, 8192] -167: model.layers.25.mlp.up_proj.weight shape: [28672, 8192] -168: model.layers.25.post_attention_layernorm.weight shape: [8192] -169: model.layers.25.self_attn.k_proj.weight shape: [1024, 8192] -170: model.layers.25.self_attn.o_proj.weight shape: [8192, 8192] -171: model.layers.25.self_attn.q_proj.weight shape: [8192, 8192] -172: model.layers.25.self_attn.v_proj.weight shape: [1024, 8192] -173: model.layers.26.input_layernorm.weight shape: [8192] -174: model.layers.26.mlp.down_proj.weight shape: [8192, 28672] -175: model.layers.26.mlp.gate_proj.weight shape: [28672, 8192] -176: model.layers.26.mlp.up_proj.weight shape: [28672, 8192] -177: model.layers.26.post_attention_layernorm.weight shape: [8192] -178: model.layers.26.self_attn.k_proj.weight shape: [1024, 8192] -179: model.layers.26.self_attn.o_proj.weight shape: [8192, 8192] -180: model.layers.26.self_attn.q_proj.weight shape: [8192, 8192] -181: model.layers.26.self_attn.v_proj.weight shape: [1024, 8192] -182: model.layers.27.input_layernorm.weight shape: [8192] -183: model.layers.27.mlp.down_proj.weight shape: [8192, 28672] -184: model.layers.27.mlp.gate_proj.weight shape: [28672, 8192] -185: model.layers.27.mlp.up_proj.weight shape: [28672, 8192] -186: model.layers.27.post_attention_layernorm.weight shape: [8192] -187: model.layers.27.self_attn.k_proj.weight shape: [1024, 8192] -188: model.layers.27.self_attn.o_proj.weight shape: [8192, 8192] -189: model.layers.27.self_attn.q_proj.weight shape: [8192, 8192] -190: model.layers.27.self_attn.v_proj.weight shape: [1024, 8192] -191: model.layers.28.input_layernorm.weight shape: [8192] -192: model.layers.28.mlp.down_proj.weight shape: [8192, 28672] -193: model.layers.28.mlp.gate_proj.weight shape: [28672, 8192] -194: model.layers.28.mlp.up_proj.weight shape: [28672, 8192] -195: model.layers.28.post_attention_layernorm.weight shape: [8192] -196: model.layers.28.self_attn.k_proj.weight shape: [1024, 8192] -197: model.layers.28.self_attn.o_proj.weight shape: [8192, 8192] -198: model.layers.28.self_attn.q_proj.weight shape: [8192, 8192] -199: model.layers.28.self_attn.v_proj.weight shape: [1024, 8192] -200: model.layers.29.input_layernorm.weight shape: [8192] -201: model.layers.29.mlp.down_proj.weight shape: [8192, 28672] -202: model.layers.29.mlp.gate_proj.weight shape: [28672, 8192] -203: model.layers.29.mlp.up_proj.weight shape: [28672, 8192] -204: model.layers.29.post_attention_layernorm.weight shape: [8192] -205: model.layers.29.self_attn.k_proj.weight shape: [1024, 8192] -206: model.layers.29.self_attn.o_proj.weight shape: [8192, 8192] -207: model.layers.29.self_attn.q_proj.weight shape: [8192, 8192] -208: model.layers.29.self_attn.v_proj.weight shape: [1024, 8192] -209: model.layers.3.input_layernorm.weight shape: [8192] -210: model.layers.3.mlp.down_proj.weight shape: [8192, 28672] -211: model.layers.3.mlp.gate_proj.weight shape: [28672, 8192] -212: model.layers.3.mlp.up_proj.weight shape: [28672, 8192] -213: model.layers.3.post_attention_layernorm.weight shape: [8192] -214: model.layers.3.self_attn.k_proj.weight shape: [1024, 8192] -215: model.layers.3.self_attn.o_proj.weight shape: [8192, 8192] -216: model.layers.3.self_attn.q_proj.weight shape: [8192, 8192] -217: model.layers.3.self_attn.v_proj.weight shape: [1024, 8192] -218: model.layers.30.input_layernorm.weight shape: [8192] -219: model.layers.30.mlp.down_proj.weight shape: [8192, 28672] -220: model.layers.30.mlp.gate_proj.weight shape: [28672, 8192] -221: model.layers.30.mlp.up_proj.weight shape: [28672, 8192] -222: model.layers.30.post_attention_layernorm.weight shape: [8192] -223: model.layers.30.self_attn.k_proj.weight shape: [1024, 8192] -224: model.layers.30.self_attn.o_proj.weight shape: [8192, 8192] -225: model.layers.30.self_attn.q_proj.weight shape: [8192, 8192] -226: model.layers.30.self_attn.v_proj.weight shape: [1024, 8192] -227: model.layers.31.input_layernorm.weight shape: [8192] -228: model.layers.31.mlp.down_proj.weight shape: [8192, 28672] -229: model.layers.31.mlp.gate_proj.weight shape: [28672, 8192] -230: model.layers.31.mlp.up_proj.weight shape: [28672, 8192] -231: model.layers.31.post_attention_layernorm.weight shape: [8192] -232: model.layers.31.self_attn.k_proj.weight shape: [1024, 8192] -233: model.layers.31.self_attn.o_proj.weight shape: [8192, 8192] -234: model.layers.31.self_attn.q_proj.weight shape: [8192, 8192] -235: model.layers.31.self_attn.v_proj.weight shape: [1024, 8192] -236: model.layers.32.input_layernorm.weight shape: [8192] -237: model.layers.32.mlp.down_proj.weight shape: [8192, 28672] -238: model.layers.32.mlp.gate_proj.weight shape: [28672, 8192] -239: model.layers.32.mlp.up_proj.weight shape: [28672, 8192] -240: model.layers.32.post_attention_layernorm.weight shape: [8192] -241: model.layers.32.self_attn.k_proj.weight shape: [1024, 8192] -242: model.layers.32.self_attn.o_proj.weight shape: [8192, 8192] -243: model.layers.32.self_attn.q_proj.weight shape: [8192, 8192] -244: model.layers.32.self_attn.v_proj.weight shape: [1024, 8192] -245: model.layers.33.input_layernorm.weight shape: [8192] -246: model.layers.33.mlp.down_proj.weight shape: [8192, 28672] -247: model.layers.33.mlp.gate_proj.weight shape: [28672, 8192] -248: model.layers.33.mlp.up_proj.weight shape: [28672, 8192] -249: model.layers.33.post_attention_layernorm.weight shape: [8192] -250: model.layers.33.self_attn.k_proj.weight shape: [1024, 8192] -251: model.layers.33.self_attn.o_proj.weight shape: [8192, 8192] -252: model.layers.33.self_attn.q_proj.weight shape: [8192, 8192] -253: model.layers.33.self_attn.v_proj.weight shape: [1024, 8192] -254: model.layers.34.input_layernorm.weight shape: [8192] -255: model.layers.34.mlp.down_proj.weight shape: [8192, 28672] -256: model.layers.34.mlp.gate_proj.weight shape: [28672, 8192] -257: model.layers.34.mlp.up_proj.weight shape: [28672, 8192] -258: model.layers.34.post_attention_layernorm.weight shape: [8192] -259: model.layers.34.self_attn.k_proj.weight shape: [1024, 8192] -260: model.layers.34.self_attn.o_proj.weight shape: [8192, 8192] -261: model.layers.34.self_attn.q_proj.weight shape: [8192, 8192] -262: model.layers.34.self_attn.v_proj.weight shape: [1024, 8192] -263: model.layers.35.input_layernorm.weight shape: [8192] -264: model.layers.35.mlp.down_proj.weight shape: [8192, 28672] -265: model.layers.35.mlp.gate_proj.weight shape: [28672, 8192] -266: model.layers.35.mlp.up_proj.weight shape: [28672, 8192] -267: model.layers.35.post_attention_layernorm.weight shape: [8192] -268: model.layers.35.self_attn.k_proj.weight shape: [1024, 8192] -269: model.layers.35.self_attn.o_proj.weight shape: [8192, 8192] -270: model.layers.35.self_attn.q_proj.weight shape: [8192, 8192] -271: model.layers.35.self_attn.v_proj.weight shape: [1024, 8192] -272: model.layers.36.input_layernorm.weight shape: [8192] -273: model.layers.36.mlp.down_proj.weight shape: [8192, 28672] -274: model.layers.36.mlp.gate_proj.weight shape: [28672, 8192] -275: model.layers.36.mlp.up_proj.weight shape: [28672, 8192] -276: model.layers.36.post_attention_layernorm.weight shape: [8192] -277: model.layers.36.self_attn.k_proj.weight shape: [1024, 8192] -278: model.layers.36.self_attn.o_proj.weight shape: [8192, 8192] -279: model.layers.36.self_attn.q_proj.weight shape: [8192, 8192] -280: model.layers.36.self_attn.v_proj.weight shape: [1024, 8192] -281: model.layers.37.input_layernorm.weight shape: [8192] -282: model.layers.37.mlp.down_proj.weight shape: [8192, 28672] -283: model.layers.37.mlp.gate_proj.weight shape: [28672, 8192] -284: model.layers.37.mlp.up_proj.weight shape: [28672, 8192] -285: model.layers.37.post_attention_layernorm.weight shape: [8192] -286: model.layers.37.self_attn.k_proj.weight shape: [1024, 8192] -287: model.layers.37.self_attn.o_proj.weight shape: [8192, 8192] -288: model.layers.37.self_attn.q_proj.weight shape: [8192, 8192] -289: model.layers.37.self_attn.v_proj.weight shape: [1024, 8192] -290: model.layers.38.input_layernorm.weight shape: [8192] -291: model.layers.38.mlp.down_proj.weight shape: [8192, 28672] -292: model.layers.38.mlp.gate_proj.weight shape: [28672, 8192] -293: model.layers.38.mlp.up_proj.weight shape: [28672, 8192] -294: model.layers.38.post_attention_layernorm.weight shape: [8192] -295: model.layers.38.self_attn.k_proj.weight shape: [1024, 8192] -296: model.layers.38.self_attn.o_proj.weight shape: [8192, 8192] -297: model.layers.38.self_attn.q_proj.weight shape: [8192, 8192] -298: model.layers.38.self_attn.v_proj.weight shape: [1024, 8192] -299: model.layers.39.input_layernorm.weight shape: [8192] -300: model.layers.39.mlp.down_proj.weight shape: [8192, 28672] -301: model.layers.39.mlp.gate_proj.weight shape: [28672, 8192] -302: model.layers.39.mlp.up_proj.weight shape: [28672, 8192] -303: model.layers.39.post_attention_layernorm.weight shape: [8192] -304: model.layers.39.self_attn.k_proj.weight shape: [1024, 8192] -305: model.layers.39.self_attn.o_proj.weight shape: [8192, 8192] -306: model.layers.39.self_attn.q_proj.weight shape: [8192, 8192] -307: model.layers.39.self_attn.v_proj.weight shape: [1024, 8192] -308: model.layers.4.input_layernorm.weight shape: [8192] -309: model.layers.4.mlp.down_proj.weight shape: [8192, 28672] -310: model.layers.4.mlp.gate_proj.weight shape: [28672, 8192] -311: model.layers.4.mlp.up_proj.weight shape: [28672, 8192] -312: model.layers.4.post_attention_layernorm.weight shape: [8192] -313: model.layers.4.self_attn.k_proj.weight shape: [1024, 8192] -314: model.layers.4.self_attn.o_proj.weight shape: [8192, 8192] -315: model.layers.4.self_attn.q_proj.weight shape: [8192, 8192] -316: model.layers.4.self_attn.v_proj.weight shape: [1024, 8192] -317: model.layers.40.input_layernorm.weight shape: [8192] -318: model.layers.40.mlp.down_proj.weight shape: [8192, 28672] -319: model.layers.40.mlp.gate_proj.weight shape: [28672, 8192] -320: model.layers.40.mlp.up_proj.weight shape: [28672, 8192] -321: model.layers.40.post_attention_layernorm.weight shape: [8192] -322: model.layers.40.self_attn.k_proj.weight shape: [1024, 8192] -323: model.layers.40.self_attn.o_proj.weight shape: [8192, 8192] -324: model.layers.40.self_attn.q_proj.weight shape: [8192, 8192] -325: model.layers.40.self_attn.v_proj.weight shape: [1024, 8192] -326: model.layers.41.input_layernorm.weight shape: [8192] -327: model.layers.41.mlp.down_proj.weight shape: [8192, 28672] -328: model.layers.41.mlp.gate_proj.weight shape: [28672, 8192] -329: model.layers.41.mlp.up_proj.weight shape: [28672, 8192] -330: model.layers.41.post_attention_layernorm.weight shape: [8192] -331: model.layers.41.self_attn.k_proj.weight shape: [1024, 8192] -332: model.layers.41.self_attn.o_proj.weight shape: [8192, 8192] -333: model.layers.41.self_attn.q_proj.weight shape: [8192, 8192] -334: model.layers.41.self_attn.v_proj.weight shape: [1024, 8192] -335: model.layers.42.input_layernorm.weight shape: [8192] -336: model.layers.42.mlp.down_proj.weight shape: [8192, 28672] -337: model.layers.42.mlp.gate_proj.weight shape: [28672, 8192] -338: model.layers.42.mlp.up_proj.weight shape: [28672, 8192] -339: model.layers.42.post_attention_layernorm.weight shape: [8192] -340: model.layers.42.self_attn.k_proj.weight shape: [1024, 8192] -341: model.layers.42.self_attn.o_proj.weight shape: [8192, 8192] -342: model.layers.42.self_attn.q_proj.weight shape: [8192, 8192] -343: model.layers.42.self_attn.v_proj.weight shape: [1024, 8192] -344: model.layers.43.input_layernorm.weight shape: [8192] -345: model.layers.43.mlp.down_proj.weight shape: [8192, 28672] -346: model.layers.43.mlp.gate_proj.weight shape: [28672, 8192] -347: model.layers.43.mlp.up_proj.weight shape: [28672, 8192] -348: model.layers.43.post_attention_layernorm.weight shape: [8192] -349: model.layers.43.self_attn.k_proj.weight shape: [1024, 8192] -350: model.layers.43.self_attn.o_proj.weight shape: [8192, 8192] -351: model.layers.43.self_attn.q_proj.weight shape: [8192, 8192] -352: model.layers.43.self_attn.v_proj.weight shape: [1024, 8192] -353: model.layers.44.input_layernorm.weight shape: [8192] -354: model.layers.44.mlp.down_proj.weight shape: [8192, 28672] -355: model.layers.44.mlp.gate_proj.weight shape: [28672, 8192] -356: model.layers.44.mlp.up_proj.weight shape: [28672, 8192] -357: model.layers.44.post_attention_layernorm.weight shape: [8192] -358: model.layers.44.self_attn.k_proj.weight shape: [1024, 8192] -359: model.layers.44.self_attn.o_proj.weight shape: [8192, 8192] -360: model.layers.44.self_attn.q_proj.weight shape: [8192, 8192] -361: model.layers.44.self_attn.v_proj.weight shape: [1024, 8192] -362: model.layers.45.input_layernorm.weight shape: [8192] -363: model.layers.45.mlp.down_proj.weight shape: [8192, 28672] -364: model.layers.45.mlp.gate_proj.weight shape: [28672, 8192] -365: model.layers.45.mlp.up_proj.weight shape: [28672, 8192] -366: model.layers.45.post_attention_layernorm.weight shape: [8192] -367: model.layers.45.self_attn.k_proj.weight shape: [1024, 8192] -368: model.layers.45.self_attn.o_proj.weight shape: [8192, 8192] -369: model.layers.45.self_attn.q_proj.weight shape: [8192, 8192] -370: model.layers.45.self_attn.v_proj.weight shape: [1024, 8192] -371: model.layers.46.input_layernorm.weight shape: [8192] -372: model.layers.46.mlp.down_proj.weight shape: [8192, 28672] -373: model.layers.46.mlp.gate_proj.weight shape: [28672, 8192] -374: model.layers.46.mlp.up_proj.weight shape: [28672, 8192] -375: model.layers.46.post_attention_layernorm.weight shape: [8192] -376: model.layers.46.self_attn.k_proj.weight shape: [1024, 8192] -377: model.layers.46.self_attn.o_proj.weight shape: [8192, 8192] -378: model.layers.46.self_attn.q_proj.weight shape: [8192, 8192] -379: model.layers.46.self_attn.v_proj.weight shape: [1024, 8192] -380: model.layers.47.input_layernorm.weight shape: [8192] -381: model.layers.47.mlp.down_proj.weight shape: [8192, 28672] -382: model.layers.47.mlp.gate_proj.weight shape: [28672, 8192] -383: model.layers.47.mlp.up_proj.weight shape: [28672, 8192] -384: model.layers.47.post_attention_layernorm.weight shape: [8192] -385: model.layers.47.self_attn.k_proj.weight shape: [1024, 8192] -386: model.layers.47.self_attn.o_proj.weight shape: [8192, 8192] -387: model.layers.47.self_attn.q_proj.weight shape: [8192, 8192] -388: model.layers.47.self_attn.v_proj.weight shape: [1024, 8192] -389: model.layers.48.input_layernorm.weight shape: [8192] -390: model.layers.48.mlp.down_proj.weight shape: [8192, 28672] -391: model.layers.48.mlp.gate_proj.weight shape: [28672, 8192] -392: model.layers.48.mlp.up_proj.weight shape: [28672, 8192] -393: model.layers.48.post_attention_layernorm.weight shape: [8192] -394: model.layers.48.self_attn.k_proj.weight shape: [1024, 8192] -395: model.layers.48.self_attn.o_proj.weight shape: [8192, 8192] -396: model.layers.48.self_attn.q_proj.weight shape: [8192, 8192] -397: model.layers.48.self_attn.v_proj.weight shape: [1024, 8192] -398: model.layers.49.input_layernorm.weight shape: [8192] -399: model.layers.49.mlp.down_proj.weight shape: [8192, 28672] -400: model.layers.49.mlp.gate_proj.weight shape: [28672, 8192] -401: model.layers.49.mlp.up_proj.weight shape: [28672, 8192] -402: model.layers.49.post_attention_layernorm.weight shape: [8192] -403: model.layers.49.self_attn.k_proj.weight shape: [1024, 8192] -404: model.layers.49.self_attn.o_proj.weight shape: [8192, 8192] -405: model.layers.49.self_attn.q_proj.weight shape: [8192, 8192] -406: model.layers.49.self_attn.v_proj.weight shape: [1024, 8192] -407: model.layers.5.input_layernorm.weight shape: [8192] -408: model.layers.5.mlp.down_proj.weight shape: [8192, 28672] -409: model.layers.5.mlp.gate_proj.weight shape: [28672, 8192] -410: model.layers.5.mlp.up_proj.weight shape: [28672, 8192] -411: model.layers.5.post_attention_layernorm.weight shape: [8192] -412: model.layers.5.self_attn.k_proj.weight shape: [1024, 8192] -413: model.layers.5.self_attn.o_proj.weight shape: [8192, 8192] -414: model.layers.5.self_attn.q_proj.weight shape: [8192, 8192] -415: model.layers.5.self_attn.v_proj.weight shape: [1024, 8192] -416: model.layers.50.input_layernorm.weight shape: [8192] -417: model.layers.50.mlp.down_proj.weight shape: [8192, 28672] -418: model.layers.50.mlp.gate_proj.weight shape: [28672, 8192] -419: model.layers.50.mlp.up_proj.weight shape: [28672, 8192] -420: model.layers.50.post_attention_layernorm.weight shape: [8192] -421: model.layers.50.self_attn.k_proj.weight shape: [1024, 8192] -422: model.layers.50.self_attn.o_proj.weight shape: [8192, 8192] -423: model.layers.50.self_attn.q_proj.weight shape: [8192, 8192] -424: model.layers.50.self_attn.v_proj.weight shape: [1024, 8192] -425: model.layers.51.input_layernorm.weight shape: [8192] -426: model.layers.51.mlp.down_proj.weight shape: [8192, 28672] -427: model.layers.51.mlp.gate_proj.weight shape: [28672, 8192] -428: model.layers.51.mlp.up_proj.weight shape: [28672, 8192] -429: model.layers.51.post_attention_layernorm.weight shape: [8192] -430: model.layers.51.self_attn.k_proj.weight shape: [1024, 8192] -431: model.layers.51.self_attn.o_proj.weight shape: [8192, 8192] -432: model.layers.51.self_attn.q_proj.weight shape: [8192, 8192] -433: model.layers.51.self_attn.v_proj.weight shape: [1024, 8192] -434: model.layers.52.input_layernorm.weight shape: [8192] -435: model.layers.52.mlp.down_proj.weight shape: [8192, 28672] -436: model.layers.52.mlp.gate_proj.weight shape: [28672, 8192] -437: model.layers.52.mlp.up_proj.weight shape: [28672, 8192] -438: model.layers.52.post_attention_layernorm.weight shape: [8192] -439: model.layers.52.self_attn.k_proj.weight shape: [1024, 8192] -440: model.layers.52.self_attn.o_proj.weight shape: [8192, 8192] -441: model.layers.52.self_attn.q_proj.weight shape: [8192, 8192] -442: model.layers.52.self_attn.v_proj.weight shape: [1024, 8192] -443: model.layers.53.input_layernorm.weight shape: [8192] -444: model.layers.53.mlp.down_proj.weight shape: [8192, 28672] -445: model.layers.53.mlp.gate_proj.weight shape: [28672, 8192] -446: model.layers.53.mlp.up_proj.weight shape: [28672, 8192] -447: model.layers.53.post_attention_layernorm.weight shape: [8192] -448: model.layers.53.self_attn.k_proj.weight shape: [1024, 8192] -449: model.layers.53.self_attn.o_proj.weight shape: [8192, 8192] -450: model.layers.53.self_attn.q_proj.weight shape: [8192, 8192] -451: model.layers.53.self_attn.v_proj.weight shape: [1024, 8192] -452: model.layers.54.input_layernorm.weight shape: [8192] -453: model.layers.54.mlp.down_proj.weight shape: [8192, 28672] -454: model.layers.54.mlp.gate_proj.weight shape: [28672, 8192] -455: model.layers.54.mlp.up_proj.weight shape: [28672, 8192] -456: model.layers.54.post_attention_layernorm.weight shape: [8192] -457: model.layers.54.self_attn.k_proj.weight shape: [1024, 8192] -458: model.layers.54.self_attn.o_proj.weight shape: [8192, 8192] -459: model.layers.54.self_attn.q_proj.weight shape: [8192, 8192] -460: model.layers.54.self_attn.v_proj.weight shape: [1024, 8192] -461: model.layers.55.input_layernorm.weight shape: [8192] -462: model.layers.55.mlp.down_proj.weight shape: [8192, 28672] -463: model.layers.55.mlp.gate_proj.weight shape: [28672, 8192] -464: model.layers.55.mlp.up_proj.weight shape: [28672, 8192] -465: model.layers.55.post_attention_layernorm.weight shape: [8192] -466: model.layers.55.self_attn.k_proj.weight shape: [1024, 8192] -467: model.layers.55.self_attn.o_proj.weight shape: [8192, 8192] -468: model.layers.55.self_attn.q_proj.weight shape: [8192, 8192] -469: model.layers.55.self_attn.v_proj.weight shape: [1024, 8192] -470: model.layers.56.input_layernorm.weight shape: [8192] -471: model.layers.56.mlp.down_proj.weight shape: [8192, 28672] -472: model.layers.56.mlp.gate_proj.weight shape: [28672, 8192] -473: model.layers.56.mlp.up_proj.weight shape: [28672, 8192] -474: model.layers.56.post_attention_layernorm.weight shape: [8192] -475: model.layers.56.self_attn.k_proj.weight shape: [1024, 8192] -476: model.layers.56.self_attn.o_proj.weight shape: [8192, 8192] -477: model.layers.56.self_attn.q_proj.weight shape: [8192, 8192] -478: model.layers.56.self_attn.v_proj.weight shape: [1024, 8192] -479: model.layers.57.input_layernorm.weight shape: [8192] -480: model.layers.57.mlp.down_proj.weight shape: [8192, 28672] -481: model.layers.57.mlp.gate_proj.weight shape: [28672, 8192] -482: model.layers.57.mlp.up_proj.weight shape: [28672, 8192] -483: model.layers.57.post_attention_layernorm.weight shape: [8192] -484: model.layers.57.self_attn.k_proj.weight shape: [1024, 8192] -485: model.layers.57.self_attn.o_proj.weight shape: [8192, 8192] -486: model.layers.57.self_attn.q_proj.weight shape: [8192, 8192] -487: model.layers.57.self_attn.v_proj.weight shape: [1024, 8192] -488: model.layers.58.input_layernorm.weight shape: [8192] -489: model.layers.58.mlp.down_proj.weight shape: [8192, 28672] -490: model.layers.58.mlp.gate_proj.weight shape: [28672, 8192] -491: model.layers.58.mlp.up_proj.weight shape: [28672, 8192] -492: model.layers.58.post_attention_layernorm.weight shape: [8192] -493: model.layers.58.self_attn.k_proj.weight shape: [1024, 8192] -494: model.layers.58.self_attn.o_proj.weight shape: [8192, 8192] -495: model.layers.58.self_attn.q_proj.weight shape: [8192, 8192] -496: model.layers.58.self_attn.v_proj.weight shape: [1024, 8192] -497: model.layers.59.input_layernorm.weight shape: [8192] -498: model.layers.59.mlp.down_proj.weight shape: [8192, 28672] -499: model.layers.59.mlp.gate_proj.weight shape: [28672, 8192] -500: model.layers.59.mlp.up_proj.weight shape: [28672, 8192] -501: model.layers.59.post_attention_layernorm.weight shape: [8192] -502: model.layers.59.self_attn.k_proj.weight shape: [1024, 8192] -503: model.layers.59.self_attn.o_proj.weight shape: [8192, 8192] -504: model.layers.59.self_attn.q_proj.weight shape: [8192, 8192] -505: model.layers.59.self_attn.v_proj.weight shape: [1024, 8192] -506: model.layers.6.input_layernorm.weight shape: [8192] -507: model.layers.6.mlp.down_proj.weight shape: [8192, 28672] -508: model.layers.6.mlp.gate_proj.weight shape: [28672, 8192] -509: model.layers.6.mlp.up_proj.weight shape: [28672, 8192] -510: model.layers.6.post_attention_layernorm.weight shape: [8192] -511: model.layers.6.self_attn.k_proj.weight shape: [1024, 8192] -512: model.layers.6.self_attn.o_proj.weight shape: [8192, 8192] -513: model.layers.6.self_attn.q_proj.weight shape: [8192, 8192] -514: model.layers.6.self_attn.v_proj.weight shape: [1024, 8192] -515: model.layers.60.input_layernorm.weight shape: [8192] -516: model.layers.60.mlp.down_proj.weight shape: [8192, 28672] -517: model.layers.60.mlp.gate_proj.weight shape: [28672, 8192] -518: model.layers.60.mlp.up_proj.weight shape: [28672, 8192] -519: model.layers.60.post_attention_layernorm.weight shape: [8192] -520: model.layers.60.self_attn.k_proj.weight shape: [1024, 8192] -521: model.layers.60.self_attn.o_proj.weight shape: [8192, 8192] -522: model.layers.60.self_attn.q_proj.weight shape: [8192, 8192] -523: model.layers.60.self_attn.v_proj.weight shape: [1024, 8192] -524: model.layers.61.input_layernorm.weight shape: [8192] -525: model.layers.61.mlp.down_proj.weight shape: [8192, 28672] -526: model.layers.61.mlp.gate_proj.weight shape: [28672, 8192] -527: model.layers.61.mlp.up_proj.weight shape: [28672, 8192] -528: model.layers.61.post_attention_layernorm.weight shape: [8192] -529: model.layers.61.self_attn.k_proj.weight shape: [1024, 8192] -530: model.layers.61.self_attn.o_proj.weight shape: [8192, 8192] -531: model.layers.61.self_attn.q_proj.weight shape: [8192, 8192] -532: model.layers.61.self_attn.v_proj.weight shape: [1024, 8192] -533: model.layers.62.input_layernorm.weight shape: [8192] -534: model.layers.62.mlp.down_proj.weight shape: [8192, 28672] -535: model.layers.62.mlp.gate_proj.weight shape: [28672, 8192] -536: model.layers.62.mlp.up_proj.weight shape: [28672, 8192] -537: model.layers.62.post_attention_layernorm.weight shape: [8192] -538: model.layers.62.self_attn.k_proj.weight shape: [1024, 8192] -539: model.layers.62.self_attn.o_proj.weight shape: [8192, 8192] -540: model.layers.62.self_attn.q_proj.weight shape: [8192, 8192] -541: model.layers.62.self_attn.v_proj.weight shape: [1024, 8192] -542: model.layers.63.input_layernorm.weight shape: [8192] -543: model.layers.63.mlp.down_proj.weight shape: [8192, 28672] -544: model.layers.63.mlp.gate_proj.weight shape: [28672, 8192] -545: model.layers.63.mlp.up_proj.weight shape: [28672, 8192] -546: model.layers.63.post_attention_layernorm.weight shape: [8192] -547: model.layers.63.self_attn.k_proj.weight shape: [1024, 8192] -548: model.layers.63.self_attn.o_proj.weight shape: [8192, 8192] -549: model.layers.63.self_attn.q_proj.weight shape: [8192, 8192] -550: model.layers.63.self_attn.v_proj.weight shape: [1024, 8192] -551: model.layers.64.input_layernorm.weight shape: [8192] -552: model.layers.64.mlp.down_proj.weight shape: [8192, 28672] -553: model.layers.64.mlp.gate_proj.weight shape: [28672, 8192] -554: model.layers.64.mlp.up_proj.weight shape: [28672, 8192] -555: model.layers.64.post_attention_layernorm.weight shape: [8192] -556: model.layers.64.self_attn.k_proj.weight shape: [1024, 8192] -557: model.layers.64.self_attn.o_proj.weight shape: [8192, 8192] -558: model.layers.64.self_attn.q_proj.weight shape: [8192, 8192] -559: model.layers.64.self_attn.v_proj.weight shape: [1024, 8192] -560: model.layers.65.input_layernorm.weight shape: [8192] -561: model.layers.65.mlp.down_proj.weight shape: [8192, 28672] -562: model.layers.65.mlp.gate_proj.weight shape: [28672, 8192] -563: model.layers.65.mlp.up_proj.weight shape: [28672, 8192] -564: model.layers.65.post_attention_layernorm.weight shape: [8192] -565: model.layers.65.self_attn.k_proj.weight shape: [1024, 8192] -566: model.layers.65.self_attn.o_proj.weight shape: [8192, 8192] -567: model.layers.65.self_attn.q_proj.weight shape: [8192, 8192] -568: model.layers.65.self_attn.v_proj.weight shape: [1024, 8192] -569: model.layers.66.input_layernorm.weight shape: [8192] -570: model.layers.66.mlp.down_proj.weight shape: [8192, 28672] -571: model.layers.66.mlp.gate_proj.weight shape: [28672, 8192] -572: model.layers.66.mlp.up_proj.weight shape: [28672, 8192] -573: model.layers.66.post_attention_layernorm.weight shape: [8192] -574: model.layers.66.self_attn.k_proj.weight shape: [1024, 8192] -575: model.layers.66.self_attn.o_proj.weight shape: [8192, 8192] -576: model.layers.66.self_attn.q_proj.weight shape: [8192, 8192] -577: model.layers.66.self_attn.v_proj.weight shape: [1024, 8192] -578: model.layers.67.input_layernorm.weight shape: [8192] -579: model.layers.67.mlp.down_proj.weight shape: [8192, 28672] -580: model.layers.67.mlp.gate_proj.weight shape: [28672, 8192] -581: model.layers.67.mlp.up_proj.weight shape: [28672, 8192] -582: model.layers.67.post_attention_layernorm.weight shape: [8192] -583: model.layers.67.self_attn.k_proj.weight shape: [1024, 8192] -584: model.layers.67.self_attn.o_proj.weight shape: [8192, 8192] -585: model.layers.67.self_attn.q_proj.weight shape: [8192, 8192] -586: model.layers.67.self_attn.v_proj.weight shape: [1024, 8192] -587: model.layers.68.input_layernorm.weight shape: [8192] -588: model.layers.68.mlp.down_proj.weight shape: [8192, 28672] -589: model.layers.68.mlp.gate_proj.weight shape: [28672, 8192] -590: model.layers.68.mlp.up_proj.weight shape: [28672, 8192] -591: model.layers.68.post_attention_layernorm.weight shape: [8192] -592: model.layers.68.self_attn.k_proj.weight shape: [1024, 8192] -593: model.layers.68.self_attn.o_proj.weight shape: [8192, 8192] -594: model.layers.68.self_attn.q_proj.weight shape: [8192, 8192] -595: model.layers.68.self_attn.v_proj.weight shape: [1024, 8192] -596: model.layers.69.input_layernorm.weight shape: [8192] -597: model.layers.69.mlp.down_proj.weight shape: [8192, 28672] -598: model.layers.69.mlp.gate_proj.weight shape: [28672, 8192] -599: model.layers.69.mlp.up_proj.weight shape: [28672, 8192] -600: model.layers.69.post_attention_layernorm.weight shape: [8192] -601: model.layers.69.self_attn.k_proj.weight shape: [1024, 8192] -602: model.layers.69.self_attn.o_proj.weight shape: [8192, 8192] -603: model.layers.69.self_attn.q_proj.weight shape: [8192, 8192] -604: model.layers.69.self_attn.v_proj.weight shape: [1024, 8192] -605: model.layers.7.input_layernorm.weight shape: [8192] -606: model.layers.7.mlp.down_proj.weight shape: [8192, 28672] -607: model.layers.7.mlp.gate_proj.weight shape: [28672, 8192] -608: model.layers.7.mlp.up_proj.weight shape: [28672, 8192] -609: model.layers.7.post_attention_layernorm.weight shape: [8192] -610: model.layers.7.self_attn.k_proj.weight shape: [1024, 8192] -611: model.layers.7.self_attn.o_proj.weight shape: [8192, 8192] -612: model.layers.7.self_attn.q_proj.weight shape: [8192, 8192] -613: model.layers.7.self_attn.v_proj.weight shape: [1024, 8192] -614: model.layers.70.input_layernorm.weight shape: [8192] -615: model.layers.70.mlp.down_proj.weight shape: [8192, 28672] -616: model.layers.70.mlp.gate_proj.weight shape: [28672, 8192] -617: model.layers.70.mlp.up_proj.weight shape: [28672, 8192] -618: model.layers.70.post_attention_layernorm.weight shape: [8192] -619: model.layers.70.self_attn.k_proj.weight shape: [1024, 8192] -620: model.layers.70.self_attn.o_proj.weight shape: [8192, 8192] -621: model.layers.70.self_attn.q_proj.weight shape: [8192, 8192] -622: model.layers.70.self_attn.v_proj.weight shape: [1024, 8192] -623: model.layers.71.input_layernorm.weight shape: [8192] -624: model.layers.71.mlp.down_proj.weight shape: [8192, 28672] -625: model.layers.71.mlp.gate_proj.weight shape: [28672, 8192] -626: model.layers.71.mlp.up_proj.weight shape: [28672, 8192] -627: model.layers.71.post_attention_layernorm.weight shape: [8192] -628: model.layers.71.self_attn.k_proj.weight shape: [1024, 8192] -629: model.layers.71.self_attn.o_proj.weight shape: [8192, 8192] -630: model.layers.71.self_attn.q_proj.weight shape: [8192, 8192] -631: model.layers.71.self_attn.v_proj.weight shape: [1024, 8192] -632: model.layers.72.input_layernorm.weight shape: [8192] -633: model.layers.72.mlp.down_proj.weight shape: [8192, 28672] -634: model.layers.72.mlp.gate_proj.weight shape: [28672, 8192] -635: model.layers.72.mlp.up_proj.weight shape: [28672, 8192] -636: model.layers.72.post_attention_layernorm.weight shape: [8192] -637: model.layers.72.self_attn.k_proj.weight shape: [1024, 8192] -638: model.layers.72.self_attn.o_proj.weight shape: [8192, 8192] -639: model.layers.72.self_attn.q_proj.weight shape: [8192, 8192] -640: model.layers.72.self_attn.v_proj.weight shape: [1024, 8192] -641: model.layers.73.input_layernorm.weight shape: [8192] -642: model.layers.73.mlp.down_proj.weight shape: [8192, 28672] -643: model.layers.73.mlp.gate_proj.weight shape: [28672, 8192] -644: model.layers.73.mlp.up_proj.weight shape: [28672, 8192] -645: model.layers.73.post_attention_layernorm.weight shape: [8192] -646: model.layers.73.self_attn.k_proj.weight shape: [1024, 8192] -647: model.layers.73.self_attn.o_proj.weight shape: [8192, 8192] -648: model.layers.73.self_attn.q_proj.weight shape: [8192, 8192] -649: model.layers.73.self_attn.v_proj.weight shape: [1024, 8192] -650: model.layers.74.input_layernorm.weight shape: [8192] -651: model.layers.74.mlp.down_proj.weight shape: [8192, 28672] -652: model.layers.74.mlp.gate_proj.weight shape: [28672, 8192] -653: model.layers.74.mlp.up_proj.weight shape: [28672, 8192] -654: model.layers.74.post_attention_layernorm.weight shape: [8192] -655: model.layers.74.self_attn.k_proj.weight shape: [1024, 8192] -656: model.layers.74.self_attn.o_proj.weight shape: [8192, 8192] -657: model.layers.74.self_attn.q_proj.weight shape: [8192, 8192] -658: model.layers.74.self_attn.v_proj.weight shape: [1024, 8192] -659: model.layers.75.input_layernorm.weight shape: [8192] -660: model.layers.75.mlp.down_proj.weight shape: [8192, 28672] -661: model.layers.75.mlp.gate_proj.weight shape: [28672, 8192] -662: model.layers.75.mlp.up_proj.weight shape: [28672, 8192] -663: model.layers.75.post_attention_layernorm.weight shape: [8192] -664: model.layers.75.self_attn.k_proj.weight shape: [1024, 8192] -665: model.layers.75.self_attn.o_proj.weight shape: [8192, 8192] -666: model.layers.75.self_attn.q_proj.weight shape: [8192, 8192] -667: model.layers.75.self_attn.v_proj.weight shape: [1024, 8192] -668: model.layers.76.input_layernorm.weight shape: [8192] -669: model.layers.76.mlp.down_proj.weight shape: [8192, 28672] -670: model.layers.76.mlp.gate_proj.weight shape: [28672, 8192] -671: model.layers.76.mlp.up_proj.weight shape: [28672, 8192] -672: model.layers.76.post_attention_layernorm.weight shape: [8192] -673: model.layers.76.self_attn.k_proj.weight shape: [1024, 8192] -674: model.layers.76.self_attn.o_proj.weight shape: [8192, 8192] -675: model.layers.76.self_attn.q_proj.weight shape: [8192, 8192] -676: model.layers.76.self_attn.v_proj.weight shape: [1024, 8192] -677: model.layers.77.input_layernorm.weight shape: [8192] -678: model.layers.77.mlp.down_proj.weight shape: [8192, 28672] -679: model.layers.77.mlp.gate_proj.weight shape: [28672, 8192] -680: model.layers.77.mlp.up_proj.weight shape: [28672, 8192] -681: model.layers.77.post_attention_layernorm.weight shape: [8192] -682: model.layers.77.self_attn.k_proj.weight shape: [1024, 8192] -683: model.layers.77.self_attn.o_proj.weight shape: [8192, 8192] -684: model.layers.77.self_attn.q_proj.weight shape: [8192, 8192] -685: model.layers.77.self_attn.v_proj.weight shape: [1024, 8192] -686: model.layers.78.input_layernorm.weight shape: [8192] -687: model.layers.78.mlp.down_proj.weight shape: [8192, 28672] -688: model.layers.78.mlp.gate_proj.weight shape: [28672, 8192] -689: model.layers.78.mlp.up_proj.weight shape: [28672, 8192] -690: model.layers.78.post_attention_layernorm.weight shape: [8192] -691: model.layers.78.self_attn.k_proj.weight shape: [1024, 8192] -692: model.layers.78.self_attn.o_proj.weight shape: [8192, 8192] -693: model.layers.78.self_attn.q_proj.weight shape: [8192, 8192] -694: model.layers.78.self_attn.v_proj.weight shape: [1024, 8192] -695: model.layers.79.input_layernorm.weight shape: [8192] -696: model.layers.79.mlp.down_proj.weight shape: [8192, 28672] -697: model.layers.79.mlp.gate_proj.weight shape: [28672, 8192] -698: model.layers.79.mlp.up_proj.weight shape: [28672, 8192] -699: model.layers.79.post_attention_layernorm.weight shape: [8192] -700: model.layers.79.self_attn.k_proj.weight shape: [1024, 8192] -701: model.layers.79.self_attn.o_proj.weight shape: [8192, 8192] -702: model.layers.79.self_attn.q_proj.weight shape: [8192, 8192] -703: model.layers.79.self_attn.v_proj.weight shape: [1024, 8192] -704: model.layers.8.input_layernorm.weight shape: [8192] -705: model.layers.8.mlp.down_proj.weight shape: [8192, 28672] -706: model.layers.8.mlp.gate_proj.weight shape: [28672, 8192] -707: model.layers.8.mlp.up_proj.weight shape: [28672, 8192] -708: model.layers.8.post_attention_layernorm.weight shape: [8192] -709: model.layers.8.self_attn.k_proj.weight shape: [1024, 8192] -710: model.layers.8.self_attn.o_proj.weight shape: [8192, 8192] -711: model.layers.8.self_attn.q_proj.weight shape: [8192, 8192] -712: model.layers.8.self_attn.v_proj.weight shape: [1024, 8192] -713: model.layers.9.input_layernorm.weight shape: [8192] -714: model.layers.9.mlp.down_proj.weight shape: [8192, 28672] -715: model.layers.9.mlp.gate_proj.weight shape: [28672, 8192] -716: model.layers.9.mlp.up_proj.weight shape: [28672, 8192] -717: model.layers.9.post_attention_layernorm.weight shape: [8192] -718: model.layers.9.self_attn.k_proj.weight shape: [1024, 8192] -719: model.layers.9.self_attn.o_proj.weight shape: [8192, 8192] -720: model.layers.9.self_attn.q_proj.weight shape: [8192, 8192] -721: model.layers.9.self_attn.v_proj.weight shape: [1024, 8192] -722: model.norm.weight shape: [8192] diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_8b_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_8b_ShapeTest.approved.txt deleted file mode 100644 index 887b49cfa6..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.Llama_3_1_8b_ShapeTest.approved.txt +++ /dev/null @@ -1,291 +0,0 @@ -0: lm_head.weight shape: [128256, 4096] -1: model.embed_tokens.weight shape: [128256, 4096] -2: model.layers.0.input_layernorm.weight shape: [4096] -3: model.layers.0.mlp.down_proj.weight shape: [4096, 14336] -4: model.layers.0.mlp.gate_proj.weight shape: [14336, 4096] -5: model.layers.0.mlp.up_proj.weight shape: [14336, 4096] -6: model.layers.0.post_attention_layernorm.weight shape: [4096] -7: model.layers.0.self_attn.k_proj.weight shape: [1024, 4096] -8: model.layers.0.self_attn.o_proj.weight shape: [4096, 4096] -9: model.layers.0.self_attn.q_proj.weight shape: [4096, 4096] -10: model.layers.0.self_attn.v_proj.weight shape: [1024, 4096] -11: model.layers.1.input_layernorm.weight shape: [4096] -12: model.layers.1.mlp.down_proj.weight shape: [4096, 14336] -13: model.layers.1.mlp.gate_proj.weight shape: [14336, 4096] -14: model.layers.1.mlp.up_proj.weight shape: [14336, 4096] -15: model.layers.1.post_attention_layernorm.weight shape: [4096] -16: model.layers.1.self_attn.k_proj.weight shape: [1024, 4096] -17: model.layers.1.self_attn.o_proj.weight shape: [4096, 4096] -18: model.layers.1.self_attn.q_proj.weight shape: [4096, 4096] -19: model.layers.1.self_attn.v_proj.weight shape: [1024, 4096] -20: model.layers.10.input_layernorm.weight shape: [4096] -21: model.layers.10.mlp.down_proj.weight shape: [4096, 14336] -22: model.layers.10.mlp.gate_proj.weight shape: [14336, 4096] -23: model.layers.10.mlp.up_proj.weight shape: [14336, 4096] -24: model.layers.10.post_attention_layernorm.weight shape: [4096] -25: model.layers.10.self_attn.k_proj.weight shape: [1024, 4096] -26: model.layers.10.self_attn.o_proj.weight shape: [4096, 4096] -27: model.layers.10.self_attn.q_proj.weight shape: [4096, 4096] -28: model.layers.10.self_attn.v_proj.weight shape: [1024, 4096] -29: model.layers.11.input_layernorm.weight shape: [4096] -30: model.layers.11.mlp.down_proj.weight shape: [4096, 14336] -31: model.layers.11.mlp.gate_proj.weight shape: [14336, 4096] -32: model.layers.11.mlp.up_proj.weight shape: [14336, 4096] -33: model.layers.11.post_attention_layernorm.weight shape: [4096] -34: model.layers.11.self_attn.k_proj.weight shape: [1024, 4096] -35: model.layers.11.self_attn.o_proj.weight shape: [4096, 4096] -36: model.layers.11.self_attn.q_proj.weight shape: [4096, 4096] -37: model.layers.11.self_attn.v_proj.weight shape: [1024, 4096] -38: model.layers.12.input_layernorm.weight shape: [4096] -39: model.layers.12.mlp.down_proj.weight shape: [4096, 14336] -40: model.layers.12.mlp.gate_proj.weight shape: [14336, 4096] -41: model.layers.12.mlp.up_proj.weight shape: [14336, 4096] -42: model.layers.12.post_attention_layernorm.weight shape: [4096] -43: model.layers.12.self_attn.k_proj.weight shape: [1024, 4096] -44: model.layers.12.self_attn.o_proj.weight shape: [4096, 4096] -45: model.layers.12.self_attn.q_proj.weight shape: [4096, 4096] -46: model.layers.12.self_attn.v_proj.weight shape: [1024, 4096] -47: model.layers.13.input_layernorm.weight shape: [4096] -48: model.layers.13.mlp.down_proj.weight shape: [4096, 14336] -49: model.layers.13.mlp.gate_proj.weight shape: [14336, 4096] -50: model.layers.13.mlp.up_proj.weight shape: [14336, 4096] -51: model.layers.13.post_attention_layernorm.weight shape: [4096] -52: model.layers.13.self_attn.k_proj.weight shape: [1024, 4096] -53: model.layers.13.self_attn.o_proj.weight shape: [4096, 4096] -54: model.layers.13.self_attn.q_proj.weight shape: [4096, 4096] -55: model.layers.13.self_attn.v_proj.weight shape: [1024, 4096] -56: model.layers.14.input_layernorm.weight shape: [4096] -57: model.layers.14.mlp.down_proj.weight shape: [4096, 14336] -58: model.layers.14.mlp.gate_proj.weight shape: [14336, 4096] -59: model.layers.14.mlp.up_proj.weight shape: [14336, 4096] -60: model.layers.14.post_attention_layernorm.weight shape: [4096] -61: model.layers.14.self_attn.k_proj.weight shape: [1024, 4096] -62: model.layers.14.self_attn.o_proj.weight shape: [4096, 4096] -63: model.layers.14.self_attn.q_proj.weight shape: [4096, 4096] -64: model.layers.14.self_attn.v_proj.weight shape: [1024, 4096] -65: model.layers.15.input_layernorm.weight shape: [4096] -66: model.layers.15.mlp.down_proj.weight shape: [4096, 14336] -67: model.layers.15.mlp.gate_proj.weight shape: [14336, 4096] -68: model.layers.15.mlp.up_proj.weight shape: [14336, 4096] -69: model.layers.15.post_attention_layernorm.weight shape: [4096] -70: model.layers.15.self_attn.k_proj.weight shape: [1024, 4096] -71: model.layers.15.self_attn.o_proj.weight shape: [4096, 4096] -72: model.layers.15.self_attn.q_proj.weight shape: [4096, 4096] -73: model.layers.15.self_attn.v_proj.weight shape: [1024, 4096] -74: model.layers.16.input_layernorm.weight shape: [4096] -75: model.layers.16.mlp.down_proj.weight shape: [4096, 14336] -76: model.layers.16.mlp.gate_proj.weight shape: [14336, 4096] -77: model.layers.16.mlp.up_proj.weight shape: [14336, 4096] -78: model.layers.16.post_attention_layernorm.weight shape: [4096] -79: model.layers.16.self_attn.k_proj.weight shape: [1024, 4096] -80: model.layers.16.self_attn.o_proj.weight shape: [4096, 4096] -81: model.layers.16.self_attn.q_proj.weight shape: [4096, 4096] -82: model.layers.16.self_attn.v_proj.weight shape: [1024, 4096] -83: model.layers.17.input_layernorm.weight shape: [4096] -84: model.layers.17.mlp.down_proj.weight shape: [4096, 14336] -85: model.layers.17.mlp.gate_proj.weight shape: [14336, 4096] -86: model.layers.17.mlp.up_proj.weight shape: [14336, 4096] -87: model.layers.17.post_attention_layernorm.weight shape: [4096] -88: model.layers.17.self_attn.k_proj.weight shape: [1024, 4096] -89: model.layers.17.self_attn.o_proj.weight shape: [4096, 4096] -90: model.layers.17.self_attn.q_proj.weight shape: [4096, 4096] -91: model.layers.17.self_attn.v_proj.weight shape: [1024, 4096] -92: model.layers.18.input_layernorm.weight shape: [4096] -93: model.layers.18.mlp.down_proj.weight shape: [4096, 14336] -94: model.layers.18.mlp.gate_proj.weight shape: [14336, 4096] -95: model.layers.18.mlp.up_proj.weight shape: [14336, 4096] -96: model.layers.18.post_attention_layernorm.weight shape: [4096] -97: model.layers.18.self_attn.k_proj.weight shape: [1024, 4096] -98: model.layers.18.self_attn.o_proj.weight shape: [4096, 4096] -99: model.layers.18.self_attn.q_proj.weight shape: [4096, 4096] -100: model.layers.18.self_attn.v_proj.weight shape: [1024, 4096] -101: model.layers.19.input_layernorm.weight shape: [4096] -102: model.layers.19.mlp.down_proj.weight shape: [4096, 14336] -103: model.layers.19.mlp.gate_proj.weight shape: [14336, 4096] -104: model.layers.19.mlp.up_proj.weight shape: [14336, 4096] -105: model.layers.19.post_attention_layernorm.weight shape: [4096] -106: model.layers.19.self_attn.k_proj.weight shape: [1024, 4096] -107: model.layers.19.self_attn.o_proj.weight shape: [4096, 4096] -108: model.layers.19.self_attn.q_proj.weight shape: [4096, 4096] -109: model.layers.19.self_attn.v_proj.weight shape: [1024, 4096] -110: model.layers.2.input_layernorm.weight shape: [4096] -111: model.layers.2.mlp.down_proj.weight shape: [4096, 14336] -112: model.layers.2.mlp.gate_proj.weight shape: [14336, 4096] -113: model.layers.2.mlp.up_proj.weight shape: [14336, 4096] -114: model.layers.2.post_attention_layernorm.weight shape: [4096] -115: model.layers.2.self_attn.k_proj.weight shape: [1024, 4096] -116: model.layers.2.self_attn.o_proj.weight shape: [4096, 4096] -117: model.layers.2.self_attn.q_proj.weight shape: [4096, 4096] -118: model.layers.2.self_attn.v_proj.weight shape: [1024, 4096] -119: model.layers.20.input_layernorm.weight shape: [4096] -120: model.layers.20.mlp.down_proj.weight shape: [4096, 14336] -121: model.layers.20.mlp.gate_proj.weight shape: [14336, 4096] -122: model.layers.20.mlp.up_proj.weight shape: [14336, 4096] -123: model.layers.20.post_attention_layernorm.weight shape: [4096] -124: model.layers.20.self_attn.k_proj.weight shape: [1024, 4096] -125: model.layers.20.self_attn.o_proj.weight shape: [4096, 4096] -126: model.layers.20.self_attn.q_proj.weight shape: [4096, 4096] -127: model.layers.20.self_attn.v_proj.weight shape: [1024, 4096] -128: model.layers.21.input_layernorm.weight shape: [4096] -129: model.layers.21.mlp.down_proj.weight shape: [4096, 14336] -130: model.layers.21.mlp.gate_proj.weight shape: [14336, 4096] -131: model.layers.21.mlp.up_proj.weight shape: [14336, 4096] -132: model.layers.21.post_attention_layernorm.weight shape: [4096] -133: model.layers.21.self_attn.k_proj.weight shape: [1024, 4096] -134: model.layers.21.self_attn.o_proj.weight shape: [4096, 4096] -135: model.layers.21.self_attn.q_proj.weight shape: [4096, 4096] -136: model.layers.21.self_attn.v_proj.weight shape: [1024, 4096] -137: model.layers.22.input_layernorm.weight shape: [4096] -138: model.layers.22.mlp.down_proj.weight shape: [4096, 14336] -139: model.layers.22.mlp.gate_proj.weight shape: [14336, 4096] -140: model.layers.22.mlp.up_proj.weight shape: [14336, 4096] -141: model.layers.22.post_attention_layernorm.weight shape: [4096] -142: model.layers.22.self_attn.k_proj.weight shape: [1024, 4096] -143: model.layers.22.self_attn.o_proj.weight shape: [4096, 4096] -144: model.layers.22.self_attn.q_proj.weight shape: [4096, 4096] -145: model.layers.22.self_attn.v_proj.weight shape: [1024, 4096] -146: model.layers.23.input_layernorm.weight shape: [4096] -147: model.layers.23.mlp.down_proj.weight shape: [4096, 14336] -148: model.layers.23.mlp.gate_proj.weight shape: [14336, 4096] -149: model.layers.23.mlp.up_proj.weight shape: [14336, 4096] -150: model.layers.23.post_attention_layernorm.weight shape: [4096] -151: model.layers.23.self_attn.k_proj.weight shape: [1024, 4096] -152: model.layers.23.self_attn.o_proj.weight shape: [4096, 4096] -153: model.layers.23.self_attn.q_proj.weight shape: [4096, 4096] -154: model.layers.23.self_attn.v_proj.weight shape: [1024, 4096] -155: model.layers.24.input_layernorm.weight shape: [4096] -156: model.layers.24.mlp.down_proj.weight shape: [4096, 14336] -157: model.layers.24.mlp.gate_proj.weight shape: [14336, 4096] -158: model.layers.24.mlp.up_proj.weight shape: [14336, 4096] -159: model.layers.24.post_attention_layernorm.weight shape: [4096] -160: model.layers.24.self_attn.k_proj.weight shape: [1024, 4096] -161: model.layers.24.self_attn.o_proj.weight shape: [4096, 4096] -162: model.layers.24.self_attn.q_proj.weight shape: [4096, 4096] -163: model.layers.24.self_attn.v_proj.weight shape: [1024, 4096] -164: model.layers.25.input_layernorm.weight shape: [4096] -165: model.layers.25.mlp.down_proj.weight shape: [4096, 14336] -166: model.layers.25.mlp.gate_proj.weight shape: [14336, 4096] -167: model.layers.25.mlp.up_proj.weight shape: [14336, 4096] -168: model.layers.25.post_attention_layernorm.weight shape: [4096] -169: model.layers.25.self_attn.k_proj.weight shape: [1024, 4096] -170: model.layers.25.self_attn.o_proj.weight shape: [4096, 4096] -171: model.layers.25.self_attn.q_proj.weight shape: [4096, 4096] -172: model.layers.25.self_attn.v_proj.weight shape: [1024, 4096] -173: model.layers.26.input_layernorm.weight shape: [4096] -174: model.layers.26.mlp.down_proj.weight shape: [4096, 14336] -175: model.layers.26.mlp.gate_proj.weight shape: [14336, 4096] -176: model.layers.26.mlp.up_proj.weight shape: [14336, 4096] -177: model.layers.26.post_attention_layernorm.weight shape: [4096] -178: model.layers.26.self_attn.k_proj.weight shape: [1024, 4096] -179: model.layers.26.self_attn.o_proj.weight shape: [4096, 4096] -180: model.layers.26.self_attn.q_proj.weight shape: [4096, 4096] -181: model.layers.26.self_attn.v_proj.weight shape: [1024, 4096] -182: model.layers.27.input_layernorm.weight shape: [4096] -183: model.layers.27.mlp.down_proj.weight shape: [4096, 14336] -184: model.layers.27.mlp.gate_proj.weight shape: [14336, 4096] -185: model.layers.27.mlp.up_proj.weight shape: [14336, 4096] -186: model.layers.27.post_attention_layernorm.weight shape: [4096] -187: model.layers.27.self_attn.k_proj.weight shape: [1024, 4096] -188: model.layers.27.self_attn.o_proj.weight shape: [4096, 4096] -189: model.layers.27.self_attn.q_proj.weight shape: [4096, 4096] -190: model.layers.27.self_attn.v_proj.weight shape: [1024, 4096] -191: model.layers.28.input_layernorm.weight shape: [4096] -192: model.layers.28.mlp.down_proj.weight shape: [4096, 14336] -193: model.layers.28.mlp.gate_proj.weight shape: [14336, 4096] -194: model.layers.28.mlp.up_proj.weight shape: [14336, 4096] -195: model.layers.28.post_attention_layernorm.weight shape: [4096] -196: model.layers.28.self_attn.k_proj.weight shape: [1024, 4096] -197: model.layers.28.self_attn.o_proj.weight shape: [4096, 4096] -198: model.layers.28.self_attn.q_proj.weight shape: [4096, 4096] -199: model.layers.28.self_attn.v_proj.weight shape: [1024, 4096] -200: model.layers.29.input_layernorm.weight shape: [4096] -201: model.layers.29.mlp.down_proj.weight shape: [4096, 14336] -202: model.layers.29.mlp.gate_proj.weight shape: [14336, 4096] -203: model.layers.29.mlp.up_proj.weight shape: [14336, 4096] -204: model.layers.29.post_attention_layernorm.weight shape: [4096] -205: model.layers.29.self_attn.k_proj.weight shape: [1024, 4096] -206: model.layers.29.self_attn.o_proj.weight shape: [4096, 4096] -207: model.layers.29.self_attn.q_proj.weight shape: [4096, 4096] -208: model.layers.29.self_attn.v_proj.weight shape: [1024, 4096] -209: model.layers.3.input_layernorm.weight shape: [4096] -210: model.layers.3.mlp.down_proj.weight shape: [4096, 14336] -211: model.layers.3.mlp.gate_proj.weight shape: [14336, 4096] -212: model.layers.3.mlp.up_proj.weight shape: [14336, 4096] -213: model.layers.3.post_attention_layernorm.weight shape: [4096] -214: model.layers.3.self_attn.k_proj.weight shape: [1024, 4096] -215: model.layers.3.self_attn.o_proj.weight shape: [4096, 4096] -216: model.layers.3.self_attn.q_proj.weight shape: [4096, 4096] -217: model.layers.3.self_attn.v_proj.weight shape: [1024, 4096] -218: model.layers.30.input_layernorm.weight shape: [4096] -219: model.layers.30.mlp.down_proj.weight shape: [4096, 14336] -220: model.layers.30.mlp.gate_proj.weight shape: [14336, 4096] -221: model.layers.30.mlp.up_proj.weight shape: [14336, 4096] -222: model.layers.30.post_attention_layernorm.weight shape: [4096] -223: model.layers.30.self_attn.k_proj.weight shape: [1024, 4096] -224: model.layers.30.self_attn.o_proj.weight shape: [4096, 4096] -225: model.layers.30.self_attn.q_proj.weight shape: [4096, 4096] -226: model.layers.30.self_attn.v_proj.weight shape: [1024, 4096] -227: model.layers.31.input_layernorm.weight shape: [4096] -228: model.layers.31.mlp.down_proj.weight shape: [4096, 14336] -229: model.layers.31.mlp.gate_proj.weight shape: [14336, 4096] -230: model.layers.31.mlp.up_proj.weight shape: [14336, 4096] -231: model.layers.31.post_attention_layernorm.weight shape: [4096] -232: model.layers.31.self_attn.k_proj.weight shape: [1024, 4096] -233: model.layers.31.self_attn.o_proj.weight shape: [4096, 4096] -234: model.layers.31.self_attn.q_proj.weight shape: [4096, 4096] -235: model.layers.31.self_attn.v_proj.weight shape: [1024, 4096] -236: model.layers.4.input_layernorm.weight shape: [4096] -237: model.layers.4.mlp.down_proj.weight shape: [4096, 14336] -238: model.layers.4.mlp.gate_proj.weight shape: [14336, 4096] -239: model.layers.4.mlp.up_proj.weight shape: [14336, 4096] -240: model.layers.4.post_attention_layernorm.weight shape: [4096] -241: model.layers.4.self_attn.k_proj.weight shape: [1024, 4096] -242: model.layers.4.self_attn.o_proj.weight shape: [4096, 4096] -243: model.layers.4.self_attn.q_proj.weight shape: [4096, 4096] -244: model.layers.4.self_attn.v_proj.weight shape: [1024, 4096] -245: model.layers.5.input_layernorm.weight shape: [4096] -246: model.layers.5.mlp.down_proj.weight shape: [4096, 14336] -247: model.layers.5.mlp.gate_proj.weight shape: [14336, 4096] -248: model.layers.5.mlp.up_proj.weight shape: [14336, 4096] -249: model.layers.5.post_attention_layernorm.weight shape: [4096] -250: model.layers.5.self_attn.k_proj.weight shape: [1024, 4096] -251: model.layers.5.self_attn.o_proj.weight shape: [4096, 4096] -252: model.layers.5.self_attn.q_proj.weight shape: [4096, 4096] -253: model.layers.5.self_attn.v_proj.weight shape: [1024, 4096] -254: model.layers.6.input_layernorm.weight shape: [4096] -255: model.layers.6.mlp.down_proj.weight shape: [4096, 14336] -256: model.layers.6.mlp.gate_proj.weight shape: [14336, 4096] -257: model.layers.6.mlp.up_proj.weight shape: [14336, 4096] -258: model.layers.6.post_attention_layernorm.weight shape: [4096] -259: model.layers.6.self_attn.k_proj.weight shape: [1024, 4096] -260: model.layers.6.self_attn.o_proj.weight shape: [4096, 4096] -261: model.layers.6.self_attn.q_proj.weight shape: [4096, 4096] -262: model.layers.6.self_attn.v_proj.weight shape: [1024, 4096] -263: model.layers.7.input_layernorm.weight shape: [4096] -264: model.layers.7.mlp.down_proj.weight shape: [4096, 14336] -265: model.layers.7.mlp.gate_proj.weight shape: [14336, 4096] -266: model.layers.7.mlp.up_proj.weight shape: [14336, 4096] -267: model.layers.7.post_attention_layernorm.weight shape: [4096] -268: model.layers.7.self_attn.k_proj.weight shape: [1024, 4096] -269: model.layers.7.self_attn.o_proj.weight shape: [4096, 4096] -270: model.layers.7.self_attn.q_proj.weight shape: [4096, 4096] -271: model.layers.7.self_attn.v_proj.weight shape: [1024, 4096] -272: model.layers.8.input_layernorm.weight shape: [4096] -273: model.layers.8.mlp.down_proj.weight shape: [4096, 14336] -274: model.layers.8.mlp.gate_proj.weight shape: [14336, 4096] -275: model.layers.8.mlp.up_proj.weight shape: [14336, 4096] -276: model.layers.8.post_attention_layernorm.weight shape: [4096] -277: model.layers.8.self_attn.k_proj.weight shape: [1024, 4096] -278: model.layers.8.self_attn.o_proj.weight shape: [4096, 4096] -279: model.layers.8.self_attn.q_proj.weight shape: [4096, 4096] -280: model.layers.8.self_attn.v_proj.weight shape: [1024, 4096] -281: model.layers.9.input_layernorm.weight shape: [4096] -282: model.layers.9.mlp.down_proj.weight shape: [4096, 14336] -283: model.layers.9.mlp.gate_proj.weight shape: [14336, 4096] -284: model.layers.9.mlp.up_proj.weight shape: [14336, 4096] -285: model.layers.9.post_attention_layernorm.weight shape: [4096] -286: model.layers.9.self_attn.k_proj.weight shape: [1024, 4096] -287: model.layers.9.self_attn.o_proj.weight shape: [4096, 4096] -288: model.layers.9.self_attn.q_proj.weight shape: [4096, 4096] -289: model.layers.9.self_attn.v_proj.weight shape: [1024, 4096] -290: model.norm.weight shape: [4096] diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.TokenizerTest.approved.txt deleted file mode 100644 index fc0568084b..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_1Tests.TokenizerTest.approved.txt +++ /dev/null @@ -1,8 +0,0 @@ -Can you provide ways to eat combinations of bananas and dragonfruits? -6854, 499, 3493, 5627, 311, 8343, 28559, 315, 68442, 323, 26161, 1658, 12059, 30 -Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey. -40914, 0, 5810, 527, 1063, 5627, 311, 8343, 68442, 323, 26161, 1658, 12059, 3871, 25, 220, 16, 13, 76924, 323, 26161, 36698, 11113, 648, 25, 55248, 68442, 323, 26161, 1658, 12059, 3871, 449, 1063, 14403, 323, 26828, 13, 220, 17, 13, 76924, 323, 26161, 36698, 33566, 25, 19771, 48715, 68442, 323, 26161, 1658, 12059, 3871, 449, 1063, 30564, 23661, 323, 26828, 13 -What about solving an 2x + 3 = 7 equation? -3923, 922, 22581, 459, 220, 17, 87, 489, 220, 18, 284, 220, 22, 24524, 30 -<|begin_of_text|>Hello World<|end_of_text|> -128000, 9906, 4435, 128001 diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_1b_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_1b_ShapeTest.approved.txt deleted file mode 100644 index 42edde3277..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_1b_ShapeTest.approved.txt +++ /dev/null @@ -1,146 +0,0 @@ -0: model.embed_tokens.weight shape: [128256, 2048] -1: model.layers.0.input_layernorm.weight shape: [2048] -2: model.layers.0.mlp.down_proj.weight shape: [2048, 8192] -3: model.layers.0.mlp.gate_proj.weight shape: [8192, 2048] -4: model.layers.0.mlp.up_proj.weight shape: [8192, 2048] -5: model.layers.0.post_attention_layernorm.weight shape: [2048] -6: model.layers.0.self_attn.k_proj.weight shape: [512, 2048] -7: model.layers.0.self_attn.o_proj.weight shape: [2048, 2048] -8: model.layers.0.self_attn.q_proj.weight shape: [2048, 2048] -9: model.layers.0.self_attn.v_proj.weight shape: [512, 2048] -10: model.layers.1.input_layernorm.weight shape: [2048] -11: model.layers.1.mlp.down_proj.weight shape: [2048, 8192] -12: model.layers.1.mlp.gate_proj.weight shape: [8192, 2048] -13: model.layers.1.mlp.up_proj.weight shape: [8192, 2048] -14: model.layers.1.post_attention_layernorm.weight shape: [2048] -15: model.layers.1.self_attn.k_proj.weight shape: [512, 2048] -16: model.layers.1.self_attn.o_proj.weight shape: [2048, 2048] -17: model.layers.1.self_attn.q_proj.weight shape: [2048, 2048] -18: model.layers.1.self_attn.v_proj.weight shape: [512, 2048] -19: model.layers.10.input_layernorm.weight shape: [2048] -20: model.layers.10.mlp.down_proj.weight shape: [2048, 8192] -21: model.layers.10.mlp.gate_proj.weight shape: [8192, 2048] -22: model.layers.10.mlp.up_proj.weight shape: [8192, 2048] -23: model.layers.10.post_attention_layernorm.weight shape: [2048] -24: model.layers.10.self_attn.k_proj.weight shape: [512, 2048] -25: model.layers.10.self_attn.o_proj.weight shape: [2048, 2048] -26: model.layers.10.self_attn.q_proj.weight shape: [2048, 2048] -27: model.layers.10.self_attn.v_proj.weight shape: [512, 2048] -28: model.layers.11.input_layernorm.weight shape: [2048] -29: model.layers.11.mlp.down_proj.weight shape: [2048, 8192] -30: model.layers.11.mlp.gate_proj.weight shape: [8192, 2048] -31: model.layers.11.mlp.up_proj.weight shape: [8192, 2048] -32: model.layers.11.post_attention_layernorm.weight shape: [2048] -33: model.layers.11.self_attn.k_proj.weight shape: [512, 2048] -34: model.layers.11.self_attn.o_proj.weight shape: [2048, 2048] -35: model.layers.11.self_attn.q_proj.weight shape: [2048, 2048] -36: model.layers.11.self_attn.v_proj.weight shape: [512, 2048] -37: model.layers.12.input_layernorm.weight shape: [2048] -38: model.layers.12.mlp.down_proj.weight shape: [2048, 8192] -39: model.layers.12.mlp.gate_proj.weight shape: [8192, 2048] -40: model.layers.12.mlp.up_proj.weight shape: [8192, 2048] -41: model.layers.12.post_attention_layernorm.weight shape: [2048] -42: model.layers.12.self_attn.k_proj.weight shape: [512, 2048] -43: model.layers.12.self_attn.o_proj.weight shape: [2048, 2048] -44: model.layers.12.self_attn.q_proj.weight shape: [2048, 2048] -45: model.layers.12.self_attn.v_proj.weight shape: [512, 2048] -46: model.layers.13.input_layernorm.weight shape: [2048] -47: model.layers.13.mlp.down_proj.weight shape: [2048, 8192] -48: model.layers.13.mlp.gate_proj.weight shape: [8192, 2048] -49: model.layers.13.mlp.up_proj.weight shape: [8192, 2048] -50: model.layers.13.post_attention_layernorm.weight shape: [2048] -51: model.layers.13.self_attn.k_proj.weight shape: [512, 2048] -52: model.layers.13.self_attn.o_proj.weight shape: [2048, 2048] -53: model.layers.13.self_attn.q_proj.weight shape: [2048, 2048] -54: model.layers.13.self_attn.v_proj.weight shape: [512, 2048] -55: model.layers.14.input_layernorm.weight shape: [2048] -56: model.layers.14.mlp.down_proj.weight shape: [2048, 8192] -57: model.layers.14.mlp.gate_proj.weight shape: [8192, 2048] -58: model.layers.14.mlp.up_proj.weight shape: [8192, 2048] -59: model.layers.14.post_attention_layernorm.weight shape: [2048] -60: model.layers.14.self_attn.k_proj.weight shape: [512, 2048] -61: model.layers.14.self_attn.o_proj.weight shape: [2048, 2048] -62: model.layers.14.self_attn.q_proj.weight shape: [2048, 2048] -63: model.layers.14.self_attn.v_proj.weight shape: [512, 2048] -64: model.layers.15.input_layernorm.weight shape: [2048] -65: model.layers.15.mlp.down_proj.weight shape: [2048, 8192] -66: model.layers.15.mlp.gate_proj.weight shape: [8192, 2048] -67: model.layers.15.mlp.up_proj.weight shape: [8192, 2048] -68: model.layers.15.post_attention_layernorm.weight shape: [2048] -69: model.layers.15.self_attn.k_proj.weight shape: [512, 2048] -70: model.layers.15.self_attn.o_proj.weight shape: [2048, 2048] -71: model.layers.15.self_attn.q_proj.weight shape: [2048, 2048] -72: model.layers.15.self_attn.v_proj.weight shape: [512, 2048] -73: model.layers.2.input_layernorm.weight shape: [2048] -74: model.layers.2.mlp.down_proj.weight shape: [2048, 8192] -75: model.layers.2.mlp.gate_proj.weight shape: [8192, 2048] -76: model.layers.2.mlp.up_proj.weight shape: [8192, 2048] -77: model.layers.2.post_attention_layernorm.weight shape: [2048] -78: model.layers.2.self_attn.k_proj.weight shape: [512, 2048] -79: model.layers.2.self_attn.o_proj.weight shape: [2048, 2048] -80: model.layers.2.self_attn.q_proj.weight shape: [2048, 2048] -81: model.layers.2.self_attn.v_proj.weight shape: [512, 2048] -82: model.layers.3.input_layernorm.weight shape: [2048] -83: model.layers.3.mlp.down_proj.weight shape: [2048, 8192] -84: model.layers.3.mlp.gate_proj.weight shape: [8192, 2048] -85: model.layers.3.mlp.up_proj.weight shape: [8192, 2048] -86: model.layers.3.post_attention_layernorm.weight shape: [2048] -87: model.layers.3.self_attn.k_proj.weight shape: [512, 2048] -88: model.layers.3.self_attn.o_proj.weight shape: [2048, 2048] -89: model.layers.3.self_attn.q_proj.weight shape: [2048, 2048] -90: model.layers.3.self_attn.v_proj.weight shape: [512, 2048] -91: model.layers.4.input_layernorm.weight shape: [2048] -92: model.layers.4.mlp.down_proj.weight shape: [2048, 8192] -93: model.layers.4.mlp.gate_proj.weight shape: [8192, 2048] -94: model.layers.4.mlp.up_proj.weight shape: [8192, 2048] -95: model.layers.4.post_attention_layernorm.weight shape: [2048] -96: model.layers.4.self_attn.k_proj.weight shape: [512, 2048] -97: model.layers.4.self_attn.o_proj.weight shape: [2048, 2048] -98: model.layers.4.self_attn.q_proj.weight shape: [2048, 2048] -99: model.layers.4.self_attn.v_proj.weight shape: [512, 2048] -100: model.layers.5.input_layernorm.weight shape: [2048] -101: model.layers.5.mlp.down_proj.weight shape: [2048, 8192] -102: model.layers.5.mlp.gate_proj.weight shape: [8192, 2048] -103: model.layers.5.mlp.up_proj.weight shape: [8192, 2048] -104: model.layers.5.post_attention_layernorm.weight shape: [2048] -105: model.layers.5.self_attn.k_proj.weight shape: [512, 2048] -106: model.layers.5.self_attn.o_proj.weight shape: [2048, 2048] -107: model.layers.5.self_attn.q_proj.weight shape: [2048, 2048] -108: model.layers.5.self_attn.v_proj.weight shape: [512, 2048] -109: model.layers.6.input_layernorm.weight shape: [2048] -110: model.layers.6.mlp.down_proj.weight shape: [2048, 8192] -111: model.layers.6.mlp.gate_proj.weight shape: [8192, 2048] -112: model.layers.6.mlp.up_proj.weight shape: [8192, 2048] -113: model.layers.6.post_attention_layernorm.weight shape: [2048] -114: model.layers.6.self_attn.k_proj.weight shape: [512, 2048] -115: model.layers.6.self_attn.o_proj.weight shape: [2048, 2048] -116: model.layers.6.self_attn.q_proj.weight shape: [2048, 2048] -117: model.layers.6.self_attn.v_proj.weight shape: [512, 2048] -118: model.layers.7.input_layernorm.weight shape: [2048] -119: model.layers.7.mlp.down_proj.weight shape: [2048, 8192] -120: model.layers.7.mlp.gate_proj.weight shape: [8192, 2048] -121: model.layers.7.mlp.up_proj.weight shape: [8192, 2048] -122: model.layers.7.post_attention_layernorm.weight shape: [2048] -123: model.layers.7.self_attn.k_proj.weight shape: [512, 2048] -124: model.layers.7.self_attn.o_proj.weight shape: [2048, 2048] -125: model.layers.7.self_attn.q_proj.weight shape: [2048, 2048] -126: model.layers.7.self_attn.v_proj.weight shape: [512, 2048] -127: model.layers.8.input_layernorm.weight shape: [2048] -128: model.layers.8.mlp.down_proj.weight shape: [2048, 8192] -129: model.layers.8.mlp.gate_proj.weight shape: [8192, 2048] -130: model.layers.8.mlp.up_proj.weight shape: [8192, 2048] -131: model.layers.8.post_attention_layernorm.weight shape: [2048] -132: model.layers.8.self_attn.k_proj.weight shape: [512, 2048] -133: model.layers.8.self_attn.o_proj.weight shape: [2048, 2048] -134: model.layers.8.self_attn.q_proj.weight shape: [2048, 2048] -135: model.layers.8.self_attn.v_proj.weight shape: [512, 2048] -136: model.layers.9.input_layernorm.weight shape: [2048] -137: model.layers.9.mlp.down_proj.weight shape: [2048, 8192] -138: model.layers.9.mlp.gate_proj.weight shape: [8192, 2048] -139: model.layers.9.mlp.up_proj.weight shape: [8192, 2048] -140: model.layers.9.post_attention_layernorm.weight shape: [2048] -141: model.layers.9.self_attn.k_proj.weight shape: [512, 2048] -142: model.layers.9.self_attn.o_proj.weight shape: [2048, 2048] -143: model.layers.9.self_attn.q_proj.weight shape: [2048, 2048] -144: model.layers.9.self_attn.v_proj.weight shape: [512, 2048] -145: model.norm.weight shape: [2048] diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_3b_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_3b_ShapeTest.approved.txt deleted file mode 100644 index f26687dcb0..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Approvals/LLaMA3_2Tests.Llama_3_2_3b_ShapeTest.approved.txt +++ /dev/null @@ -1,254 +0,0 @@ -0: model.embed_tokens.weight shape: [128256, 3072] -1: model.layers.0.input_layernorm.weight shape: [3072] -2: model.layers.0.mlp.down_proj.weight shape: [3072, 8192] -3: model.layers.0.mlp.gate_proj.weight shape: [8192, 3072] -4: model.layers.0.mlp.up_proj.weight shape: [8192, 3072] -5: model.layers.0.post_attention_layernorm.weight shape: [3072] -6: model.layers.0.self_attn.k_proj.weight shape: [1024, 3072] -7: model.layers.0.self_attn.o_proj.weight shape: [3072, 3072] -8: model.layers.0.self_attn.q_proj.weight shape: [3072, 3072] -9: model.layers.0.self_attn.v_proj.weight shape: [1024, 3072] -10: model.layers.1.input_layernorm.weight shape: [3072] -11: model.layers.1.mlp.down_proj.weight shape: [3072, 8192] -12: model.layers.1.mlp.gate_proj.weight shape: [8192, 3072] -13: model.layers.1.mlp.up_proj.weight shape: [8192, 3072] -14: model.layers.1.post_attention_layernorm.weight shape: [3072] -15: model.layers.1.self_attn.k_proj.weight shape: [1024, 3072] -16: model.layers.1.self_attn.o_proj.weight shape: [3072, 3072] -17: model.layers.1.self_attn.q_proj.weight shape: [3072, 3072] -18: model.layers.1.self_attn.v_proj.weight shape: [1024, 3072] -19: model.layers.10.input_layernorm.weight shape: [3072] -20: model.layers.10.mlp.down_proj.weight shape: [3072, 8192] -21: model.layers.10.mlp.gate_proj.weight shape: [8192, 3072] -22: model.layers.10.mlp.up_proj.weight shape: [8192, 3072] -23: model.layers.10.post_attention_layernorm.weight shape: [3072] -24: model.layers.10.self_attn.k_proj.weight shape: [1024, 3072] -25: model.layers.10.self_attn.o_proj.weight shape: [3072, 3072] -26: model.layers.10.self_attn.q_proj.weight shape: [3072, 3072] -27: model.layers.10.self_attn.v_proj.weight shape: [1024, 3072] -28: model.layers.11.input_layernorm.weight shape: [3072] -29: model.layers.11.mlp.down_proj.weight shape: [3072, 8192] -30: model.layers.11.mlp.gate_proj.weight shape: [8192, 3072] -31: model.layers.11.mlp.up_proj.weight shape: [8192, 3072] -32: model.layers.11.post_attention_layernorm.weight shape: [3072] -33: model.layers.11.self_attn.k_proj.weight shape: [1024, 3072] -34: model.layers.11.self_attn.o_proj.weight shape: [3072, 3072] -35: model.layers.11.self_attn.q_proj.weight shape: [3072, 3072] -36: model.layers.11.self_attn.v_proj.weight shape: [1024, 3072] -37: model.layers.12.input_layernorm.weight shape: [3072] -38: model.layers.12.mlp.down_proj.weight shape: [3072, 8192] -39: model.layers.12.mlp.gate_proj.weight shape: [8192, 3072] -40: model.layers.12.mlp.up_proj.weight shape: [8192, 3072] -41: model.layers.12.post_attention_layernorm.weight shape: [3072] -42: model.layers.12.self_attn.k_proj.weight shape: [1024, 3072] -43: model.layers.12.self_attn.o_proj.weight shape: [3072, 3072] -44: model.layers.12.self_attn.q_proj.weight shape: [3072, 3072] -45: model.layers.12.self_attn.v_proj.weight shape: [1024, 3072] -46: model.layers.13.input_layernorm.weight shape: [3072] -47: model.layers.13.mlp.down_proj.weight shape: [3072, 8192] -48: model.layers.13.mlp.gate_proj.weight shape: [8192, 3072] -49: model.layers.13.mlp.up_proj.weight shape: [8192, 3072] -50: model.layers.13.post_attention_layernorm.weight shape: [3072] -51: model.layers.13.self_attn.k_proj.weight shape: [1024, 3072] -52: model.layers.13.self_attn.o_proj.weight shape: [3072, 3072] -53: model.layers.13.self_attn.q_proj.weight shape: [3072, 3072] -54: model.layers.13.self_attn.v_proj.weight shape: [1024, 3072] -55: model.layers.14.input_layernorm.weight shape: [3072] -56: model.layers.14.mlp.down_proj.weight shape: [3072, 8192] -57: model.layers.14.mlp.gate_proj.weight shape: [8192, 3072] -58: model.layers.14.mlp.up_proj.weight shape: [8192, 3072] -59: model.layers.14.post_attention_layernorm.weight shape: [3072] -60: model.layers.14.self_attn.k_proj.weight shape: [1024, 3072] -61: model.layers.14.self_attn.o_proj.weight shape: [3072, 3072] -62: model.layers.14.self_attn.q_proj.weight shape: [3072, 3072] -63: model.layers.14.self_attn.v_proj.weight shape: [1024, 3072] -64: model.layers.15.input_layernorm.weight shape: [3072] -65: model.layers.15.mlp.down_proj.weight shape: [3072, 8192] -66: model.layers.15.mlp.gate_proj.weight shape: [8192, 3072] -67: model.layers.15.mlp.up_proj.weight shape: [8192, 3072] -68: model.layers.15.post_attention_layernorm.weight shape: [3072] -69: model.layers.15.self_attn.k_proj.weight shape: [1024, 3072] -70: model.layers.15.self_attn.o_proj.weight shape: [3072, 3072] -71: model.layers.15.self_attn.q_proj.weight shape: [3072, 3072] -72: model.layers.15.self_attn.v_proj.weight shape: [1024, 3072] -73: model.layers.16.input_layernorm.weight shape: [3072] -74: model.layers.16.mlp.down_proj.weight shape: [3072, 8192] -75: model.layers.16.mlp.gate_proj.weight shape: [8192, 3072] -76: model.layers.16.mlp.up_proj.weight shape: [8192, 3072] -77: model.layers.16.post_attention_layernorm.weight shape: [3072] -78: model.layers.16.self_attn.k_proj.weight shape: [1024, 3072] -79: model.layers.16.self_attn.o_proj.weight shape: [3072, 3072] -80: model.layers.16.self_attn.q_proj.weight shape: [3072, 3072] -81: model.layers.16.self_attn.v_proj.weight shape: [1024, 3072] -82: model.layers.17.input_layernorm.weight shape: [3072] -83: model.layers.17.mlp.down_proj.weight shape: [3072, 8192] -84: model.layers.17.mlp.gate_proj.weight shape: [8192, 3072] -85: model.layers.17.mlp.up_proj.weight shape: [8192, 3072] -86: model.layers.17.post_attention_layernorm.weight shape: [3072] -87: model.layers.17.self_attn.k_proj.weight shape: [1024, 3072] -88: model.layers.17.self_attn.o_proj.weight shape: [3072, 3072] -89: model.layers.17.self_attn.q_proj.weight shape: [3072, 3072] -90: model.layers.17.self_attn.v_proj.weight shape: [1024, 3072] -91: model.layers.18.input_layernorm.weight shape: [3072] -92: model.layers.18.mlp.down_proj.weight shape: [3072, 8192] -93: model.layers.18.mlp.gate_proj.weight shape: [8192, 3072] -94: model.layers.18.mlp.up_proj.weight shape: [8192, 3072] -95: model.layers.18.post_attention_layernorm.weight shape: [3072] -96: model.layers.18.self_attn.k_proj.weight shape: [1024, 3072] -97: model.layers.18.self_attn.o_proj.weight shape: [3072, 3072] -98: model.layers.18.self_attn.q_proj.weight shape: [3072, 3072] -99: model.layers.18.self_attn.v_proj.weight shape: [1024, 3072] -100: model.layers.19.input_layernorm.weight shape: [3072] -101: model.layers.19.mlp.down_proj.weight shape: [3072, 8192] -102: model.layers.19.mlp.gate_proj.weight shape: [8192, 3072] -103: model.layers.19.mlp.up_proj.weight shape: [8192, 3072] -104: model.layers.19.post_attention_layernorm.weight shape: [3072] -105: model.layers.19.self_attn.k_proj.weight shape: [1024, 3072] -106: model.layers.19.self_attn.o_proj.weight shape: [3072, 3072] -107: model.layers.19.self_attn.q_proj.weight shape: [3072, 3072] -108: model.layers.19.self_attn.v_proj.weight shape: [1024, 3072] -109: model.layers.2.input_layernorm.weight shape: [3072] -110: model.layers.2.mlp.down_proj.weight shape: [3072, 8192] -111: model.layers.2.mlp.gate_proj.weight shape: [8192, 3072] -112: model.layers.2.mlp.up_proj.weight shape: [8192, 3072] -113: model.layers.2.post_attention_layernorm.weight shape: [3072] -114: model.layers.2.self_attn.k_proj.weight shape: [1024, 3072] -115: model.layers.2.self_attn.o_proj.weight shape: [3072, 3072] -116: model.layers.2.self_attn.q_proj.weight shape: [3072, 3072] -117: model.layers.2.self_attn.v_proj.weight shape: [1024, 3072] -118: model.layers.20.input_layernorm.weight shape: [3072] -119: model.layers.20.mlp.down_proj.weight shape: [3072, 8192] -120: model.layers.20.mlp.gate_proj.weight shape: [8192, 3072] -121: model.layers.20.mlp.up_proj.weight shape: [8192, 3072] -122: model.layers.20.post_attention_layernorm.weight shape: [3072] -123: model.layers.20.self_attn.k_proj.weight shape: [1024, 3072] -124: model.layers.20.self_attn.o_proj.weight shape: [3072, 3072] -125: model.layers.20.self_attn.q_proj.weight shape: [3072, 3072] -126: model.layers.20.self_attn.v_proj.weight shape: [1024, 3072] -127: model.layers.21.input_layernorm.weight shape: [3072] -128: model.layers.21.mlp.down_proj.weight shape: [3072, 8192] -129: model.layers.21.mlp.gate_proj.weight shape: [8192, 3072] -130: model.layers.21.mlp.up_proj.weight shape: [8192, 3072] -131: model.layers.21.post_attention_layernorm.weight shape: [3072] -132: model.layers.21.self_attn.k_proj.weight shape: [1024, 3072] -133: model.layers.21.self_attn.o_proj.weight shape: [3072, 3072] -134: model.layers.21.self_attn.q_proj.weight shape: [3072, 3072] -135: model.layers.21.self_attn.v_proj.weight shape: [1024, 3072] -136: model.layers.22.input_layernorm.weight shape: [3072] -137: model.layers.22.mlp.down_proj.weight shape: [3072, 8192] -138: model.layers.22.mlp.gate_proj.weight shape: [8192, 3072] -139: model.layers.22.mlp.up_proj.weight shape: [8192, 3072] -140: model.layers.22.post_attention_layernorm.weight shape: [3072] -141: model.layers.22.self_attn.k_proj.weight shape: [1024, 3072] -142: model.layers.22.self_attn.o_proj.weight shape: [3072, 3072] -143: model.layers.22.self_attn.q_proj.weight shape: [3072, 3072] -144: model.layers.22.self_attn.v_proj.weight shape: [1024, 3072] -145: model.layers.23.input_layernorm.weight shape: [3072] -146: model.layers.23.mlp.down_proj.weight shape: [3072, 8192] -147: model.layers.23.mlp.gate_proj.weight shape: [8192, 3072] -148: model.layers.23.mlp.up_proj.weight shape: [8192, 3072] -149: model.layers.23.post_attention_layernorm.weight shape: [3072] -150: model.layers.23.self_attn.k_proj.weight shape: [1024, 3072] -151: model.layers.23.self_attn.o_proj.weight shape: [3072, 3072] -152: model.layers.23.self_attn.q_proj.weight shape: [3072, 3072] -153: model.layers.23.self_attn.v_proj.weight shape: [1024, 3072] -154: model.layers.24.input_layernorm.weight shape: [3072] -155: model.layers.24.mlp.down_proj.weight shape: [3072, 8192] -156: model.layers.24.mlp.gate_proj.weight shape: [8192, 3072] -157: model.layers.24.mlp.up_proj.weight shape: [8192, 3072] -158: model.layers.24.post_attention_layernorm.weight shape: [3072] -159: model.layers.24.self_attn.k_proj.weight shape: [1024, 3072] -160: model.layers.24.self_attn.o_proj.weight shape: [3072, 3072] -161: model.layers.24.self_attn.q_proj.weight shape: [3072, 3072] -162: model.layers.24.self_attn.v_proj.weight shape: [1024, 3072] -163: model.layers.25.input_layernorm.weight shape: [3072] -164: model.layers.25.mlp.down_proj.weight shape: [3072, 8192] -165: model.layers.25.mlp.gate_proj.weight shape: [8192, 3072] -166: model.layers.25.mlp.up_proj.weight shape: [8192, 3072] -167: model.layers.25.post_attention_layernorm.weight shape: [3072] -168: model.layers.25.self_attn.k_proj.weight shape: [1024, 3072] -169: model.layers.25.self_attn.o_proj.weight shape: [3072, 3072] -170: model.layers.25.self_attn.q_proj.weight shape: [3072, 3072] -171: model.layers.25.self_attn.v_proj.weight shape: [1024, 3072] -172: model.layers.26.input_layernorm.weight shape: [3072] -173: model.layers.26.mlp.down_proj.weight shape: [3072, 8192] -174: model.layers.26.mlp.gate_proj.weight shape: [8192, 3072] -175: model.layers.26.mlp.up_proj.weight shape: [8192, 3072] -176: model.layers.26.post_attention_layernorm.weight shape: [3072] -177: model.layers.26.self_attn.k_proj.weight shape: [1024, 3072] -178: model.layers.26.self_attn.o_proj.weight shape: [3072, 3072] -179: model.layers.26.self_attn.q_proj.weight shape: [3072, 3072] -180: model.layers.26.self_attn.v_proj.weight shape: [1024, 3072] -181: model.layers.27.input_layernorm.weight shape: [3072] -182: model.layers.27.mlp.down_proj.weight shape: [3072, 8192] -183: model.layers.27.mlp.gate_proj.weight shape: [8192, 3072] -184: model.layers.27.mlp.up_proj.weight shape: [8192, 3072] -185: model.layers.27.post_attention_layernorm.weight shape: [3072] -186: model.layers.27.self_attn.k_proj.weight shape: [1024, 3072] -187: model.layers.27.self_attn.o_proj.weight shape: [3072, 3072] -188: model.layers.27.self_attn.q_proj.weight shape: [3072, 3072] -189: model.layers.27.self_attn.v_proj.weight shape: [1024, 3072] -190: model.layers.3.input_layernorm.weight shape: [3072] -191: model.layers.3.mlp.down_proj.weight shape: [3072, 8192] -192: model.layers.3.mlp.gate_proj.weight shape: [8192, 3072] -193: model.layers.3.mlp.up_proj.weight shape: [8192, 3072] -194: model.layers.3.post_attention_layernorm.weight shape: [3072] -195: model.layers.3.self_attn.k_proj.weight shape: [1024, 3072] -196: model.layers.3.self_attn.o_proj.weight shape: [3072, 3072] -197: model.layers.3.self_attn.q_proj.weight shape: [3072, 3072] -198: model.layers.3.self_attn.v_proj.weight shape: [1024, 3072] -199: model.layers.4.input_layernorm.weight shape: [3072] -200: model.layers.4.mlp.down_proj.weight shape: [3072, 8192] -201: model.layers.4.mlp.gate_proj.weight shape: [8192, 3072] -202: model.layers.4.mlp.up_proj.weight shape: [8192, 3072] -203: model.layers.4.post_attention_layernorm.weight shape: [3072] -204: model.layers.4.self_attn.k_proj.weight shape: [1024, 3072] -205: model.layers.4.self_attn.o_proj.weight shape: [3072, 3072] -206: model.layers.4.self_attn.q_proj.weight shape: [3072, 3072] -207: model.layers.4.self_attn.v_proj.weight shape: [1024, 3072] -208: model.layers.5.input_layernorm.weight shape: [3072] -209: model.layers.5.mlp.down_proj.weight shape: [3072, 8192] -210: model.layers.5.mlp.gate_proj.weight shape: [8192, 3072] -211: model.layers.5.mlp.up_proj.weight shape: [8192, 3072] -212: model.layers.5.post_attention_layernorm.weight shape: [3072] -213: model.layers.5.self_attn.k_proj.weight shape: [1024, 3072] -214: model.layers.5.self_attn.o_proj.weight shape: [3072, 3072] -215: model.layers.5.self_attn.q_proj.weight shape: [3072, 3072] -216: model.layers.5.self_attn.v_proj.weight shape: [1024, 3072] -217: model.layers.6.input_layernorm.weight shape: [3072] -218: model.layers.6.mlp.down_proj.weight shape: [3072, 8192] -219: model.layers.6.mlp.gate_proj.weight shape: [8192, 3072] -220: model.layers.6.mlp.up_proj.weight shape: [8192, 3072] -221: model.layers.6.post_attention_layernorm.weight shape: [3072] -222: model.layers.6.self_attn.k_proj.weight shape: [1024, 3072] -223: model.layers.6.self_attn.o_proj.weight shape: [3072, 3072] -224: model.layers.6.self_attn.q_proj.weight shape: [3072, 3072] -225: model.layers.6.self_attn.v_proj.weight shape: [1024, 3072] -226: model.layers.7.input_layernorm.weight shape: [3072] -227: model.layers.7.mlp.down_proj.weight shape: [3072, 8192] -228: model.layers.7.mlp.gate_proj.weight shape: [8192, 3072] -229: model.layers.7.mlp.up_proj.weight shape: [8192, 3072] -230: model.layers.7.post_attention_layernorm.weight shape: [3072] -231: model.layers.7.self_attn.k_proj.weight shape: [1024, 3072] -232: model.layers.7.self_attn.o_proj.weight shape: [3072, 3072] -233: model.layers.7.self_attn.q_proj.weight shape: [3072, 3072] -234: model.layers.7.self_attn.v_proj.weight shape: [1024, 3072] -235: model.layers.8.input_layernorm.weight shape: [3072] -236: model.layers.8.mlp.down_proj.weight shape: [3072, 8192] -237: model.layers.8.mlp.gate_proj.weight shape: [8192, 3072] -238: model.layers.8.mlp.up_proj.weight shape: [8192, 3072] -239: model.layers.8.post_attention_layernorm.weight shape: [3072] -240: model.layers.8.self_attn.k_proj.weight shape: [1024, 3072] -241: model.layers.8.self_attn.o_proj.weight shape: [3072, 3072] -242: model.layers.8.self_attn.q_proj.weight shape: [3072, 3072] -243: model.layers.8.self_attn.v_proj.weight shape: [1024, 3072] -244: model.layers.9.input_layernorm.weight shape: [3072] -245: model.layers.9.mlp.down_proj.weight shape: [3072, 8192] -246: model.layers.9.mlp.gate_proj.weight shape: [8192, 3072] -247: model.layers.9.mlp.up_proj.weight shape: [8192, 3072] -248: model.layers.9.post_attention_layernorm.weight shape: [3072] -249: model.layers.9.self_attn.k_proj.weight shape: [1024, 3072] -250: model.layers.9.self_attn.o_proj.weight shape: [3072, 3072] -251: model.layers.9.self_attn.q_proj.weight shape: [3072, 3072] -252: model.layers.9.self_attn.v_proj.weight shape: [1024, 3072] -253: model.norm.weight shape: [3072] diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.TokenizerTest.received.txt b/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.TokenizerTest.received.txt deleted file mode 100644 index 9bb3220214..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.TokenizerTest.received.txt +++ /dev/null @@ -1,6 +0,0 @@ -Can you provide ways to eat combinations of bananas and dragonfruits? -6854, 499, 3493, 5627, 311, 8343, 28559, 315, 68442, 323, 26161, 1658, 12059, 30 -Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey. -40914, 0, 5810, 527, 1063, 5627, 311, 8343, 68442, 323, 26161, 1658, 12059, 3871, 25, 220, 16, 13, 76924, 323, 26161, 36698, 11113, 648, 25, 55248, 68442, 323, 26161, 1658, 12059, 3871, 449, 1063, 14403, 323, 26828, 13, 220, 17, 13, 76924, 323, 26161, 36698, 33566, 25, 19771, 48715, 68442, 323, 26161, 1658, 12059, 3871, 449, 1063, 30564, 23661, 323, 26828, 13 -What about solving an 2x + 3 = 7 equation? -3923, 922, 22581, 459, 220, 17, 87, 489, 220, 18, 284, 220, 22, 24524, 30 diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.cs b/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.cs deleted file mode 100644 index 453bbcc283..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_1Tests.cs +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using ApprovalTests; -using ApprovalTests.Namers; -using ApprovalTests.Reporters; -using AutoGen.Core; -using Microsoft.Extensions.AI; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using TorchSharp; -using Xunit; - -namespace Microsoft.ML.GenAI.LLaMA.Tests; - -[Collection("NoParallelization")] -public class LLaMA3_1Tests -{ - public LLaMA3_1Tests() - { - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } - - torch.set_default_device("meta"); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Llama_3_1_8b_ShapeTest() - { - var model = new LlamaForCausalLM(LlamaConfig.Llama3_1_8B_Instruct, "meta"); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [WindowsOnlyFact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Llama_3_1_70b_ShapeTest() - { - var model = new LlamaForCausalLM(LlamaConfig.Llama3_1_70B_Instruct, "meta"); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [WindowsOnlyFact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Llama_3_1_405b_ShapeTest() - { - var model = new LlamaForCausalLM(LlamaConfig.Llama3_1_405B_Instruct, "meta"); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void TokenizerTest() - { - var modelWeightFolder = Path.Join("Llama-3.1"); - var tokenizer = LlamaTokenizerHelper.FromPretrained(modelWeightFolder); - - var messages = new string[] - { - "Can you provide ways to eat combinations of bananas and dragonfruits?", - "Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey.", - "What about solving an 2x + 3 = 7 equation?", - """ - <|begin_of_text|>Hello World<|end_of_text|> - """ - }; - - var sb = new StringBuilder(); - foreach (var message in messages) - { - var tokenizeIds = tokenizer.EncodeToIds(message, true, false); - var decodeToString = tokenizer.Decode(tokenizeIds); - sb.AppendLine(decodeToString); - var tokenizedStr = string.Join(", ", tokenizeIds.Select(x => x.ToString())); - - sb.AppendLine(tokenizedStr); - } - Approvals.Verify(sb.ToString()); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void ItBuildChatTemplateFromAutoGenChatHistory() - { - var chatHistory = new List - { - new TextMessage(Role.System, "You are a helpful AI assistant."), - new TextMessage(Role.User, "Hello?"), - new TextMessage(Role.Assistant, "World!"), - }; - - var prompt = Llama3_1ChatTemplateBuilder.Instance.BuildPrompt(chatHistory); - - Approvals.Verify(prompt); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void ItBuildChatTemplateFromSemanticKernelChatHistory() - { - var chatHistory = new ChatHistory - { - new ChatMessageContent(AuthorRole.System, "You are a helpful AI assistant."), - new ChatMessageContent(AuthorRole.User, "Hello?"), - new ChatMessageContent(AuthorRole.Assistant, "World!"), - }; - - var prompt = Llama3_1ChatTemplateBuilder.Instance.BuildPrompt(chatHistory); - - Approvals.Verify(prompt); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void ItBuildChatTemplateFromMEAIChatHistory() - { - var chatHistory = new[] - { - new ChatMessage(ChatRole.System, "You are a helpful AI assistant."), - new ChatMessage(ChatRole.User, "Hello?"), - new ChatMessage(ChatRole.Assistant, "World!"), - }; - - var prompt = Llama3_1ChatTemplateBuilder.Instance.BuildPrompt(chatHistory); - - Approvals.Verify(prompt); - } -} diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_2Tests.cs b/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_2Tests.cs deleted file mode 100644 index 6babd14910..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/LLaMA3_2Tests.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using ApprovalTests; -using ApprovalTests.Namers; -using ApprovalTests.Reporters; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp; -using Xunit; - -namespace Microsoft.ML.GenAI.LLaMA.Tests; - -[Collection("NoParallelization")] -public class LLaMA3_2Tests -{ - public LLaMA3_2Tests() - { - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } - - torch.set_default_device("meta"); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Llama_3_2_1b_ShapeTest() - { - var model = new LlamaForCausalLM(LlamaConfig.Llama3_2_1B_Instruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [WindowsOnlyFact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Llama_3_2_3b_ShapeTest() - { - var model = new LlamaForCausalLM(LlamaConfig.Llama_3_2_3B_Instruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } -} diff --git a/test/Microsoft.ML.GenAI.LLaMA.Tests/Microsoft.ML.GenAI.LLaMA.Tests.csproj b/test/Microsoft.ML.GenAI.LLaMA.Tests/Microsoft.ML.GenAI.LLaMA.Tests.csproj deleted file mode 100644 index 75b3c41c0b..0000000000 --- a/test/Microsoft.ML.GenAI.LLaMA.Tests/Microsoft.ML.GenAI.LLaMA.Tests.csproj +++ /dev/null @@ -1,47 +0,0 @@ - - - - net8.0 - enable - $(NoWarn);MSML_ExtendBaseTestClass - enable - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - - - - - - - - - diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt b/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt deleted file mode 100644 index 493b07d9ec..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateFromAutoGenChatHistory.approved.txt +++ /dev/null @@ -1,3 +0,0 @@ -[INST]You are a helpful AI assistant. - -Hello?[/INST]World! \ No newline at end of file diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateWithToolsFromAutoGenChatHistory.approved.txt b/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateWithToolsFromAutoGenChatHistory.approved.txt deleted file mode 100644 index 4731561ae7..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.ItBuildChatTemplateWithToolsFromAutoGenChatHistory.approved.txt +++ /dev/null @@ -1,3 +0,0 @@ -[INST]What's the weather in Seattle?[/INST][TOOL_CALLS][{"name":"get_current_weather","arguments":{"location":"Seattle, WA"},"id":"9Ae3bDc2F"}][TOOL_RESULTS][{"id":"9Ae3bDc2F","content":"sunny"}][/TOOL_RESULTS]The current temperature in Seattle is 22.0 degrees celsius.[INST]What's the weather in New York?[/INST][TOOL_CALLS][{"name":"get_current_weather","arguments":{"location":"Seattle, WA"},"id":"9Ae3bDc2F"}][TOOL_RESULTS][{"id":"9Ae3bDc2F","content":"sunny"}][/TOOL_RESULTS]The current temperature in New York is 22.0 degrees celsius.[AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}}, "required": ["location"]}}}][/AVAILABLE_TOOLS][INST]You are a helpful AI assistant. - -What's the weather in Paris?[/INST] \ No newline at end of file diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.Mistral_7B_Instruct_V0_3_ShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.Mistral_7B_Instruct_V0_3_ShapeTest.approved.txt deleted file mode 100644 index 4bad35f7d7..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.Mistral_7B_Instruct_V0_3_ShapeTest.approved.txt +++ /dev/null @@ -1,291 +0,0 @@ -0: lm_head.weight shape: [32768, 4096] -1: model.embed_tokens.weight shape: [32768, 4096] -2: model.layers.0.input_layernorm.weight shape: [4096] -3: model.layers.0.mlp.down_proj.weight shape: [4096, 14336] -4: model.layers.0.mlp.gate_proj.weight shape: [14336, 4096] -5: model.layers.0.mlp.up_proj.weight shape: [14336, 4096] -6: model.layers.0.post_attention_layernorm.weight shape: [4096] -7: model.layers.0.self_attn.k_proj.weight shape: [1024, 4096] -8: model.layers.0.self_attn.o_proj.weight shape: [4096, 4096] -9: model.layers.0.self_attn.q_proj.weight shape: [4096, 4096] -10: model.layers.0.self_attn.v_proj.weight shape: [1024, 4096] -11: model.layers.1.input_layernorm.weight shape: [4096] -12: model.layers.1.mlp.down_proj.weight shape: [4096, 14336] -13: model.layers.1.mlp.gate_proj.weight shape: [14336, 4096] -14: model.layers.1.mlp.up_proj.weight shape: [14336, 4096] -15: model.layers.1.post_attention_layernorm.weight shape: [4096] -16: model.layers.1.self_attn.k_proj.weight shape: [1024, 4096] -17: model.layers.1.self_attn.o_proj.weight shape: [4096, 4096] -18: model.layers.1.self_attn.q_proj.weight shape: [4096, 4096] -19: model.layers.1.self_attn.v_proj.weight shape: [1024, 4096] -20: model.layers.10.input_layernorm.weight shape: [4096] -21: model.layers.10.mlp.down_proj.weight shape: [4096, 14336] -22: model.layers.10.mlp.gate_proj.weight shape: [14336, 4096] -23: model.layers.10.mlp.up_proj.weight shape: [14336, 4096] -24: model.layers.10.post_attention_layernorm.weight shape: [4096] -25: model.layers.10.self_attn.k_proj.weight shape: [1024, 4096] -26: model.layers.10.self_attn.o_proj.weight shape: [4096, 4096] -27: model.layers.10.self_attn.q_proj.weight shape: [4096, 4096] -28: model.layers.10.self_attn.v_proj.weight shape: [1024, 4096] -29: model.layers.11.input_layernorm.weight shape: [4096] -30: model.layers.11.mlp.down_proj.weight shape: [4096, 14336] -31: model.layers.11.mlp.gate_proj.weight shape: [14336, 4096] -32: model.layers.11.mlp.up_proj.weight shape: [14336, 4096] -33: model.layers.11.post_attention_layernorm.weight shape: [4096] -34: model.layers.11.self_attn.k_proj.weight shape: [1024, 4096] -35: model.layers.11.self_attn.o_proj.weight shape: [4096, 4096] -36: model.layers.11.self_attn.q_proj.weight shape: [4096, 4096] -37: model.layers.11.self_attn.v_proj.weight shape: [1024, 4096] -38: model.layers.12.input_layernorm.weight shape: [4096] -39: model.layers.12.mlp.down_proj.weight shape: [4096, 14336] -40: model.layers.12.mlp.gate_proj.weight shape: [14336, 4096] -41: model.layers.12.mlp.up_proj.weight shape: [14336, 4096] -42: model.layers.12.post_attention_layernorm.weight shape: [4096] -43: model.layers.12.self_attn.k_proj.weight shape: [1024, 4096] -44: model.layers.12.self_attn.o_proj.weight shape: [4096, 4096] -45: model.layers.12.self_attn.q_proj.weight shape: [4096, 4096] -46: model.layers.12.self_attn.v_proj.weight shape: [1024, 4096] -47: model.layers.13.input_layernorm.weight shape: [4096] -48: model.layers.13.mlp.down_proj.weight shape: [4096, 14336] -49: model.layers.13.mlp.gate_proj.weight shape: [14336, 4096] -50: model.layers.13.mlp.up_proj.weight shape: [14336, 4096] -51: model.layers.13.post_attention_layernorm.weight shape: [4096] -52: model.layers.13.self_attn.k_proj.weight shape: [1024, 4096] -53: model.layers.13.self_attn.o_proj.weight shape: [4096, 4096] -54: model.layers.13.self_attn.q_proj.weight shape: [4096, 4096] -55: model.layers.13.self_attn.v_proj.weight shape: [1024, 4096] -56: model.layers.14.input_layernorm.weight shape: [4096] -57: model.layers.14.mlp.down_proj.weight shape: [4096, 14336] -58: model.layers.14.mlp.gate_proj.weight shape: [14336, 4096] -59: model.layers.14.mlp.up_proj.weight shape: [14336, 4096] -60: model.layers.14.post_attention_layernorm.weight shape: [4096] -61: model.layers.14.self_attn.k_proj.weight shape: [1024, 4096] -62: model.layers.14.self_attn.o_proj.weight shape: [4096, 4096] -63: model.layers.14.self_attn.q_proj.weight shape: [4096, 4096] -64: model.layers.14.self_attn.v_proj.weight shape: [1024, 4096] -65: model.layers.15.input_layernorm.weight shape: [4096] -66: model.layers.15.mlp.down_proj.weight shape: [4096, 14336] -67: model.layers.15.mlp.gate_proj.weight shape: [14336, 4096] -68: model.layers.15.mlp.up_proj.weight shape: [14336, 4096] -69: model.layers.15.post_attention_layernorm.weight shape: [4096] -70: model.layers.15.self_attn.k_proj.weight shape: [1024, 4096] -71: model.layers.15.self_attn.o_proj.weight shape: [4096, 4096] -72: model.layers.15.self_attn.q_proj.weight shape: [4096, 4096] -73: model.layers.15.self_attn.v_proj.weight shape: [1024, 4096] -74: model.layers.16.input_layernorm.weight shape: [4096] -75: model.layers.16.mlp.down_proj.weight shape: [4096, 14336] -76: model.layers.16.mlp.gate_proj.weight shape: [14336, 4096] -77: model.layers.16.mlp.up_proj.weight shape: [14336, 4096] -78: model.layers.16.post_attention_layernorm.weight shape: [4096] -79: model.layers.16.self_attn.k_proj.weight shape: [1024, 4096] -80: model.layers.16.self_attn.o_proj.weight shape: [4096, 4096] -81: model.layers.16.self_attn.q_proj.weight shape: [4096, 4096] -82: model.layers.16.self_attn.v_proj.weight shape: [1024, 4096] -83: model.layers.17.input_layernorm.weight shape: [4096] -84: model.layers.17.mlp.down_proj.weight shape: [4096, 14336] -85: model.layers.17.mlp.gate_proj.weight shape: [14336, 4096] -86: model.layers.17.mlp.up_proj.weight shape: [14336, 4096] -87: model.layers.17.post_attention_layernorm.weight shape: [4096] -88: model.layers.17.self_attn.k_proj.weight shape: [1024, 4096] -89: model.layers.17.self_attn.o_proj.weight shape: [4096, 4096] -90: model.layers.17.self_attn.q_proj.weight shape: [4096, 4096] -91: model.layers.17.self_attn.v_proj.weight shape: [1024, 4096] -92: model.layers.18.input_layernorm.weight shape: [4096] -93: model.layers.18.mlp.down_proj.weight shape: [4096, 14336] -94: model.layers.18.mlp.gate_proj.weight shape: [14336, 4096] -95: model.layers.18.mlp.up_proj.weight shape: [14336, 4096] -96: model.layers.18.post_attention_layernorm.weight shape: [4096] -97: model.layers.18.self_attn.k_proj.weight shape: [1024, 4096] -98: model.layers.18.self_attn.o_proj.weight shape: [4096, 4096] -99: model.layers.18.self_attn.q_proj.weight shape: [4096, 4096] -100: model.layers.18.self_attn.v_proj.weight shape: [1024, 4096] -101: model.layers.19.input_layernorm.weight shape: [4096] -102: model.layers.19.mlp.down_proj.weight shape: [4096, 14336] -103: model.layers.19.mlp.gate_proj.weight shape: [14336, 4096] -104: model.layers.19.mlp.up_proj.weight shape: [14336, 4096] -105: model.layers.19.post_attention_layernorm.weight shape: [4096] -106: model.layers.19.self_attn.k_proj.weight shape: [1024, 4096] -107: model.layers.19.self_attn.o_proj.weight shape: [4096, 4096] -108: model.layers.19.self_attn.q_proj.weight shape: [4096, 4096] -109: model.layers.19.self_attn.v_proj.weight shape: [1024, 4096] -110: model.layers.2.input_layernorm.weight shape: [4096] -111: model.layers.2.mlp.down_proj.weight shape: [4096, 14336] -112: model.layers.2.mlp.gate_proj.weight shape: [14336, 4096] -113: model.layers.2.mlp.up_proj.weight shape: [14336, 4096] -114: model.layers.2.post_attention_layernorm.weight shape: [4096] -115: model.layers.2.self_attn.k_proj.weight shape: [1024, 4096] -116: model.layers.2.self_attn.o_proj.weight shape: [4096, 4096] -117: model.layers.2.self_attn.q_proj.weight shape: [4096, 4096] -118: model.layers.2.self_attn.v_proj.weight shape: [1024, 4096] -119: model.layers.20.input_layernorm.weight shape: [4096] -120: model.layers.20.mlp.down_proj.weight shape: [4096, 14336] -121: model.layers.20.mlp.gate_proj.weight shape: [14336, 4096] -122: model.layers.20.mlp.up_proj.weight shape: [14336, 4096] -123: model.layers.20.post_attention_layernorm.weight shape: [4096] -124: model.layers.20.self_attn.k_proj.weight shape: [1024, 4096] -125: model.layers.20.self_attn.o_proj.weight shape: [4096, 4096] -126: model.layers.20.self_attn.q_proj.weight shape: [4096, 4096] -127: model.layers.20.self_attn.v_proj.weight shape: [1024, 4096] -128: model.layers.21.input_layernorm.weight shape: [4096] -129: model.layers.21.mlp.down_proj.weight shape: [4096, 14336] -130: model.layers.21.mlp.gate_proj.weight shape: [14336, 4096] -131: model.layers.21.mlp.up_proj.weight shape: [14336, 4096] -132: model.layers.21.post_attention_layernorm.weight shape: [4096] -133: model.layers.21.self_attn.k_proj.weight shape: [1024, 4096] -134: model.layers.21.self_attn.o_proj.weight shape: [4096, 4096] -135: model.layers.21.self_attn.q_proj.weight shape: [4096, 4096] -136: model.layers.21.self_attn.v_proj.weight shape: [1024, 4096] -137: model.layers.22.input_layernorm.weight shape: [4096] -138: model.layers.22.mlp.down_proj.weight shape: [4096, 14336] -139: model.layers.22.mlp.gate_proj.weight shape: [14336, 4096] -140: model.layers.22.mlp.up_proj.weight shape: [14336, 4096] -141: model.layers.22.post_attention_layernorm.weight shape: [4096] -142: model.layers.22.self_attn.k_proj.weight shape: [1024, 4096] -143: model.layers.22.self_attn.o_proj.weight shape: [4096, 4096] -144: model.layers.22.self_attn.q_proj.weight shape: [4096, 4096] -145: model.layers.22.self_attn.v_proj.weight shape: [1024, 4096] -146: model.layers.23.input_layernorm.weight shape: [4096] -147: model.layers.23.mlp.down_proj.weight shape: [4096, 14336] -148: model.layers.23.mlp.gate_proj.weight shape: [14336, 4096] -149: model.layers.23.mlp.up_proj.weight shape: [14336, 4096] -150: model.layers.23.post_attention_layernorm.weight shape: [4096] -151: model.layers.23.self_attn.k_proj.weight shape: [1024, 4096] -152: model.layers.23.self_attn.o_proj.weight shape: [4096, 4096] -153: model.layers.23.self_attn.q_proj.weight shape: [4096, 4096] -154: model.layers.23.self_attn.v_proj.weight shape: [1024, 4096] -155: model.layers.24.input_layernorm.weight shape: [4096] -156: model.layers.24.mlp.down_proj.weight shape: [4096, 14336] -157: model.layers.24.mlp.gate_proj.weight shape: [14336, 4096] -158: model.layers.24.mlp.up_proj.weight shape: [14336, 4096] -159: model.layers.24.post_attention_layernorm.weight shape: [4096] -160: model.layers.24.self_attn.k_proj.weight shape: [1024, 4096] -161: model.layers.24.self_attn.o_proj.weight shape: [4096, 4096] -162: model.layers.24.self_attn.q_proj.weight shape: [4096, 4096] -163: model.layers.24.self_attn.v_proj.weight shape: [1024, 4096] -164: model.layers.25.input_layernorm.weight shape: [4096] -165: model.layers.25.mlp.down_proj.weight shape: [4096, 14336] -166: model.layers.25.mlp.gate_proj.weight shape: [14336, 4096] -167: model.layers.25.mlp.up_proj.weight shape: [14336, 4096] -168: model.layers.25.post_attention_layernorm.weight shape: [4096] -169: model.layers.25.self_attn.k_proj.weight shape: [1024, 4096] -170: model.layers.25.self_attn.o_proj.weight shape: [4096, 4096] -171: model.layers.25.self_attn.q_proj.weight shape: [4096, 4096] -172: model.layers.25.self_attn.v_proj.weight shape: [1024, 4096] -173: model.layers.26.input_layernorm.weight shape: [4096] -174: model.layers.26.mlp.down_proj.weight shape: [4096, 14336] -175: model.layers.26.mlp.gate_proj.weight shape: [14336, 4096] -176: model.layers.26.mlp.up_proj.weight shape: [14336, 4096] -177: model.layers.26.post_attention_layernorm.weight shape: [4096] -178: model.layers.26.self_attn.k_proj.weight shape: [1024, 4096] -179: model.layers.26.self_attn.o_proj.weight shape: [4096, 4096] -180: model.layers.26.self_attn.q_proj.weight shape: [4096, 4096] -181: model.layers.26.self_attn.v_proj.weight shape: [1024, 4096] -182: model.layers.27.input_layernorm.weight shape: [4096] -183: model.layers.27.mlp.down_proj.weight shape: [4096, 14336] -184: model.layers.27.mlp.gate_proj.weight shape: [14336, 4096] -185: model.layers.27.mlp.up_proj.weight shape: [14336, 4096] -186: model.layers.27.post_attention_layernorm.weight shape: [4096] -187: model.layers.27.self_attn.k_proj.weight shape: [1024, 4096] -188: model.layers.27.self_attn.o_proj.weight shape: [4096, 4096] -189: model.layers.27.self_attn.q_proj.weight shape: [4096, 4096] -190: model.layers.27.self_attn.v_proj.weight shape: [1024, 4096] -191: model.layers.28.input_layernorm.weight shape: [4096] -192: model.layers.28.mlp.down_proj.weight shape: [4096, 14336] -193: model.layers.28.mlp.gate_proj.weight shape: [14336, 4096] -194: model.layers.28.mlp.up_proj.weight shape: [14336, 4096] -195: model.layers.28.post_attention_layernorm.weight shape: [4096] -196: model.layers.28.self_attn.k_proj.weight shape: [1024, 4096] -197: model.layers.28.self_attn.o_proj.weight shape: [4096, 4096] -198: model.layers.28.self_attn.q_proj.weight shape: [4096, 4096] -199: model.layers.28.self_attn.v_proj.weight shape: [1024, 4096] -200: model.layers.29.input_layernorm.weight shape: [4096] -201: model.layers.29.mlp.down_proj.weight shape: [4096, 14336] -202: model.layers.29.mlp.gate_proj.weight shape: [14336, 4096] -203: model.layers.29.mlp.up_proj.weight shape: [14336, 4096] -204: model.layers.29.post_attention_layernorm.weight shape: [4096] -205: model.layers.29.self_attn.k_proj.weight shape: [1024, 4096] -206: model.layers.29.self_attn.o_proj.weight shape: [4096, 4096] -207: model.layers.29.self_attn.q_proj.weight shape: [4096, 4096] -208: model.layers.29.self_attn.v_proj.weight shape: [1024, 4096] -209: model.layers.3.input_layernorm.weight shape: [4096] -210: model.layers.3.mlp.down_proj.weight shape: [4096, 14336] -211: model.layers.3.mlp.gate_proj.weight shape: [14336, 4096] -212: model.layers.3.mlp.up_proj.weight shape: [14336, 4096] -213: model.layers.3.post_attention_layernorm.weight shape: [4096] -214: model.layers.3.self_attn.k_proj.weight shape: [1024, 4096] -215: model.layers.3.self_attn.o_proj.weight shape: [4096, 4096] -216: model.layers.3.self_attn.q_proj.weight shape: [4096, 4096] -217: model.layers.3.self_attn.v_proj.weight shape: [1024, 4096] -218: model.layers.30.input_layernorm.weight shape: [4096] -219: model.layers.30.mlp.down_proj.weight shape: [4096, 14336] -220: model.layers.30.mlp.gate_proj.weight shape: [14336, 4096] -221: model.layers.30.mlp.up_proj.weight shape: [14336, 4096] -222: model.layers.30.post_attention_layernorm.weight shape: [4096] -223: model.layers.30.self_attn.k_proj.weight shape: [1024, 4096] -224: model.layers.30.self_attn.o_proj.weight shape: [4096, 4096] -225: model.layers.30.self_attn.q_proj.weight shape: [4096, 4096] -226: model.layers.30.self_attn.v_proj.weight shape: [1024, 4096] -227: model.layers.31.input_layernorm.weight shape: [4096] -228: model.layers.31.mlp.down_proj.weight shape: [4096, 14336] -229: model.layers.31.mlp.gate_proj.weight shape: [14336, 4096] -230: model.layers.31.mlp.up_proj.weight shape: [14336, 4096] -231: model.layers.31.post_attention_layernorm.weight shape: [4096] -232: model.layers.31.self_attn.k_proj.weight shape: [1024, 4096] -233: model.layers.31.self_attn.o_proj.weight shape: [4096, 4096] -234: model.layers.31.self_attn.q_proj.weight shape: [4096, 4096] -235: model.layers.31.self_attn.v_proj.weight shape: [1024, 4096] -236: model.layers.4.input_layernorm.weight shape: [4096] -237: model.layers.4.mlp.down_proj.weight shape: [4096, 14336] -238: model.layers.4.mlp.gate_proj.weight shape: [14336, 4096] -239: model.layers.4.mlp.up_proj.weight shape: [14336, 4096] -240: model.layers.4.post_attention_layernorm.weight shape: [4096] -241: model.layers.4.self_attn.k_proj.weight shape: [1024, 4096] -242: model.layers.4.self_attn.o_proj.weight shape: [4096, 4096] -243: model.layers.4.self_attn.q_proj.weight shape: [4096, 4096] -244: model.layers.4.self_attn.v_proj.weight shape: [1024, 4096] -245: model.layers.5.input_layernorm.weight shape: [4096] -246: model.layers.5.mlp.down_proj.weight shape: [4096, 14336] -247: model.layers.5.mlp.gate_proj.weight shape: [14336, 4096] -248: model.layers.5.mlp.up_proj.weight shape: [14336, 4096] -249: model.layers.5.post_attention_layernorm.weight shape: [4096] -250: model.layers.5.self_attn.k_proj.weight shape: [1024, 4096] -251: model.layers.5.self_attn.o_proj.weight shape: [4096, 4096] -252: model.layers.5.self_attn.q_proj.weight shape: [4096, 4096] -253: model.layers.5.self_attn.v_proj.weight shape: [1024, 4096] -254: model.layers.6.input_layernorm.weight shape: [4096] -255: model.layers.6.mlp.down_proj.weight shape: [4096, 14336] -256: model.layers.6.mlp.gate_proj.weight shape: [14336, 4096] -257: model.layers.6.mlp.up_proj.weight shape: [14336, 4096] -258: model.layers.6.post_attention_layernorm.weight shape: [4096] -259: model.layers.6.self_attn.k_proj.weight shape: [1024, 4096] -260: model.layers.6.self_attn.o_proj.weight shape: [4096, 4096] -261: model.layers.6.self_attn.q_proj.weight shape: [4096, 4096] -262: model.layers.6.self_attn.v_proj.weight shape: [1024, 4096] -263: model.layers.7.input_layernorm.weight shape: [4096] -264: model.layers.7.mlp.down_proj.weight shape: [4096, 14336] -265: model.layers.7.mlp.gate_proj.weight shape: [14336, 4096] -266: model.layers.7.mlp.up_proj.weight shape: [14336, 4096] -267: model.layers.7.post_attention_layernorm.weight shape: [4096] -268: model.layers.7.self_attn.k_proj.weight shape: [1024, 4096] -269: model.layers.7.self_attn.o_proj.weight shape: [4096, 4096] -270: model.layers.7.self_attn.q_proj.weight shape: [4096, 4096] -271: model.layers.7.self_attn.v_proj.weight shape: [1024, 4096] -272: model.layers.8.input_layernorm.weight shape: [4096] -273: model.layers.8.mlp.down_proj.weight shape: [4096, 14336] -274: model.layers.8.mlp.gate_proj.weight shape: [14336, 4096] -275: model.layers.8.mlp.up_proj.weight shape: [14336, 4096] -276: model.layers.8.post_attention_layernorm.weight shape: [4096] -277: model.layers.8.self_attn.k_proj.weight shape: [1024, 4096] -278: model.layers.8.self_attn.o_proj.weight shape: [4096, 4096] -279: model.layers.8.self_attn.q_proj.weight shape: [4096, 4096] -280: model.layers.8.self_attn.v_proj.weight shape: [1024, 4096] -281: model.layers.9.input_layernorm.weight shape: [4096] -282: model.layers.9.mlp.down_proj.weight shape: [4096, 14336] -283: model.layers.9.mlp.gate_proj.weight shape: [14336, 4096] -284: model.layers.9.mlp.up_proj.weight shape: [14336, 4096] -285: model.layers.9.post_attention_layernorm.weight shape: [4096] -286: model.layers.9.self_attn.k_proj.weight shape: [1024, 4096] -287: model.layers.9.self_attn.o_proj.weight shape: [4096, 4096] -288: model.layers.9.self_attn.q_proj.weight shape: [4096, 4096] -289: model.layers.9.self_attn.v_proj.weight shape: [1024, 4096] -290: model.norm.weight shape: [4096] diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.TokenizerTest.approved.txt deleted file mode 100644 index 0287bd2f22..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_7B_Instruct_V0_3Tests.TokenizerTest.approved.txt +++ /dev/null @@ -1,2 +0,0 @@ -[AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "format": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit to use. Infer this from the users location."}}, "required": ["location", "format"]}}}][/AVAILABLE_TOOLS][INST] What's the weather like in Paris?[/INST][TOOL_CALLS] [{"name": "get_current_temperature", "arguments": {"location": "Paris, France", "unit": "celsius"}, "id": "9Ae3bDc2F"}][TOOL_RESULTS] {"content": 22.0, "call_id": "9Ae3bDc2F"}[/TOOL_RESULTS] The current temperature in Paris is 22.0 degrees celsius. -1, 6, 1501, 7567, 1891, 2032, 1113, 3396, 1316, 1113, 3396, 2032, 10598, 1629, 2032, 1113, 1295, 29498, 3790, 29498, 1537, 1991, 1316, 1113, 7286, 2032, 1113, 2226, 1040, 2636, 8854, 1316, 1113, 12206, 2032, 10598, 1891, 2032, 1113, 3582, 1316, 1113, 11491, 2032, 10598, 3501, 2032, 10598, 1891, 2032, 1113, 2195, 1316, 1113, 7286, 2032, 1113, 1782, 3758, 1072, 2433, 29493, 1085, 29491, 29489, 29491, 4420, 10454, 29493, 10229, 8474, 1113, 4530, 2032, 10598, 1891, 2032, 1113, 2195, 1316, 1113, 10825, 2032, 8135, 29485, 1958, 3938, 1316, 1113, 29490, 19425, 13075, 9651, 1113, 7286, 2032, 1113, 1782, 8409, 5796, 1066, 1706, 29491, 1328, 1410, 1224, 1245, 1040, 6211, 5491, 1379, 11549, 1113, 11661, 2032, 8135, 3501, 1316, 1113, 4530, 3010, 14879, 29561, 7, 3, 2592, 29510, 29481, 1040, 8854, 1505, 1065, 6233, 29572, 4, 5, 1501, 7567, 1629, 2032, 1113, 1295, 29498, 3790, 29498, 29475, 17329, 1316, 1113, 17452, 2032, 10598, 3501, 2032, 1113, 4684, 1046, 29493, 5611, 1316, 1113, 6074, 2032, 1113, 29485, 1958, 3938, 8474, 1113, 1081, 2032, 1113, 29542, 29509, 29474, 29538, 29494, 29525, 29485, 29518, 29533, 29507, 10925, 2, 8, 10598, 4557, 2032, 29473, 29518, 29518, 29491, 29502, 29493, 1113, 3613, 29498, 1081, 2032, 1113, 29542, 29509, 29474, 29538, 29494, 29525, 29485, 29518, 29533, 18163, 9, 1183, 2636, 8409, 1065, 6233, 1117, 29473, 29518, 29518, 29491, 29502, 11950, 1045, 1958, 3938, 29491, 2 diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_V0_3Tests.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_V0_3Tests.TokenizerTest.approved.txt deleted file mode 100644 index fc8562c9e9..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Approvals/Mistral_V0_3Tests.TokenizerTest.approved.txt +++ /dev/null @@ -1,2 +0,0 @@ - [{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "format": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit to use. Infer this from the users location."}}, "required": ["location", "format"]}}}] What's the weather like in Paris? [{"name": "get_current_temperature", "arguments": {"location": "Paris, France", "unit": "celsius"}, "id": "9Ae3bDc2F"}] {"content": 22.0, "call_id": "9Ae3bDc2F"} The current temperature in Paris is 22.0 degrees celsius. -1, 1, 6, 1501, 7567, 1891, 2032, 1113, 3396, 1316, 1113, 3396, 2032, 10598, 1629, 2032, 1113, 1295, 29498, 3790, 29498, 1537, 1991, 1316, 1113, 7286, 2032, 1113, 2226, 1040, 2636, 8854, 1316, 1113, 12206, 2032, 10598, 1891, 2032, 1113, 3582, 1316, 1113, 11491, 2032, 10598, 3501, 2032, 10598, 1891, 2032, 1113, 2195, 1316, 1113, 7286, 2032, 1113, 1782, 3758, 1072, 2433, 29493, 1085, 29491, 29489, 29491, 4420, 10454, 29493, 10229, 8474, 1113, 4530, 2032, 10598, 1891, 2032, 1113, 2195, 1316, 1113, 10825, 2032, 8135, 29485, 1958, 3938, 1316, 1113, 29490, 19425, 13075, 9651, 1113, 7286, 2032, 1113, 1782, 8409, 5796, 1066, 1706, 29491, 1328, 1410, 1224, 1245, 1040, 6211, 5491, 1379, 11549, 1113, 11661, 2032, 8135, 3501, 1316, 1113, 4530, 3010, 14879, 29561, 7, 3, 2592, 29510, 29481, 1040, 8854, 1505, 1065, 6233, 29572, 4, 5, 1501, 7567, 1629, 2032, 1113, 1295, 29498, 3790, 29498, 29475, 17329, 1316, 1113, 17452, 2032, 10598, 3501, 2032, 1113, 4684, 1046, 29493, 5611, 1316, 1113, 6074, 2032, 1113, 29485, 1958, 3938, 8474, 1113, 1081, 2032, 1113, 29542, 29509, 29474, 29538, 29494, 29525, 29485, 29518, 29533, 29507, 10925, 2, 8, 10598, 4557, 2032, 29473, 29518, 29518, 29491, 29502, 29493, 1113, 3613, 29498, 1081, 2032, 1113, 29542, 29509, 29474, 29538, 29494, 29525, 29485, 29518, 29533, 18163, 9, 1183, 2636, 8409, 1065, 6233, 1117, 29473, 29518, 29518, 29491, 29502, 11950, 1045, 1958, 3938, 29491, 2 diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Microsoft.ML.GenAI.Mistral.Tests.csproj b/test/Microsoft.ML.GenAI.Mistral.Tests/Microsoft.ML.GenAI.Mistral.Tests.csproj deleted file mode 100644 index 4c4b492447..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Microsoft.ML.GenAI.Mistral.Tests.csproj +++ /dev/null @@ -1,45 +0,0 @@ - - - - net8.0 - enable - $(NoWarn);MSML_ExtendBaseTestClass - enable - true - - - - - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - - - - - - - - - diff --git a/test/Microsoft.ML.GenAI.Mistral.Tests/Mistral_7B_Instruct_V0_3Tests.cs b/test/Microsoft.ML.GenAI.Mistral.Tests/Mistral_7B_Instruct_V0_3Tests.cs deleted file mode 100644 index 0aa80e8880..0000000000 --- a/test/Microsoft.ML.GenAI.Mistral.Tests/Mistral_7B_Instruct_V0_3Tests.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using ApprovalTests; -using ApprovalTests.Namers; -using ApprovalTests.Reporters; -using AutoGen.Core; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp; -using Xunit; - -namespace Microsoft.ML.GenAI.Mistral.Tests; - -[Collection("NoParallelization")] -public class Mistral_7B_Instruct_V0_3Tests -{ - public Mistral_7B_Instruct_V0_3Tests() - { - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } - - torch.set_default_device("meta"); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Mistral_7B_Instruct_V0_3_ShapeTest() - { - var model = new MistralForCausalLM(MistralConfig.Mistral_7B_Instruct_v0_3); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void ItBuildChatTemplateFromAutoGenChatHistory() - { - var chatHistory = new List - { - new TextMessage(Role.System, "You are a helpful AI assistant."), - new TextMessage(Role.User, "Hello?"), - new TextMessage(Role.Assistant, "World!"), - }; - - var prompt = Mistral_7B_0_3ChatTemplateBuilder.Instance.BuildPrompt(chatHistory); - - Approvals.Verify(prompt); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void ItBuildChatTemplateWithToolsFromAutoGenChatHistory() - { - var getWeatherTool = new FunctionContract - { - Name = "get_current_weather", - Namespace = "weather", - Description = "Get the current weather", - Parameters = [ - new FunctionParameterContract - { - Name = "location", - ParameterType = typeof(string), - Description = "The city and state, e.g. San Francisco, CA", - IsRequired = true - } - ] - }; - - var getWeatherToolCall = new ToolCall("get_current_weather", "{\"location\": \"Seattle, WA\"}") { ToolCallId = "9Ae3bDc2F" }; - var getWeatherToolCallResult = new ToolCall("get_current_weather", "{\"temperature\": 22.0}", "sunny") { ToolCallId = "9Ae3bDc2F" }; - var toolCallMessage = new ToolCallMessage([getWeatherToolCall]); - var toolCallResultMessage = new ToolCallResultMessage([getWeatherToolCallResult]); - var aggregateToolCallMessage = new ToolCallAggregateMessage(toolCallMessage, toolCallResultMessage); - - var chatHistory = new List - { - new TextMessage(Role.System, "You are a helpful AI assistant."), - new TextMessage(Role.User, "What's the weather in Seattle?"), - toolCallMessage, - toolCallResultMessage, - new TextMessage(Role.Assistant, "The current temperature in Seattle is 22.0 degrees celsius."), - - // test tool call aggregate message for immediate tool call execution - new TextMessage(Role.User, "What's the weather in New York?"), - aggregateToolCallMessage, - new TextMessage(Role.Assistant, "The current temperature in New York is 22.0 degrees celsius."), - - new TextMessage(Role.User, "What's the weather in Paris?"), - }; - - var prompt = Mistral_7B_0_3ChatTemplateBuilder.Instance.BuildPrompt(chatHistory, [getWeatherTool]); - - Approvals.Verify(prompt); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void TokenizerTest() - { - var modelWeightFolder = "Mistral"; - var tokenizer = MistralTokenizerHelper.FromPretrained(modelWeightFolder); - - var messages = new string[] - { - // system : You are a helpful assistant that can answer questions about the weather. - // tool: [get-weather-tool-call] - // user : What's the weather like in Paris? - // assistant: // get-weather-tool-call - // tool: get-weather-tool-call-result - // assistant: The current temperature in Paris is 22.0 degrees celsius. - """ - [AVAILABLE_TOOLS][{"type": "function", "function": {"name": "get_current_weather", "description": "Get the current weather", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city and state, e.g. San Francisco, CA"}, "format": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "The temperature unit to use. Infer this from the users location."}}, "required": ["location", "format"]}}}][/AVAILABLE_TOOLS][INST] What's the weather like in Paris?[/INST][TOOL_CALLS] [{"name": "get_current_temperature", "arguments": {"location": "Paris, France", "unit": "celsius"}, "id": "9Ae3bDc2F"}][TOOL_RESULTS] {"content": 22.0, "call_id": "9Ae3bDc2F"}[/TOOL_RESULTS] The current temperature in Paris is 22.0 degrees celsius. - """ - }; - - var sb = new StringBuilder(); - foreach (var message in messages) - { - var tokenizeIds = tokenizer.EncodeToIds(message, false, false); - var decodeToString = tokenizer.Decode(tokenizeIds, considerSpecialTokens: true); - sb.AppendLine(decodeToString); - var tokenizedStr = string.Join(", ", tokenizeIds.Select(x => x.ToString())); - - sb.AppendLine(tokenizedStr); - } - Approvals.Verify(sb.ToString()); - } -} diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.LoadSafeTensorShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.LoadSafeTensorShapeTest.approved.txt deleted file mode 100644 index 75e17ad1a6..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.LoadSafeTensorShapeTest.approved.txt +++ /dev/null @@ -1,453 +0,0 @@ -0: lm_head.bias shape: [51200] -1: lm_head.weight shape: [51200, 2560] -2: model.embed_tokens.weight shape: [51200, 2560] -3: model.final_layernorm.bias shape: [2560] -4: model.final_layernorm.weight shape: [2560] -5: model.layers.0.input_layernorm.bias shape: [2560] -6: model.layers.0.input_layernorm.weight shape: [2560] -7: model.layers.0.mlp.fc1.bias shape: [10240] -8: model.layers.0.mlp.fc1.weight shape: [10240, 2560] -9: model.layers.0.mlp.fc2.bias shape: [2560] -10: model.layers.0.mlp.fc2.weight shape: [2560, 10240] -11: model.layers.0.self_attn.dense.bias shape: [2560] -12: model.layers.0.self_attn.dense.weight shape: [2560, 2560] -13: model.layers.0.self_attn.k_proj.bias shape: [2560] -14: model.layers.0.self_attn.k_proj.weight shape: [2560, 2560] -15: model.layers.0.self_attn.q_proj.bias shape: [2560] -16: model.layers.0.self_attn.q_proj.weight shape: [2560, 2560] -17: model.layers.0.self_attn.v_proj.bias shape: [2560] -18: model.layers.0.self_attn.v_proj.weight shape: [2560, 2560] -19: model.layers.1.input_layernorm.bias shape: [2560] -20: model.layers.1.input_layernorm.weight shape: [2560] -21: model.layers.1.mlp.fc1.bias shape: [10240] -22: model.layers.1.mlp.fc1.weight shape: [10240, 2560] -23: model.layers.1.mlp.fc2.bias shape: [2560] -24: model.layers.1.mlp.fc2.weight shape: [2560, 10240] -25: model.layers.1.self_attn.dense.bias shape: [2560] -26: model.layers.1.self_attn.dense.weight shape: [2560, 2560] -27: model.layers.1.self_attn.k_proj.bias shape: [2560] -28: model.layers.1.self_attn.k_proj.weight shape: [2560, 2560] -29: model.layers.1.self_attn.q_proj.bias shape: [2560] -30: model.layers.1.self_attn.q_proj.weight shape: [2560, 2560] -31: model.layers.1.self_attn.v_proj.bias shape: [2560] -32: model.layers.1.self_attn.v_proj.weight shape: [2560, 2560] -33: model.layers.10.input_layernorm.bias shape: [2560] -34: model.layers.10.input_layernorm.weight shape: [2560] -35: model.layers.10.mlp.fc1.bias shape: [10240] -36: model.layers.10.mlp.fc1.weight shape: [10240, 2560] -37: model.layers.10.mlp.fc2.bias shape: [2560] -38: model.layers.10.mlp.fc2.weight shape: [2560, 10240] -39: model.layers.10.self_attn.dense.bias shape: [2560] -40: model.layers.10.self_attn.dense.weight shape: [2560, 2560] -41: model.layers.10.self_attn.k_proj.bias shape: [2560] -42: model.layers.10.self_attn.k_proj.weight shape: [2560, 2560] -43: model.layers.10.self_attn.q_proj.bias shape: [2560] -44: model.layers.10.self_attn.q_proj.weight shape: [2560, 2560] -45: model.layers.10.self_attn.v_proj.bias shape: [2560] -46: model.layers.10.self_attn.v_proj.weight shape: [2560, 2560] -47: model.layers.11.input_layernorm.bias shape: [2560] -48: model.layers.11.input_layernorm.weight shape: [2560] -49: model.layers.11.mlp.fc1.bias shape: [10240] -50: model.layers.11.mlp.fc1.weight shape: [10240, 2560] -51: model.layers.11.mlp.fc2.bias shape: [2560] -52: model.layers.11.mlp.fc2.weight shape: [2560, 10240] -53: model.layers.11.self_attn.dense.bias shape: [2560] -54: model.layers.11.self_attn.dense.weight shape: [2560, 2560] -55: model.layers.11.self_attn.k_proj.bias shape: [2560] -56: model.layers.11.self_attn.k_proj.weight shape: [2560, 2560] -57: model.layers.11.self_attn.q_proj.bias shape: [2560] -58: model.layers.11.self_attn.q_proj.weight shape: [2560, 2560] -59: model.layers.11.self_attn.v_proj.bias shape: [2560] -60: model.layers.11.self_attn.v_proj.weight shape: [2560, 2560] -61: model.layers.12.input_layernorm.bias shape: [2560] -62: model.layers.12.input_layernorm.weight shape: [2560] -63: model.layers.12.mlp.fc1.bias shape: [10240] -64: model.layers.12.mlp.fc1.weight shape: [10240, 2560] -65: model.layers.12.mlp.fc2.bias shape: [2560] -66: model.layers.12.mlp.fc2.weight shape: [2560, 10240] -67: model.layers.12.self_attn.dense.bias shape: [2560] -68: model.layers.12.self_attn.dense.weight shape: [2560, 2560] -69: model.layers.12.self_attn.k_proj.bias shape: [2560] -70: model.layers.12.self_attn.k_proj.weight shape: [2560, 2560] -71: model.layers.12.self_attn.q_proj.bias shape: [2560] -72: model.layers.12.self_attn.q_proj.weight shape: [2560, 2560] -73: model.layers.12.self_attn.v_proj.bias shape: [2560] -74: model.layers.12.self_attn.v_proj.weight shape: [2560, 2560] -75: model.layers.13.input_layernorm.bias shape: [2560] -76: model.layers.13.input_layernorm.weight shape: [2560] -77: model.layers.13.mlp.fc1.bias shape: [10240] -78: model.layers.13.mlp.fc1.weight shape: [10240, 2560] -79: model.layers.13.mlp.fc2.bias shape: [2560] -80: model.layers.13.mlp.fc2.weight shape: [2560, 10240] -81: model.layers.13.self_attn.dense.bias shape: [2560] -82: model.layers.13.self_attn.dense.weight shape: [2560, 2560] -83: model.layers.13.self_attn.k_proj.bias shape: [2560] -84: model.layers.13.self_attn.k_proj.weight shape: [2560, 2560] -85: model.layers.13.self_attn.q_proj.bias shape: [2560] -86: model.layers.13.self_attn.q_proj.weight shape: [2560, 2560] -87: model.layers.13.self_attn.v_proj.bias shape: [2560] -88: model.layers.13.self_attn.v_proj.weight shape: [2560, 2560] -89: model.layers.14.input_layernorm.bias shape: [2560] -90: model.layers.14.input_layernorm.weight shape: [2560] -91: model.layers.14.mlp.fc1.bias shape: [10240] -92: model.layers.14.mlp.fc1.weight shape: [10240, 2560] -93: model.layers.14.mlp.fc2.bias shape: [2560] -94: model.layers.14.mlp.fc2.weight shape: [2560, 10240] -95: model.layers.14.self_attn.dense.bias shape: [2560] -96: model.layers.14.self_attn.dense.weight shape: [2560, 2560] -97: model.layers.14.self_attn.k_proj.bias shape: [2560] -98: model.layers.14.self_attn.k_proj.weight shape: [2560, 2560] -99: model.layers.14.self_attn.q_proj.bias shape: [2560] -100: model.layers.14.self_attn.q_proj.weight shape: [2560, 2560] -101: model.layers.14.self_attn.v_proj.bias shape: [2560] -102: model.layers.14.self_attn.v_proj.weight shape: [2560, 2560] -103: model.layers.15.input_layernorm.bias shape: [2560] -104: model.layers.15.input_layernorm.weight shape: [2560] -105: model.layers.15.mlp.fc1.bias shape: [10240] -106: model.layers.15.mlp.fc1.weight shape: [10240, 2560] -107: model.layers.15.mlp.fc2.bias shape: [2560] -108: model.layers.15.mlp.fc2.weight shape: [2560, 10240] -109: model.layers.15.self_attn.dense.bias shape: [2560] -110: model.layers.15.self_attn.dense.weight shape: [2560, 2560] -111: model.layers.15.self_attn.k_proj.bias shape: [2560] -112: model.layers.15.self_attn.k_proj.weight shape: [2560, 2560] -113: model.layers.15.self_attn.q_proj.bias shape: [2560] -114: model.layers.15.self_attn.q_proj.weight shape: [2560, 2560] -115: model.layers.15.self_attn.v_proj.bias shape: [2560] -116: model.layers.15.self_attn.v_proj.weight shape: [2560, 2560] -117: model.layers.16.input_layernorm.bias shape: [2560] -118: model.layers.16.input_layernorm.weight shape: [2560] -119: model.layers.16.mlp.fc1.bias shape: [10240] -120: model.layers.16.mlp.fc1.weight shape: [10240, 2560] -121: model.layers.16.mlp.fc2.bias shape: [2560] -122: model.layers.16.mlp.fc2.weight shape: [2560, 10240] -123: model.layers.16.self_attn.dense.bias shape: [2560] -124: model.layers.16.self_attn.dense.weight shape: [2560, 2560] -125: model.layers.16.self_attn.k_proj.bias shape: [2560] -126: model.layers.16.self_attn.k_proj.weight shape: [2560, 2560] -127: model.layers.16.self_attn.q_proj.bias shape: [2560] -128: model.layers.16.self_attn.q_proj.weight shape: [2560, 2560] -129: model.layers.16.self_attn.v_proj.bias shape: [2560] -130: model.layers.16.self_attn.v_proj.weight shape: [2560, 2560] -131: model.layers.17.input_layernorm.bias shape: [2560] -132: model.layers.17.input_layernorm.weight shape: [2560] -133: model.layers.17.mlp.fc1.bias shape: [10240] -134: model.layers.17.mlp.fc1.weight shape: [10240, 2560] -135: model.layers.17.mlp.fc2.bias shape: [2560] -136: model.layers.17.mlp.fc2.weight shape: [2560, 10240] -137: model.layers.17.self_attn.dense.bias shape: [2560] -138: model.layers.17.self_attn.dense.weight shape: [2560, 2560] -139: model.layers.17.self_attn.k_proj.bias shape: [2560] -140: model.layers.17.self_attn.k_proj.weight shape: [2560, 2560] -141: model.layers.17.self_attn.q_proj.bias shape: [2560] -142: model.layers.17.self_attn.q_proj.weight shape: [2560, 2560] -143: model.layers.17.self_attn.v_proj.bias shape: [2560] -144: model.layers.17.self_attn.v_proj.weight shape: [2560, 2560] -145: model.layers.18.input_layernorm.bias shape: [2560] -146: model.layers.18.input_layernorm.weight shape: [2560] -147: model.layers.18.mlp.fc1.bias shape: [10240] -148: model.layers.18.mlp.fc1.weight shape: [10240, 2560] -149: model.layers.18.mlp.fc2.bias shape: [2560] -150: model.layers.18.mlp.fc2.weight shape: [2560, 10240] -151: model.layers.18.self_attn.dense.bias shape: [2560] -152: model.layers.18.self_attn.dense.weight shape: [2560, 2560] -153: model.layers.18.self_attn.k_proj.bias shape: [2560] -154: model.layers.18.self_attn.k_proj.weight shape: [2560, 2560] -155: model.layers.18.self_attn.q_proj.bias shape: [2560] -156: model.layers.18.self_attn.q_proj.weight shape: [2560, 2560] -157: model.layers.18.self_attn.v_proj.bias shape: [2560] -158: model.layers.18.self_attn.v_proj.weight shape: [2560, 2560] -159: model.layers.19.input_layernorm.bias shape: [2560] -160: model.layers.19.input_layernorm.weight shape: [2560] -161: model.layers.19.mlp.fc1.bias shape: [10240] -162: model.layers.19.mlp.fc1.weight shape: [10240, 2560] -163: model.layers.19.mlp.fc2.bias shape: [2560] -164: model.layers.19.mlp.fc2.weight shape: [2560, 10240] -165: model.layers.19.self_attn.dense.bias shape: [2560] -166: model.layers.19.self_attn.dense.weight shape: [2560, 2560] -167: model.layers.19.self_attn.k_proj.bias shape: [2560] -168: model.layers.19.self_attn.k_proj.weight shape: [2560, 2560] -169: model.layers.19.self_attn.q_proj.bias shape: [2560] -170: model.layers.19.self_attn.q_proj.weight shape: [2560, 2560] -171: model.layers.19.self_attn.v_proj.bias shape: [2560] -172: model.layers.19.self_attn.v_proj.weight shape: [2560, 2560] -173: model.layers.2.input_layernorm.bias shape: [2560] -174: model.layers.2.input_layernorm.weight shape: [2560] -175: model.layers.2.mlp.fc1.bias shape: [10240] -176: model.layers.2.mlp.fc1.weight shape: [10240, 2560] -177: model.layers.2.mlp.fc2.bias shape: [2560] -178: model.layers.2.mlp.fc2.weight shape: [2560, 10240] -179: model.layers.2.self_attn.dense.bias shape: [2560] -180: model.layers.2.self_attn.dense.weight shape: [2560, 2560] -181: model.layers.2.self_attn.k_proj.bias shape: [2560] -182: model.layers.2.self_attn.k_proj.weight shape: [2560, 2560] -183: model.layers.2.self_attn.q_proj.bias shape: [2560] -184: model.layers.2.self_attn.q_proj.weight shape: [2560, 2560] -185: model.layers.2.self_attn.v_proj.bias shape: [2560] -186: model.layers.2.self_attn.v_proj.weight shape: [2560, 2560] -187: model.layers.20.input_layernorm.bias shape: [2560] -188: model.layers.20.input_layernorm.weight shape: [2560] -189: model.layers.20.mlp.fc1.bias shape: [10240] -190: model.layers.20.mlp.fc1.weight shape: [10240, 2560] -191: model.layers.20.mlp.fc2.bias shape: [2560] -192: model.layers.20.mlp.fc2.weight shape: [2560, 10240] -193: model.layers.20.self_attn.dense.bias shape: [2560] -194: model.layers.20.self_attn.dense.weight shape: [2560, 2560] -195: model.layers.20.self_attn.k_proj.bias shape: [2560] -196: model.layers.20.self_attn.k_proj.weight shape: [2560, 2560] -197: model.layers.20.self_attn.q_proj.bias shape: [2560] -198: model.layers.20.self_attn.q_proj.weight shape: [2560, 2560] -199: model.layers.20.self_attn.v_proj.bias shape: [2560] -200: model.layers.20.self_attn.v_proj.weight shape: [2560, 2560] -201: model.layers.21.input_layernorm.bias shape: [2560] -202: model.layers.21.input_layernorm.weight shape: [2560] -203: model.layers.21.mlp.fc1.bias shape: [10240] -204: model.layers.21.mlp.fc1.weight shape: [10240, 2560] -205: model.layers.21.mlp.fc2.bias shape: [2560] -206: model.layers.21.mlp.fc2.weight shape: [2560, 10240] -207: model.layers.21.self_attn.dense.bias shape: [2560] -208: model.layers.21.self_attn.dense.weight shape: [2560, 2560] -209: model.layers.21.self_attn.k_proj.bias shape: [2560] -210: model.layers.21.self_attn.k_proj.weight shape: [2560, 2560] -211: model.layers.21.self_attn.q_proj.bias shape: [2560] -212: model.layers.21.self_attn.q_proj.weight shape: [2560, 2560] -213: model.layers.21.self_attn.v_proj.bias shape: [2560] -214: model.layers.21.self_attn.v_proj.weight shape: [2560, 2560] -215: model.layers.22.input_layernorm.bias shape: [2560] -216: model.layers.22.input_layernorm.weight shape: [2560] -217: model.layers.22.mlp.fc1.bias shape: [10240] -218: model.layers.22.mlp.fc1.weight shape: [10240, 2560] -219: model.layers.22.mlp.fc2.bias shape: [2560] -220: model.layers.22.mlp.fc2.weight shape: [2560, 10240] -221: model.layers.22.self_attn.dense.bias shape: [2560] -222: model.layers.22.self_attn.dense.weight shape: [2560, 2560] -223: model.layers.22.self_attn.k_proj.bias shape: [2560] -224: model.layers.22.self_attn.k_proj.weight shape: [2560, 2560] -225: model.layers.22.self_attn.q_proj.bias shape: [2560] -226: model.layers.22.self_attn.q_proj.weight shape: [2560, 2560] -227: model.layers.22.self_attn.v_proj.bias shape: [2560] -228: model.layers.22.self_attn.v_proj.weight shape: [2560, 2560] -229: model.layers.23.input_layernorm.bias shape: [2560] -230: model.layers.23.input_layernorm.weight shape: [2560] -231: model.layers.23.mlp.fc1.bias shape: [10240] -232: model.layers.23.mlp.fc1.weight shape: [10240, 2560] -233: model.layers.23.mlp.fc2.bias shape: [2560] -234: model.layers.23.mlp.fc2.weight shape: [2560, 10240] -235: model.layers.23.self_attn.dense.bias shape: [2560] -236: model.layers.23.self_attn.dense.weight shape: [2560, 2560] -237: model.layers.23.self_attn.k_proj.bias shape: [2560] -238: model.layers.23.self_attn.k_proj.weight shape: [2560, 2560] -239: model.layers.23.self_attn.q_proj.bias shape: [2560] -240: model.layers.23.self_attn.q_proj.weight shape: [2560, 2560] -241: model.layers.23.self_attn.v_proj.bias shape: [2560] -242: model.layers.23.self_attn.v_proj.weight shape: [2560, 2560] -243: model.layers.24.input_layernorm.bias shape: [2560] -244: model.layers.24.input_layernorm.weight shape: [2560] -245: model.layers.24.mlp.fc1.bias shape: [10240] -246: model.layers.24.mlp.fc1.weight shape: [10240, 2560] -247: model.layers.24.mlp.fc2.bias shape: [2560] -248: model.layers.24.mlp.fc2.weight shape: [2560, 10240] -249: model.layers.24.self_attn.dense.bias shape: [2560] -250: model.layers.24.self_attn.dense.weight shape: [2560, 2560] -251: model.layers.24.self_attn.k_proj.bias shape: [2560] -252: model.layers.24.self_attn.k_proj.weight shape: [2560, 2560] -253: model.layers.24.self_attn.q_proj.bias shape: [2560] -254: model.layers.24.self_attn.q_proj.weight shape: [2560, 2560] -255: model.layers.24.self_attn.v_proj.bias shape: [2560] -256: model.layers.24.self_attn.v_proj.weight shape: [2560, 2560] -257: model.layers.25.input_layernorm.bias shape: [2560] -258: model.layers.25.input_layernorm.weight shape: [2560] -259: model.layers.25.mlp.fc1.bias shape: [10240] -260: model.layers.25.mlp.fc1.weight shape: [10240, 2560] -261: model.layers.25.mlp.fc2.bias shape: [2560] -262: model.layers.25.mlp.fc2.weight shape: [2560, 10240] -263: model.layers.25.self_attn.dense.bias shape: [2560] -264: model.layers.25.self_attn.dense.weight shape: [2560, 2560] -265: model.layers.25.self_attn.k_proj.bias shape: [2560] -266: model.layers.25.self_attn.k_proj.weight shape: [2560, 2560] -267: model.layers.25.self_attn.q_proj.bias shape: [2560] -268: model.layers.25.self_attn.q_proj.weight shape: [2560, 2560] -269: model.layers.25.self_attn.v_proj.bias shape: [2560] -270: model.layers.25.self_attn.v_proj.weight shape: [2560, 2560] -271: model.layers.26.input_layernorm.bias shape: [2560] -272: model.layers.26.input_layernorm.weight shape: [2560] -273: model.layers.26.mlp.fc1.bias shape: [10240] -274: model.layers.26.mlp.fc1.weight shape: [10240, 2560] -275: model.layers.26.mlp.fc2.bias shape: [2560] -276: model.layers.26.mlp.fc2.weight shape: [2560, 10240] -277: model.layers.26.self_attn.dense.bias shape: [2560] -278: model.layers.26.self_attn.dense.weight shape: [2560, 2560] -279: model.layers.26.self_attn.k_proj.bias shape: [2560] -280: model.layers.26.self_attn.k_proj.weight shape: [2560, 2560] -281: model.layers.26.self_attn.q_proj.bias shape: [2560] -282: model.layers.26.self_attn.q_proj.weight shape: [2560, 2560] -283: model.layers.26.self_attn.v_proj.bias shape: [2560] -284: model.layers.26.self_attn.v_proj.weight shape: [2560, 2560] -285: model.layers.27.input_layernorm.bias shape: [2560] -286: model.layers.27.input_layernorm.weight shape: [2560] -287: model.layers.27.mlp.fc1.bias shape: [10240] -288: model.layers.27.mlp.fc1.weight shape: [10240, 2560] -289: model.layers.27.mlp.fc2.bias shape: [2560] -290: model.layers.27.mlp.fc2.weight shape: [2560, 10240] -291: model.layers.27.self_attn.dense.bias shape: [2560] -292: model.layers.27.self_attn.dense.weight shape: [2560, 2560] -293: model.layers.27.self_attn.k_proj.bias shape: [2560] -294: model.layers.27.self_attn.k_proj.weight shape: [2560, 2560] -295: model.layers.27.self_attn.q_proj.bias shape: [2560] -296: model.layers.27.self_attn.q_proj.weight shape: [2560, 2560] -297: model.layers.27.self_attn.v_proj.bias shape: [2560] -298: model.layers.27.self_attn.v_proj.weight shape: [2560, 2560] -299: model.layers.28.input_layernorm.bias shape: [2560] -300: model.layers.28.input_layernorm.weight shape: [2560] -301: model.layers.28.mlp.fc1.bias shape: [10240] -302: model.layers.28.mlp.fc1.weight shape: [10240, 2560] -303: model.layers.28.mlp.fc2.bias shape: [2560] -304: model.layers.28.mlp.fc2.weight shape: [2560, 10240] -305: model.layers.28.self_attn.dense.bias shape: [2560] -306: model.layers.28.self_attn.dense.weight shape: [2560, 2560] -307: model.layers.28.self_attn.k_proj.bias shape: [2560] -308: model.layers.28.self_attn.k_proj.weight shape: [2560, 2560] -309: model.layers.28.self_attn.q_proj.bias shape: [2560] -310: model.layers.28.self_attn.q_proj.weight shape: [2560, 2560] -311: model.layers.28.self_attn.v_proj.bias shape: [2560] -312: model.layers.28.self_attn.v_proj.weight shape: [2560, 2560] -313: model.layers.29.input_layernorm.bias shape: [2560] -314: model.layers.29.input_layernorm.weight shape: [2560] -315: model.layers.29.mlp.fc1.bias shape: [10240] -316: model.layers.29.mlp.fc1.weight shape: [10240, 2560] -317: model.layers.29.mlp.fc2.bias shape: [2560] -318: model.layers.29.mlp.fc2.weight shape: [2560, 10240] -319: model.layers.29.self_attn.dense.bias shape: [2560] -320: model.layers.29.self_attn.dense.weight shape: [2560, 2560] -321: model.layers.29.self_attn.k_proj.bias shape: [2560] -322: model.layers.29.self_attn.k_proj.weight shape: [2560, 2560] -323: model.layers.29.self_attn.q_proj.bias shape: [2560] -324: model.layers.29.self_attn.q_proj.weight shape: [2560, 2560] -325: model.layers.29.self_attn.v_proj.bias shape: [2560] -326: model.layers.29.self_attn.v_proj.weight shape: [2560, 2560] -327: model.layers.3.input_layernorm.bias shape: [2560] -328: model.layers.3.input_layernorm.weight shape: [2560] -329: model.layers.3.mlp.fc1.bias shape: [10240] -330: model.layers.3.mlp.fc1.weight shape: [10240, 2560] -331: model.layers.3.mlp.fc2.bias shape: [2560] -332: model.layers.3.mlp.fc2.weight shape: [2560, 10240] -333: model.layers.3.self_attn.dense.bias shape: [2560] -334: model.layers.3.self_attn.dense.weight shape: [2560, 2560] -335: model.layers.3.self_attn.k_proj.bias shape: [2560] -336: model.layers.3.self_attn.k_proj.weight shape: [2560, 2560] -337: model.layers.3.self_attn.q_proj.bias shape: [2560] -338: model.layers.3.self_attn.q_proj.weight shape: [2560, 2560] -339: model.layers.3.self_attn.v_proj.bias shape: [2560] -340: model.layers.3.self_attn.v_proj.weight shape: [2560, 2560] -341: model.layers.30.input_layernorm.bias shape: [2560] -342: model.layers.30.input_layernorm.weight shape: [2560] -343: model.layers.30.mlp.fc1.bias shape: [10240] -344: model.layers.30.mlp.fc1.weight shape: [10240, 2560] -345: model.layers.30.mlp.fc2.bias shape: [2560] -346: model.layers.30.mlp.fc2.weight shape: [2560, 10240] -347: model.layers.30.self_attn.dense.bias shape: [2560] -348: model.layers.30.self_attn.dense.weight shape: [2560, 2560] -349: model.layers.30.self_attn.k_proj.bias shape: [2560] -350: model.layers.30.self_attn.k_proj.weight shape: [2560, 2560] -351: model.layers.30.self_attn.q_proj.bias shape: [2560] -352: model.layers.30.self_attn.q_proj.weight shape: [2560, 2560] -353: model.layers.30.self_attn.v_proj.bias shape: [2560] -354: model.layers.30.self_attn.v_proj.weight shape: [2560, 2560] -355: model.layers.31.input_layernorm.bias shape: [2560] -356: model.layers.31.input_layernorm.weight shape: [2560] -357: model.layers.31.mlp.fc1.bias shape: [10240] -358: model.layers.31.mlp.fc1.weight shape: [10240, 2560] -359: model.layers.31.mlp.fc2.bias shape: [2560] -360: model.layers.31.mlp.fc2.weight shape: [2560, 10240] -361: model.layers.31.self_attn.dense.bias shape: [2560] -362: model.layers.31.self_attn.dense.weight shape: [2560, 2560] -363: model.layers.31.self_attn.k_proj.bias shape: [2560] -364: model.layers.31.self_attn.k_proj.weight shape: [2560, 2560] -365: model.layers.31.self_attn.q_proj.bias shape: [2560] -366: model.layers.31.self_attn.q_proj.weight shape: [2560, 2560] -367: model.layers.31.self_attn.v_proj.bias shape: [2560] -368: model.layers.31.self_attn.v_proj.weight shape: [2560, 2560] -369: model.layers.4.input_layernorm.bias shape: [2560] -370: model.layers.4.input_layernorm.weight shape: [2560] -371: model.layers.4.mlp.fc1.bias shape: [10240] -372: model.layers.4.mlp.fc1.weight shape: [10240, 2560] -373: model.layers.4.mlp.fc2.bias shape: [2560] -374: model.layers.4.mlp.fc2.weight shape: [2560, 10240] -375: model.layers.4.self_attn.dense.bias shape: [2560] -376: model.layers.4.self_attn.dense.weight shape: [2560, 2560] -377: model.layers.4.self_attn.k_proj.bias shape: [2560] -378: model.layers.4.self_attn.k_proj.weight shape: [2560, 2560] -379: model.layers.4.self_attn.q_proj.bias shape: [2560] -380: model.layers.4.self_attn.q_proj.weight shape: [2560, 2560] -381: model.layers.4.self_attn.v_proj.bias shape: [2560] -382: model.layers.4.self_attn.v_proj.weight shape: [2560, 2560] -383: model.layers.5.input_layernorm.bias shape: [2560] -384: model.layers.5.input_layernorm.weight shape: [2560] -385: model.layers.5.mlp.fc1.bias shape: [10240] -386: model.layers.5.mlp.fc1.weight shape: [10240, 2560] -387: model.layers.5.mlp.fc2.bias shape: [2560] -388: model.layers.5.mlp.fc2.weight shape: [2560, 10240] -389: model.layers.5.self_attn.dense.bias shape: [2560] -390: model.layers.5.self_attn.dense.weight shape: [2560, 2560] -391: model.layers.5.self_attn.k_proj.bias shape: [2560] -392: model.layers.5.self_attn.k_proj.weight shape: [2560, 2560] -393: model.layers.5.self_attn.q_proj.bias shape: [2560] -394: model.layers.5.self_attn.q_proj.weight shape: [2560, 2560] -395: model.layers.5.self_attn.v_proj.bias shape: [2560] -396: model.layers.5.self_attn.v_proj.weight shape: [2560, 2560] -397: model.layers.6.input_layernorm.bias shape: [2560] -398: model.layers.6.input_layernorm.weight shape: [2560] -399: model.layers.6.mlp.fc1.bias shape: [10240] -400: model.layers.6.mlp.fc1.weight shape: [10240, 2560] -401: model.layers.6.mlp.fc2.bias shape: [2560] -402: model.layers.6.mlp.fc2.weight shape: [2560, 10240] -403: model.layers.6.self_attn.dense.bias shape: [2560] -404: model.layers.6.self_attn.dense.weight shape: [2560, 2560] -405: model.layers.6.self_attn.k_proj.bias shape: [2560] -406: model.layers.6.self_attn.k_proj.weight shape: [2560, 2560] -407: model.layers.6.self_attn.q_proj.bias shape: [2560] -408: model.layers.6.self_attn.q_proj.weight shape: [2560, 2560] -409: model.layers.6.self_attn.v_proj.bias shape: [2560] -410: model.layers.6.self_attn.v_proj.weight shape: [2560, 2560] -411: model.layers.7.input_layernorm.bias shape: [2560] -412: model.layers.7.input_layernorm.weight shape: [2560] -413: model.layers.7.mlp.fc1.bias shape: [10240] -414: model.layers.7.mlp.fc1.weight shape: [10240, 2560] -415: model.layers.7.mlp.fc2.bias shape: [2560] -416: model.layers.7.mlp.fc2.weight shape: [2560, 10240] -417: model.layers.7.self_attn.dense.bias shape: [2560] -418: model.layers.7.self_attn.dense.weight shape: [2560, 2560] -419: model.layers.7.self_attn.k_proj.bias shape: [2560] -420: model.layers.7.self_attn.k_proj.weight shape: [2560, 2560] -421: model.layers.7.self_attn.q_proj.bias shape: [2560] -422: model.layers.7.self_attn.q_proj.weight shape: [2560, 2560] -423: model.layers.7.self_attn.v_proj.bias shape: [2560] -424: model.layers.7.self_attn.v_proj.weight shape: [2560, 2560] -425: model.layers.8.input_layernorm.bias shape: [2560] -426: model.layers.8.input_layernorm.weight shape: [2560] -427: model.layers.8.mlp.fc1.bias shape: [10240] -428: model.layers.8.mlp.fc1.weight shape: [10240, 2560] -429: model.layers.8.mlp.fc2.bias shape: [2560] -430: model.layers.8.mlp.fc2.weight shape: [2560, 10240] -431: model.layers.8.self_attn.dense.bias shape: [2560] -432: model.layers.8.self_attn.dense.weight shape: [2560, 2560] -433: model.layers.8.self_attn.k_proj.bias shape: [2560] -434: model.layers.8.self_attn.k_proj.weight shape: [2560, 2560] -435: model.layers.8.self_attn.q_proj.bias shape: [2560] -436: model.layers.8.self_attn.q_proj.weight shape: [2560, 2560] -437: model.layers.8.self_attn.v_proj.bias shape: [2560] -438: model.layers.8.self_attn.v_proj.weight shape: [2560, 2560] -439: model.layers.9.input_layernorm.bias shape: [2560] -440: model.layers.9.input_layernorm.weight shape: [2560] -441: model.layers.9.mlp.fc1.bias shape: [10240] -442: model.layers.9.mlp.fc1.weight shape: [10240, 2560] -443: model.layers.9.mlp.fc2.bias shape: [2560] -444: model.layers.9.mlp.fc2.weight shape: [2560, 10240] -445: model.layers.9.self_attn.dense.bias shape: [2560] -446: model.layers.9.self_attn.dense.weight shape: [2560, 2560] -447: model.layers.9.self_attn.k_proj.bias shape: [2560] -448: model.layers.9.self_attn.k_proj.weight shape: [2560, 2560] -449: model.layers.9.self_attn.q_proj.bias shape: [2560] -450: model.layers.9.self_attn.q_proj.weight shape: [2560, 2560] -451: model.layers.9.self_attn.v_proj.bias shape: [2560] -452: model.layers.9.self_attn.v_proj.weight shape: [2560, 2560] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.TokenizerTest.approved.txt deleted file mode 100644 index 7338548917..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Test.TokenizerTest.approved.txt +++ /dev/null @@ -1,3 +0,0 @@ -50256, 6090, 345, 2148, 2842, 284, 4483, 17790, 286, 35484, 290, 10441, 69, 50187, 30 -50256, 19457, 0, 3423, 389, 617, 2842, 284, 4483, 35484, 290, 10441, 69, 50187, 1978, 25, 352, 13, 40058, 290, 10441, 34711, 7209, 494, 25, 41198, 35484, 290, 10441, 69, 50187, 1978, 351, 617, 7545, 290, 12498, 13, 362, 13, 40058, 290, 10441, 34711, 20698, 25, 15561, 26790, 35484, 290, 10441, 69, 50187, 1978, 351, 617, 18873, 13135, 290, 12498, 13 -50256, 2061, 546, 18120, 281, 362, 87, 1343, 513, 796, 767, 16022, 30 diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.LoadSafeTensorShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.LoadSafeTensorShapeTest.approved.txt deleted file mode 100644 index 75e17ad1a6..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.LoadSafeTensorShapeTest.approved.txt +++ /dev/null @@ -1,453 +0,0 @@ -0: lm_head.bias shape: [51200] -1: lm_head.weight shape: [51200, 2560] -2: model.embed_tokens.weight shape: [51200, 2560] -3: model.final_layernorm.bias shape: [2560] -4: model.final_layernorm.weight shape: [2560] -5: model.layers.0.input_layernorm.bias shape: [2560] -6: model.layers.0.input_layernorm.weight shape: [2560] -7: model.layers.0.mlp.fc1.bias shape: [10240] -8: model.layers.0.mlp.fc1.weight shape: [10240, 2560] -9: model.layers.0.mlp.fc2.bias shape: [2560] -10: model.layers.0.mlp.fc2.weight shape: [2560, 10240] -11: model.layers.0.self_attn.dense.bias shape: [2560] -12: model.layers.0.self_attn.dense.weight shape: [2560, 2560] -13: model.layers.0.self_attn.k_proj.bias shape: [2560] -14: model.layers.0.self_attn.k_proj.weight shape: [2560, 2560] -15: model.layers.0.self_attn.q_proj.bias shape: [2560] -16: model.layers.0.self_attn.q_proj.weight shape: [2560, 2560] -17: model.layers.0.self_attn.v_proj.bias shape: [2560] -18: model.layers.0.self_attn.v_proj.weight shape: [2560, 2560] -19: model.layers.1.input_layernorm.bias shape: [2560] -20: model.layers.1.input_layernorm.weight shape: [2560] -21: model.layers.1.mlp.fc1.bias shape: [10240] -22: model.layers.1.mlp.fc1.weight shape: [10240, 2560] -23: model.layers.1.mlp.fc2.bias shape: [2560] -24: model.layers.1.mlp.fc2.weight shape: [2560, 10240] -25: model.layers.1.self_attn.dense.bias shape: [2560] -26: model.layers.1.self_attn.dense.weight shape: [2560, 2560] -27: model.layers.1.self_attn.k_proj.bias shape: [2560] -28: model.layers.1.self_attn.k_proj.weight shape: [2560, 2560] -29: model.layers.1.self_attn.q_proj.bias shape: [2560] -30: model.layers.1.self_attn.q_proj.weight shape: [2560, 2560] -31: model.layers.1.self_attn.v_proj.bias shape: [2560] -32: model.layers.1.self_attn.v_proj.weight shape: [2560, 2560] -33: model.layers.10.input_layernorm.bias shape: [2560] -34: model.layers.10.input_layernorm.weight shape: [2560] -35: model.layers.10.mlp.fc1.bias shape: [10240] -36: model.layers.10.mlp.fc1.weight shape: [10240, 2560] -37: model.layers.10.mlp.fc2.bias shape: [2560] -38: model.layers.10.mlp.fc2.weight shape: [2560, 10240] -39: model.layers.10.self_attn.dense.bias shape: [2560] -40: model.layers.10.self_attn.dense.weight shape: [2560, 2560] -41: model.layers.10.self_attn.k_proj.bias shape: [2560] -42: model.layers.10.self_attn.k_proj.weight shape: [2560, 2560] -43: model.layers.10.self_attn.q_proj.bias shape: [2560] -44: model.layers.10.self_attn.q_proj.weight shape: [2560, 2560] -45: model.layers.10.self_attn.v_proj.bias shape: [2560] -46: model.layers.10.self_attn.v_proj.weight shape: [2560, 2560] -47: model.layers.11.input_layernorm.bias shape: [2560] -48: model.layers.11.input_layernorm.weight shape: [2560] -49: model.layers.11.mlp.fc1.bias shape: [10240] -50: model.layers.11.mlp.fc1.weight shape: [10240, 2560] -51: model.layers.11.mlp.fc2.bias shape: [2560] -52: model.layers.11.mlp.fc2.weight shape: [2560, 10240] -53: model.layers.11.self_attn.dense.bias shape: [2560] -54: model.layers.11.self_attn.dense.weight shape: [2560, 2560] -55: model.layers.11.self_attn.k_proj.bias shape: [2560] -56: model.layers.11.self_attn.k_proj.weight shape: [2560, 2560] -57: model.layers.11.self_attn.q_proj.bias shape: [2560] -58: model.layers.11.self_attn.q_proj.weight shape: [2560, 2560] -59: model.layers.11.self_attn.v_proj.bias shape: [2560] -60: model.layers.11.self_attn.v_proj.weight shape: [2560, 2560] -61: model.layers.12.input_layernorm.bias shape: [2560] -62: model.layers.12.input_layernorm.weight shape: [2560] -63: model.layers.12.mlp.fc1.bias shape: [10240] -64: model.layers.12.mlp.fc1.weight shape: [10240, 2560] -65: model.layers.12.mlp.fc2.bias shape: [2560] -66: model.layers.12.mlp.fc2.weight shape: [2560, 10240] -67: model.layers.12.self_attn.dense.bias shape: [2560] -68: model.layers.12.self_attn.dense.weight shape: [2560, 2560] -69: model.layers.12.self_attn.k_proj.bias shape: [2560] -70: model.layers.12.self_attn.k_proj.weight shape: [2560, 2560] -71: model.layers.12.self_attn.q_proj.bias shape: [2560] -72: model.layers.12.self_attn.q_proj.weight shape: [2560, 2560] -73: model.layers.12.self_attn.v_proj.bias shape: [2560] -74: model.layers.12.self_attn.v_proj.weight shape: [2560, 2560] -75: model.layers.13.input_layernorm.bias shape: [2560] -76: model.layers.13.input_layernorm.weight shape: [2560] -77: model.layers.13.mlp.fc1.bias shape: [10240] -78: model.layers.13.mlp.fc1.weight shape: [10240, 2560] -79: model.layers.13.mlp.fc2.bias shape: [2560] -80: model.layers.13.mlp.fc2.weight shape: [2560, 10240] -81: model.layers.13.self_attn.dense.bias shape: [2560] -82: model.layers.13.self_attn.dense.weight shape: [2560, 2560] -83: model.layers.13.self_attn.k_proj.bias shape: [2560] -84: model.layers.13.self_attn.k_proj.weight shape: [2560, 2560] -85: model.layers.13.self_attn.q_proj.bias shape: [2560] -86: model.layers.13.self_attn.q_proj.weight shape: [2560, 2560] -87: model.layers.13.self_attn.v_proj.bias shape: [2560] -88: model.layers.13.self_attn.v_proj.weight shape: [2560, 2560] -89: model.layers.14.input_layernorm.bias shape: [2560] -90: model.layers.14.input_layernorm.weight shape: [2560] -91: model.layers.14.mlp.fc1.bias shape: [10240] -92: model.layers.14.mlp.fc1.weight shape: [10240, 2560] -93: model.layers.14.mlp.fc2.bias shape: [2560] -94: model.layers.14.mlp.fc2.weight shape: [2560, 10240] -95: model.layers.14.self_attn.dense.bias shape: [2560] -96: model.layers.14.self_attn.dense.weight shape: [2560, 2560] -97: model.layers.14.self_attn.k_proj.bias shape: [2560] -98: model.layers.14.self_attn.k_proj.weight shape: [2560, 2560] -99: model.layers.14.self_attn.q_proj.bias shape: [2560] -100: model.layers.14.self_attn.q_proj.weight shape: [2560, 2560] -101: model.layers.14.self_attn.v_proj.bias shape: [2560] -102: model.layers.14.self_attn.v_proj.weight shape: [2560, 2560] -103: model.layers.15.input_layernorm.bias shape: [2560] -104: model.layers.15.input_layernorm.weight shape: [2560] -105: model.layers.15.mlp.fc1.bias shape: [10240] -106: model.layers.15.mlp.fc1.weight shape: [10240, 2560] -107: model.layers.15.mlp.fc2.bias shape: [2560] -108: model.layers.15.mlp.fc2.weight shape: [2560, 10240] -109: model.layers.15.self_attn.dense.bias shape: [2560] -110: model.layers.15.self_attn.dense.weight shape: [2560, 2560] -111: model.layers.15.self_attn.k_proj.bias shape: [2560] -112: model.layers.15.self_attn.k_proj.weight shape: [2560, 2560] -113: model.layers.15.self_attn.q_proj.bias shape: [2560] -114: model.layers.15.self_attn.q_proj.weight shape: [2560, 2560] -115: model.layers.15.self_attn.v_proj.bias shape: [2560] -116: model.layers.15.self_attn.v_proj.weight shape: [2560, 2560] -117: model.layers.16.input_layernorm.bias shape: [2560] -118: model.layers.16.input_layernorm.weight shape: [2560] -119: model.layers.16.mlp.fc1.bias shape: [10240] -120: model.layers.16.mlp.fc1.weight shape: [10240, 2560] -121: model.layers.16.mlp.fc2.bias shape: [2560] -122: model.layers.16.mlp.fc2.weight shape: [2560, 10240] -123: model.layers.16.self_attn.dense.bias shape: [2560] -124: model.layers.16.self_attn.dense.weight shape: [2560, 2560] -125: model.layers.16.self_attn.k_proj.bias shape: [2560] -126: model.layers.16.self_attn.k_proj.weight shape: [2560, 2560] -127: model.layers.16.self_attn.q_proj.bias shape: [2560] -128: model.layers.16.self_attn.q_proj.weight shape: [2560, 2560] -129: model.layers.16.self_attn.v_proj.bias shape: [2560] -130: model.layers.16.self_attn.v_proj.weight shape: [2560, 2560] -131: model.layers.17.input_layernorm.bias shape: [2560] -132: model.layers.17.input_layernorm.weight shape: [2560] -133: model.layers.17.mlp.fc1.bias shape: [10240] -134: model.layers.17.mlp.fc1.weight shape: [10240, 2560] -135: model.layers.17.mlp.fc2.bias shape: [2560] -136: model.layers.17.mlp.fc2.weight shape: [2560, 10240] -137: model.layers.17.self_attn.dense.bias shape: [2560] -138: model.layers.17.self_attn.dense.weight shape: [2560, 2560] -139: model.layers.17.self_attn.k_proj.bias shape: [2560] -140: model.layers.17.self_attn.k_proj.weight shape: [2560, 2560] -141: model.layers.17.self_attn.q_proj.bias shape: [2560] -142: model.layers.17.self_attn.q_proj.weight shape: [2560, 2560] -143: model.layers.17.self_attn.v_proj.bias shape: [2560] -144: model.layers.17.self_attn.v_proj.weight shape: [2560, 2560] -145: model.layers.18.input_layernorm.bias shape: [2560] -146: model.layers.18.input_layernorm.weight shape: [2560] -147: model.layers.18.mlp.fc1.bias shape: [10240] -148: model.layers.18.mlp.fc1.weight shape: [10240, 2560] -149: model.layers.18.mlp.fc2.bias shape: [2560] -150: model.layers.18.mlp.fc2.weight shape: [2560, 10240] -151: model.layers.18.self_attn.dense.bias shape: [2560] -152: model.layers.18.self_attn.dense.weight shape: [2560, 2560] -153: model.layers.18.self_attn.k_proj.bias shape: [2560] -154: model.layers.18.self_attn.k_proj.weight shape: [2560, 2560] -155: model.layers.18.self_attn.q_proj.bias shape: [2560] -156: model.layers.18.self_attn.q_proj.weight shape: [2560, 2560] -157: model.layers.18.self_attn.v_proj.bias shape: [2560] -158: model.layers.18.self_attn.v_proj.weight shape: [2560, 2560] -159: model.layers.19.input_layernorm.bias shape: [2560] -160: model.layers.19.input_layernorm.weight shape: [2560] -161: model.layers.19.mlp.fc1.bias shape: [10240] -162: model.layers.19.mlp.fc1.weight shape: [10240, 2560] -163: model.layers.19.mlp.fc2.bias shape: [2560] -164: model.layers.19.mlp.fc2.weight shape: [2560, 10240] -165: model.layers.19.self_attn.dense.bias shape: [2560] -166: model.layers.19.self_attn.dense.weight shape: [2560, 2560] -167: model.layers.19.self_attn.k_proj.bias shape: [2560] -168: model.layers.19.self_attn.k_proj.weight shape: [2560, 2560] -169: model.layers.19.self_attn.q_proj.bias shape: [2560] -170: model.layers.19.self_attn.q_proj.weight shape: [2560, 2560] -171: model.layers.19.self_attn.v_proj.bias shape: [2560] -172: model.layers.19.self_attn.v_proj.weight shape: [2560, 2560] -173: model.layers.2.input_layernorm.bias shape: [2560] -174: model.layers.2.input_layernorm.weight shape: [2560] -175: model.layers.2.mlp.fc1.bias shape: [10240] -176: model.layers.2.mlp.fc1.weight shape: [10240, 2560] -177: model.layers.2.mlp.fc2.bias shape: [2560] -178: model.layers.2.mlp.fc2.weight shape: [2560, 10240] -179: model.layers.2.self_attn.dense.bias shape: [2560] -180: model.layers.2.self_attn.dense.weight shape: [2560, 2560] -181: model.layers.2.self_attn.k_proj.bias shape: [2560] -182: model.layers.2.self_attn.k_proj.weight shape: [2560, 2560] -183: model.layers.2.self_attn.q_proj.bias shape: [2560] -184: model.layers.2.self_attn.q_proj.weight shape: [2560, 2560] -185: model.layers.2.self_attn.v_proj.bias shape: [2560] -186: model.layers.2.self_attn.v_proj.weight shape: [2560, 2560] -187: model.layers.20.input_layernorm.bias shape: [2560] -188: model.layers.20.input_layernorm.weight shape: [2560] -189: model.layers.20.mlp.fc1.bias shape: [10240] -190: model.layers.20.mlp.fc1.weight shape: [10240, 2560] -191: model.layers.20.mlp.fc2.bias shape: [2560] -192: model.layers.20.mlp.fc2.weight shape: [2560, 10240] -193: model.layers.20.self_attn.dense.bias shape: [2560] -194: model.layers.20.self_attn.dense.weight shape: [2560, 2560] -195: model.layers.20.self_attn.k_proj.bias shape: [2560] -196: model.layers.20.self_attn.k_proj.weight shape: [2560, 2560] -197: model.layers.20.self_attn.q_proj.bias shape: [2560] -198: model.layers.20.self_attn.q_proj.weight shape: [2560, 2560] -199: model.layers.20.self_attn.v_proj.bias shape: [2560] -200: model.layers.20.self_attn.v_proj.weight shape: [2560, 2560] -201: model.layers.21.input_layernorm.bias shape: [2560] -202: model.layers.21.input_layernorm.weight shape: [2560] -203: model.layers.21.mlp.fc1.bias shape: [10240] -204: model.layers.21.mlp.fc1.weight shape: [10240, 2560] -205: model.layers.21.mlp.fc2.bias shape: [2560] -206: model.layers.21.mlp.fc2.weight shape: [2560, 10240] -207: model.layers.21.self_attn.dense.bias shape: [2560] -208: model.layers.21.self_attn.dense.weight shape: [2560, 2560] -209: model.layers.21.self_attn.k_proj.bias shape: [2560] -210: model.layers.21.self_attn.k_proj.weight shape: [2560, 2560] -211: model.layers.21.self_attn.q_proj.bias shape: [2560] -212: model.layers.21.self_attn.q_proj.weight shape: [2560, 2560] -213: model.layers.21.self_attn.v_proj.bias shape: [2560] -214: model.layers.21.self_attn.v_proj.weight shape: [2560, 2560] -215: model.layers.22.input_layernorm.bias shape: [2560] -216: model.layers.22.input_layernorm.weight shape: [2560] -217: model.layers.22.mlp.fc1.bias shape: [10240] -218: model.layers.22.mlp.fc1.weight shape: [10240, 2560] -219: model.layers.22.mlp.fc2.bias shape: [2560] -220: model.layers.22.mlp.fc2.weight shape: [2560, 10240] -221: model.layers.22.self_attn.dense.bias shape: [2560] -222: model.layers.22.self_attn.dense.weight shape: [2560, 2560] -223: model.layers.22.self_attn.k_proj.bias shape: [2560] -224: model.layers.22.self_attn.k_proj.weight shape: [2560, 2560] -225: model.layers.22.self_attn.q_proj.bias shape: [2560] -226: model.layers.22.self_attn.q_proj.weight shape: [2560, 2560] -227: model.layers.22.self_attn.v_proj.bias shape: [2560] -228: model.layers.22.self_attn.v_proj.weight shape: [2560, 2560] -229: model.layers.23.input_layernorm.bias shape: [2560] -230: model.layers.23.input_layernorm.weight shape: [2560] -231: model.layers.23.mlp.fc1.bias shape: [10240] -232: model.layers.23.mlp.fc1.weight shape: [10240, 2560] -233: model.layers.23.mlp.fc2.bias shape: [2560] -234: model.layers.23.mlp.fc2.weight shape: [2560, 10240] -235: model.layers.23.self_attn.dense.bias shape: [2560] -236: model.layers.23.self_attn.dense.weight shape: [2560, 2560] -237: model.layers.23.self_attn.k_proj.bias shape: [2560] -238: model.layers.23.self_attn.k_proj.weight shape: [2560, 2560] -239: model.layers.23.self_attn.q_proj.bias shape: [2560] -240: model.layers.23.self_attn.q_proj.weight shape: [2560, 2560] -241: model.layers.23.self_attn.v_proj.bias shape: [2560] -242: model.layers.23.self_attn.v_proj.weight shape: [2560, 2560] -243: model.layers.24.input_layernorm.bias shape: [2560] -244: model.layers.24.input_layernorm.weight shape: [2560] -245: model.layers.24.mlp.fc1.bias shape: [10240] -246: model.layers.24.mlp.fc1.weight shape: [10240, 2560] -247: model.layers.24.mlp.fc2.bias shape: [2560] -248: model.layers.24.mlp.fc2.weight shape: [2560, 10240] -249: model.layers.24.self_attn.dense.bias shape: [2560] -250: model.layers.24.self_attn.dense.weight shape: [2560, 2560] -251: model.layers.24.self_attn.k_proj.bias shape: [2560] -252: model.layers.24.self_attn.k_proj.weight shape: [2560, 2560] -253: model.layers.24.self_attn.q_proj.bias shape: [2560] -254: model.layers.24.self_attn.q_proj.weight shape: [2560, 2560] -255: model.layers.24.self_attn.v_proj.bias shape: [2560] -256: model.layers.24.self_attn.v_proj.weight shape: [2560, 2560] -257: model.layers.25.input_layernorm.bias shape: [2560] -258: model.layers.25.input_layernorm.weight shape: [2560] -259: model.layers.25.mlp.fc1.bias shape: [10240] -260: model.layers.25.mlp.fc1.weight shape: [10240, 2560] -261: model.layers.25.mlp.fc2.bias shape: [2560] -262: model.layers.25.mlp.fc2.weight shape: [2560, 10240] -263: model.layers.25.self_attn.dense.bias shape: [2560] -264: model.layers.25.self_attn.dense.weight shape: [2560, 2560] -265: model.layers.25.self_attn.k_proj.bias shape: [2560] -266: model.layers.25.self_attn.k_proj.weight shape: [2560, 2560] -267: model.layers.25.self_attn.q_proj.bias shape: [2560] -268: model.layers.25.self_attn.q_proj.weight shape: [2560, 2560] -269: model.layers.25.self_attn.v_proj.bias shape: [2560] -270: model.layers.25.self_attn.v_proj.weight shape: [2560, 2560] -271: model.layers.26.input_layernorm.bias shape: [2560] -272: model.layers.26.input_layernorm.weight shape: [2560] -273: model.layers.26.mlp.fc1.bias shape: [10240] -274: model.layers.26.mlp.fc1.weight shape: [10240, 2560] -275: model.layers.26.mlp.fc2.bias shape: [2560] -276: model.layers.26.mlp.fc2.weight shape: [2560, 10240] -277: model.layers.26.self_attn.dense.bias shape: [2560] -278: model.layers.26.self_attn.dense.weight shape: [2560, 2560] -279: model.layers.26.self_attn.k_proj.bias shape: [2560] -280: model.layers.26.self_attn.k_proj.weight shape: [2560, 2560] -281: model.layers.26.self_attn.q_proj.bias shape: [2560] -282: model.layers.26.self_attn.q_proj.weight shape: [2560, 2560] -283: model.layers.26.self_attn.v_proj.bias shape: [2560] -284: model.layers.26.self_attn.v_proj.weight shape: [2560, 2560] -285: model.layers.27.input_layernorm.bias shape: [2560] -286: model.layers.27.input_layernorm.weight shape: [2560] -287: model.layers.27.mlp.fc1.bias shape: [10240] -288: model.layers.27.mlp.fc1.weight shape: [10240, 2560] -289: model.layers.27.mlp.fc2.bias shape: [2560] -290: model.layers.27.mlp.fc2.weight shape: [2560, 10240] -291: model.layers.27.self_attn.dense.bias shape: [2560] -292: model.layers.27.self_attn.dense.weight shape: [2560, 2560] -293: model.layers.27.self_attn.k_proj.bias shape: [2560] -294: model.layers.27.self_attn.k_proj.weight shape: [2560, 2560] -295: model.layers.27.self_attn.q_proj.bias shape: [2560] -296: model.layers.27.self_attn.q_proj.weight shape: [2560, 2560] -297: model.layers.27.self_attn.v_proj.bias shape: [2560] -298: model.layers.27.self_attn.v_proj.weight shape: [2560, 2560] -299: model.layers.28.input_layernorm.bias shape: [2560] -300: model.layers.28.input_layernorm.weight shape: [2560] -301: model.layers.28.mlp.fc1.bias shape: [10240] -302: model.layers.28.mlp.fc1.weight shape: [10240, 2560] -303: model.layers.28.mlp.fc2.bias shape: [2560] -304: model.layers.28.mlp.fc2.weight shape: [2560, 10240] -305: model.layers.28.self_attn.dense.bias shape: [2560] -306: model.layers.28.self_attn.dense.weight shape: [2560, 2560] -307: model.layers.28.self_attn.k_proj.bias shape: [2560] -308: model.layers.28.self_attn.k_proj.weight shape: [2560, 2560] -309: model.layers.28.self_attn.q_proj.bias shape: [2560] -310: model.layers.28.self_attn.q_proj.weight shape: [2560, 2560] -311: model.layers.28.self_attn.v_proj.bias shape: [2560] -312: model.layers.28.self_attn.v_proj.weight shape: [2560, 2560] -313: model.layers.29.input_layernorm.bias shape: [2560] -314: model.layers.29.input_layernorm.weight shape: [2560] -315: model.layers.29.mlp.fc1.bias shape: [10240] -316: model.layers.29.mlp.fc1.weight shape: [10240, 2560] -317: model.layers.29.mlp.fc2.bias shape: [2560] -318: model.layers.29.mlp.fc2.weight shape: [2560, 10240] -319: model.layers.29.self_attn.dense.bias shape: [2560] -320: model.layers.29.self_attn.dense.weight shape: [2560, 2560] -321: model.layers.29.self_attn.k_proj.bias shape: [2560] -322: model.layers.29.self_attn.k_proj.weight shape: [2560, 2560] -323: model.layers.29.self_attn.q_proj.bias shape: [2560] -324: model.layers.29.self_attn.q_proj.weight shape: [2560, 2560] -325: model.layers.29.self_attn.v_proj.bias shape: [2560] -326: model.layers.29.self_attn.v_proj.weight shape: [2560, 2560] -327: model.layers.3.input_layernorm.bias shape: [2560] -328: model.layers.3.input_layernorm.weight shape: [2560] -329: model.layers.3.mlp.fc1.bias shape: [10240] -330: model.layers.3.mlp.fc1.weight shape: [10240, 2560] -331: model.layers.3.mlp.fc2.bias shape: [2560] -332: model.layers.3.mlp.fc2.weight shape: [2560, 10240] -333: model.layers.3.self_attn.dense.bias shape: [2560] -334: model.layers.3.self_attn.dense.weight shape: [2560, 2560] -335: model.layers.3.self_attn.k_proj.bias shape: [2560] -336: model.layers.3.self_attn.k_proj.weight shape: [2560, 2560] -337: model.layers.3.self_attn.q_proj.bias shape: [2560] -338: model.layers.3.self_attn.q_proj.weight shape: [2560, 2560] -339: model.layers.3.self_attn.v_proj.bias shape: [2560] -340: model.layers.3.self_attn.v_proj.weight shape: [2560, 2560] -341: model.layers.30.input_layernorm.bias shape: [2560] -342: model.layers.30.input_layernorm.weight shape: [2560] -343: model.layers.30.mlp.fc1.bias shape: [10240] -344: model.layers.30.mlp.fc1.weight shape: [10240, 2560] -345: model.layers.30.mlp.fc2.bias shape: [2560] -346: model.layers.30.mlp.fc2.weight shape: [2560, 10240] -347: model.layers.30.self_attn.dense.bias shape: [2560] -348: model.layers.30.self_attn.dense.weight shape: [2560, 2560] -349: model.layers.30.self_attn.k_proj.bias shape: [2560] -350: model.layers.30.self_attn.k_proj.weight shape: [2560, 2560] -351: model.layers.30.self_attn.q_proj.bias shape: [2560] -352: model.layers.30.self_attn.q_proj.weight shape: [2560, 2560] -353: model.layers.30.self_attn.v_proj.bias shape: [2560] -354: model.layers.30.self_attn.v_proj.weight shape: [2560, 2560] -355: model.layers.31.input_layernorm.bias shape: [2560] -356: model.layers.31.input_layernorm.weight shape: [2560] -357: model.layers.31.mlp.fc1.bias shape: [10240] -358: model.layers.31.mlp.fc1.weight shape: [10240, 2560] -359: model.layers.31.mlp.fc2.bias shape: [2560] -360: model.layers.31.mlp.fc2.weight shape: [2560, 10240] -361: model.layers.31.self_attn.dense.bias shape: [2560] -362: model.layers.31.self_attn.dense.weight shape: [2560, 2560] -363: model.layers.31.self_attn.k_proj.bias shape: [2560] -364: model.layers.31.self_attn.k_proj.weight shape: [2560, 2560] -365: model.layers.31.self_attn.q_proj.bias shape: [2560] -366: model.layers.31.self_attn.q_proj.weight shape: [2560, 2560] -367: model.layers.31.self_attn.v_proj.bias shape: [2560] -368: model.layers.31.self_attn.v_proj.weight shape: [2560, 2560] -369: model.layers.4.input_layernorm.bias shape: [2560] -370: model.layers.4.input_layernorm.weight shape: [2560] -371: model.layers.4.mlp.fc1.bias shape: [10240] -372: model.layers.4.mlp.fc1.weight shape: [10240, 2560] -373: model.layers.4.mlp.fc2.bias shape: [2560] -374: model.layers.4.mlp.fc2.weight shape: [2560, 10240] -375: model.layers.4.self_attn.dense.bias shape: [2560] -376: model.layers.4.self_attn.dense.weight shape: [2560, 2560] -377: model.layers.4.self_attn.k_proj.bias shape: [2560] -378: model.layers.4.self_attn.k_proj.weight shape: [2560, 2560] -379: model.layers.4.self_attn.q_proj.bias shape: [2560] -380: model.layers.4.self_attn.q_proj.weight shape: [2560, 2560] -381: model.layers.4.self_attn.v_proj.bias shape: [2560] -382: model.layers.4.self_attn.v_proj.weight shape: [2560, 2560] -383: model.layers.5.input_layernorm.bias shape: [2560] -384: model.layers.5.input_layernorm.weight shape: [2560] -385: model.layers.5.mlp.fc1.bias shape: [10240] -386: model.layers.5.mlp.fc1.weight shape: [10240, 2560] -387: model.layers.5.mlp.fc2.bias shape: [2560] -388: model.layers.5.mlp.fc2.weight shape: [2560, 10240] -389: model.layers.5.self_attn.dense.bias shape: [2560] -390: model.layers.5.self_attn.dense.weight shape: [2560, 2560] -391: model.layers.5.self_attn.k_proj.bias shape: [2560] -392: model.layers.5.self_attn.k_proj.weight shape: [2560, 2560] -393: model.layers.5.self_attn.q_proj.bias shape: [2560] -394: model.layers.5.self_attn.q_proj.weight shape: [2560, 2560] -395: model.layers.5.self_attn.v_proj.bias shape: [2560] -396: model.layers.5.self_attn.v_proj.weight shape: [2560, 2560] -397: model.layers.6.input_layernorm.bias shape: [2560] -398: model.layers.6.input_layernorm.weight shape: [2560] -399: model.layers.6.mlp.fc1.bias shape: [10240] -400: model.layers.6.mlp.fc1.weight shape: [10240, 2560] -401: model.layers.6.mlp.fc2.bias shape: [2560] -402: model.layers.6.mlp.fc2.weight shape: [2560, 10240] -403: model.layers.6.self_attn.dense.bias shape: [2560] -404: model.layers.6.self_attn.dense.weight shape: [2560, 2560] -405: model.layers.6.self_attn.k_proj.bias shape: [2560] -406: model.layers.6.self_attn.k_proj.weight shape: [2560, 2560] -407: model.layers.6.self_attn.q_proj.bias shape: [2560] -408: model.layers.6.self_attn.q_proj.weight shape: [2560, 2560] -409: model.layers.6.self_attn.v_proj.bias shape: [2560] -410: model.layers.6.self_attn.v_proj.weight shape: [2560, 2560] -411: model.layers.7.input_layernorm.bias shape: [2560] -412: model.layers.7.input_layernorm.weight shape: [2560] -413: model.layers.7.mlp.fc1.bias shape: [10240] -414: model.layers.7.mlp.fc1.weight shape: [10240, 2560] -415: model.layers.7.mlp.fc2.bias shape: [2560] -416: model.layers.7.mlp.fc2.weight shape: [2560, 10240] -417: model.layers.7.self_attn.dense.bias shape: [2560] -418: model.layers.7.self_attn.dense.weight shape: [2560, 2560] -419: model.layers.7.self_attn.k_proj.bias shape: [2560] -420: model.layers.7.self_attn.k_proj.weight shape: [2560, 2560] -421: model.layers.7.self_attn.q_proj.bias shape: [2560] -422: model.layers.7.self_attn.q_proj.weight shape: [2560, 2560] -423: model.layers.7.self_attn.v_proj.bias shape: [2560] -424: model.layers.7.self_attn.v_proj.weight shape: [2560, 2560] -425: model.layers.8.input_layernorm.bias shape: [2560] -426: model.layers.8.input_layernorm.weight shape: [2560] -427: model.layers.8.mlp.fc1.bias shape: [10240] -428: model.layers.8.mlp.fc1.weight shape: [10240, 2560] -429: model.layers.8.mlp.fc2.bias shape: [2560] -430: model.layers.8.mlp.fc2.weight shape: [2560, 10240] -431: model.layers.8.self_attn.dense.bias shape: [2560] -432: model.layers.8.self_attn.dense.weight shape: [2560, 2560] -433: model.layers.8.self_attn.k_proj.bias shape: [2560] -434: model.layers.8.self_attn.k_proj.weight shape: [2560, 2560] -435: model.layers.8.self_attn.q_proj.bias shape: [2560] -436: model.layers.8.self_attn.q_proj.weight shape: [2560, 2560] -437: model.layers.8.self_attn.v_proj.bias shape: [2560] -438: model.layers.8.self_attn.v_proj.weight shape: [2560, 2560] -439: model.layers.9.input_layernorm.bias shape: [2560] -440: model.layers.9.input_layernorm.weight shape: [2560] -441: model.layers.9.mlp.fc1.bias shape: [10240] -442: model.layers.9.mlp.fc1.weight shape: [10240, 2560] -443: model.layers.9.mlp.fc2.bias shape: [2560] -444: model.layers.9.mlp.fc2.weight shape: [2560, 10240] -445: model.layers.9.self_attn.dense.bias shape: [2560] -446: model.layers.9.self_attn.dense.weight shape: [2560, 2560] -447: model.layers.9.self_attn.k_proj.bias shape: [2560] -448: model.layers.9.self_attn.k_proj.weight shape: [2560, 2560] -449: model.layers.9.self_attn.q_proj.bias shape: [2560] -450: model.layers.9.self_attn.q_proj.weight shape: [2560, 2560] -451: model.layers.9.self_attn.v_proj.bias shape: [2560] -452: model.layers.9.self_attn.v_proj.weight shape: [2560, 2560] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.TokenizerTest.approved.txt deleted file mode 100644 index 7338548917..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi2Tests.TokenizerTest.approved.txt +++ /dev/null @@ -1,3 +0,0 @@ -50256, 6090, 345, 2148, 2842, 284, 4483, 17790, 286, 35484, 290, 10441, 69, 50187, 30 -50256, 19457, 0, 3423, 389, 617, 2842, 284, 4483, 35484, 290, 10441, 69, 50187, 1978, 25, 352, 13, 40058, 290, 10441, 34711, 7209, 494, 25, 41198, 35484, 290, 10441, 69, 50187, 1978, 351, 617, 7545, 290, 12498, 13, 362, 13, 40058, 290, 10441, 34711, 20698, 25, 15561, 26790, 35484, 290, 10441, 69, 50187, 1978, 351, 617, 18873, 13135, 290, 12498, 13 -50256, 2061, 546, 18120, 281, 362, 87, 1343, 513, 796, 767, 16022, 30 diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium128KShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium128KShapeTest.approved.txt deleted file mode 100644 index 277f686aa7..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium128KShapeTest.approved.txt +++ /dev/null @@ -1,243 +0,0 @@ -0: lm_head.weight shape: [32064, 5120] -1: model.embed_tokens.weight shape: [32064, 5120] -2: model.layers.0.input_layernorm.weight shape: [5120] -3: model.layers.0.mlp.down_proj.weight shape: [5120, 17920] -4: model.layers.0.mlp.gate_up_proj.weight shape: [35840, 5120] -5: model.layers.0.post_attention_layernorm.weight shape: [5120] -6: model.layers.0.self_attn.o_proj.weight shape: [5120, 5120] -7: model.layers.0.self_attn.qkv_proj.weight shape: [7680, 5120] -8: model.layers.1.input_layernorm.weight shape: [5120] -9: model.layers.1.mlp.down_proj.weight shape: [5120, 17920] -10: model.layers.1.mlp.gate_up_proj.weight shape: [35840, 5120] -11: model.layers.1.post_attention_layernorm.weight shape: [5120] -12: model.layers.1.self_attn.o_proj.weight shape: [5120, 5120] -13: model.layers.1.self_attn.qkv_proj.weight shape: [7680, 5120] -14: model.layers.10.input_layernorm.weight shape: [5120] -15: model.layers.10.mlp.down_proj.weight shape: [5120, 17920] -16: model.layers.10.mlp.gate_up_proj.weight shape: [35840, 5120] -17: model.layers.10.post_attention_layernorm.weight shape: [5120] -18: model.layers.10.self_attn.o_proj.weight shape: [5120, 5120] -19: model.layers.10.self_attn.qkv_proj.weight shape: [7680, 5120] -20: model.layers.11.input_layernorm.weight shape: [5120] -21: model.layers.11.mlp.down_proj.weight shape: [5120, 17920] -22: model.layers.11.mlp.gate_up_proj.weight shape: [35840, 5120] -23: model.layers.11.post_attention_layernorm.weight shape: [5120] -24: model.layers.11.self_attn.o_proj.weight shape: [5120, 5120] -25: model.layers.11.self_attn.qkv_proj.weight shape: [7680, 5120] -26: model.layers.12.input_layernorm.weight shape: [5120] -27: model.layers.12.mlp.down_proj.weight shape: [5120, 17920] -28: model.layers.12.mlp.gate_up_proj.weight shape: [35840, 5120] -29: model.layers.12.post_attention_layernorm.weight shape: [5120] -30: model.layers.12.self_attn.o_proj.weight shape: [5120, 5120] -31: model.layers.12.self_attn.qkv_proj.weight shape: [7680, 5120] -32: model.layers.13.input_layernorm.weight shape: [5120] -33: model.layers.13.mlp.down_proj.weight shape: [5120, 17920] -34: model.layers.13.mlp.gate_up_proj.weight shape: [35840, 5120] -35: model.layers.13.post_attention_layernorm.weight shape: [5120] -36: model.layers.13.self_attn.o_proj.weight shape: [5120, 5120] -37: model.layers.13.self_attn.qkv_proj.weight shape: [7680, 5120] -38: model.layers.14.input_layernorm.weight shape: [5120] -39: model.layers.14.mlp.down_proj.weight shape: [5120, 17920] -40: model.layers.14.mlp.gate_up_proj.weight shape: [35840, 5120] -41: model.layers.14.post_attention_layernorm.weight shape: [5120] -42: model.layers.14.self_attn.o_proj.weight shape: [5120, 5120] -43: model.layers.14.self_attn.qkv_proj.weight shape: [7680, 5120] -44: model.layers.15.input_layernorm.weight shape: [5120] -45: model.layers.15.mlp.down_proj.weight shape: [5120, 17920] -46: model.layers.15.mlp.gate_up_proj.weight shape: [35840, 5120] -47: model.layers.15.post_attention_layernorm.weight shape: [5120] -48: model.layers.15.self_attn.o_proj.weight shape: [5120, 5120] -49: model.layers.15.self_attn.qkv_proj.weight shape: [7680, 5120] -50: model.layers.16.input_layernorm.weight shape: [5120] -51: model.layers.16.mlp.down_proj.weight shape: [5120, 17920] -52: model.layers.16.mlp.gate_up_proj.weight shape: [35840, 5120] -53: model.layers.16.post_attention_layernorm.weight shape: [5120] -54: model.layers.16.self_attn.o_proj.weight shape: [5120, 5120] -55: model.layers.16.self_attn.qkv_proj.weight shape: [7680, 5120] -56: model.layers.17.input_layernorm.weight shape: [5120] -57: model.layers.17.mlp.down_proj.weight shape: [5120, 17920] -58: model.layers.17.mlp.gate_up_proj.weight shape: [35840, 5120] -59: model.layers.17.post_attention_layernorm.weight shape: [5120] -60: model.layers.17.self_attn.o_proj.weight shape: [5120, 5120] -61: model.layers.17.self_attn.qkv_proj.weight shape: [7680, 5120] -62: model.layers.18.input_layernorm.weight shape: [5120] -63: model.layers.18.mlp.down_proj.weight shape: [5120, 17920] -64: model.layers.18.mlp.gate_up_proj.weight shape: [35840, 5120] -65: model.layers.18.post_attention_layernorm.weight shape: [5120] -66: model.layers.18.self_attn.o_proj.weight shape: [5120, 5120] -67: model.layers.18.self_attn.qkv_proj.weight shape: [7680, 5120] -68: model.layers.19.input_layernorm.weight shape: [5120] -69: model.layers.19.mlp.down_proj.weight shape: [5120, 17920] -70: model.layers.19.mlp.gate_up_proj.weight shape: [35840, 5120] -71: model.layers.19.post_attention_layernorm.weight shape: [5120] -72: model.layers.19.self_attn.o_proj.weight shape: [5120, 5120] -73: model.layers.19.self_attn.qkv_proj.weight shape: [7680, 5120] -74: model.layers.2.input_layernorm.weight shape: [5120] -75: model.layers.2.mlp.down_proj.weight shape: [5120, 17920] -76: model.layers.2.mlp.gate_up_proj.weight shape: [35840, 5120] -77: model.layers.2.post_attention_layernorm.weight shape: [5120] -78: model.layers.2.self_attn.o_proj.weight shape: [5120, 5120] -79: model.layers.2.self_attn.qkv_proj.weight shape: [7680, 5120] -80: model.layers.20.input_layernorm.weight shape: [5120] -81: model.layers.20.mlp.down_proj.weight shape: [5120, 17920] -82: model.layers.20.mlp.gate_up_proj.weight shape: [35840, 5120] -83: model.layers.20.post_attention_layernorm.weight shape: [5120] -84: model.layers.20.self_attn.o_proj.weight shape: [5120, 5120] -85: model.layers.20.self_attn.qkv_proj.weight shape: [7680, 5120] -86: model.layers.21.input_layernorm.weight shape: [5120] -87: model.layers.21.mlp.down_proj.weight shape: [5120, 17920] -88: model.layers.21.mlp.gate_up_proj.weight shape: [35840, 5120] -89: model.layers.21.post_attention_layernorm.weight shape: [5120] -90: model.layers.21.self_attn.o_proj.weight shape: [5120, 5120] -91: model.layers.21.self_attn.qkv_proj.weight shape: [7680, 5120] -92: model.layers.22.input_layernorm.weight shape: [5120] -93: model.layers.22.mlp.down_proj.weight shape: [5120, 17920] -94: model.layers.22.mlp.gate_up_proj.weight shape: [35840, 5120] -95: model.layers.22.post_attention_layernorm.weight shape: [5120] -96: model.layers.22.self_attn.o_proj.weight shape: [5120, 5120] -97: model.layers.22.self_attn.qkv_proj.weight shape: [7680, 5120] -98: model.layers.23.input_layernorm.weight shape: [5120] -99: model.layers.23.mlp.down_proj.weight shape: [5120, 17920] -100: model.layers.23.mlp.gate_up_proj.weight shape: [35840, 5120] -101: model.layers.23.post_attention_layernorm.weight shape: [5120] -102: model.layers.23.self_attn.o_proj.weight shape: [5120, 5120] -103: model.layers.23.self_attn.qkv_proj.weight shape: [7680, 5120] -104: model.layers.24.input_layernorm.weight shape: [5120] -105: model.layers.24.mlp.down_proj.weight shape: [5120, 17920] -106: model.layers.24.mlp.gate_up_proj.weight shape: [35840, 5120] -107: model.layers.24.post_attention_layernorm.weight shape: [5120] -108: model.layers.24.self_attn.o_proj.weight shape: [5120, 5120] -109: model.layers.24.self_attn.qkv_proj.weight shape: [7680, 5120] -110: model.layers.25.input_layernorm.weight shape: [5120] -111: model.layers.25.mlp.down_proj.weight shape: [5120, 17920] -112: model.layers.25.mlp.gate_up_proj.weight shape: [35840, 5120] -113: model.layers.25.post_attention_layernorm.weight shape: [5120] -114: model.layers.25.self_attn.o_proj.weight shape: [5120, 5120] -115: model.layers.25.self_attn.qkv_proj.weight shape: [7680, 5120] -116: model.layers.26.input_layernorm.weight shape: [5120] -117: model.layers.26.mlp.down_proj.weight shape: [5120, 17920] -118: model.layers.26.mlp.gate_up_proj.weight shape: [35840, 5120] -119: model.layers.26.post_attention_layernorm.weight shape: [5120] -120: model.layers.26.self_attn.o_proj.weight shape: [5120, 5120] -121: model.layers.26.self_attn.qkv_proj.weight shape: [7680, 5120] -122: model.layers.27.input_layernorm.weight shape: [5120] -123: model.layers.27.mlp.down_proj.weight shape: [5120, 17920] -124: model.layers.27.mlp.gate_up_proj.weight shape: [35840, 5120] -125: model.layers.27.post_attention_layernorm.weight shape: [5120] -126: model.layers.27.self_attn.o_proj.weight shape: [5120, 5120] -127: model.layers.27.self_attn.qkv_proj.weight shape: [7680, 5120] -128: model.layers.28.input_layernorm.weight shape: [5120] -129: model.layers.28.mlp.down_proj.weight shape: [5120, 17920] -130: model.layers.28.mlp.gate_up_proj.weight shape: [35840, 5120] -131: model.layers.28.post_attention_layernorm.weight shape: [5120] -132: model.layers.28.self_attn.o_proj.weight shape: [5120, 5120] -133: model.layers.28.self_attn.qkv_proj.weight shape: [7680, 5120] -134: model.layers.29.input_layernorm.weight shape: [5120] -135: model.layers.29.mlp.down_proj.weight shape: [5120, 17920] -136: model.layers.29.mlp.gate_up_proj.weight shape: [35840, 5120] -137: model.layers.29.post_attention_layernorm.weight shape: [5120] -138: model.layers.29.self_attn.o_proj.weight shape: [5120, 5120] -139: model.layers.29.self_attn.qkv_proj.weight shape: [7680, 5120] -140: model.layers.3.input_layernorm.weight shape: [5120] -141: model.layers.3.mlp.down_proj.weight shape: [5120, 17920] -142: model.layers.3.mlp.gate_up_proj.weight shape: [35840, 5120] -143: model.layers.3.post_attention_layernorm.weight shape: [5120] -144: model.layers.3.self_attn.o_proj.weight shape: [5120, 5120] -145: model.layers.3.self_attn.qkv_proj.weight shape: [7680, 5120] -146: model.layers.30.input_layernorm.weight shape: [5120] -147: model.layers.30.mlp.down_proj.weight shape: [5120, 17920] -148: model.layers.30.mlp.gate_up_proj.weight shape: [35840, 5120] -149: model.layers.30.post_attention_layernorm.weight shape: [5120] -150: model.layers.30.self_attn.o_proj.weight shape: [5120, 5120] -151: model.layers.30.self_attn.qkv_proj.weight shape: [7680, 5120] -152: model.layers.31.input_layernorm.weight shape: [5120] -153: model.layers.31.mlp.down_proj.weight shape: [5120, 17920] -154: model.layers.31.mlp.gate_up_proj.weight shape: [35840, 5120] -155: model.layers.31.post_attention_layernorm.weight shape: [5120] -156: model.layers.31.self_attn.o_proj.weight shape: [5120, 5120] -157: model.layers.31.self_attn.qkv_proj.weight shape: [7680, 5120] -158: model.layers.32.input_layernorm.weight shape: [5120] -159: model.layers.32.mlp.down_proj.weight shape: [5120, 17920] -160: model.layers.32.mlp.gate_up_proj.weight shape: [35840, 5120] -161: model.layers.32.post_attention_layernorm.weight shape: [5120] -162: model.layers.32.self_attn.o_proj.weight shape: [5120, 5120] -163: model.layers.32.self_attn.qkv_proj.weight shape: [7680, 5120] -164: model.layers.33.input_layernorm.weight shape: [5120] -165: model.layers.33.mlp.down_proj.weight shape: [5120, 17920] -166: model.layers.33.mlp.gate_up_proj.weight shape: [35840, 5120] -167: model.layers.33.post_attention_layernorm.weight shape: [5120] -168: model.layers.33.self_attn.o_proj.weight shape: [5120, 5120] -169: model.layers.33.self_attn.qkv_proj.weight shape: [7680, 5120] -170: model.layers.34.input_layernorm.weight shape: [5120] -171: model.layers.34.mlp.down_proj.weight shape: [5120, 17920] -172: model.layers.34.mlp.gate_up_proj.weight shape: [35840, 5120] -173: model.layers.34.post_attention_layernorm.weight shape: [5120] -174: model.layers.34.self_attn.o_proj.weight shape: [5120, 5120] -175: model.layers.34.self_attn.qkv_proj.weight shape: [7680, 5120] -176: model.layers.35.input_layernorm.weight shape: [5120] -177: model.layers.35.mlp.down_proj.weight shape: [5120, 17920] -178: model.layers.35.mlp.gate_up_proj.weight shape: [35840, 5120] -179: model.layers.35.post_attention_layernorm.weight shape: [5120] -180: model.layers.35.self_attn.o_proj.weight shape: [5120, 5120] -181: model.layers.35.self_attn.qkv_proj.weight shape: [7680, 5120] -182: model.layers.36.input_layernorm.weight shape: [5120] -183: model.layers.36.mlp.down_proj.weight shape: [5120, 17920] -184: model.layers.36.mlp.gate_up_proj.weight shape: [35840, 5120] -185: model.layers.36.post_attention_layernorm.weight shape: [5120] -186: model.layers.36.self_attn.o_proj.weight shape: [5120, 5120] -187: model.layers.36.self_attn.qkv_proj.weight shape: [7680, 5120] -188: model.layers.37.input_layernorm.weight shape: [5120] -189: model.layers.37.mlp.down_proj.weight shape: [5120, 17920] -190: model.layers.37.mlp.gate_up_proj.weight shape: [35840, 5120] -191: model.layers.37.post_attention_layernorm.weight shape: [5120] -192: model.layers.37.self_attn.o_proj.weight shape: [5120, 5120] -193: model.layers.37.self_attn.qkv_proj.weight shape: [7680, 5120] -194: model.layers.38.input_layernorm.weight shape: [5120] -195: model.layers.38.mlp.down_proj.weight shape: [5120, 17920] -196: model.layers.38.mlp.gate_up_proj.weight shape: [35840, 5120] -197: model.layers.38.post_attention_layernorm.weight shape: [5120] -198: model.layers.38.self_attn.o_proj.weight shape: [5120, 5120] -199: model.layers.38.self_attn.qkv_proj.weight shape: [7680, 5120] -200: model.layers.39.input_layernorm.weight shape: [5120] -201: model.layers.39.mlp.down_proj.weight shape: [5120, 17920] -202: model.layers.39.mlp.gate_up_proj.weight shape: [35840, 5120] -203: model.layers.39.post_attention_layernorm.weight shape: [5120] -204: model.layers.39.self_attn.o_proj.weight shape: [5120, 5120] -205: model.layers.39.self_attn.qkv_proj.weight shape: [7680, 5120] -206: model.layers.4.input_layernorm.weight shape: [5120] -207: model.layers.4.mlp.down_proj.weight shape: [5120, 17920] -208: model.layers.4.mlp.gate_up_proj.weight shape: [35840, 5120] -209: model.layers.4.post_attention_layernorm.weight shape: [5120] -210: model.layers.4.self_attn.o_proj.weight shape: [5120, 5120] -211: model.layers.4.self_attn.qkv_proj.weight shape: [7680, 5120] -212: model.layers.5.input_layernorm.weight shape: [5120] -213: model.layers.5.mlp.down_proj.weight shape: [5120, 17920] -214: model.layers.5.mlp.gate_up_proj.weight shape: [35840, 5120] -215: model.layers.5.post_attention_layernorm.weight shape: [5120] -216: model.layers.5.self_attn.o_proj.weight shape: [5120, 5120] -217: model.layers.5.self_attn.qkv_proj.weight shape: [7680, 5120] -218: model.layers.6.input_layernorm.weight shape: [5120] -219: model.layers.6.mlp.down_proj.weight shape: [5120, 17920] -220: model.layers.6.mlp.gate_up_proj.weight shape: [35840, 5120] -221: model.layers.6.post_attention_layernorm.weight shape: [5120] -222: model.layers.6.self_attn.o_proj.weight shape: [5120, 5120] -223: model.layers.6.self_attn.qkv_proj.weight shape: [7680, 5120] -224: model.layers.7.input_layernorm.weight shape: [5120] -225: model.layers.7.mlp.down_proj.weight shape: [5120, 17920] -226: model.layers.7.mlp.gate_up_proj.weight shape: [35840, 5120] -227: model.layers.7.post_attention_layernorm.weight shape: [5120] -228: model.layers.7.self_attn.o_proj.weight shape: [5120, 5120] -229: model.layers.7.self_attn.qkv_proj.weight shape: [7680, 5120] -230: model.layers.8.input_layernorm.weight shape: [5120] -231: model.layers.8.mlp.down_proj.weight shape: [5120, 17920] -232: model.layers.8.mlp.gate_up_proj.weight shape: [35840, 5120] -233: model.layers.8.post_attention_layernorm.weight shape: [5120] -234: model.layers.8.self_attn.o_proj.weight shape: [5120, 5120] -235: model.layers.8.self_attn.qkv_proj.weight shape: [7680, 5120] -236: model.layers.9.input_layernorm.weight shape: [5120] -237: model.layers.9.mlp.down_proj.weight shape: [5120, 17920] -238: model.layers.9.mlp.gate_up_proj.weight shape: [35840, 5120] -239: model.layers.9.post_attention_layernorm.weight shape: [5120] -240: model.layers.9.self_attn.o_proj.weight shape: [5120, 5120] -241: model.layers.9.self_attn.qkv_proj.weight shape: [7680, 5120] -242: model.norm.weight shape: [5120] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium4KShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium4KShapeTest.approved.txt deleted file mode 100644 index 277f686aa7..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Medium4KShapeTest.approved.txt +++ /dev/null @@ -1,243 +0,0 @@ -0: lm_head.weight shape: [32064, 5120] -1: model.embed_tokens.weight shape: [32064, 5120] -2: model.layers.0.input_layernorm.weight shape: [5120] -3: model.layers.0.mlp.down_proj.weight shape: [5120, 17920] -4: model.layers.0.mlp.gate_up_proj.weight shape: [35840, 5120] -5: model.layers.0.post_attention_layernorm.weight shape: [5120] -6: model.layers.0.self_attn.o_proj.weight shape: [5120, 5120] -7: model.layers.0.self_attn.qkv_proj.weight shape: [7680, 5120] -8: model.layers.1.input_layernorm.weight shape: [5120] -9: model.layers.1.mlp.down_proj.weight shape: [5120, 17920] -10: model.layers.1.mlp.gate_up_proj.weight shape: [35840, 5120] -11: model.layers.1.post_attention_layernorm.weight shape: [5120] -12: model.layers.1.self_attn.o_proj.weight shape: [5120, 5120] -13: model.layers.1.self_attn.qkv_proj.weight shape: [7680, 5120] -14: model.layers.10.input_layernorm.weight shape: [5120] -15: model.layers.10.mlp.down_proj.weight shape: [5120, 17920] -16: model.layers.10.mlp.gate_up_proj.weight shape: [35840, 5120] -17: model.layers.10.post_attention_layernorm.weight shape: [5120] -18: model.layers.10.self_attn.o_proj.weight shape: [5120, 5120] -19: model.layers.10.self_attn.qkv_proj.weight shape: [7680, 5120] -20: model.layers.11.input_layernorm.weight shape: [5120] -21: model.layers.11.mlp.down_proj.weight shape: [5120, 17920] -22: model.layers.11.mlp.gate_up_proj.weight shape: [35840, 5120] -23: model.layers.11.post_attention_layernorm.weight shape: [5120] -24: model.layers.11.self_attn.o_proj.weight shape: [5120, 5120] -25: model.layers.11.self_attn.qkv_proj.weight shape: [7680, 5120] -26: model.layers.12.input_layernorm.weight shape: [5120] -27: model.layers.12.mlp.down_proj.weight shape: [5120, 17920] -28: model.layers.12.mlp.gate_up_proj.weight shape: [35840, 5120] -29: model.layers.12.post_attention_layernorm.weight shape: [5120] -30: model.layers.12.self_attn.o_proj.weight shape: [5120, 5120] -31: model.layers.12.self_attn.qkv_proj.weight shape: [7680, 5120] -32: model.layers.13.input_layernorm.weight shape: [5120] -33: model.layers.13.mlp.down_proj.weight shape: [5120, 17920] -34: model.layers.13.mlp.gate_up_proj.weight shape: [35840, 5120] -35: model.layers.13.post_attention_layernorm.weight shape: [5120] -36: model.layers.13.self_attn.o_proj.weight shape: [5120, 5120] -37: model.layers.13.self_attn.qkv_proj.weight shape: [7680, 5120] -38: model.layers.14.input_layernorm.weight shape: [5120] -39: model.layers.14.mlp.down_proj.weight shape: [5120, 17920] -40: model.layers.14.mlp.gate_up_proj.weight shape: [35840, 5120] -41: model.layers.14.post_attention_layernorm.weight shape: [5120] -42: model.layers.14.self_attn.o_proj.weight shape: [5120, 5120] -43: model.layers.14.self_attn.qkv_proj.weight shape: [7680, 5120] -44: model.layers.15.input_layernorm.weight shape: [5120] -45: model.layers.15.mlp.down_proj.weight shape: [5120, 17920] -46: model.layers.15.mlp.gate_up_proj.weight shape: [35840, 5120] -47: model.layers.15.post_attention_layernorm.weight shape: [5120] -48: model.layers.15.self_attn.o_proj.weight shape: [5120, 5120] -49: model.layers.15.self_attn.qkv_proj.weight shape: [7680, 5120] -50: model.layers.16.input_layernorm.weight shape: [5120] -51: model.layers.16.mlp.down_proj.weight shape: [5120, 17920] -52: model.layers.16.mlp.gate_up_proj.weight shape: [35840, 5120] -53: model.layers.16.post_attention_layernorm.weight shape: [5120] -54: model.layers.16.self_attn.o_proj.weight shape: [5120, 5120] -55: model.layers.16.self_attn.qkv_proj.weight shape: [7680, 5120] -56: model.layers.17.input_layernorm.weight shape: [5120] -57: model.layers.17.mlp.down_proj.weight shape: [5120, 17920] -58: model.layers.17.mlp.gate_up_proj.weight shape: [35840, 5120] -59: model.layers.17.post_attention_layernorm.weight shape: [5120] -60: model.layers.17.self_attn.o_proj.weight shape: [5120, 5120] -61: model.layers.17.self_attn.qkv_proj.weight shape: [7680, 5120] -62: model.layers.18.input_layernorm.weight shape: [5120] -63: model.layers.18.mlp.down_proj.weight shape: [5120, 17920] -64: model.layers.18.mlp.gate_up_proj.weight shape: [35840, 5120] -65: model.layers.18.post_attention_layernorm.weight shape: [5120] -66: model.layers.18.self_attn.o_proj.weight shape: [5120, 5120] -67: model.layers.18.self_attn.qkv_proj.weight shape: [7680, 5120] -68: model.layers.19.input_layernorm.weight shape: [5120] -69: model.layers.19.mlp.down_proj.weight shape: [5120, 17920] -70: model.layers.19.mlp.gate_up_proj.weight shape: [35840, 5120] -71: model.layers.19.post_attention_layernorm.weight shape: [5120] -72: model.layers.19.self_attn.o_proj.weight shape: [5120, 5120] -73: model.layers.19.self_attn.qkv_proj.weight shape: [7680, 5120] -74: model.layers.2.input_layernorm.weight shape: [5120] -75: model.layers.2.mlp.down_proj.weight shape: [5120, 17920] -76: model.layers.2.mlp.gate_up_proj.weight shape: [35840, 5120] -77: model.layers.2.post_attention_layernorm.weight shape: [5120] -78: model.layers.2.self_attn.o_proj.weight shape: [5120, 5120] -79: model.layers.2.self_attn.qkv_proj.weight shape: [7680, 5120] -80: model.layers.20.input_layernorm.weight shape: [5120] -81: model.layers.20.mlp.down_proj.weight shape: [5120, 17920] -82: model.layers.20.mlp.gate_up_proj.weight shape: [35840, 5120] -83: model.layers.20.post_attention_layernorm.weight shape: [5120] -84: model.layers.20.self_attn.o_proj.weight shape: [5120, 5120] -85: model.layers.20.self_attn.qkv_proj.weight shape: [7680, 5120] -86: model.layers.21.input_layernorm.weight shape: [5120] -87: model.layers.21.mlp.down_proj.weight shape: [5120, 17920] -88: model.layers.21.mlp.gate_up_proj.weight shape: [35840, 5120] -89: model.layers.21.post_attention_layernorm.weight shape: [5120] -90: model.layers.21.self_attn.o_proj.weight shape: [5120, 5120] -91: model.layers.21.self_attn.qkv_proj.weight shape: [7680, 5120] -92: model.layers.22.input_layernorm.weight shape: [5120] -93: model.layers.22.mlp.down_proj.weight shape: [5120, 17920] -94: model.layers.22.mlp.gate_up_proj.weight shape: [35840, 5120] -95: model.layers.22.post_attention_layernorm.weight shape: [5120] -96: model.layers.22.self_attn.o_proj.weight shape: [5120, 5120] -97: model.layers.22.self_attn.qkv_proj.weight shape: [7680, 5120] -98: model.layers.23.input_layernorm.weight shape: [5120] -99: model.layers.23.mlp.down_proj.weight shape: [5120, 17920] -100: model.layers.23.mlp.gate_up_proj.weight shape: [35840, 5120] -101: model.layers.23.post_attention_layernorm.weight shape: [5120] -102: model.layers.23.self_attn.o_proj.weight shape: [5120, 5120] -103: model.layers.23.self_attn.qkv_proj.weight shape: [7680, 5120] -104: model.layers.24.input_layernorm.weight shape: [5120] -105: model.layers.24.mlp.down_proj.weight shape: [5120, 17920] -106: model.layers.24.mlp.gate_up_proj.weight shape: [35840, 5120] -107: model.layers.24.post_attention_layernorm.weight shape: [5120] -108: model.layers.24.self_attn.o_proj.weight shape: [5120, 5120] -109: model.layers.24.self_attn.qkv_proj.weight shape: [7680, 5120] -110: model.layers.25.input_layernorm.weight shape: [5120] -111: model.layers.25.mlp.down_proj.weight shape: [5120, 17920] -112: model.layers.25.mlp.gate_up_proj.weight shape: [35840, 5120] -113: model.layers.25.post_attention_layernorm.weight shape: [5120] -114: model.layers.25.self_attn.o_proj.weight shape: [5120, 5120] -115: model.layers.25.self_attn.qkv_proj.weight shape: [7680, 5120] -116: model.layers.26.input_layernorm.weight shape: [5120] -117: model.layers.26.mlp.down_proj.weight shape: [5120, 17920] -118: model.layers.26.mlp.gate_up_proj.weight shape: [35840, 5120] -119: model.layers.26.post_attention_layernorm.weight shape: [5120] -120: model.layers.26.self_attn.o_proj.weight shape: [5120, 5120] -121: model.layers.26.self_attn.qkv_proj.weight shape: [7680, 5120] -122: model.layers.27.input_layernorm.weight shape: [5120] -123: model.layers.27.mlp.down_proj.weight shape: [5120, 17920] -124: model.layers.27.mlp.gate_up_proj.weight shape: [35840, 5120] -125: model.layers.27.post_attention_layernorm.weight shape: [5120] -126: model.layers.27.self_attn.o_proj.weight shape: [5120, 5120] -127: model.layers.27.self_attn.qkv_proj.weight shape: [7680, 5120] -128: model.layers.28.input_layernorm.weight shape: [5120] -129: model.layers.28.mlp.down_proj.weight shape: [5120, 17920] -130: model.layers.28.mlp.gate_up_proj.weight shape: [35840, 5120] -131: model.layers.28.post_attention_layernorm.weight shape: [5120] -132: model.layers.28.self_attn.o_proj.weight shape: [5120, 5120] -133: model.layers.28.self_attn.qkv_proj.weight shape: [7680, 5120] -134: model.layers.29.input_layernorm.weight shape: [5120] -135: model.layers.29.mlp.down_proj.weight shape: [5120, 17920] -136: model.layers.29.mlp.gate_up_proj.weight shape: [35840, 5120] -137: model.layers.29.post_attention_layernorm.weight shape: [5120] -138: model.layers.29.self_attn.o_proj.weight shape: [5120, 5120] -139: model.layers.29.self_attn.qkv_proj.weight shape: [7680, 5120] -140: model.layers.3.input_layernorm.weight shape: [5120] -141: model.layers.3.mlp.down_proj.weight shape: [5120, 17920] -142: model.layers.3.mlp.gate_up_proj.weight shape: [35840, 5120] -143: model.layers.3.post_attention_layernorm.weight shape: [5120] -144: model.layers.3.self_attn.o_proj.weight shape: [5120, 5120] -145: model.layers.3.self_attn.qkv_proj.weight shape: [7680, 5120] -146: model.layers.30.input_layernorm.weight shape: [5120] -147: model.layers.30.mlp.down_proj.weight shape: [5120, 17920] -148: model.layers.30.mlp.gate_up_proj.weight shape: [35840, 5120] -149: model.layers.30.post_attention_layernorm.weight shape: [5120] -150: model.layers.30.self_attn.o_proj.weight shape: [5120, 5120] -151: model.layers.30.self_attn.qkv_proj.weight shape: [7680, 5120] -152: model.layers.31.input_layernorm.weight shape: [5120] -153: model.layers.31.mlp.down_proj.weight shape: [5120, 17920] -154: model.layers.31.mlp.gate_up_proj.weight shape: [35840, 5120] -155: model.layers.31.post_attention_layernorm.weight shape: [5120] -156: model.layers.31.self_attn.o_proj.weight shape: [5120, 5120] -157: model.layers.31.self_attn.qkv_proj.weight shape: [7680, 5120] -158: model.layers.32.input_layernorm.weight shape: [5120] -159: model.layers.32.mlp.down_proj.weight shape: [5120, 17920] -160: model.layers.32.mlp.gate_up_proj.weight shape: [35840, 5120] -161: model.layers.32.post_attention_layernorm.weight shape: [5120] -162: model.layers.32.self_attn.o_proj.weight shape: [5120, 5120] -163: model.layers.32.self_attn.qkv_proj.weight shape: [7680, 5120] -164: model.layers.33.input_layernorm.weight shape: [5120] -165: model.layers.33.mlp.down_proj.weight shape: [5120, 17920] -166: model.layers.33.mlp.gate_up_proj.weight shape: [35840, 5120] -167: model.layers.33.post_attention_layernorm.weight shape: [5120] -168: model.layers.33.self_attn.o_proj.weight shape: [5120, 5120] -169: model.layers.33.self_attn.qkv_proj.weight shape: [7680, 5120] -170: model.layers.34.input_layernorm.weight shape: [5120] -171: model.layers.34.mlp.down_proj.weight shape: [5120, 17920] -172: model.layers.34.mlp.gate_up_proj.weight shape: [35840, 5120] -173: model.layers.34.post_attention_layernorm.weight shape: [5120] -174: model.layers.34.self_attn.o_proj.weight shape: [5120, 5120] -175: model.layers.34.self_attn.qkv_proj.weight shape: [7680, 5120] -176: model.layers.35.input_layernorm.weight shape: [5120] -177: model.layers.35.mlp.down_proj.weight shape: [5120, 17920] -178: model.layers.35.mlp.gate_up_proj.weight shape: [35840, 5120] -179: model.layers.35.post_attention_layernorm.weight shape: [5120] -180: model.layers.35.self_attn.o_proj.weight shape: [5120, 5120] -181: model.layers.35.self_attn.qkv_proj.weight shape: [7680, 5120] -182: model.layers.36.input_layernorm.weight shape: [5120] -183: model.layers.36.mlp.down_proj.weight shape: [5120, 17920] -184: model.layers.36.mlp.gate_up_proj.weight shape: [35840, 5120] -185: model.layers.36.post_attention_layernorm.weight shape: [5120] -186: model.layers.36.self_attn.o_proj.weight shape: [5120, 5120] -187: model.layers.36.self_attn.qkv_proj.weight shape: [7680, 5120] -188: model.layers.37.input_layernorm.weight shape: [5120] -189: model.layers.37.mlp.down_proj.weight shape: [5120, 17920] -190: model.layers.37.mlp.gate_up_proj.weight shape: [35840, 5120] -191: model.layers.37.post_attention_layernorm.weight shape: [5120] -192: model.layers.37.self_attn.o_proj.weight shape: [5120, 5120] -193: model.layers.37.self_attn.qkv_proj.weight shape: [7680, 5120] -194: model.layers.38.input_layernorm.weight shape: [5120] -195: model.layers.38.mlp.down_proj.weight shape: [5120, 17920] -196: model.layers.38.mlp.gate_up_proj.weight shape: [35840, 5120] -197: model.layers.38.post_attention_layernorm.weight shape: [5120] -198: model.layers.38.self_attn.o_proj.weight shape: [5120, 5120] -199: model.layers.38.self_attn.qkv_proj.weight shape: [7680, 5120] -200: model.layers.39.input_layernorm.weight shape: [5120] -201: model.layers.39.mlp.down_proj.weight shape: [5120, 17920] -202: model.layers.39.mlp.gate_up_proj.weight shape: [35840, 5120] -203: model.layers.39.post_attention_layernorm.weight shape: [5120] -204: model.layers.39.self_attn.o_proj.weight shape: [5120, 5120] -205: model.layers.39.self_attn.qkv_proj.weight shape: [7680, 5120] -206: model.layers.4.input_layernorm.weight shape: [5120] -207: model.layers.4.mlp.down_proj.weight shape: [5120, 17920] -208: model.layers.4.mlp.gate_up_proj.weight shape: [35840, 5120] -209: model.layers.4.post_attention_layernorm.weight shape: [5120] -210: model.layers.4.self_attn.o_proj.weight shape: [5120, 5120] -211: model.layers.4.self_attn.qkv_proj.weight shape: [7680, 5120] -212: model.layers.5.input_layernorm.weight shape: [5120] -213: model.layers.5.mlp.down_proj.weight shape: [5120, 17920] -214: model.layers.5.mlp.gate_up_proj.weight shape: [35840, 5120] -215: model.layers.5.post_attention_layernorm.weight shape: [5120] -216: model.layers.5.self_attn.o_proj.weight shape: [5120, 5120] -217: model.layers.5.self_attn.qkv_proj.weight shape: [7680, 5120] -218: model.layers.6.input_layernorm.weight shape: [5120] -219: model.layers.6.mlp.down_proj.weight shape: [5120, 17920] -220: model.layers.6.mlp.gate_up_proj.weight shape: [35840, 5120] -221: model.layers.6.post_attention_layernorm.weight shape: [5120] -222: model.layers.6.self_attn.o_proj.weight shape: [5120, 5120] -223: model.layers.6.self_attn.qkv_proj.weight shape: [7680, 5120] -224: model.layers.7.input_layernorm.weight shape: [5120] -225: model.layers.7.mlp.down_proj.weight shape: [5120, 17920] -226: model.layers.7.mlp.gate_up_proj.weight shape: [35840, 5120] -227: model.layers.7.post_attention_layernorm.weight shape: [5120] -228: model.layers.7.self_attn.o_proj.weight shape: [5120, 5120] -229: model.layers.7.self_attn.qkv_proj.weight shape: [7680, 5120] -230: model.layers.8.input_layernorm.weight shape: [5120] -231: model.layers.8.mlp.down_proj.weight shape: [5120, 17920] -232: model.layers.8.mlp.gate_up_proj.weight shape: [35840, 5120] -233: model.layers.8.post_attention_layernorm.weight shape: [5120] -234: model.layers.8.self_attn.o_proj.weight shape: [5120, 5120] -235: model.layers.8.self_attn.qkv_proj.weight shape: [7680, 5120] -236: model.layers.9.input_layernorm.weight shape: [5120] -237: model.layers.9.mlp.down_proj.weight shape: [5120, 17920] -238: model.layers.9.mlp.gate_up_proj.weight shape: [35840, 5120] -239: model.layers.9.post_attention_layernorm.weight shape: [5120] -240: model.layers.9.self_attn.o_proj.weight shape: [5120, 5120] -241: model.layers.9.self_attn.qkv_proj.weight shape: [7680, 5120] -242: model.norm.weight shape: [5120] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KLayerSizeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KLayerSizeTest.approved.txt deleted file mode 100644 index 4f711239a7..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KLayerSizeTest.approved.txt +++ /dev/null @@ -1,34 +0,0 @@ -{ - "model.layers.0": 216, - "model.layers.1": 216, - "model.layers.2": 216, - "model.layers.3": 216, - "model.layers.4": 216, - "model.layers.5": 216, - "model.layers.6": 216, - "model.layers.7": 216, - "model.layers.8": 216, - "model.layers.9": 216, - "model.layers.10": 216, - "model.layers.11": 216, - "model.layers.12": 216, - "model.layers.13": 216, - "model.layers.14": 216, - "model.layers.15": 216, - "model.layers.16": 216, - "model.layers.17": 216, - "model.layers.18": 216, - "model.layers.19": 216, - "model.layers.20": 216, - "model.layers.21": 216, - "model.layers.22": 216, - "model.layers.23": 216, - "model.layers.24": 216, - "model.layers.25": 216, - "model.layers.26": 216, - "model.layers.27": 216, - "model.layers.28": 216, - "model.layers.29": 216, - "model.layers.30": 216, - "model.layers.31": 216 -} \ No newline at end of file diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KShapeTest.approved.txt deleted file mode 100644 index 2278f3b67d..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini128KShapeTest.approved.txt +++ /dev/null @@ -1,195 +0,0 @@ -0: lm_head.weight shape: [32064, 3072] -1: model.embed_tokens.weight shape: [32064, 3072] -2: model.layers.0.input_layernorm.weight shape: [3072] -3: model.layers.0.mlp.down_proj.weight shape: [3072, 8192] -4: model.layers.0.mlp.gate_up_proj.weight shape: [16384, 3072] -5: model.layers.0.post_attention_layernorm.weight shape: [3072] -6: model.layers.0.self_attn.o_proj.weight shape: [3072, 3072] -7: model.layers.0.self_attn.qkv_proj.weight shape: [9216, 3072] -8: model.layers.1.input_layernorm.weight shape: [3072] -9: model.layers.1.mlp.down_proj.weight shape: [3072, 8192] -10: model.layers.1.mlp.gate_up_proj.weight shape: [16384, 3072] -11: model.layers.1.post_attention_layernorm.weight shape: [3072] -12: model.layers.1.self_attn.o_proj.weight shape: [3072, 3072] -13: model.layers.1.self_attn.qkv_proj.weight shape: [9216, 3072] -14: model.layers.10.input_layernorm.weight shape: [3072] -15: model.layers.10.mlp.down_proj.weight shape: [3072, 8192] -16: model.layers.10.mlp.gate_up_proj.weight shape: [16384, 3072] -17: model.layers.10.post_attention_layernorm.weight shape: [3072] -18: model.layers.10.self_attn.o_proj.weight shape: [3072, 3072] -19: model.layers.10.self_attn.qkv_proj.weight shape: [9216, 3072] -20: model.layers.11.input_layernorm.weight shape: [3072] -21: model.layers.11.mlp.down_proj.weight shape: [3072, 8192] -22: model.layers.11.mlp.gate_up_proj.weight shape: [16384, 3072] -23: model.layers.11.post_attention_layernorm.weight shape: [3072] -24: model.layers.11.self_attn.o_proj.weight shape: [3072, 3072] -25: model.layers.11.self_attn.qkv_proj.weight shape: [9216, 3072] -26: model.layers.12.input_layernorm.weight shape: [3072] -27: model.layers.12.mlp.down_proj.weight shape: [3072, 8192] -28: model.layers.12.mlp.gate_up_proj.weight shape: [16384, 3072] -29: model.layers.12.post_attention_layernorm.weight shape: [3072] -30: model.layers.12.self_attn.o_proj.weight shape: [3072, 3072] -31: model.layers.12.self_attn.qkv_proj.weight shape: [9216, 3072] -32: model.layers.13.input_layernorm.weight shape: [3072] -33: model.layers.13.mlp.down_proj.weight shape: [3072, 8192] -34: model.layers.13.mlp.gate_up_proj.weight shape: [16384, 3072] -35: model.layers.13.post_attention_layernorm.weight shape: [3072] -36: model.layers.13.self_attn.o_proj.weight shape: [3072, 3072] -37: model.layers.13.self_attn.qkv_proj.weight shape: [9216, 3072] -38: model.layers.14.input_layernorm.weight shape: [3072] -39: model.layers.14.mlp.down_proj.weight shape: [3072, 8192] -40: model.layers.14.mlp.gate_up_proj.weight shape: [16384, 3072] -41: model.layers.14.post_attention_layernorm.weight shape: [3072] -42: model.layers.14.self_attn.o_proj.weight shape: [3072, 3072] -43: model.layers.14.self_attn.qkv_proj.weight shape: [9216, 3072] -44: model.layers.15.input_layernorm.weight shape: [3072] -45: model.layers.15.mlp.down_proj.weight shape: [3072, 8192] -46: model.layers.15.mlp.gate_up_proj.weight shape: [16384, 3072] -47: model.layers.15.post_attention_layernorm.weight shape: [3072] -48: model.layers.15.self_attn.o_proj.weight shape: [3072, 3072] -49: model.layers.15.self_attn.qkv_proj.weight shape: [9216, 3072] -50: model.layers.16.input_layernorm.weight shape: [3072] -51: model.layers.16.mlp.down_proj.weight shape: [3072, 8192] -52: model.layers.16.mlp.gate_up_proj.weight shape: [16384, 3072] -53: model.layers.16.post_attention_layernorm.weight shape: [3072] -54: model.layers.16.self_attn.o_proj.weight shape: [3072, 3072] -55: model.layers.16.self_attn.qkv_proj.weight shape: [9216, 3072] -56: model.layers.17.input_layernorm.weight shape: [3072] -57: model.layers.17.mlp.down_proj.weight shape: [3072, 8192] -58: model.layers.17.mlp.gate_up_proj.weight shape: [16384, 3072] -59: model.layers.17.post_attention_layernorm.weight shape: [3072] -60: model.layers.17.self_attn.o_proj.weight shape: [3072, 3072] -61: model.layers.17.self_attn.qkv_proj.weight shape: [9216, 3072] -62: model.layers.18.input_layernorm.weight shape: [3072] -63: model.layers.18.mlp.down_proj.weight shape: [3072, 8192] -64: model.layers.18.mlp.gate_up_proj.weight shape: [16384, 3072] -65: model.layers.18.post_attention_layernorm.weight shape: [3072] -66: model.layers.18.self_attn.o_proj.weight shape: [3072, 3072] -67: model.layers.18.self_attn.qkv_proj.weight shape: [9216, 3072] -68: model.layers.19.input_layernorm.weight shape: [3072] -69: model.layers.19.mlp.down_proj.weight shape: [3072, 8192] -70: model.layers.19.mlp.gate_up_proj.weight shape: [16384, 3072] -71: model.layers.19.post_attention_layernorm.weight shape: [3072] -72: model.layers.19.self_attn.o_proj.weight shape: [3072, 3072] -73: model.layers.19.self_attn.qkv_proj.weight shape: [9216, 3072] -74: model.layers.2.input_layernorm.weight shape: [3072] -75: model.layers.2.mlp.down_proj.weight shape: [3072, 8192] -76: model.layers.2.mlp.gate_up_proj.weight shape: [16384, 3072] -77: model.layers.2.post_attention_layernorm.weight shape: [3072] -78: model.layers.2.self_attn.o_proj.weight shape: [3072, 3072] -79: model.layers.2.self_attn.qkv_proj.weight shape: [9216, 3072] -80: model.layers.20.input_layernorm.weight shape: [3072] -81: model.layers.20.mlp.down_proj.weight shape: [3072, 8192] -82: model.layers.20.mlp.gate_up_proj.weight shape: [16384, 3072] -83: model.layers.20.post_attention_layernorm.weight shape: [3072] -84: model.layers.20.self_attn.o_proj.weight shape: [3072, 3072] -85: model.layers.20.self_attn.qkv_proj.weight shape: [9216, 3072] -86: model.layers.21.input_layernorm.weight shape: [3072] -87: model.layers.21.mlp.down_proj.weight shape: [3072, 8192] -88: model.layers.21.mlp.gate_up_proj.weight shape: [16384, 3072] -89: model.layers.21.post_attention_layernorm.weight shape: [3072] -90: model.layers.21.self_attn.o_proj.weight shape: [3072, 3072] -91: model.layers.21.self_attn.qkv_proj.weight shape: [9216, 3072] -92: model.layers.22.input_layernorm.weight shape: [3072] -93: model.layers.22.mlp.down_proj.weight shape: [3072, 8192] -94: model.layers.22.mlp.gate_up_proj.weight shape: [16384, 3072] -95: model.layers.22.post_attention_layernorm.weight shape: [3072] -96: model.layers.22.self_attn.o_proj.weight shape: [3072, 3072] -97: model.layers.22.self_attn.qkv_proj.weight shape: [9216, 3072] -98: model.layers.23.input_layernorm.weight shape: [3072] -99: model.layers.23.mlp.down_proj.weight shape: [3072, 8192] -100: model.layers.23.mlp.gate_up_proj.weight shape: [16384, 3072] -101: model.layers.23.post_attention_layernorm.weight shape: [3072] -102: model.layers.23.self_attn.o_proj.weight shape: [3072, 3072] -103: model.layers.23.self_attn.qkv_proj.weight shape: [9216, 3072] -104: model.layers.24.input_layernorm.weight shape: [3072] -105: model.layers.24.mlp.down_proj.weight shape: [3072, 8192] -106: model.layers.24.mlp.gate_up_proj.weight shape: [16384, 3072] -107: model.layers.24.post_attention_layernorm.weight shape: [3072] -108: model.layers.24.self_attn.o_proj.weight shape: [3072, 3072] -109: model.layers.24.self_attn.qkv_proj.weight shape: [9216, 3072] -110: model.layers.25.input_layernorm.weight shape: [3072] -111: model.layers.25.mlp.down_proj.weight shape: [3072, 8192] -112: model.layers.25.mlp.gate_up_proj.weight shape: [16384, 3072] -113: model.layers.25.post_attention_layernorm.weight shape: [3072] -114: model.layers.25.self_attn.o_proj.weight shape: [3072, 3072] -115: model.layers.25.self_attn.qkv_proj.weight shape: [9216, 3072] -116: model.layers.26.input_layernorm.weight shape: [3072] -117: model.layers.26.mlp.down_proj.weight shape: [3072, 8192] -118: model.layers.26.mlp.gate_up_proj.weight shape: [16384, 3072] -119: model.layers.26.post_attention_layernorm.weight shape: [3072] -120: model.layers.26.self_attn.o_proj.weight shape: [3072, 3072] -121: model.layers.26.self_attn.qkv_proj.weight shape: [9216, 3072] -122: model.layers.27.input_layernorm.weight shape: [3072] -123: model.layers.27.mlp.down_proj.weight shape: [3072, 8192] -124: model.layers.27.mlp.gate_up_proj.weight shape: [16384, 3072] -125: model.layers.27.post_attention_layernorm.weight shape: [3072] -126: model.layers.27.self_attn.o_proj.weight shape: [3072, 3072] -127: model.layers.27.self_attn.qkv_proj.weight shape: [9216, 3072] -128: model.layers.28.input_layernorm.weight shape: [3072] -129: model.layers.28.mlp.down_proj.weight shape: [3072, 8192] -130: model.layers.28.mlp.gate_up_proj.weight shape: [16384, 3072] -131: model.layers.28.post_attention_layernorm.weight shape: [3072] -132: model.layers.28.self_attn.o_proj.weight shape: [3072, 3072] -133: model.layers.28.self_attn.qkv_proj.weight shape: [9216, 3072] -134: model.layers.29.input_layernorm.weight shape: [3072] -135: model.layers.29.mlp.down_proj.weight shape: [3072, 8192] -136: model.layers.29.mlp.gate_up_proj.weight shape: [16384, 3072] -137: model.layers.29.post_attention_layernorm.weight shape: [3072] -138: model.layers.29.self_attn.o_proj.weight shape: [3072, 3072] -139: model.layers.29.self_attn.qkv_proj.weight shape: [9216, 3072] -140: model.layers.3.input_layernorm.weight shape: [3072] -141: model.layers.3.mlp.down_proj.weight shape: [3072, 8192] -142: model.layers.3.mlp.gate_up_proj.weight shape: [16384, 3072] -143: model.layers.3.post_attention_layernorm.weight shape: [3072] -144: model.layers.3.self_attn.o_proj.weight shape: [3072, 3072] -145: model.layers.3.self_attn.qkv_proj.weight shape: [9216, 3072] -146: model.layers.30.input_layernorm.weight shape: [3072] -147: model.layers.30.mlp.down_proj.weight shape: [3072, 8192] -148: model.layers.30.mlp.gate_up_proj.weight shape: [16384, 3072] -149: model.layers.30.post_attention_layernorm.weight shape: [3072] -150: model.layers.30.self_attn.o_proj.weight shape: [3072, 3072] -151: model.layers.30.self_attn.qkv_proj.weight shape: [9216, 3072] -152: model.layers.31.input_layernorm.weight shape: [3072] -153: model.layers.31.mlp.down_proj.weight shape: [3072, 8192] -154: model.layers.31.mlp.gate_up_proj.weight shape: [16384, 3072] -155: model.layers.31.post_attention_layernorm.weight shape: [3072] -156: model.layers.31.self_attn.o_proj.weight shape: [3072, 3072] -157: model.layers.31.self_attn.qkv_proj.weight shape: [9216, 3072] -158: model.layers.4.input_layernorm.weight shape: [3072] -159: model.layers.4.mlp.down_proj.weight shape: [3072, 8192] -160: model.layers.4.mlp.gate_up_proj.weight shape: [16384, 3072] -161: model.layers.4.post_attention_layernorm.weight shape: [3072] -162: model.layers.4.self_attn.o_proj.weight shape: [3072, 3072] -163: model.layers.4.self_attn.qkv_proj.weight shape: [9216, 3072] -164: model.layers.5.input_layernorm.weight shape: [3072] -165: model.layers.5.mlp.down_proj.weight shape: [3072, 8192] -166: model.layers.5.mlp.gate_up_proj.weight shape: [16384, 3072] -167: model.layers.5.post_attention_layernorm.weight shape: [3072] -168: model.layers.5.self_attn.o_proj.weight shape: [3072, 3072] -169: model.layers.5.self_attn.qkv_proj.weight shape: [9216, 3072] -170: model.layers.6.input_layernorm.weight shape: [3072] -171: model.layers.6.mlp.down_proj.weight shape: [3072, 8192] -172: model.layers.6.mlp.gate_up_proj.weight shape: [16384, 3072] -173: model.layers.6.post_attention_layernorm.weight shape: [3072] -174: model.layers.6.self_attn.o_proj.weight shape: [3072, 3072] -175: model.layers.6.self_attn.qkv_proj.weight shape: [9216, 3072] -176: model.layers.7.input_layernorm.weight shape: [3072] -177: model.layers.7.mlp.down_proj.weight shape: [3072, 8192] -178: model.layers.7.mlp.gate_up_proj.weight shape: [16384, 3072] -179: model.layers.7.post_attention_layernorm.weight shape: [3072] -180: model.layers.7.self_attn.o_proj.weight shape: [3072, 3072] -181: model.layers.7.self_attn.qkv_proj.weight shape: [9216, 3072] -182: model.layers.8.input_layernorm.weight shape: [3072] -183: model.layers.8.mlp.down_proj.weight shape: [3072, 8192] -184: model.layers.8.mlp.gate_up_proj.weight shape: [16384, 3072] -185: model.layers.8.post_attention_layernorm.weight shape: [3072] -186: model.layers.8.self_attn.o_proj.weight shape: [3072, 3072] -187: model.layers.8.self_attn.qkv_proj.weight shape: [9216, 3072] -188: model.layers.9.input_layernorm.weight shape: [3072] -189: model.layers.9.mlp.down_proj.weight shape: [3072, 8192] -190: model.layers.9.mlp.gate_up_proj.weight shape: [16384, 3072] -191: model.layers.9.post_attention_layernorm.weight shape: [3072] -192: model.layers.9.self_attn.o_proj.weight shape: [3072, 3072] -193: model.layers.9.self_attn.qkv_proj.weight shape: [9216, 3072] -194: model.norm.weight shape: [3072] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt4QuantizeShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt4QuantizeShapeTest.approved.txt deleted file mode 100644 index 1855e6396e..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt4QuantizeShapeTest.approved.txt +++ /dev/null @@ -1,451 +0,0 @@ -0: lm_head.weight shape: [32064, 3072] -1: model.embed_tokens.weight shape: [32064, 3072] -2: model.layers.0.input_layernorm.weight shape: [3072] -3: model.layers.0.mlp.down_proj.4bit_weight shape: [12582912] -4: model.layers.0.mlp.down_proj.scale shape: [3072] -5: model.layers.0.mlp.down_proj.zeroPoint shape: [1536] -6: model.layers.0.mlp.gate_up_proj.4bit_weight shape: [25165824] -7: model.layers.0.mlp.gate_up_proj.scale shape: [16384] -8: model.layers.0.mlp.gate_up_proj.zeroPoint shape: [8192] -9: model.layers.0.post_attention_layernorm.weight shape: [3072] -10: model.layers.0.self_attn.o_proj.4bit_weight shape: [4718592] -11: model.layers.0.self_attn.o_proj.scale shape: [3072] -12: model.layers.0.self_attn.o_proj.zeroPoint shape: [1536] -13: model.layers.0.self_attn.qkv_proj.4bit_weight shape: [14155776] -14: model.layers.0.self_attn.qkv_proj.scale shape: [9216] -15: model.layers.0.self_attn.qkv_proj.zeroPoint shape: [4608] -16: model.layers.1.input_layernorm.weight shape: [3072] -17: model.layers.1.mlp.down_proj.4bit_weight shape: [12582912] -18: model.layers.1.mlp.down_proj.scale shape: [3072] -19: model.layers.1.mlp.down_proj.zeroPoint shape: [1536] -20: model.layers.1.mlp.gate_up_proj.4bit_weight shape: [25165824] -21: model.layers.1.mlp.gate_up_proj.scale shape: [16384] -22: model.layers.1.mlp.gate_up_proj.zeroPoint shape: [8192] -23: model.layers.1.post_attention_layernorm.weight shape: [3072] -24: model.layers.1.self_attn.o_proj.4bit_weight shape: [4718592] -25: model.layers.1.self_attn.o_proj.scale shape: [3072] -26: model.layers.1.self_attn.o_proj.zeroPoint shape: [1536] -27: model.layers.1.self_attn.qkv_proj.4bit_weight shape: [14155776] -28: model.layers.1.self_attn.qkv_proj.scale shape: [9216] -29: model.layers.1.self_attn.qkv_proj.zeroPoint shape: [4608] -30: model.layers.10.input_layernorm.weight shape: [3072] -31: model.layers.10.mlp.down_proj.4bit_weight shape: [12582912] -32: model.layers.10.mlp.down_proj.scale shape: [3072] -33: model.layers.10.mlp.down_proj.zeroPoint shape: [1536] -34: model.layers.10.mlp.gate_up_proj.4bit_weight shape: [25165824] -35: model.layers.10.mlp.gate_up_proj.scale shape: [16384] -36: model.layers.10.mlp.gate_up_proj.zeroPoint shape: [8192] -37: model.layers.10.post_attention_layernorm.weight shape: [3072] -38: model.layers.10.self_attn.o_proj.4bit_weight shape: [4718592] -39: model.layers.10.self_attn.o_proj.scale shape: [3072] -40: model.layers.10.self_attn.o_proj.zeroPoint shape: [1536] -41: model.layers.10.self_attn.qkv_proj.4bit_weight shape: [14155776] -42: model.layers.10.self_attn.qkv_proj.scale shape: [9216] -43: model.layers.10.self_attn.qkv_proj.zeroPoint shape: [4608] -44: model.layers.11.input_layernorm.weight shape: [3072] -45: model.layers.11.mlp.down_proj.4bit_weight shape: [12582912] -46: model.layers.11.mlp.down_proj.scale shape: [3072] -47: model.layers.11.mlp.down_proj.zeroPoint shape: [1536] -48: model.layers.11.mlp.gate_up_proj.4bit_weight shape: [25165824] -49: model.layers.11.mlp.gate_up_proj.scale shape: [16384] -50: model.layers.11.mlp.gate_up_proj.zeroPoint shape: [8192] -51: model.layers.11.post_attention_layernorm.weight shape: [3072] -52: model.layers.11.self_attn.o_proj.4bit_weight shape: [4718592] -53: model.layers.11.self_attn.o_proj.scale shape: [3072] -54: model.layers.11.self_attn.o_proj.zeroPoint shape: [1536] -55: model.layers.11.self_attn.qkv_proj.4bit_weight shape: [14155776] -56: model.layers.11.self_attn.qkv_proj.scale shape: [9216] -57: model.layers.11.self_attn.qkv_proj.zeroPoint shape: [4608] -58: model.layers.12.input_layernorm.weight shape: [3072] -59: model.layers.12.mlp.down_proj.4bit_weight shape: [12582912] -60: model.layers.12.mlp.down_proj.scale shape: [3072] -61: model.layers.12.mlp.down_proj.zeroPoint shape: [1536] -62: model.layers.12.mlp.gate_up_proj.4bit_weight shape: [25165824] -63: model.layers.12.mlp.gate_up_proj.scale shape: [16384] -64: model.layers.12.mlp.gate_up_proj.zeroPoint shape: [8192] -65: model.layers.12.post_attention_layernorm.weight shape: [3072] -66: model.layers.12.self_attn.o_proj.4bit_weight shape: [4718592] -67: model.layers.12.self_attn.o_proj.scale shape: [3072] -68: model.layers.12.self_attn.o_proj.zeroPoint shape: [1536] -69: model.layers.12.self_attn.qkv_proj.4bit_weight shape: [14155776] -70: model.layers.12.self_attn.qkv_proj.scale shape: [9216] -71: model.layers.12.self_attn.qkv_proj.zeroPoint shape: [4608] -72: model.layers.13.input_layernorm.weight shape: [3072] -73: model.layers.13.mlp.down_proj.4bit_weight shape: [12582912] -74: model.layers.13.mlp.down_proj.scale shape: [3072] -75: model.layers.13.mlp.down_proj.zeroPoint shape: [1536] -76: model.layers.13.mlp.gate_up_proj.4bit_weight shape: [25165824] -77: model.layers.13.mlp.gate_up_proj.scale shape: [16384] -78: model.layers.13.mlp.gate_up_proj.zeroPoint shape: [8192] -79: model.layers.13.post_attention_layernorm.weight shape: [3072] -80: model.layers.13.self_attn.o_proj.4bit_weight shape: [4718592] -81: model.layers.13.self_attn.o_proj.scale shape: [3072] -82: model.layers.13.self_attn.o_proj.zeroPoint shape: [1536] -83: model.layers.13.self_attn.qkv_proj.4bit_weight shape: [14155776] -84: model.layers.13.self_attn.qkv_proj.scale shape: [9216] -85: model.layers.13.self_attn.qkv_proj.zeroPoint shape: [4608] -86: model.layers.14.input_layernorm.weight shape: [3072] -87: model.layers.14.mlp.down_proj.4bit_weight shape: [12582912] -88: model.layers.14.mlp.down_proj.scale shape: [3072] -89: model.layers.14.mlp.down_proj.zeroPoint shape: [1536] -90: model.layers.14.mlp.gate_up_proj.4bit_weight shape: [25165824] -91: model.layers.14.mlp.gate_up_proj.scale shape: [16384] -92: model.layers.14.mlp.gate_up_proj.zeroPoint shape: [8192] -93: model.layers.14.post_attention_layernorm.weight shape: [3072] -94: model.layers.14.self_attn.o_proj.4bit_weight shape: [4718592] -95: model.layers.14.self_attn.o_proj.scale shape: [3072] -96: model.layers.14.self_attn.o_proj.zeroPoint shape: [1536] -97: model.layers.14.self_attn.qkv_proj.4bit_weight shape: [14155776] -98: model.layers.14.self_attn.qkv_proj.scale shape: [9216] -99: model.layers.14.self_attn.qkv_proj.zeroPoint shape: [4608] -100: model.layers.15.input_layernorm.weight shape: [3072] -101: model.layers.15.mlp.down_proj.4bit_weight shape: [12582912] -102: model.layers.15.mlp.down_proj.scale shape: [3072] -103: model.layers.15.mlp.down_proj.zeroPoint shape: [1536] -104: model.layers.15.mlp.gate_up_proj.4bit_weight shape: [25165824] -105: model.layers.15.mlp.gate_up_proj.scale shape: [16384] -106: model.layers.15.mlp.gate_up_proj.zeroPoint shape: [8192] -107: model.layers.15.post_attention_layernorm.weight shape: [3072] -108: model.layers.15.self_attn.o_proj.4bit_weight shape: [4718592] -109: model.layers.15.self_attn.o_proj.scale shape: [3072] -110: model.layers.15.self_attn.o_proj.zeroPoint shape: [1536] -111: model.layers.15.self_attn.qkv_proj.4bit_weight shape: [14155776] -112: model.layers.15.self_attn.qkv_proj.scale shape: [9216] -113: model.layers.15.self_attn.qkv_proj.zeroPoint shape: [4608] -114: model.layers.16.input_layernorm.weight shape: [3072] -115: model.layers.16.mlp.down_proj.4bit_weight shape: [12582912] -116: model.layers.16.mlp.down_proj.scale shape: [3072] -117: model.layers.16.mlp.down_proj.zeroPoint shape: [1536] -118: model.layers.16.mlp.gate_up_proj.4bit_weight shape: [25165824] -119: model.layers.16.mlp.gate_up_proj.scale shape: [16384] -120: model.layers.16.mlp.gate_up_proj.zeroPoint shape: [8192] -121: model.layers.16.post_attention_layernorm.weight shape: [3072] -122: model.layers.16.self_attn.o_proj.4bit_weight shape: [4718592] -123: model.layers.16.self_attn.o_proj.scale shape: [3072] -124: model.layers.16.self_attn.o_proj.zeroPoint shape: [1536] -125: model.layers.16.self_attn.qkv_proj.4bit_weight shape: [14155776] -126: model.layers.16.self_attn.qkv_proj.scale shape: [9216] -127: model.layers.16.self_attn.qkv_proj.zeroPoint shape: [4608] -128: model.layers.17.input_layernorm.weight shape: [3072] -129: model.layers.17.mlp.down_proj.4bit_weight shape: [12582912] -130: model.layers.17.mlp.down_proj.scale shape: [3072] -131: model.layers.17.mlp.down_proj.zeroPoint shape: [1536] -132: model.layers.17.mlp.gate_up_proj.4bit_weight shape: [25165824] -133: model.layers.17.mlp.gate_up_proj.scale shape: [16384] -134: model.layers.17.mlp.gate_up_proj.zeroPoint shape: [8192] -135: model.layers.17.post_attention_layernorm.weight shape: [3072] -136: model.layers.17.self_attn.o_proj.4bit_weight shape: [4718592] -137: model.layers.17.self_attn.o_proj.scale shape: [3072] -138: model.layers.17.self_attn.o_proj.zeroPoint shape: [1536] -139: model.layers.17.self_attn.qkv_proj.4bit_weight shape: [14155776] -140: model.layers.17.self_attn.qkv_proj.scale shape: [9216] -141: model.layers.17.self_attn.qkv_proj.zeroPoint shape: [4608] -142: model.layers.18.input_layernorm.weight shape: [3072] -143: model.layers.18.mlp.down_proj.4bit_weight shape: [12582912] -144: model.layers.18.mlp.down_proj.scale shape: [3072] -145: model.layers.18.mlp.down_proj.zeroPoint shape: [1536] -146: model.layers.18.mlp.gate_up_proj.4bit_weight shape: [25165824] -147: model.layers.18.mlp.gate_up_proj.scale shape: [16384] -148: model.layers.18.mlp.gate_up_proj.zeroPoint shape: [8192] -149: model.layers.18.post_attention_layernorm.weight shape: [3072] -150: model.layers.18.self_attn.o_proj.4bit_weight shape: [4718592] -151: model.layers.18.self_attn.o_proj.scale shape: [3072] -152: model.layers.18.self_attn.o_proj.zeroPoint shape: [1536] -153: model.layers.18.self_attn.qkv_proj.4bit_weight shape: [14155776] -154: model.layers.18.self_attn.qkv_proj.scale shape: [9216] -155: model.layers.18.self_attn.qkv_proj.zeroPoint shape: [4608] -156: model.layers.19.input_layernorm.weight shape: [3072] -157: model.layers.19.mlp.down_proj.4bit_weight shape: [12582912] -158: model.layers.19.mlp.down_proj.scale shape: [3072] -159: model.layers.19.mlp.down_proj.zeroPoint shape: [1536] -160: model.layers.19.mlp.gate_up_proj.4bit_weight shape: [25165824] -161: model.layers.19.mlp.gate_up_proj.scale shape: [16384] -162: model.layers.19.mlp.gate_up_proj.zeroPoint shape: [8192] -163: model.layers.19.post_attention_layernorm.weight shape: [3072] -164: model.layers.19.self_attn.o_proj.4bit_weight shape: [4718592] -165: model.layers.19.self_attn.o_proj.scale shape: [3072] -166: model.layers.19.self_attn.o_proj.zeroPoint shape: [1536] -167: model.layers.19.self_attn.qkv_proj.4bit_weight shape: [14155776] -168: model.layers.19.self_attn.qkv_proj.scale shape: [9216] -169: model.layers.19.self_attn.qkv_proj.zeroPoint shape: [4608] -170: model.layers.2.input_layernorm.weight shape: [3072] -171: model.layers.2.mlp.down_proj.4bit_weight shape: [12582912] -172: model.layers.2.mlp.down_proj.scale shape: [3072] -173: model.layers.2.mlp.down_proj.zeroPoint shape: [1536] -174: model.layers.2.mlp.gate_up_proj.4bit_weight shape: [25165824] -175: model.layers.2.mlp.gate_up_proj.scale shape: [16384] -176: model.layers.2.mlp.gate_up_proj.zeroPoint shape: [8192] -177: model.layers.2.post_attention_layernorm.weight shape: [3072] -178: model.layers.2.self_attn.o_proj.4bit_weight shape: [4718592] -179: model.layers.2.self_attn.o_proj.scale shape: [3072] -180: model.layers.2.self_attn.o_proj.zeroPoint shape: [1536] -181: model.layers.2.self_attn.qkv_proj.4bit_weight shape: [14155776] -182: model.layers.2.self_attn.qkv_proj.scale shape: [9216] -183: model.layers.2.self_attn.qkv_proj.zeroPoint shape: [4608] -184: model.layers.20.input_layernorm.weight shape: [3072] -185: model.layers.20.mlp.down_proj.4bit_weight shape: [12582912] -186: model.layers.20.mlp.down_proj.scale shape: [3072] -187: model.layers.20.mlp.down_proj.zeroPoint shape: [1536] -188: model.layers.20.mlp.gate_up_proj.4bit_weight shape: [25165824] -189: model.layers.20.mlp.gate_up_proj.scale shape: [16384] -190: model.layers.20.mlp.gate_up_proj.zeroPoint shape: [8192] -191: model.layers.20.post_attention_layernorm.weight shape: [3072] -192: model.layers.20.self_attn.o_proj.4bit_weight shape: [4718592] -193: model.layers.20.self_attn.o_proj.scale shape: [3072] -194: model.layers.20.self_attn.o_proj.zeroPoint shape: [1536] -195: model.layers.20.self_attn.qkv_proj.4bit_weight shape: [14155776] -196: model.layers.20.self_attn.qkv_proj.scale shape: [9216] -197: model.layers.20.self_attn.qkv_proj.zeroPoint shape: [4608] -198: model.layers.21.input_layernorm.weight shape: [3072] -199: model.layers.21.mlp.down_proj.4bit_weight shape: [12582912] -200: model.layers.21.mlp.down_proj.scale shape: [3072] -201: model.layers.21.mlp.down_proj.zeroPoint shape: [1536] -202: model.layers.21.mlp.gate_up_proj.4bit_weight shape: [25165824] -203: model.layers.21.mlp.gate_up_proj.scale shape: [16384] -204: model.layers.21.mlp.gate_up_proj.zeroPoint shape: [8192] -205: model.layers.21.post_attention_layernorm.weight shape: [3072] -206: model.layers.21.self_attn.o_proj.4bit_weight shape: [4718592] -207: model.layers.21.self_attn.o_proj.scale shape: [3072] -208: model.layers.21.self_attn.o_proj.zeroPoint shape: [1536] -209: model.layers.21.self_attn.qkv_proj.4bit_weight shape: [14155776] -210: model.layers.21.self_attn.qkv_proj.scale shape: [9216] -211: model.layers.21.self_attn.qkv_proj.zeroPoint shape: [4608] -212: model.layers.22.input_layernorm.weight shape: [3072] -213: model.layers.22.mlp.down_proj.4bit_weight shape: [12582912] -214: model.layers.22.mlp.down_proj.scale shape: [3072] -215: model.layers.22.mlp.down_proj.zeroPoint shape: [1536] -216: model.layers.22.mlp.gate_up_proj.4bit_weight shape: [25165824] -217: model.layers.22.mlp.gate_up_proj.scale shape: [16384] -218: model.layers.22.mlp.gate_up_proj.zeroPoint shape: [8192] -219: model.layers.22.post_attention_layernorm.weight shape: [3072] -220: model.layers.22.self_attn.o_proj.4bit_weight shape: [4718592] -221: model.layers.22.self_attn.o_proj.scale shape: [3072] -222: model.layers.22.self_attn.o_proj.zeroPoint shape: [1536] -223: model.layers.22.self_attn.qkv_proj.4bit_weight shape: [14155776] -224: model.layers.22.self_attn.qkv_proj.scale shape: [9216] -225: model.layers.22.self_attn.qkv_proj.zeroPoint shape: [4608] -226: model.layers.23.input_layernorm.weight shape: [3072] -227: model.layers.23.mlp.down_proj.4bit_weight shape: [12582912] -228: model.layers.23.mlp.down_proj.scale shape: [3072] -229: model.layers.23.mlp.down_proj.zeroPoint shape: [1536] -230: model.layers.23.mlp.gate_up_proj.4bit_weight shape: [25165824] -231: model.layers.23.mlp.gate_up_proj.scale shape: [16384] -232: model.layers.23.mlp.gate_up_proj.zeroPoint shape: [8192] -233: model.layers.23.post_attention_layernorm.weight shape: [3072] -234: model.layers.23.self_attn.o_proj.4bit_weight shape: [4718592] -235: model.layers.23.self_attn.o_proj.scale shape: [3072] -236: model.layers.23.self_attn.o_proj.zeroPoint shape: [1536] -237: model.layers.23.self_attn.qkv_proj.4bit_weight shape: [14155776] -238: model.layers.23.self_attn.qkv_proj.scale shape: [9216] -239: model.layers.23.self_attn.qkv_proj.zeroPoint shape: [4608] -240: model.layers.24.input_layernorm.weight shape: [3072] -241: model.layers.24.mlp.down_proj.4bit_weight shape: [12582912] -242: model.layers.24.mlp.down_proj.scale shape: [3072] -243: model.layers.24.mlp.down_proj.zeroPoint shape: [1536] -244: model.layers.24.mlp.gate_up_proj.4bit_weight shape: [25165824] -245: model.layers.24.mlp.gate_up_proj.scale shape: [16384] -246: model.layers.24.mlp.gate_up_proj.zeroPoint shape: [8192] -247: model.layers.24.post_attention_layernorm.weight shape: [3072] -248: model.layers.24.self_attn.o_proj.4bit_weight shape: [4718592] -249: model.layers.24.self_attn.o_proj.scale shape: [3072] -250: model.layers.24.self_attn.o_proj.zeroPoint shape: [1536] -251: model.layers.24.self_attn.qkv_proj.4bit_weight shape: [14155776] -252: model.layers.24.self_attn.qkv_proj.scale shape: [9216] -253: model.layers.24.self_attn.qkv_proj.zeroPoint shape: [4608] -254: model.layers.25.input_layernorm.weight shape: [3072] -255: model.layers.25.mlp.down_proj.4bit_weight shape: [12582912] -256: model.layers.25.mlp.down_proj.scale shape: [3072] -257: model.layers.25.mlp.down_proj.zeroPoint shape: [1536] -258: model.layers.25.mlp.gate_up_proj.4bit_weight shape: [25165824] -259: model.layers.25.mlp.gate_up_proj.scale shape: [16384] -260: model.layers.25.mlp.gate_up_proj.zeroPoint shape: [8192] -261: model.layers.25.post_attention_layernorm.weight shape: [3072] -262: model.layers.25.self_attn.o_proj.4bit_weight shape: [4718592] -263: model.layers.25.self_attn.o_proj.scale shape: [3072] -264: model.layers.25.self_attn.o_proj.zeroPoint shape: [1536] -265: model.layers.25.self_attn.qkv_proj.4bit_weight shape: [14155776] -266: model.layers.25.self_attn.qkv_proj.scale shape: [9216] -267: model.layers.25.self_attn.qkv_proj.zeroPoint shape: [4608] -268: model.layers.26.input_layernorm.weight shape: [3072] -269: model.layers.26.mlp.down_proj.4bit_weight shape: [12582912] -270: model.layers.26.mlp.down_proj.scale shape: [3072] -271: model.layers.26.mlp.down_proj.zeroPoint shape: [1536] -272: model.layers.26.mlp.gate_up_proj.4bit_weight shape: [25165824] -273: model.layers.26.mlp.gate_up_proj.scale shape: [16384] -274: model.layers.26.mlp.gate_up_proj.zeroPoint shape: [8192] -275: model.layers.26.post_attention_layernorm.weight shape: [3072] -276: model.layers.26.self_attn.o_proj.4bit_weight shape: [4718592] -277: model.layers.26.self_attn.o_proj.scale shape: [3072] -278: model.layers.26.self_attn.o_proj.zeroPoint shape: [1536] -279: model.layers.26.self_attn.qkv_proj.4bit_weight shape: [14155776] -280: model.layers.26.self_attn.qkv_proj.scale shape: [9216] -281: model.layers.26.self_attn.qkv_proj.zeroPoint shape: [4608] -282: model.layers.27.input_layernorm.weight shape: [3072] -283: model.layers.27.mlp.down_proj.4bit_weight shape: [12582912] -284: model.layers.27.mlp.down_proj.scale shape: [3072] -285: model.layers.27.mlp.down_proj.zeroPoint shape: [1536] -286: model.layers.27.mlp.gate_up_proj.4bit_weight shape: [25165824] -287: model.layers.27.mlp.gate_up_proj.scale shape: [16384] -288: model.layers.27.mlp.gate_up_proj.zeroPoint shape: [8192] -289: model.layers.27.post_attention_layernorm.weight shape: [3072] -290: model.layers.27.self_attn.o_proj.4bit_weight shape: [4718592] -291: model.layers.27.self_attn.o_proj.scale shape: [3072] -292: model.layers.27.self_attn.o_proj.zeroPoint shape: [1536] -293: model.layers.27.self_attn.qkv_proj.4bit_weight shape: [14155776] -294: model.layers.27.self_attn.qkv_proj.scale shape: [9216] -295: model.layers.27.self_attn.qkv_proj.zeroPoint shape: [4608] -296: model.layers.28.input_layernorm.weight shape: [3072] -297: model.layers.28.mlp.down_proj.4bit_weight shape: [12582912] -298: model.layers.28.mlp.down_proj.scale shape: [3072] -299: model.layers.28.mlp.down_proj.zeroPoint shape: [1536] -300: model.layers.28.mlp.gate_up_proj.4bit_weight shape: [25165824] -301: model.layers.28.mlp.gate_up_proj.scale shape: [16384] -302: model.layers.28.mlp.gate_up_proj.zeroPoint shape: [8192] -303: model.layers.28.post_attention_layernorm.weight shape: [3072] -304: model.layers.28.self_attn.o_proj.4bit_weight shape: [4718592] -305: model.layers.28.self_attn.o_proj.scale shape: [3072] -306: model.layers.28.self_attn.o_proj.zeroPoint shape: [1536] -307: model.layers.28.self_attn.qkv_proj.4bit_weight shape: [14155776] -308: model.layers.28.self_attn.qkv_proj.scale shape: [9216] -309: model.layers.28.self_attn.qkv_proj.zeroPoint shape: [4608] -310: model.layers.29.input_layernorm.weight shape: [3072] -311: model.layers.29.mlp.down_proj.4bit_weight shape: [12582912] -312: model.layers.29.mlp.down_proj.scale shape: [3072] -313: model.layers.29.mlp.down_proj.zeroPoint shape: [1536] -314: model.layers.29.mlp.gate_up_proj.4bit_weight shape: [25165824] -315: model.layers.29.mlp.gate_up_proj.scale shape: [16384] -316: model.layers.29.mlp.gate_up_proj.zeroPoint shape: [8192] -317: model.layers.29.post_attention_layernorm.weight shape: [3072] -318: model.layers.29.self_attn.o_proj.4bit_weight shape: [4718592] -319: model.layers.29.self_attn.o_proj.scale shape: [3072] -320: model.layers.29.self_attn.o_proj.zeroPoint shape: [1536] -321: model.layers.29.self_attn.qkv_proj.4bit_weight shape: [14155776] -322: model.layers.29.self_attn.qkv_proj.scale shape: [9216] -323: model.layers.29.self_attn.qkv_proj.zeroPoint shape: [4608] -324: model.layers.3.input_layernorm.weight shape: [3072] -325: model.layers.3.mlp.down_proj.4bit_weight shape: [12582912] -326: model.layers.3.mlp.down_proj.scale shape: [3072] -327: model.layers.3.mlp.down_proj.zeroPoint shape: [1536] -328: model.layers.3.mlp.gate_up_proj.4bit_weight shape: [25165824] -329: model.layers.3.mlp.gate_up_proj.scale shape: [16384] -330: model.layers.3.mlp.gate_up_proj.zeroPoint shape: [8192] -331: model.layers.3.post_attention_layernorm.weight shape: [3072] -332: model.layers.3.self_attn.o_proj.4bit_weight shape: [4718592] -333: model.layers.3.self_attn.o_proj.scale shape: [3072] -334: model.layers.3.self_attn.o_proj.zeroPoint shape: [1536] -335: model.layers.3.self_attn.qkv_proj.4bit_weight shape: [14155776] -336: model.layers.3.self_attn.qkv_proj.scale shape: [9216] -337: model.layers.3.self_attn.qkv_proj.zeroPoint shape: [4608] -338: model.layers.30.input_layernorm.weight shape: [3072] -339: model.layers.30.mlp.down_proj.4bit_weight shape: [12582912] -340: model.layers.30.mlp.down_proj.scale shape: [3072] -341: model.layers.30.mlp.down_proj.zeroPoint shape: [1536] -342: model.layers.30.mlp.gate_up_proj.4bit_weight shape: [25165824] -343: model.layers.30.mlp.gate_up_proj.scale shape: [16384] -344: model.layers.30.mlp.gate_up_proj.zeroPoint shape: [8192] -345: model.layers.30.post_attention_layernorm.weight shape: [3072] -346: model.layers.30.self_attn.o_proj.4bit_weight shape: [4718592] -347: model.layers.30.self_attn.o_proj.scale shape: [3072] -348: model.layers.30.self_attn.o_proj.zeroPoint shape: [1536] -349: model.layers.30.self_attn.qkv_proj.4bit_weight shape: [14155776] -350: model.layers.30.self_attn.qkv_proj.scale shape: [9216] -351: model.layers.30.self_attn.qkv_proj.zeroPoint shape: [4608] -352: model.layers.31.input_layernorm.weight shape: [3072] -353: model.layers.31.mlp.down_proj.4bit_weight shape: [12582912] -354: model.layers.31.mlp.down_proj.scale shape: [3072] -355: model.layers.31.mlp.down_proj.zeroPoint shape: [1536] -356: model.layers.31.mlp.gate_up_proj.4bit_weight shape: [25165824] -357: model.layers.31.mlp.gate_up_proj.scale shape: [16384] -358: model.layers.31.mlp.gate_up_proj.zeroPoint shape: [8192] -359: model.layers.31.post_attention_layernorm.weight shape: [3072] -360: model.layers.31.self_attn.o_proj.4bit_weight shape: [4718592] -361: model.layers.31.self_attn.o_proj.scale shape: [3072] -362: model.layers.31.self_attn.o_proj.zeroPoint shape: [1536] -363: model.layers.31.self_attn.qkv_proj.4bit_weight shape: [14155776] -364: model.layers.31.self_attn.qkv_proj.scale shape: [9216] -365: model.layers.31.self_attn.qkv_proj.zeroPoint shape: [4608] -366: model.layers.4.input_layernorm.weight shape: [3072] -367: model.layers.4.mlp.down_proj.4bit_weight shape: [12582912] -368: model.layers.4.mlp.down_proj.scale shape: [3072] -369: model.layers.4.mlp.down_proj.zeroPoint shape: [1536] -370: model.layers.4.mlp.gate_up_proj.4bit_weight shape: [25165824] -371: model.layers.4.mlp.gate_up_proj.scale shape: [16384] -372: model.layers.4.mlp.gate_up_proj.zeroPoint shape: [8192] -373: model.layers.4.post_attention_layernorm.weight shape: [3072] -374: model.layers.4.self_attn.o_proj.4bit_weight shape: [4718592] -375: model.layers.4.self_attn.o_proj.scale shape: [3072] -376: model.layers.4.self_attn.o_proj.zeroPoint shape: [1536] -377: model.layers.4.self_attn.qkv_proj.4bit_weight shape: [14155776] -378: model.layers.4.self_attn.qkv_proj.scale shape: [9216] -379: model.layers.4.self_attn.qkv_proj.zeroPoint shape: [4608] -380: model.layers.5.input_layernorm.weight shape: [3072] -381: model.layers.5.mlp.down_proj.4bit_weight shape: [12582912] -382: model.layers.5.mlp.down_proj.scale shape: [3072] -383: model.layers.5.mlp.down_proj.zeroPoint shape: [1536] -384: model.layers.5.mlp.gate_up_proj.4bit_weight shape: [25165824] -385: model.layers.5.mlp.gate_up_proj.scale shape: [16384] -386: model.layers.5.mlp.gate_up_proj.zeroPoint shape: [8192] -387: model.layers.5.post_attention_layernorm.weight shape: [3072] -388: model.layers.5.self_attn.o_proj.4bit_weight shape: [4718592] -389: model.layers.5.self_attn.o_proj.scale shape: [3072] -390: model.layers.5.self_attn.o_proj.zeroPoint shape: [1536] -391: model.layers.5.self_attn.qkv_proj.4bit_weight shape: [14155776] -392: model.layers.5.self_attn.qkv_proj.scale shape: [9216] -393: model.layers.5.self_attn.qkv_proj.zeroPoint shape: [4608] -394: model.layers.6.input_layernorm.weight shape: [3072] -395: model.layers.6.mlp.down_proj.4bit_weight shape: [12582912] -396: model.layers.6.mlp.down_proj.scale shape: [3072] -397: model.layers.6.mlp.down_proj.zeroPoint shape: [1536] -398: model.layers.6.mlp.gate_up_proj.4bit_weight shape: [25165824] -399: model.layers.6.mlp.gate_up_proj.scale shape: [16384] -400: model.layers.6.mlp.gate_up_proj.zeroPoint shape: [8192] -401: model.layers.6.post_attention_layernorm.weight shape: [3072] -402: model.layers.6.self_attn.o_proj.4bit_weight shape: [4718592] -403: model.layers.6.self_attn.o_proj.scale shape: [3072] -404: model.layers.6.self_attn.o_proj.zeroPoint shape: [1536] -405: model.layers.6.self_attn.qkv_proj.4bit_weight shape: [14155776] -406: model.layers.6.self_attn.qkv_proj.scale shape: [9216] -407: model.layers.6.self_attn.qkv_proj.zeroPoint shape: [4608] -408: model.layers.7.input_layernorm.weight shape: [3072] -409: model.layers.7.mlp.down_proj.4bit_weight shape: [12582912] -410: model.layers.7.mlp.down_proj.scale shape: [3072] -411: model.layers.7.mlp.down_proj.zeroPoint shape: [1536] -412: model.layers.7.mlp.gate_up_proj.4bit_weight shape: [25165824] -413: model.layers.7.mlp.gate_up_proj.scale shape: [16384] -414: model.layers.7.mlp.gate_up_proj.zeroPoint shape: [8192] -415: model.layers.7.post_attention_layernorm.weight shape: [3072] -416: model.layers.7.self_attn.o_proj.4bit_weight shape: [4718592] -417: model.layers.7.self_attn.o_proj.scale shape: [3072] -418: model.layers.7.self_attn.o_proj.zeroPoint shape: [1536] -419: model.layers.7.self_attn.qkv_proj.4bit_weight shape: [14155776] -420: model.layers.7.self_attn.qkv_proj.scale shape: [9216] -421: model.layers.7.self_attn.qkv_proj.zeroPoint shape: [4608] -422: model.layers.8.input_layernorm.weight shape: [3072] -423: model.layers.8.mlp.down_proj.4bit_weight shape: [12582912] -424: model.layers.8.mlp.down_proj.scale shape: [3072] -425: model.layers.8.mlp.down_proj.zeroPoint shape: [1536] -426: model.layers.8.mlp.gate_up_proj.4bit_weight shape: [25165824] -427: model.layers.8.mlp.gate_up_proj.scale shape: [16384] -428: model.layers.8.mlp.gate_up_proj.zeroPoint shape: [8192] -429: model.layers.8.post_attention_layernorm.weight shape: [3072] -430: model.layers.8.self_attn.o_proj.4bit_weight shape: [4718592] -431: model.layers.8.self_attn.o_proj.scale shape: [3072] -432: model.layers.8.self_attn.o_proj.zeroPoint shape: [1536] -433: model.layers.8.self_attn.qkv_proj.4bit_weight shape: [14155776] -434: model.layers.8.self_attn.qkv_proj.scale shape: [9216] -435: model.layers.8.self_attn.qkv_proj.zeroPoint shape: [4608] -436: model.layers.9.input_layernorm.weight shape: [3072] -437: model.layers.9.mlp.down_proj.4bit_weight shape: [12582912] -438: model.layers.9.mlp.down_proj.scale shape: [3072] -439: model.layers.9.mlp.down_proj.zeroPoint shape: [1536] -440: model.layers.9.mlp.gate_up_proj.4bit_weight shape: [25165824] -441: model.layers.9.mlp.gate_up_proj.scale shape: [16384] -442: model.layers.9.mlp.gate_up_proj.zeroPoint shape: [8192] -443: model.layers.9.post_attention_layernorm.weight shape: [3072] -444: model.layers.9.self_attn.o_proj.4bit_weight shape: [4718592] -445: model.layers.9.self_attn.o_proj.scale shape: [3072] -446: model.layers.9.self_attn.o_proj.zeroPoint shape: [1536] -447: model.layers.9.self_attn.qkv_proj.4bit_weight shape: [14155776] -448: model.layers.9.self_attn.qkv_proj.scale shape: [9216] -449: model.layers.9.self_attn.qkv_proj.zeroPoint shape: [4608] -450: model.norm.weight shape: [3072] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt8QuantizeShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt8QuantizeShapeTest.approved.txt deleted file mode 100644 index d3ab1d8010..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KInt8QuantizeShapeTest.approved.txt +++ /dev/null @@ -1,451 +0,0 @@ -0: lm_head.weight shape: [32064, 3072] -1: model.embed_tokens.weight shape: [32064, 3072] -2: model.layers.0.input_layernorm.weight shape: [3072] -3: model.layers.0.mlp.down_proj.8bit_weight shape: [3072, 8192] -4: model.layers.0.mlp.down_proj.scale shape: [3072] -5: model.layers.0.mlp.down_proj.zeroPoint shape: [3072] -6: model.layers.0.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -7: model.layers.0.mlp.gate_up_proj.scale shape: [16384] -8: model.layers.0.mlp.gate_up_proj.zeroPoint shape: [16384] -9: model.layers.0.post_attention_layernorm.weight shape: [3072] -10: model.layers.0.self_attn.o_proj.8bit_weight shape: [3072, 3072] -11: model.layers.0.self_attn.o_proj.scale shape: [3072] -12: model.layers.0.self_attn.o_proj.zeroPoint shape: [3072] -13: model.layers.0.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -14: model.layers.0.self_attn.qkv_proj.scale shape: [9216] -15: model.layers.0.self_attn.qkv_proj.zeroPoint shape: [9216] -16: model.layers.1.input_layernorm.weight shape: [3072] -17: model.layers.1.mlp.down_proj.8bit_weight shape: [3072, 8192] -18: model.layers.1.mlp.down_proj.scale shape: [3072] -19: model.layers.1.mlp.down_proj.zeroPoint shape: [3072] -20: model.layers.1.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -21: model.layers.1.mlp.gate_up_proj.scale shape: [16384] -22: model.layers.1.mlp.gate_up_proj.zeroPoint shape: [16384] -23: model.layers.1.post_attention_layernorm.weight shape: [3072] -24: model.layers.1.self_attn.o_proj.8bit_weight shape: [3072, 3072] -25: model.layers.1.self_attn.o_proj.scale shape: [3072] -26: model.layers.1.self_attn.o_proj.zeroPoint shape: [3072] -27: model.layers.1.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -28: model.layers.1.self_attn.qkv_proj.scale shape: [9216] -29: model.layers.1.self_attn.qkv_proj.zeroPoint shape: [9216] -30: model.layers.10.input_layernorm.weight shape: [3072] -31: model.layers.10.mlp.down_proj.8bit_weight shape: [3072, 8192] -32: model.layers.10.mlp.down_proj.scale shape: [3072] -33: model.layers.10.mlp.down_proj.zeroPoint shape: [3072] -34: model.layers.10.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -35: model.layers.10.mlp.gate_up_proj.scale shape: [16384] -36: model.layers.10.mlp.gate_up_proj.zeroPoint shape: [16384] -37: model.layers.10.post_attention_layernorm.weight shape: [3072] -38: model.layers.10.self_attn.o_proj.8bit_weight shape: [3072, 3072] -39: model.layers.10.self_attn.o_proj.scale shape: [3072] -40: model.layers.10.self_attn.o_proj.zeroPoint shape: [3072] -41: model.layers.10.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -42: model.layers.10.self_attn.qkv_proj.scale shape: [9216] -43: model.layers.10.self_attn.qkv_proj.zeroPoint shape: [9216] -44: model.layers.11.input_layernorm.weight shape: [3072] -45: model.layers.11.mlp.down_proj.8bit_weight shape: [3072, 8192] -46: model.layers.11.mlp.down_proj.scale shape: [3072] -47: model.layers.11.mlp.down_proj.zeroPoint shape: [3072] -48: model.layers.11.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -49: model.layers.11.mlp.gate_up_proj.scale shape: [16384] -50: model.layers.11.mlp.gate_up_proj.zeroPoint shape: [16384] -51: model.layers.11.post_attention_layernorm.weight shape: [3072] -52: model.layers.11.self_attn.o_proj.8bit_weight shape: [3072, 3072] -53: model.layers.11.self_attn.o_proj.scale shape: [3072] -54: model.layers.11.self_attn.o_proj.zeroPoint shape: [3072] -55: model.layers.11.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -56: model.layers.11.self_attn.qkv_proj.scale shape: [9216] -57: model.layers.11.self_attn.qkv_proj.zeroPoint shape: [9216] -58: model.layers.12.input_layernorm.weight shape: [3072] -59: model.layers.12.mlp.down_proj.8bit_weight shape: [3072, 8192] -60: model.layers.12.mlp.down_proj.scale shape: [3072] -61: model.layers.12.mlp.down_proj.zeroPoint shape: [3072] -62: model.layers.12.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -63: model.layers.12.mlp.gate_up_proj.scale shape: [16384] -64: model.layers.12.mlp.gate_up_proj.zeroPoint shape: [16384] -65: model.layers.12.post_attention_layernorm.weight shape: [3072] -66: model.layers.12.self_attn.o_proj.8bit_weight shape: [3072, 3072] -67: model.layers.12.self_attn.o_proj.scale shape: [3072] -68: model.layers.12.self_attn.o_proj.zeroPoint shape: [3072] -69: model.layers.12.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -70: model.layers.12.self_attn.qkv_proj.scale shape: [9216] -71: model.layers.12.self_attn.qkv_proj.zeroPoint shape: [9216] -72: model.layers.13.input_layernorm.weight shape: [3072] -73: model.layers.13.mlp.down_proj.8bit_weight shape: [3072, 8192] -74: model.layers.13.mlp.down_proj.scale shape: [3072] -75: model.layers.13.mlp.down_proj.zeroPoint shape: [3072] -76: model.layers.13.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -77: model.layers.13.mlp.gate_up_proj.scale shape: [16384] -78: model.layers.13.mlp.gate_up_proj.zeroPoint shape: [16384] -79: model.layers.13.post_attention_layernorm.weight shape: [3072] -80: model.layers.13.self_attn.o_proj.8bit_weight shape: [3072, 3072] -81: model.layers.13.self_attn.o_proj.scale shape: [3072] -82: model.layers.13.self_attn.o_proj.zeroPoint shape: [3072] -83: model.layers.13.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -84: model.layers.13.self_attn.qkv_proj.scale shape: [9216] -85: model.layers.13.self_attn.qkv_proj.zeroPoint shape: [9216] -86: model.layers.14.input_layernorm.weight shape: [3072] -87: model.layers.14.mlp.down_proj.8bit_weight shape: [3072, 8192] -88: model.layers.14.mlp.down_proj.scale shape: [3072] -89: model.layers.14.mlp.down_proj.zeroPoint shape: [3072] -90: model.layers.14.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -91: model.layers.14.mlp.gate_up_proj.scale shape: [16384] -92: model.layers.14.mlp.gate_up_proj.zeroPoint shape: [16384] -93: model.layers.14.post_attention_layernorm.weight shape: [3072] -94: model.layers.14.self_attn.o_proj.8bit_weight shape: [3072, 3072] -95: model.layers.14.self_attn.o_proj.scale shape: [3072] -96: model.layers.14.self_attn.o_proj.zeroPoint shape: [3072] -97: model.layers.14.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -98: model.layers.14.self_attn.qkv_proj.scale shape: [9216] -99: model.layers.14.self_attn.qkv_proj.zeroPoint shape: [9216] -100: model.layers.15.input_layernorm.weight shape: [3072] -101: model.layers.15.mlp.down_proj.8bit_weight shape: [3072, 8192] -102: model.layers.15.mlp.down_proj.scale shape: [3072] -103: model.layers.15.mlp.down_proj.zeroPoint shape: [3072] -104: model.layers.15.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -105: model.layers.15.mlp.gate_up_proj.scale shape: [16384] -106: model.layers.15.mlp.gate_up_proj.zeroPoint shape: [16384] -107: model.layers.15.post_attention_layernorm.weight shape: [3072] -108: model.layers.15.self_attn.o_proj.8bit_weight shape: [3072, 3072] -109: model.layers.15.self_attn.o_proj.scale shape: [3072] -110: model.layers.15.self_attn.o_proj.zeroPoint shape: [3072] -111: model.layers.15.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -112: model.layers.15.self_attn.qkv_proj.scale shape: [9216] -113: model.layers.15.self_attn.qkv_proj.zeroPoint shape: [9216] -114: model.layers.16.input_layernorm.weight shape: [3072] -115: model.layers.16.mlp.down_proj.8bit_weight shape: [3072, 8192] -116: model.layers.16.mlp.down_proj.scale shape: [3072] -117: model.layers.16.mlp.down_proj.zeroPoint shape: [3072] -118: model.layers.16.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -119: model.layers.16.mlp.gate_up_proj.scale shape: [16384] -120: model.layers.16.mlp.gate_up_proj.zeroPoint shape: [16384] -121: model.layers.16.post_attention_layernorm.weight shape: [3072] -122: model.layers.16.self_attn.o_proj.8bit_weight shape: [3072, 3072] -123: model.layers.16.self_attn.o_proj.scale shape: [3072] -124: model.layers.16.self_attn.o_proj.zeroPoint shape: [3072] -125: model.layers.16.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -126: model.layers.16.self_attn.qkv_proj.scale shape: [9216] -127: model.layers.16.self_attn.qkv_proj.zeroPoint shape: [9216] -128: model.layers.17.input_layernorm.weight shape: [3072] -129: model.layers.17.mlp.down_proj.8bit_weight shape: [3072, 8192] -130: model.layers.17.mlp.down_proj.scale shape: [3072] -131: model.layers.17.mlp.down_proj.zeroPoint shape: [3072] -132: model.layers.17.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -133: model.layers.17.mlp.gate_up_proj.scale shape: [16384] -134: model.layers.17.mlp.gate_up_proj.zeroPoint shape: [16384] -135: model.layers.17.post_attention_layernorm.weight shape: [3072] -136: model.layers.17.self_attn.o_proj.8bit_weight shape: [3072, 3072] -137: model.layers.17.self_attn.o_proj.scale shape: [3072] -138: model.layers.17.self_attn.o_proj.zeroPoint shape: [3072] -139: model.layers.17.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -140: model.layers.17.self_attn.qkv_proj.scale shape: [9216] -141: model.layers.17.self_attn.qkv_proj.zeroPoint shape: [9216] -142: model.layers.18.input_layernorm.weight shape: [3072] -143: model.layers.18.mlp.down_proj.8bit_weight shape: [3072, 8192] -144: model.layers.18.mlp.down_proj.scale shape: [3072] -145: model.layers.18.mlp.down_proj.zeroPoint shape: [3072] -146: model.layers.18.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -147: model.layers.18.mlp.gate_up_proj.scale shape: [16384] -148: model.layers.18.mlp.gate_up_proj.zeroPoint shape: [16384] -149: model.layers.18.post_attention_layernorm.weight shape: [3072] -150: model.layers.18.self_attn.o_proj.8bit_weight shape: [3072, 3072] -151: model.layers.18.self_attn.o_proj.scale shape: [3072] -152: model.layers.18.self_attn.o_proj.zeroPoint shape: [3072] -153: model.layers.18.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -154: model.layers.18.self_attn.qkv_proj.scale shape: [9216] -155: model.layers.18.self_attn.qkv_proj.zeroPoint shape: [9216] -156: model.layers.19.input_layernorm.weight shape: [3072] -157: model.layers.19.mlp.down_proj.8bit_weight shape: [3072, 8192] -158: model.layers.19.mlp.down_proj.scale shape: [3072] -159: model.layers.19.mlp.down_proj.zeroPoint shape: [3072] -160: model.layers.19.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -161: model.layers.19.mlp.gate_up_proj.scale shape: [16384] -162: model.layers.19.mlp.gate_up_proj.zeroPoint shape: [16384] -163: model.layers.19.post_attention_layernorm.weight shape: [3072] -164: model.layers.19.self_attn.o_proj.8bit_weight shape: [3072, 3072] -165: model.layers.19.self_attn.o_proj.scale shape: [3072] -166: model.layers.19.self_attn.o_proj.zeroPoint shape: [3072] -167: model.layers.19.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -168: model.layers.19.self_attn.qkv_proj.scale shape: [9216] -169: model.layers.19.self_attn.qkv_proj.zeroPoint shape: [9216] -170: model.layers.2.input_layernorm.weight shape: [3072] -171: model.layers.2.mlp.down_proj.8bit_weight shape: [3072, 8192] -172: model.layers.2.mlp.down_proj.scale shape: [3072] -173: model.layers.2.mlp.down_proj.zeroPoint shape: [3072] -174: model.layers.2.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -175: model.layers.2.mlp.gate_up_proj.scale shape: [16384] -176: model.layers.2.mlp.gate_up_proj.zeroPoint shape: [16384] -177: model.layers.2.post_attention_layernorm.weight shape: [3072] -178: model.layers.2.self_attn.o_proj.8bit_weight shape: [3072, 3072] -179: model.layers.2.self_attn.o_proj.scale shape: [3072] -180: model.layers.2.self_attn.o_proj.zeroPoint shape: [3072] -181: model.layers.2.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -182: model.layers.2.self_attn.qkv_proj.scale shape: [9216] -183: model.layers.2.self_attn.qkv_proj.zeroPoint shape: [9216] -184: model.layers.20.input_layernorm.weight shape: [3072] -185: model.layers.20.mlp.down_proj.8bit_weight shape: [3072, 8192] -186: model.layers.20.mlp.down_proj.scale shape: [3072] -187: model.layers.20.mlp.down_proj.zeroPoint shape: [3072] -188: model.layers.20.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -189: model.layers.20.mlp.gate_up_proj.scale shape: [16384] -190: model.layers.20.mlp.gate_up_proj.zeroPoint shape: [16384] -191: model.layers.20.post_attention_layernorm.weight shape: [3072] -192: model.layers.20.self_attn.o_proj.8bit_weight shape: [3072, 3072] -193: model.layers.20.self_attn.o_proj.scale shape: [3072] -194: model.layers.20.self_attn.o_proj.zeroPoint shape: [3072] -195: model.layers.20.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -196: model.layers.20.self_attn.qkv_proj.scale shape: [9216] -197: model.layers.20.self_attn.qkv_proj.zeroPoint shape: [9216] -198: model.layers.21.input_layernorm.weight shape: [3072] -199: model.layers.21.mlp.down_proj.8bit_weight shape: [3072, 8192] -200: model.layers.21.mlp.down_proj.scale shape: [3072] -201: model.layers.21.mlp.down_proj.zeroPoint shape: [3072] -202: model.layers.21.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -203: model.layers.21.mlp.gate_up_proj.scale shape: [16384] -204: model.layers.21.mlp.gate_up_proj.zeroPoint shape: [16384] -205: model.layers.21.post_attention_layernorm.weight shape: [3072] -206: model.layers.21.self_attn.o_proj.8bit_weight shape: [3072, 3072] -207: model.layers.21.self_attn.o_proj.scale shape: [3072] -208: model.layers.21.self_attn.o_proj.zeroPoint shape: [3072] -209: model.layers.21.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -210: model.layers.21.self_attn.qkv_proj.scale shape: [9216] -211: model.layers.21.self_attn.qkv_proj.zeroPoint shape: [9216] -212: model.layers.22.input_layernorm.weight shape: [3072] -213: model.layers.22.mlp.down_proj.8bit_weight shape: [3072, 8192] -214: model.layers.22.mlp.down_proj.scale shape: [3072] -215: model.layers.22.mlp.down_proj.zeroPoint shape: [3072] -216: model.layers.22.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -217: model.layers.22.mlp.gate_up_proj.scale shape: [16384] -218: model.layers.22.mlp.gate_up_proj.zeroPoint shape: [16384] -219: model.layers.22.post_attention_layernorm.weight shape: [3072] -220: model.layers.22.self_attn.o_proj.8bit_weight shape: [3072, 3072] -221: model.layers.22.self_attn.o_proj.scale shape: [3072] -222: model.layers.22.self_attn.o_proj.zeroPoint shape: [3072] -223: model.layers.22.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -224: model.layers.22.self_attn.qkv_proj.scale shape: [9216] -225: model.layers.22.self_attn.qkv_proj.zeroPoint shape: [9216] -226: model.layers.23.input_layernorm.weight shape: [3072] -227: model.layers.23.mlp.down_proj.8bit_weight shape: [3072, 8192] -228: model.layers.23.mlp.down_proj.scale shape: [3072] -229: model.layers.23.mlp.down_proj.zeroPoint shape: [3072] -230: model.layers.23.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -231: model.layers.23.mlp.gate_up_proj.scale shape: [16384] -232: model.layers.23.mlp.gate_up_proj.zeroPoint shape: [16384] -233: model.layers.23.post_attention_layernorm.weight shape: [3072] -234: model.layers.23.self_attn.o_proj.8bit_weight shape: [3072, 3072] -235: model.layers.23.self_attn.o_proj.scale shape: [3072] -236: model.layers.23.self_attn.o_proj.zeroPoint shape: [3072] -237: model.layers.23.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -238: model.layers.23.self_attn.qkv_proj.scale shape: [9216] -239: model.layers.23.self_attn.qkv_proj.zeroPoint shape: [9216] -240: model.layers.24.input_layernorm.weight shape: [3072] -241: model.layers.24.mlp.down_proj.8bit_weight shape: [3072, 8192] -242: model.layers.24.mlp.down_proj.scale shape: [3072] -243: model.layers.24.mlp.down_proj.zeroPoint shape: [3072] -244: model.layers.24.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -245: model.layers.24.mlp.gate_up_proj.scale shape: [16384] -246: model.layers.24.mlp.gate_up_proj.zeroPoint shape: [16384] -247: model.layers.24.post_attention_layernorm.weight shape: [3072] -248: model.layers.24.self_attn.o_proj.8bit_weight shape: [3072, 3072] -249: model.layers.24.self_attn.o_proj.scale shape: [3072] -250: model.layers.24.self_attn.o_proj.zeroPoint shape: [3072] -251: model.layers.24.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -252: model.layers.24.self_attn.qkv_proj.scale shape: [9216] -253: model.layers.24.self_attn.qkv_proj.zeroPoint shape: [9216] -254: model.layers.25.input_layernorm.weight shape: [3072] -255: model.layers.25.mlp.down_proj.8bit_weight shape: [3072, 8192] -256: model.layers.25.mlp.down_proj.scale shape: [3072] -257: model.layers.25.mlp.down_proj.zeroPoint shape: [3072] -258: model.layers.25.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -259: model.layers.25.mlp.gate_up_proj.scale shape: [16384] -260: model.layers.25.mlp.gate_up_proj.zeroPoint shape: [16384] -261: model.layers.25.post_attention_layernorm.weight shape: [3072] -262: model.layers.25.self_attn.o_proj.8bit_weight shape: [3072, 3072] -263: model.layers.25.self_attn.o_proj.scale shape: [3072] -264: model.layers.25.self_attn.o_proj.zeroPoint shape: [3072] -265: model.layers.25.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -266: model.layers.25.self_attn.qkv_proj.scale shape: [9216] -267: model.layers.25.self_attn.qkv_proj.zeroPoint shape: [9216] -268: model.layers.26.input_layernorm.weight shape: [3072] -269: model.layers.26.mlp.down_proj.8bit_weight shape: [3072, 8192] -270: model.layers.26.mlp.down_proj.scale shape: [3072] -271: model.layers.26.mlp.down_proj.zeroPoint shape: [3072] -272: model.layers.26.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -273: model.layers.26.mlp.gate_up_proj.scale shape: [16384] -274: model.layers.26.mlp.gate_up_proj.zeroPoint shape: [16384] -275: model.layers.26.post_attention_layernorm.weight shape: [3072] -276: model.layers.26.self_attn.o_proj.8bit_weight shape: [3072, 3072] -277: model.layers.26.self_attn.o_proj.scale shape: [3072] -278: model.layers.26.self_attn.o_proj.zeroPoint shape: [3072] -279: model.layers.26.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -280: model.layers.26.self_attn.qkv_proj.scale shape: [9216] -281: model.layers.26.self_attn.qkv_proj.zeroPoint shape: [9216] -282: model.layers.27.input_layernorm.weight shape: [3072] -283: model.layers.27.mlp.down_proj.8bit_weight shape: [3072, 8192] -284: model.layers.27.mlp.down_proj.scale shape: [3072] -285: model.layers.27.mlp.down_proj.zeroPoint shape: [3072] -286: model.layers.27.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -287: model.layers.27.mlp.gate_up_proj.scale shape: [16384] -288: model.layers.27.mlp.gate_up_proj.zeroPoint shape: [16384] -289: model.layers.27.post_attention_layernorm.weight shape: [3072] -290: model.layers.27.self_attn.o_proj.8bit_weight shape: [3072, 3072] -291: model.layers.27.self_attn.o_proj.scale shape: [3072] -292: model.layers.27.self_attn.o_proj.zeroPoint shape: [3072] -293: model.layers.27.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -294: model.layers.27.self_attn.qkv_proj.scale shape: [9216] -295: model.layers.27.self_attn.qkv_proj.zeroPoint shape: [9216] -296: model.layers.28.input_layernorm.weight shape: [3072] -297: model.layers.28.mlp.down_proj.8bit_weight shape: [3072, 8192] -298: model.layers.28.mlp.down_proj.scale shape: [3072] -299: model.layers.28.mlp.down_proj.zeroPoint shape: [3072] -300: model.layers.28.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -301: model.layers.28.mlp.gate_up_proj.scale shape: [16384] -302: model.layers.28.mlp.gate_up_proj.zeroPoint shape: [16384] -303: model.layers.28.post_attention_layernorm.weight shape: [3072] -304: model.layers.28.self_attn.o_proj.8bit_weight shape: [3072, 3072] -305: model.layers.28.self_attn.o_proj.scale shape: [3072] -306: model.layers.28.self_attn.o_proj.zeroPoint shape: [3072] -307: model.layers.28.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -308: model.layers.28.self_attn.qkv_proj.scale shape: [9216] -309: model.layers.28.self_attn.qkv_proj.zeroPoint shape: [9216] -310: model.layers.29.input_layernorm.weight shape: [3072] -311: model.layers.29.mlp.down_proj.8bit_weight shape: [3072, 8192] -312: model.layers.29.mlp.down_proj.scale shape: [3072] -313: model.layers.29.mlp.down_proj.zeroPoint shape: [3072] -314: model.layers.29.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -315: model.layers.29.mlp.gate_up_proj.scale shape: [16384] -316: model.layers.29.mlp.gate_up_proj.zeroPoint shape: [16384] -317: model.layers.29.post_attention_layernorm.weight shape: [3072] -318: model.layers.29.self_attn.o_proj.8bit_weight shape: [3072, 3072] -319: model.layers.29.self_attn.o_proj.scale shape: [3072] -320: model.layers.29.self_attn.o_proj.zeroPoint shape: [3072] -321: model.layers.29.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -322: model.layers.29.self_attn.qkv_proj.scale shape: [9216] -323: model.layers.29.self_attn.qkv_proj.zeroPoint shape: [9216] -324: model.layers.3.input_layernorm.weight shape: [3072] -325: model.layers.3.mlp.down_proj.8bit_weight shape: [3072, 8192] -326: model.layers.3.mlp.down_proj.scale shape: [3072] -327: model.layers.3.mlp.down_proj.zeroPoint shape: [3072] -328: model.layers.3.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -329: model.layers.3.mlp.gate_up_proj.scale shape: [16384] -330: model.layers.3.mlp.gate_up_proj.zeroPoint shape: [16384] -331: model.layers.3.post_attention_layernorm.weight shape: [3072] -332: model.layers.3.self_attn.o_proj.8bit_weight shape: [3072, 3072] -333: model.layers.3.self_attn.o_proj.scale shape: [3072] -334: model.layers.3.self_attn.o_proj.zeroPoint shape: [3072] -335: model.layers.3.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -336: model.layers.3.self_attn.qkv_proj.scale shape: [9216] -337: model.layers.3.self_attn.qkv_proj.zeroPoint shape: [9216] -338: model.layers.30.input_layernorm.weight shape: [3072] -339: model.layers.30.mlp.down_proj.8bit_weight shape: [3072, 8192] -340: model.layers.30.mlp.down_proj.scale shape: [3072] -341: model.layers.30.mlp.down_proj.zeroPoint shape: [3072] -342: model.layers.30.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -343: model.layers.30.mlp.gate_up_proj.scale shape: [16384] -344: model.layers.30.mlp.gate_up_proj.zeroPoint shape: [16384] -345: model.layers.30.post_attention_layernorm.weight shape: [3072] -346: model.layers.30.self_attn.o_proj.8bit_weight shape: [3072, 3072] -347: model.layers.30.self_attn.o_proj.scale shape: [3072] -348: model.layers.30.self_attn.o_proj.zeroPoint shape: [3072] -349: model.layers.30.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -350: model.layers.30.self_attn.qkv_proj.scale shape: [9216] -351: model.layers.30.self_attn.qkv_proj.zeroPoint shape: [9216] -352: model.layers.31.input_layernorm.weight shape: [3072] -353: model.layers.31.mlp.down_proj.8bit_weight shape: [3072, 8192] -354: model.layers.31.mlp.down_proj.scale shape: [3072] -355: model.layers.31.mlp.down_proj.zeroPoint shape: [3072] -356: model.layers.31.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -357: model.layers.31.mlp.gate_up_proj.scale shape: [16384] -358: model.layers.31.mlp.gate_up_proj.zeroPoint shape: [16384] -359: model.layers.31.post_attention_layernorm.weight shape: [3072] -360: model.layers.31.self_attn.o_proj.8bit_weight shape: [3072, 3072] -361: model.layers.31.self_attn.o_proj.scale shape: [3072] -362: model.layers.31.self_attn.o_proj.zeroPoint shape: [3072] -363: model.layers.31.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -364: model.layers.31.self_attn.qkv_proj.scale shape: [9216] -365: model.layers.31.self_attn.qkv_proj.zeroPoint shape: [9216] -366: model.layers.4.input_layernorm.weight shape: [3072] -367: model.layers.4.mlp.down_proj.8bit_weight shape: [3072, 8192] -368: model.layers.4.mlp.down_proj.scale shape: [3072] -369: model.layers.4.mlp.down_proj.zeroPoint shape: [3072] -370: model.layers.4.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -371: model.layers.4.mlp.gate_up_proj.scale shape: [16384] -372: model.layers.4.mlp.gate_up_proj.zeroPoint shape: [16384] -373: model.layers.4.post_attention_layernorm.weight shape: [3072] -374: model.layers.4.self_attn.o_proj.8bit_weight shape: [3072, 3072] -375: model.layers.4.self_attn.o_proj.scale shape: [3072] -376: model.layers.4.self_attn.o_proj.zeroPoint shape: [3072] -377: model.layers.4.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -378: model.layers.4.self_attn.qkv_proj.scale shape: [9216] -379: model.layers.4.self_attn.qkv_proj.zeroPoint shape: [9216] -380: model.layers.5.input_layernorm.weight shape: [3072] -381: model.layers.5.mlp.down_proj.8bit_weight shape: [3072, 8192] -382: model.layers.5.mlp.down_proj.scale shape: [3072] -383: model.layers.5.mlp.down_proj.zeroPoint shape: [3072] -384: model.layers.5.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -385: model.layers.5.mlp.gate_up_proj.scale shape: [16384] -386: model.layers.5.mlp.gate_up_proj.zeroPoint shape: [16384] -387: model.layers.5.post_attention_layernorm.weight shape: [3072] -388: model.layers.5.self_attn.o_proj.8bit_weight shape: [3072, 3072] -389: model.layers.5.self_attn.o_proj.scale shape: [3072] -390: model.layers.5.self_attn.o_proj.zeroPoint shape: [3072] -391: model.layers.5.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -392: model.layers.5.self_attn.qkv_proj.scale shape: [9216] -393: model.layers.5.self_attn.qkv_proj.zeroPoint shape: [9216] -394: model.layers.6.input_layernorm.weight shape: [3072] -395: model.layers.6.mlp.down_proj.8bit_weight shape: [3072, 8192] -396: model.layers.6.mlp.down_proj.scale shape: [3072] -397: model.layers.6.mlp.down_proj.zeroPoint shape: [3072] -398: model.layers.6.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -399: model.layers.6.mlp.gate_up_proj.scale shape: [16384] -400: model.layers.6.mlp.gate_up_proj.zeroPoint shape: [16384] -401: model.layers.6.post_attention_layernorm.weight shape: [3072] -402: model.layers.6.self_attn.o_proj.8bit_weight shape: [3072, 3072] -403: model.layers.6.self_attn.o_proj.scale shape: [3072] -404: model.layers.6.self_attn.o_proj.zeroPoint shape: [3072] -405: model.layers.6.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -406: model.layers.6.self_attn.qkv_proj.scale shape: [9216] -407: model.layers.6.self_attn.qkv_proj.zeroPoint shape: [9216] -408: model.layers.7.input_layernorm.weight shape: [3072] -409: model.layers.7.mlp.down_proj.8bit_weight shape: [3072, 8192] -410: model.layers.7.mlp.down_proj.scale shape: [3072] -411: model.layers.7.mlp.down_proj.zeroPoint shape: [3072] -412: model.layers.7.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -413: model.layers.7.mlp.gate_up_proj.scale shape: [16384] -414: model.layers.7.mlp.gate_up_proj.zeroPoint shape: [16384] -415: model.layers.7.post_attention_layernorm.weight shape: [3072] -416: model.layers.7.self_attn.o_proj.8bit_weight shape: [3072, 3072] -417: model.layers.7.self_attn.o_proj.scale shape: [3072] -418: model.layers.7.self_attn.o_proj.zeroPoint shape: [3072] -419: model.layers.7.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -420: model.layers.7.self_attn.qkv_proj.scale shape: [9216] -421: model.layers.7.self_attn.qkv_proj.zeroPoint shape: [9216] -422: model.layers.8.input_layernorm.weight shape: [3072] -423: model.layers.8.mlp.down_proj.8bit_weight shape: [3072, 8192] -424: model.layers.8.mlp.down_proj.scale shape: [3072] -425: model.layers.8.mlp.down_proj.zeroPoint shape: [3072] -426: model.layers.8.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -427: model.layers.8.mlp.gate_up_proj.scale shape: [16384] -428: model.layers.8.mlp.gate_up_proj.zeroPoint shape: [16384] -429: model.layers.8.post_attention_layernorm.weight shape: [3072] -430: model.layers.8.self_attn.o_proj.8bit_weight shape: [3072, 3072] -431: model.layers.8.self_attn.o_proj.scale shape: [3072] -432: model.layers.8.self_attn.o_proj.zeroPoint shape: [3072] -433: model.layers.8.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -434: model.layers.8.self_attn.qkv_proj.scale shape: [9216] -435: model.layers.8.self_attn.qkv_proj.zeroPoint shape: [9216] -436: model.layers.9.input_layernorm.weight shape: [3072] -437: model.layers.9.mlp.down_proj.8bit_weight shape: [3072, 8192] -438: model.layers.9.mlp.down_proj.scale shape: [3072] -439: model.layers.9.mlp.down_proj.zeroPoint shape: [3072] -440: model.layers.9.mlp.gate_up_proj.8bit_weight shape: [16384, 3072] -441: model.layers.9.mlp.gate_up_proj.scale shape: [16384] -442: model.layers.9.mlp.gate_up_proj.zeroPoint shape: [16384] -443: model.layers.9.post_attention_layernorm.weight shape: [3072] -444: model.layers.9.self_attn.o_proj.8bit_weight shape: [3072, 3072] -445: model.layers.9.self_attn.o_proj.scale shape: [3072] -446: model.layers.9.self_attn.o_proj.zeroPoint shape: [3072] -447: model.layers.9.self_attn.qkv_proj.8bit_weight shape: [9216, 3072] -448: model.layers.9.self_attn.qkv_proj.scale shape: [9216] -449: model.layers.9.self_attn.qkv_proj.zeroPoint shape: [9216] -450: model.norm.weight shape: [3072] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KShapeTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KShapeTest.approved.txt deleted file mode 100644 index 2278f3b67d..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.Phi3Mini4KShapeTest.approved.txt +++ /dev/null @@ -1,195 +0,0 @@ -0: lm_head.weight shape: [32064, 3072] -1: model.embed_tokens.weight shape: [32064, 3072] -2: model.layers.0.input_layernorm.weight shape: [3072] -3: model.layers.0.mlp.down_proj.weight shape: [3072, 8192] -4: model.layers.0.mlp.gate_up_proj.weight shape: [16384, 3072] -5: model.layers.0.post_attention_layernorm.weight shape: [3072] -6: model.layers.0.self_attn.o_proj.weight shape: [3072, 3072] -7: model.layers.0.self_attn.qkv_proj.weight shape: [9216, 3072] -8: model.layers.1.input_layernorm.weight shape: [3072] -9: model.layers.1.mlp.down_proj.weight shape: [3072, 8192] -10: model.layers.1.mlp.gate_up_proj.weight shape: [16384, 3072] -11: model.layers.1.post_attention_layernorm.weight shape: [3072] -12: model.layers.1.self_attn.o_proj.weight shape: [3072, 3072] -13: model.layers.1.self_attn.qkv_proj.weight shape: [9216, 3072] -14: model.layers.10.input_layernorm.weight shape: [3072] -15: model.layers.10.mlp.down_proj.weight shape: [3072, 8192] -16: model.layers.10.mlp.gate_up_proj.weight shape: [16384, 3072] -17: model.layers.10.post_attention_layernorm.weight shape: [3072] -18: model.layers.10.self_attn.o_proj.weight shape: [3072, 3072] -19: model.layers.10.self_attn.qkv_proj.weight shape: [9216, 3072] -20: model.layers.11.input_layernorm.weight shape: [3072] -21: model.layers.11.mlp.down_proj.weight shape: [3072, 8192] -22: model.layers.11.mlp.gate_up_proj.weight shape: [16384, 3072] -23: model.layers.11.post_attention_layernorm.weight shape: [3072] -24: model.layers.11.self_attn.o_proj.weight shape: [3072, 3072] -25: model.layers.11.self_attn.qkv_proj.weight shape: [9216, 3072] -26: model.layers.12.input_layernorm.weight shape: [3072] -27: model.layers.12.mlp.down_proj.weight shape: [3072, 8192] -28: model.layers.12.mlp.gate_up_proj.weight shape: [16384, 3072] -29: model.layers.12.post_attention_layernorm.weight shape: [3072] -30: model.layers.12.self_attn.o_proj.weight shape: [3072, 3072] -31: model.layers.12.self_attn.qkv_proj.weight shape: [9216, 3072] -32: model.layers.13.input_layernorm.weight shape: [3072] -33: model.layers.13.mlp.down_proj.weight shape: [3072, 8192] -34: model.layers.13.mlp.gate_up_proj.weight shape: [16384, 3072] -35: model.layers.13.post_attention_layernorm.weight shape: [3072] -36: model.layers.13.self_attn.o_proj.weight shape: [3072, 3072] -37: model.layers.13.self_attn.qkv_proj.weight shape: [9216, 3072] -38: model.layers.14.input_layernorm.weight shape: [3072] -39: model.layers.14.mlp.down_proj.weight shape: [3072, 8192] -40: model.layers.14.mlp.gate_up_proj.weight shape: [16384, 3072] -41: model.layers.14.post_attention_layernorm.weight shape: [3072] -42: model.layers.14.self_attn.o_proj.weight shape: [3072, 3072] -43: model.layers.14.self_attn.qkv_proj.weight shape: [9216, 3072] -44: model.layers.15.input_layernorm.weight shape: [3072] -45: model.layers.15.mlp.down_proj.weight shape: [3072, 8192] -46: model.layers.15.mlp.gate_up_proj.weight shape: [16384, 3072] -47: model.layers.15.post_attention_layernorm.weight shape: [3072] -48: model.layers.15.self_attn.o_proj.weight shape: [3072, 3072] -49: model.layers.15.self_attn.qkv_proj.weight shape: [9216, 3072] -50: model.layers.16.input_layernorm.weight shape: [3072] -51: model.layers.16.mlp.down_proj.weight shape: [3072, 8192] -52: model.layers.16.mlp.gate_up_proj.weight shape: [16384, 3072] -53: model.layers.16.post_attention_layernorm.weight shape: [3072] -54: model.layers.16.self_attn.o_proj.weight shape: [3072, 3072] -55: model.layers.16.self_attn.qkv_proj.weight shape: [9216, 3072] -56: model.layers.17.input_layernorm.weight shape: [3072] -57: model.layers.17.mlp.down_proj.weight shape: [3072, 8192] -58: model.layers.17.mlp.gate_up_proj.weight shape: [16384, 3072] -59: model.layers.17.post_attention_layernorm.weight shape: [3072] -60: model.layers.17.self_attn.o_proj.weight shape: [3072, 3072] -61: model.layers.17.self_attn.qkv_proj.weight shape: [9216, 3072] -62: model.layers.18.input_layernorm.weight shape: [3072] -63: model.layers.18.mlp.down_proj.weight shape: [3072, 8192] -64: model.layers.18.mlp.gate_up_proj.weight shape: [16384, 3072] -65: model.layers.18.post_attention_layernorm.weight shape: [3072] -66: model.layers.18.self_attn.o_proj.weight shape: [3072, 3072] -67: model.layers.18.self_attn.qkv_proj.weight shape: [9216, 3072] -68: model.layers.19.input_layernorm.weight shape: [3072] -69: model.layers.19.mlp.down_proj.weight shape: [3072, 8192] -70: model.layers.19.mlp.gate_up_proj.weight shape: [16384, 3072] -71: model.layers.19.post_attention_layernorm.weight shape: [3072] -72: model.layers.19.self_attn.o_proj.weight shape: [3072, 3072] -73: model.layers.19.self_attn.qkv_proj.weight shape: [9216, 3072] -74: model.layers.2.input_layernorm.weight shape: [3072] -75: model.layers.2.mlp.down_proj.weight shape: [3072, 8192] -76: model.layers.2.mlp.gate_up_proj.weight shape: [16384, 3072] -77: model.layers.2.post_attention_layernorm.weight shape: [3072] -78: model.layers.2.self_attn.o_proj.weight shape: [3072, 3072] -79: model.layers.2.self_attn.qkv_proj.weight shape: [9216, 3072] -80: model.layers.20.input_layernorm.weight shape: [3072] -81: model.layers.20.mlp.down_proj.weight shape: [3072, 8192] -82: model.layers.20.mlp.gate_up_proj.weight shape: [16384, 3072] -83: model.layers.20.post_attention_layernorm.weight shape: [3072] -84: model.layers.20.self_attn.o_proj.weight shape: [3072, 3072] -85: model.layers.20.self_attn.qkv_proj.weight shape: [9216, 3072] -86: model.layers.21.input_layernorm.weight shape: [3072] -87: model.layers.21.mlp.down_proj.weight shape: [3072, 8192] -88: model.layers.21.mlp.gate_up_proj.weight shape: [16384, 3072] -89: model.layers.21.post_attention_layernorm.weight shape: [3072] -90: model.layers.21.self_attn.o_proj.weight shape: [3072, 3072] -91: model.layers.21.self_attn.qkv_proj.weight shape: [9216, 3072] -92: model.layers.22.input_layernorm.weight shape: [3072] -93: model.layers.22.mlp.down_proj.weight shape: [3072, 8192] -94: model.layers.22.mlp.gate_up_proj.weight shape: [16384, 3072] -95: model.layers.22.post_attention_layernorm.weight shape: [3072] -96: model.layers.22.self_attn.o_proj.weight shape: [3072, 3072] -97: model.layers.22.self_attn.qkv_proj.weight shape: [9216, 3072] -98: model.layers.23.input_layernorm.weight shape: [3072] -99: model.layers.23.mlp.down_proj.weight shape: [3072, 8192] -100: model.layers.23.mlp.gate_up_proj.weight shape: [16384, 3072] -101: model.layers.23.post_attention_layernorm.weight shape: [3072] -102: model.layers.23.self_attn.o_proj.weight shape: [3072, 3072] -103: model.layers.23.self_attn.qkv_proj.weight shape: [9216, 3072] -104: model.layers.24.input_layernorm.weight shape: [3072] -105: model.layers.24.mlp.down_proj.weight shape: [3072, 8192] -106: model.layers.24.mlp.gate_up_proj.weight shape: [16384, 3072] -107: model.layers.24.post_attention_layernorm.weight shape: [3072] -108: model.layers.24.self_attn.o_proj.weight shape: [3072, 3072] -109: model.layers.24.self_attn.qkv_proj.weight shape: [9216, 3072] -110: model.layers.25.input_layernorm.weight shape: [3072] -111: model.layers.25.mlp.down_proj.weight shape: [3072, 8192] -112: model.layers.25.mlp.gate_up_proj.weight shape: [16384, 3072] -113: model.layers.25.post_attention_layernorm.weight shape: [3072] -114: model.layers.25.self_attn.o_proj.weight shape: [3072, 3072] -115: model.layers.25.self_attn.qkv_proj.weight shape: [9216, 3072] -116: model.layers.26.input_layernorm.weight shape: [3072] -117: model.layers.26.mlp.down_proj.weight shape: [3072, 8192] -118: model.layers.26.mlp.gate_up_proj.weight shape: [16384, 3072] -119: model.layers.26.post_attention_layernorm.weight shape: [3072] -120: model.layers.26.self_attn.o_proj.weight shape: [3072, 3072] -121: model.layers.26.self_attn.qkv_proj.weight shape: [9216, 3072] -122: model.layers.27.input_layernorm.weight shape: [3072] -123: model.layers.27.mlp.down_proj.weight shape: [3072, 8192] -124: model.layers.27.mlp.gate_up_proj.weight shape: [16384, 3072] -125: model.layers.27.post_attention_layernorm.weight shape: [3072] -126: model.layers.27.self_attn.o_proj.weight shape: [3072, 3072] -127: model.layers.27.self_attn.qkv_proj.weight shape: [9216, 3072] -128: model.layers.28.input_layernorm.weight shape: [3072] -129: model.layers.28.mlp.down_proj.weight shape: [3072, 8192] -130: model.layers.28.mlp.gate_up_proj.weight shape: [16384, 3072] -131: model.layers.28.post_attention_layernorm.weight shape: [3072] -132: model.layers.28.self_attn.o_proj.weight shape: [3072, 3072] -133: model.layers.28.self_attn.qkv_proj.weight shape: [9216, 3072] -134: model.layers.29.input_layernorm.weight shape: [3072] -135: model.layers.29.mlp.down_proj.weight shape: [3072, 8192] -136: model.layers.29.mlp.gate_up_proj.weight shape: [16384, 3072] -137: model.layers.29.post_attention_layernorm.weight shape: [3072] -138: model.layers.29.self_attn.o_proj.weight shape: [3072, 3072] -139: model.layers.29.self_attn.qkv_proj.weight shape: [9216, 3072] -140: model.layers.3.input_layernorm.weight shape: [3072] -141: model.layers.3.mlp.down_proj.weight shape: [3072, 8192] -142: model.layers.3.mlp.gate_up_proj.weight shape: [16384, 3072] -143: model.layers.3.post_attention_layernorm.weight shape: [3072] -144: model.layers.3.self_attn.o_proj.weight shape: [3072, 3072] -145: model.layers.3.self_attn.qkv_proj.weight shape: [9216, 3072] -146: model.layers.30.input_layernorm.weight shape: [3072] -147: model.layers.30.mlp.down_proj.weight shape: [3072, 8192] -148: model.layers.30.mlp.gate_up_proj.weight shape: [16384, 3072] -149: model.layers.30.post_attention_layernorm.weight shape: [3072] -150: model.layers.30.self_attn.o_proj.weight shape: [3072, 3072] -151: model.layers.30.self_attn.qkv_proj.weight shape: [9216, 3072] -152: model.layers.31.input_layernorm.weight shape: [3072] -153: model.layers.31.mlp.down_proj.weight shape: [3072, 8192] -154: model.layers.31.mlp.gate_up_proj.weight shape: [16384, 3072] -155: model.layers.31.post_attention_layernorm.weight shape: [3072] -156: model.layers.31.self_attn.o_proj.weight shape: [3072, 3072] -157: model.layers.31.self_attn.qkv_proj.weight shape: [9216, 3072] -158: model.layers.4.input_layernorm.weight shape: [3072] -159: model.layers.4.mlp.down_proj.weight shape: [3072, 8192] -160: model.layers.4.mlp.gate_up_proj.weight shape: [16384, 3072] -161: model.layers.4.post_attention_layernorm.weight shape: [3072] -162: model.layers.4.self_attn.o_proj.weight shape: [3072, 3072] -163: model.layers.4.self_attn.qkv_proj.weight shape: [9216, 3072] -164: model.layers.5.input_layernorm.weight shape: [3072] -165: model.layers.5.mlp.down_proj.weight shape: [3072, 8192] -166: model.layers.5.mlp.gate_up_proj.weight shape: [16384, 3072] -167: model.layers.5.post_attention_layernorm.weight shape: [3072] -168: model.layers.5.self_attn.o_proj.weight shape: [3072, 3072] -169: model.layers.5.self_attn.qkv_proj.weight shape: [9216, 3072] -170: model.layers.6.input_layernorm.weight shape: [3072] -171: model.layers.6.mlp.down_proj.weight shape: [3072, 8192] -172: model.layers.6.mlp.gate_up_proj.weight shape: [16384, 3072] -173: model.layers.6.post_attention_layernorm.weight shape: [3072] -174: model.layers.6.self_attn.o_proj.weight shape: [3072, 3072] -175: model.layers.6.self_attn.qkv_proj.weight shape: [9216, 3072] -176: model.layers.7.input_layernorm.weight shape: [3072] -177: model.layers.7.mlp.down_proj.weight shape: [3072, 8192] -178: model.layers.7.mlp.gate_up_proj.weight shape: [16384, 3072] -179: model.layers.7.post_attention_layernorm.weight shape: [3072] -180: model.layers.7.self_attn.o_proj.weight shape: [3072, 3072] -181: model.layers.7.self_attn.qkv_proj.weight shape: [9216, 3072] -182: model.layers.8.input_layernorm.weight shape: [3072] -183: model.layers.8.mlp.down_proj.weight shape: [3072, 8192] -184: model.layers.8.mlp.gate_up_proj.weight shape: [16384, 3072] -185: model.layers.8.post_attention_layernorm.weight shape: [3072] -186: model.layers.8.self_attn.o_proj.weight shape: [3072, 3072] -187: model.layers.8.self_attn.qkv_proj.weight shape: [9216, 3072] -188: model.layers.9.input_layernorm.weight shape: [3072] -189: model.layers.9.mlp.down_proj.weight shape: [3072, 8192] -190: model.layers.9.mlp.gate_up_proj.weight shape: [16384, 3072] -191: model.layers.9.post_attention_layernorm.weight shape: [3072] -192: model.layers.9.self_attn.o_proj.weight shape: [3072, 3072] -193: model.layers.9.self_attn.qkv_proj.weight shape: [9216, 3072] -194: model.norm.weight shape: [3072] diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.TokenizerTest.approved.txt b/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.TokenizerTest.approved.txt deleted file mode 100644 index 70624d24df..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Approvals/Phi3Tests.TokenizerTest.approved.txt +++ /dev/null @@ -1,20 +0,0 @@ -Can you provide ways to eat combinations of bananas and dragonfruits? -1, 1815, 366, 3867, 5837, 304, 17545, 18240, 310, 9892, 16397, 322, 8338, 265, 29888, 21211, 29973 -Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey. -1, 18585, 29991, 2266, 526, 777, 5837, 304, 17545, 9892, 16397, 322, 8338, 265, 29888, 21211, 4208, 29901, 29871, 29896, 29889, 10765, 1648, 322, 8338, 265, 29888, 9216, 10597, 347, 29901, 3164, 355, 9892, 16397, 322, 8338, 265, 29888, 21211, 4208, 411, 777, 27274, 322, 298, 4992, 29889, 29871, 29906, 29889, 10765, 1648, 322, 8338, 265, 29888, 9216, 4497, 328, 29901, 23478, 269, 506, 287, 9892, 16397, 322, 8338, 265, 29888, 21211, 4208, 411, 777, 454, 3712, 3623, 625, 322, 298, 4992, 29889 -What about solving an 2x + 3 = 7 equation? -1, 1724, 1048, 17069, 385, 29871, 29906, 29916, 718, 29871, 29941, 353, 29871, 29955, 6306, 29973 - -Count to 3 - -1, 29871, 13, 3981, 304, 29871, 29941, 13 -<|user|> -1, 32010 -<|end|> -1, 32007 -<|assistant|> -1, 32001 -<|user|> -Count to 3<|end|> -<|assistant|> -1, 32010, 29871, 13, 3981, 304, 29871, 29941, 32007, 13, 32001 diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/AutoGenTests.cs b/test/Microsoft.ML.GenAI.Phi.Tests/AutoGenTests.cs deleted file mode 100644 index 6f0928f3c0..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/AutoGenTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using AutoGen.Core; -using FluentAssertions; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.Tokenizers; -using Moq; -using Xunit; - -namespace Microsoft.ML.GenAI.Phi.Tests; - -public class AutoGenTests -{ - [Fact] - public async Task ItGenerateTextReply() - { - var pipeline = Mock.Of>(); - // mock generate api - Mock.Get(pipeline).Setup(p => p.Generate( - It.IsAny(), // prompt - It.IsAny(), // max length - It.IsAny(), // temperature - It.IsAny(), // top_p - It.IsAny())) // stop sequence - .Callback((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => - { - // check prompt - prompt.Should().Be("<|system|>\nyou are a helpful assistant<|end|>\n<|user|>\nhey<|end|>\n<|assistant|>"); - }) - .Returns((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => "hello"); - - var agent = new Phi3Agent(pipeline, "assistant"); - var reply = await agent.SendAsync("hey"); - - reply.GetContent().Should().Be("hello"); - reply.From.Should().Be(agent.Name); - } -} diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Microsoft.ML.GenAI.Phi.Tests.csproj b/test/Microsoft.ML.GenAI.Phi.Tests/Microsoft.ML.GenAI.Phi.Tests.csproj deleted file mode 100644 index 62d6260523..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Microsoft.ML.GenAI.Phi.Tests.csproj +++ /dev/null @@ -1,46 +0,0 @@ - - - - net8.0 - enable - $(NoWarn);MSML_ExtendBaseTestClass - enable - true - true - - - - - - - - - - - - - - - - - - - - - - PreserveNewest - - - - - - - - - - - - - - - diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Phi2Tests.cs b/test/Microsoft.ML.GenAI.Phi.Tests/Phi2Tests.cs deleted file mode 100644 index a196d8ab81..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Phi2Tests.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using ApprovalTests; -using ApprovalTests.Namers; -using ApprovalTests.Reporters; -using FluentAssertions; -using Microsoft.ML.GenAI.Core.Extension; -using Microsoft.ML.Tokenizers; -using TorchSharp; -using Xunit; -namespace Microsoft.ML.GenAI.Phi.Tests; - -[Collection("NoParallelization")] -public class Phi2Tests -{ - public Phi2Tests() - { - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } - torch.set_default_device("meta"); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void LoadSafeTensorShapeTest() - { - var model = new Phi2ForCausalLM(Phi2Config.Phi2); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void TokenizerTest() - { - var modelWeightFolder = Path.Join("Phi-2"); - var tokenizer = Phi2TokenizerHelper.Create(modelWeightFolder, addBeginOfSentence: true); - tokenizer.EndOfSentenceId.Should().Be(50256); - tokenizer.BeginningOfSentenceId.Should().Be(50256); - var messages = new string[] - { - "Can you provide ways to eat combinations of bananas and dragonfruits?", - "Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey.", - "What about solving an 2x + 3 = 7 equation?" - }; - var sb = new StringBuilder(); - foreach (var message in messages) - { - var tokenized = tokenizer.EncodeToIds(message, true, false); - var tokenizedStr = string.Join(", ", tokenized.Select(x => x.ToString())); - - sb.AppendLine(tokenizedStr); - } - Approvals.Verify(sb.ToString()); - } -} diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/Phi3Tests.cs b/test/Microsoft.ML.GenAI.Phi.Tests/Phi3Tests.cs deleted file mode 100644 index aec4016fd7..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/Phi3Tests.cs +++ /dev/null @@ -1,151 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System.Text; -using System.Text.Json; -using ApprovalTests; -using ApprovalTests.Namers; -using ApprovalTests.Reporters; -using FluentAssertions; -using Microsoft.ML.GenAI.Core.Extension; -using TorchSharp; -using Xunit; - -namespace Microsoft.ML.GenAI.Phi.Tests; - -[Collection("NoParallelization")] -public class Phi3Tests -{ - public Phi3Tests() - { - if (Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID") != null) - { - Approvals.UseAssemblyLocationForApprovedFiles(); - } - - torch.set_default_device("meta"); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Mini4KShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Mini4kInstruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Mini4KInt8QuantizeShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Mini4kInstruct); - model.ToInt8QuantizeModule(); - var size = model.GetSizeInBytes(); - var stateDictStr = model.PeekShape(); - var sizeInGB = size / 1024 / 1024 / 1024; - sizeInGB.Should().Be(3); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Mini4KInt4QuantizeShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Mini4kInstruct); - model.ToInt4QuantizeModule(); - var size = model.GetSizeInBytes(); - var stateDictStr = model.PeekShape(); - var sizeInGB = size / 1024 / 1024 / 1024; - sizeInGB.Should().Be(2); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Medium4KShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Medium4kInstruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Medium128KShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Medium128kInstruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Mini128KShapeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Mini128kInstruct); - var stateDictStr = model.PeekShape(); - Approvals.Verify(stateDictStr); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void Phi3Mini128KLayerSizeTest() - { - var model = new Phi3ForCausalLM(Phi3Config.Phi3Mini128kInstruct); - var size = model.GetSizeForEachDynamicLayerInBytes(); - // convert size to MB - var sizeInMB = size.ToDictionary(x => x.Key, x => x.Value / 1024 / 1024); - - var json = JsonSerializer.Serialize(sizeInMB, new JsonSerializerOptions { WriteIndented = true }); - Approvals.Verify(json); - } - - [Fact] - [UseReporter(typeof(DiffReporter))] - [UseApprovalSubdirectory("Approvals")] - public void TokenizerTest() - { - var modelWeightFolder = Path.Join("Llama"); - var modelPath = Path.Join(modelWeightFolder, "tokenizer.model"); - var tokenizer = Phi3TokenizerHelper.FromPretrained(modelPath); - tokenizer.BeginningOfSentenceId.Should().Be(1); - tokenizer.EndOfSentenceId.Should().Be(2); - - // test <|end|> - var endIds = tokenizer.EncodeToIds("<|end|>", addBeginningOfSentence: false, addEndOfSentence: false, considerPreTokenization: false, considerNormalization: false); - endIds.Should().BeEquivalentTo(new int[] { 32007 }); - - var messages = new string[] - { - "Can you provide ways to eat combinations of bananas and dragonfruits?", - "Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey.", - "What about solving an 2x + 3 = 7 equation?", - "\nCount to 3\n", - "<|user|>", - "<|end|>", - "<|assistant|>", - "<|user|>\nCount to 3<|end|>\n<|assistant|>", - }; - var sb = new StringBuilder(); - foreach (var message in messages) - { - var tokenizeIds = tokenizer.EncodeToIds(message, true, false, considerPreTokenization: true); - var decodeToString = tokenizer.Decode(tokenizeIds, considerSpecialTokens: true); - sb.AppendLine(decodeToString); - var tokenizedStr = string.Join(", ", tokenizeIds.Select(x => x.ToString())); - - sb.AppendLine(tokenizedStr); - } - Approvals.Verify(sb.ToString()); - } -} diff --git a/test/Microsoft.ML.GenAI.Phi.Tests/SemanticKernelTests.cs b/test/Microsoft.ML.GenAI.Phi.Tests/SemanticKernelTests.cs deleted file mode 100644 index 63777bc3bf..0000000000 --- a/test/Microsoft.ML.GenAI.Phi.Tests/SemanticKernelTests.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.ML.GenAI.Core; -using Microsoft.ML.GenAI.Phi.Extension; -using Microsoft.ML.Tokenizers; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.ChatCompletion; -using Moq; -using Xunit; - -namespace Microsoft.ML.GenAI.Phi.Tests; - -public class SemanticKernelTests -{ - [Fact] - public async Task ItAddPhi3CausalLMChatCompletionServiceTestAsync() - { - var pipeline = Mock.Of>(); - // mock generate api - Mock.Get(pipeline).Setup(p => p.Generate( - It.IsAny(), // prompt - It.IsAny(), // max length - It.IsAny(), // temperature - It.IsAny(), // top_p - It.IsAny())) // stop sequence - .Callback((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => - { - // check prompt - prompt.Should().Be("<|system|>\nyou are a helpful assistant<|end|>\n<|user|>\nhey<|end|>\n<|assistant|>"); - }) - .Returns((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => "hello"); - - var kernel = Kernel.CreateBuilder() - .AddGenAIChatCompletion(pipeline) - .Build(); - - var chatService = kernel.Services.GetRequiredService(); - - var chatHistory = new ChatHistory(); - chatHistory.AddSystemMessage("you are a helpful assistant"); - chatHistory.AddUserMessage("hey"); - var responses = await chatService.GetChatMessageContentsAsync(chatHistory); - responses.Count().Should().Be(1); - var response = responses.First(); - response.Role.Should().Be(AuthorRole.Assistant); - response.Items.Count().Should().Be(1); - var textContent = response.Items.First() as TextContent; - textContent!.Text.Should().Be("hello"); - } - - [Fact] - public async Task ItAddPhi3CausalLMTextGenerationServiceTestAsync() - { - var pipeline = Mock.Of>(); - // mock generate api - Mock.Get(pipeline).Setup(p => p.Generate( - It.IsAny(), // prompt - It.IsAny(), // max length - It.IsAny(), // temperature - It.IsAny(), // top_p - It.IsAny())) // stop sequence - .Callback((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => - { - // check prompt - prompt.Should().Be("test"); - }) - .Returns((string prompt, int maxLen, float temperature, float topP, string[] stopSequences) => "hello"); - - var kernel = Kernel.CreateBuilder() - .AddGenAITextGeneration(pipeline) - .Build(); - - var response = await kernel.InvokePromptAsync("test"); - } -} diff --git a/test/Microsoft.ML.TorchSharp.Tests/Microsoft.ML.TorchSharp.Tests.csproj b/test/Microsoft.ML.TorchSharp.Tests/Microsoft.ML.TorchSharp.Tests.csproj index 7e28551e01..d53f849413 100644 --- a/test/Microsoft.ML.TorchSharp.Tests/Microsoft.ML.TorchSharp.Tests.csproj +++ b/test/Microsoft.ML.TorchSharp.Tests/Microsoft.ML.TorchSharp.Tests.csproj @@ -30,7 +30,6 @@ -