diff --git a/python/stt/whisperService/README.md b/python/stt/whisperService/README.md index c3bc6883c6..1e33f9068a 100644 --- a/python/stt/whisperService/README.md +++ b/python/stt/whisperService/README.md @@ -7,16 +7,16 @@ You can launch the service by running e.g. `python faster-whisper.py`. ### Prerequisites -- [Python 3](https://wiki.python.org/moin/BeginnersGuide/Download) +- [Python 3.9 or newer](https://wiki.python.org/moin/BeginnersGuide/Download) - [Pip](https://pip.pypa.io/en/stable/installation/) - **Windows** - [FFMpeg](https://www.gyan.dev/ffmpeg/builds/) - you can run `winget install ffmpeg` to install the package - **Linux** - `apt-get -y install python3-pyaudio portaudio19-dev` - needed to install `pyaudio` - - `apt-get -y install ffmpeg libavcodec-extra` - needed for pydub + - `apt-get -y install ffmpeg libavcodec-extra` - needed for audio decoding - `apt-get -y install cudnn9-cuda-12`- Follow instruction in [NVIDIA cuDNN on Linux](https://docs.nvidia.com/deeplearning/cudnn/latest/installation/linux.html) to enable NVIDIA network repository. - **MacOS** - - `brew install portaudio` - needed to install `pyaudio` + - `brew install ffmpeg portaudio` - needed for audio decoding and `pyaudio` ### Install @@ -46,3 +46,20 @@ You can connect to the whisper service using the example "whisperClient" project - Start the web UI using `pnpm run start` This web client will capture audio from microphone, send to the local service for transcription and show the result. + +## Input and processing limits + +The service accepts audio files up to **30 MiB** and recordings up to **five minutes**. The complete multipart request has a separate 30 MiB + 64 KiB limit for form headers. Both declared and streamed request sizes are checked before multipart parsing. Once a request reaches the front of the queue, its complete body must arrive within **30 seconds**, including continuously streamed uploads. An upload timeout returns HTTP 408 with an `error` message and releases capacity for the next request; queueing and transcription do not count toward this deadline. + +FFmpeg decodes only the first audio stream into 16 kHz mono 16-bit PCM, with bounded output and a 30-second decoding timeout. Oversized files or recordings return HTTP 413, invalid audio returns HTTP 400, and a decoding timeout returns HTTP 422. These responses contain an `error` message instead of transcription text. + +One request is processed at a time, and one additional request can wait. Further requests receive HTTP 503 with `Retry-After: 1`. Processing remains reserved until the worker finishes, including when its client cancels. The Shell and example client display failures and stop the failed recording; start a new recording to retry. Delayed results or errors from an earlier recording do not affect a new recording. + +## Offline tests + +The tests use the installed FFmpeg executable and a small model stand-in, without downloading a Whisper model or requiring a GPU: + +```shell +pip install -r requirements-test.txt +python -m unittest -v test_audio_limits +``` diff --git a/python/stt/whisperService/faster-whisper.py b/python/stt/whisperService/faster-whisper.py index 63cab985f5..90f35929ad 100644 --- a/python/stt/whisperService/faster-whisper.py +++ b/python/stt/whisperService/faster-whisper.py @@ -1,25 +1,122 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -from io import BytesIO -from pydub import AudioSegment +import asyncio +from pathlib import Path +import subprocess +import tempfile import uvicorn import numpy as np import torch -from fastapi import FastAPI, UploadFile, File +from fastapi import FastAPI, UploadFile, File, HTTPException from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from starlette.datastructures import Headers, State +from starlette.types import ASGIApp, Message, Receive, Scope, Send import logging from faster_whisper import WhisperModel import os -os.environ['KMP_DUPLICATE_LIB_OK']='True' +os.environ["KMP_DUPLICATE_LIB_OK"] = "True" # Configure logging -logging.basicConfig(level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig( + level=logging.ERROR, + format="%(asctime)s - %(levelname)s - %(message)s", +) + +MAX_UPLOAD_BYTES = 30 * 1024 * 1024 +MAX_REQUEST_BYTES = MAX_UPLOAD_BYTES + 64 * 1024 +MAX_AUDIO_SECONDS = 300 +SAMPLE_RATE = 16000 +UPLOAD_TIMEOUT_SECONDS = 30 +DECODE_TIMEOUT_SECONDS = 30 +READ_CHUNK_BYTES = 64 * 1024 + + +class TranscriptionInputMiddleware: + def __init__(self, app: ASGIApp, state: State) -> None: + self.app = app + self.state = state + self.lock = asyncio.Lock() + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if ( + scope["type"] != "http" + or scope["method"] != "POST" + or scope["path"].rstrip("/") != "/transcribe" + ): + await self.app(scope, receive, send) + return + + async def reject(status: int, message: str) -> None: + response = JSONResponse( + {"error": message}, + status_code=status, + headers={"Retry-After": "1"} if status == 503 else None, + ) + await response(scope, receive, send) + + content_length = Headers(scope=scope).get("content-length") + if content_length is not None: + if not content_length.isdecimal() or len(content_length) > 20: + await reject(400, "Invalid request size.") + return + if int(content_length) > MAX_REQUEST_BYTES: + await reject( + 413, "The audio upload exceeds the request size limit." + ) + return + if self.state.pending_transcriptions >= 2: + await reject(503, "The transcription service is busy. Try again shortly.") + return + + self.state.pending_transcriptions += 1 + try: + async with self.lock: + with tempfile.SpooledTemporaryFile(max_size=1024 * 1024) as body: + size = 0 + loop = asyncio.get_running_loop() + upload_deadline = loop.time() + UPLOAD_TIMEOUT_SECONDS + while True: + try: + message = await asyncio.wait_for( + receive(), timeout=upload_deadline - loop.time() + ) + except asyncio.TimeoutError: + await reject(408, "The audio upload exceeded the time limit.") + return + if message["type"] == "http.disconnect": + return + if message["type"] != "http.request": + raise RuntimeError("Unexpected transcription request message.") + chunk = message.get("body", b"") + size += len(chunk) + if size > MAX_REQUEST_BYTES: + await reject(413, "The audio upload exceeds the request size limit.") + return + body.write(chunk) + if not message.get("more_body", False): + break + body.seek(0) + + async def replay() -> Message: + chunk = body.read(READ_CHUNK_BYTES) + return { + "type": "http.request", + "body": chunk, + "more_body": body.tell() < size, + } + + await self.app(scope, replay, send) + finally: + self.state.pending_transcriptions -= 1 + # Initialize the app app = FastAPI() +app.state.pending_transcriptions = 0 +app.add_middleware(TranscriptionInputMiddleware, state=app.state) # allow all cors app.add_middleware( @@ -38,30 +135,98 @@ print("Model loaded!") -@app.post("/transcribe/") -async def transcription(file: UploadFile = File(...)): - try: - audio_bytes = await file.read() +def decode_audio(audio_bytes: bytes) -> np.ndarray: + max_samples = int(MAX_AUDIO_SECONDS * SAMPLE_RATE) + with tempfile.TemporaryDirectory(prefix="typeagent-whisper-") as directory: + source = Path(directory) / "audio" + source.write_bytes(audio_bytes) + try: + result = subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-nostdin", + "-loglevel", + "fatal", + "-max_alloc", + str(64 * 1024 * 1024), + "-threads", + "1", + "-protocol_whitelist", + "file", + "-i", + str(source), + "-map", + "0:a:0", + "-vn", + "-sn", + "-dn", + "-ac", + "1", + "-ar", + str(SAMPLE_RATE), + "-t", + str((max_samples + 1) / SAMPLE_RATE), + "-fs", + str((max_samples + 1) * 2), + "-f", + "s16le", + "pipe:1", + ], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=DECODE_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as error: + raise HTTPException( + 422, "Audio decoding exceeded the processing time limit." + ) from error + if result.returncode != 0 or not result.stdout or len(result.stdout) % 2: + raise HTTPException(400, "Audio could not be decoded.") + if len(result.stdout) > max_samples * 2: + raise HTTPException(413, "The audio recording exceeds the duration limit.") + audio = np.frombuffer(result.stdout, dtype=" str: + segments, _ = model.transcribe(decode_audio(audio_bytes)) + return "\n".join(segment.text for segment in segments) - # faster_whisper returns a generator. Iterate though it to get the transcription - segments, _ = model.transcribe(audio_np) - segments = list(segments) - transcription = "\n".join(i.text for i in segments) - return JSONResponse(content={"transcription": transcription}, status_code=200) - except Exception as e: +@app.post("/transcribe/") +async def transcription(file: UploadFile = File(...)) -> JSONResponse: + try: + audio_bytes = await file.read(MAX_UPLOAD_BYTES + 1) + if len(audio_bytes) > MAX_UPLOAD_BYTES: + raise HTTPException(413, "The audio file exceeds the upload size limit.") + worker = asyncio.create_task(asyncio.to_thread(transcribe_audio, audio_bytes)) + try: + text = await asyncio.shield(worker) + except asyncio.CancelledError: + cleanup = asyncio.gather(worker, return_exceptions=True) + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + results = cleanup.result() + if isinstance(results[0], Exception): + logging.error("Cancelled transcription failed", exc_info=results[0]) + raise + return JSONResponse(content={"transcription": text}, status_code=200) + except HTTPException as error: + logging.warning("Audio input rejected: %s", error.detail) + return JSONResponse( + content={"error": error.detail}, status_code=error.status_code + ) + except Exception: logging.error("An error occurred during transcription", exc_info=True) - return JSONResponse(content={"error": "An internal error has occurred!"}, status_code=500) + return JSONResponse( + content={"error": "An internal error has occurred!"}, status_code=500 + ) if __name__ == "__main__": diff --git a/python/stt/whisperService/requirements-test.txt b/python/stt/whisperService/requirements-test.txt new file mode 100644 index 0000000000..ecfeefa239 --- /dev/null +++ b/python/stt/whisperService/requirements-test.txt @@ -0,0 +1,5 @@ +fastapi +httpx +numpy +python-multipart +uvicorn diff --git a/python/stt/whisperService/requirements.txt b/python/stt/whisperService/requirements.txt index 7fafb5b016..7210ff96d3 100644 --- a/python/stt/whisperService/requirements.txt +++ b/python/stt/whisperService/requirements.txt @@ -1,7 +1,6 @@ pyaudio torch numpy -pydub fastapi uvicorn python-multipart diff --git a/python/stt/whisperService/test_audio_limits.py b/python/stt/whisperService/test_audio_limits.py new file mode 100644 index 0000000000..217674124d --- /dev/null +++ b/python/stt/whisperService/test_audio_limits.py @@ -0,0 +1,308 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import asyncio +import importlib.util +import io +from contextlib import redirect_stdout +from pathlib import Path +import subprocess +import threading +import types +import unittest +from unittest.mock import patch +import wave + +import httpx +import numpy as np + + +def audio_bytes(samples: int = 320) -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as audio: + audio.setnchannels(1) + audio.setsampwidth(2) + audio.setframerate(16000) + audio.writeframes(np.full(samples, 16384, dtype=" None: + self.inputs: list[np.ndarray] = [] + self.started = threading.Event() + self.release = threading.Event() + self.block = False + + def transcribe(self, audio: np.ndarray): + self.inputs.append(audio.copy()) + self.started.set() + if self.block and not self.release.wait(5): + raise RuntimeError("Test transcription was not released") + return iter([types.SimpleNamespace(text="hello")]), None + + +def load_service(model: Model): + torch = types.ModuleType("torch") + torch.cuda = types.SimpleNamespace(is_available=lambda: False) + whisper = types.ModuleType("faster_whisper") + whisper.WhisperModel = lambda *args, **kwargs: model + + spec = importlib.util.spec_from_file_location( + "whisper_service_test", Path(__file__).with_name("faster-whisper.py") + ) + if spec is None or spec.loader is None: + raise RuntimeError("Unable to load the service") + service = importlib.util.module_from_spec(spec) + with patch.dict( + "sys.modules", {"torch": torch, "faster_whisper": whisper} + ), redirect_stdout(io.StringIO()): + spec.loader.exec_module(service) + return service + + +class AudioLimitsTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self) -> None: + self.model = Model() + self.service = load_service(self.model) + self.service.MAX_UPLOAD_BYTES = len(audio_bytes()) + self.service.MAX_REQUEST_BYTES = self.service.MAX_UPLOAD_BYTES + 65536 + self.service.MAX_AUDIO_SECONDS = 0.02 + self.client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=self.service.app), + base_url="http://test", + ) + + async def asyncTearDown(self) -> None: + self.model.release.set() + await self.client.aclose() + + async def upload(self, content: bytes): + return await self.client.post( + "/transcribe/", + files={"file": ("recording.wav", content, "audio/wav")}, + ) + + async def test_exact_upload_and_duration_limits_preserve_output(self) -> None: + response = await self.upload(audio_bytes()) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {"transcription": "hello"}) + self.assertEqual(len(self.model.inputs), 1) + self.assertEqual(self.model.inputs[0].dtype, np.float32) + np.testing.assert_array_equal(self.model.inputs[0], np.full(320, 0.5)) + + async def test_upload_one_byte_over_limit_stops_before_inference(self) -> None: + response = await self.upload(audio_bytes() + b"x") + self.assertEqual(response.status_code, 413) + self.assertIn("error", response.json()) + self.assertEqual(self.model.inputs, []) + + async def test_decoded_audio_one_sample_over_limit_is_rejected(self) -> None: + self.service.MAX_UPLOAD_BYTES = 4096 + response = await self.upload(audio_bytes(321)) + self.assertEqual(response.status_code, 413) + self.assertEqual(self.model.inputs, []) + + async def test_audio_below_duration_limit_is_accepted(self) -> None: + response = await self.upload(audio_bytes(319)) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(self.model.inputs[0]), 319) + + async def test_compressed_audio_is_bounded_by_decoded_samples(self) -> None: + compressed = subprocess.run( + [ + "ffmpeg", "-hide_banner", "-loglevel", "error", + "-f", "wav", "-i", "pipe:0", "-c:a", "flac", + "-f", "flac", "pipe:1", + ], + input=audio_bytes(16000), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + timeout=10, + ).stdout + self.assertLess(len(compressed), len(audio_bytes(16000))) + self.service.MAX_UPLOAD_BYTES = len(compressed) + response = await self.upload(compressed) + self.assertEqual(response.status_code, 413) + self.assertEqual(self.model.inputs, []) + + async def test_decoder_timeout_is_reported_and_releases_capacity(self) -> None: + with patch.object( + self.service.subprocess, + "run", + side_effect=subprocess.TimeoutExpired("ffmpeg", 30), + ): + response = await self.upload(audio_bytes()) + self.assertEqual(response.status_code, 422) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + self.assertEqual(self.model.inputs, []) + self.assertEqual((await self.upload(audio_bytes())).status_code, 200) + + async def test_invalid_audio_is_an_explicit_input_error(self) -> None: + response = await self.upload(b"not audio") + self.assertEqual(response.status_code, 400) + self.assertIn("error", response.json()) + self.assertEqual(self.model.inputs, []) + + async def test_declared_request_limit_rejects_before_reading_body(self) -> None: + self.service.MAX_REQUEST_BYTES = 100 + response = await self.client.post( + "/transcribe/", + content=b"x" * 101, + headers={"origin": "http://example.test"}, + ) + self.assertEqual(response.status_code, 413) + self.assertIsNotNone(response.headers.get("access-control-allow-origin")) + self.assertEqual(self.model.inputs, []) + + async def test_chunked_request_limit_does_not_depend_on_content_length(self) -> None: + self.service.MAX_REQUEST_BYTES = 100 + + async def content(): + yield b"x" * 50 + yield b"x" * 51 + + response = await self.client.post("/transcribe/", content=content()) + self.assertEqual(response.status_code, 413) + self.assertEqual(self.model.inputs, []) + + async def test_stalled_upload_times_out_and_releases_waiting_request(self) -> None: + self.service.UPLOAD_TIMEOUT_SECONDS = 0.25 + started = asyncio.Event() + + async def content(): + started.set() + await asyncio.Event().wait() + yield b"" + + tasks = [asyncio.create_task(self.client.post( + "/transcribe/", + content=content(), + headers={"origin": "http://example.test"}, + ))] + try: + await asyncio.wait_for(started.wait(), 2) + tasks.append(asyncio.create_task(self.upload(audio_bytes()))) + for _ in range(100): + if self.service.app.state.pending_transcriptions == 2: + break + await asyncio.sleep(0.001) + self.assertEqual(self.service.app.state.pending_transcriptions, 2) + self.assertEqual((await self.upload(audio_bytes())).status_code, 503) + stalled, waiting = await asyncio.wait_for(asyncio.gather(*tasks), 5) + self.assertEqual(stalled.status_code, 408) + self.assertEqual(stalled.json(), { + "error": "The audio upload exceeded the time limit." + }) + self.assertIsNotNone( + stalled.headers.get("access-control-allow-origin") + ) + self.assertEqual(waiting.status_code, 200) + self.assertEqual(waiting.json(), {"transcription": "hello"}) + self.assertEqual(len(self.model.inputs), 1) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + async def test_upload_stalled_after_a_chunk_times_out(self) -> None: + self.service.UPLOAD_TIMEOUT_SECONDS = 0.05 + + async def content(): + yield b"unfinished audio" + await asyncio.Event().wait() + + response = await asyncio.wait_for( + self.client.post("/transcribe/", content=content()), 2 + ) + self.assertEqual(response.status_code, 408) + self.assertEqual(response.json(), { + "error": "The audio upload exceeded the time limit." + }) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + self.assertEqual(self.model.inputs, []) + self.assertEqual((await self.upload(audio_bytes())).status_code, 200) + + async def test_upload_timeout_bounds_total_time_not_each_chunk(self) -> None: + self.service.UPLOAD_TIMEOUT_SECONDS = 0.1 + sent_chunks = 0 + + async def content(): + nonlocal sent_chunks + for _ in range(10): + await asyncio.sleep(0.025) + sent_chunks += 1 + yield b"x" + + response = await asyncio.wait_for( + self.client.post("/transcribe/", content=content()), 2 + ) + self.assertEqual(response.status_code, 408) + self.assertLess(sent_chunks, 10) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + self.assertEqual(self.model.inputs, []) + + async def test_one_waiting_request_and_explicit_busy_response(self) -> None: + self.service.UPLOAD_TIMEOUT_SECONDS = 0.1 + self.model.block = True + first = asyncio.create_task(self.upload(audio_bytes())) + try: + started = await asyncio.to_thread(self.model.started.wait, 2) + self.assertTrue(started) + second = asyncio.create_task(self.upload(audio_bytes())) + try: + for _ in range(100): + if self.service.app.state.pending_transcriptions == 2: + break + await asyncio.sleep(0.01) + self.assertEqual(self.service.app.state.pending_transcriptions, 2) + response = await self.upload(audio_bytes()) + self.assertEqual(response.status_code, 503) + self.assertEqual(response.headers["retry-after"], "1") + self.assertEqual(len(self.model.inputs), 1) + await asyncio.sleep(0.15) + self.assertFalse(first.done()) + self.assertFalse(second.done()) + self.model.release.set() + responses = await asyncio.gather(first, second) + self.assertTrue( + all(result.status_code == 200 for result in responses) + ) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + finally: + self.model.release.set() + await second + finally: + self.model.release.set() + await first + + async def test_cancelled_request_keeps_capacity_until_worker_finishes(self) -> None: + self.model.block = True + first = asyncio.create_task(self.upload(audio_bytes())) + self.assertTrue(await asyncio.to_thread(self.model.started.wait, 2)) + first.cancel() + second = asyncio.create_task(self.upload(audio_bytes())) + try: + for _ in range(100): + if self.service.app.state.pending_transcriptions == 2: + break + await asyncio.sleep(0.01) + self.assertEqual(self.service.app.state.pending_transcriptions, 2) + first.cancel() + await asyncio.sleep(0.01) + self.assertEqual((await self.upload(audio_bytes())).status_code, 503) + self.assertEqual(len(self.model.inputs), 1) + self.model.release.set() + with self.assertRaises(asyncio.CancelledError): + await first + self.assertEqual((await second).status_code, 200) + self.assertEqual(self.service.app.state.pending_transcriptions, 0) + finally: + self.model.release.set() + await asyncio.gather(first, second, return_exceptions=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/ts/examples/whisperClient/package.json b/ts/examples/whisperClient/package.json index ab341d6187..880642b396 100644 --- a/ts/examples/whisperClient/package.json +++ b/ts/examples/whisperClient/package.json @@ -23,6 +23,7 @@ "webpack": "webpack" }, "dependencies": { + "@typeagent/common-utils": "workspace:*", "chalk": "^5.4.1", "debug": "^4.4.0" }, diff --git a/ts/examples/whisperClient/public/index.html b/ts/examples/whisperClient/public/index.html index f47e69f452..e444780b68 100644 --- a/ts/examples/whisperClient/public/index.html +++ b/ts/examples/whisperClient/public/index.html @@ -9,11 +9,16 @@
- +