[
{
"path": ".dockerignore",
"content": "__pycache__\n.cache\n.dockerignore\ndocker-compose.yml\nDockerfile"
},
{
"path": ".github/FUNDING.yml",
"content": "# These are supported funding model platforms\n\ngithub: KoljaB\npatreon: # Replace with a single Patreon username\nopen_collective: # Replace with a single Open Collective username\nko_fi: koljab\ntidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel\ncommunity_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry\nliberapay: # Replace with a single Liberapay username\nissuehunt: # Replace with a single IssueHunt username\nlfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry\npolar: # Replace with a single Polar username\nbuy_me_a_coffee: koljab\nthanks_dev: # Replace with a single thanks.dev username\ncustom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']\n"
},
{
"path": "Dockerfile",
"content": "FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04 as gpu\n\nWORKDIR /app\n\nRUN apt-get update -y && \\\n apt-get install -y python3 python3-pip portaudio19-dev\n\nRUN pip3 install torch==2.7.1+cu128 torchaudio==2.7.1+cu128 --index-url https://download.pytorch.org/whl/cu128\n\nCOPY requirements-gpu* /app/\nRUN pip3 install -r /app/requirements-gpu-torch.txt && \\\n pip3 install -r /app/requirements-gpu.txt\n\nRUN mkdir example_browserclient\nCOPY example_browserclient/server.py /app/example_browserclient/server.py\nCOPY RealtimeSTT /app/RealtimeSTT\n\nEXPOSE 9001\nENV PYTHONPATH \"${PYTHONPATH}:/app\"\nRUN export PYTHONPATH=\"${PYTHONPATH}:/app\"\nCMD [\"python3\", \"example_browserclient/server.py\"]\n\n# --------------------------------------------\n\nFROM ubuntu:24.04 as cpu\n\nWORKDIR /app\n\nRUN apt-get update -y && \\\n apt-get install -y python3 python3-pip portaudio19-dev\n\nRUN pip3 install torch==2.7.1 torchaudio==2.7.1\n\nCOPY requirements.txt /app/requirements.txt\nRUN pip3 install -r /app/requirements.txt\n\nEXPOSE 9001\nENV PYTHONPATH \"${PYTHONPATH}:/app\"\nRUN export PYTHONPATH=\"${PYTHONPATH}:/app\"\nCMD [\"python3\", \"example_browserclient/server.py\"]\n"
},
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2023 Kolja Beigel\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "MANIFEST.in",
"content": "include requirements.txt\ninclude README.md\ninclude LICENSE\n"
},
{
"path": "README.md",
"content": "# RealtimeSTT\n[](https://pypi.org/project/RealtimeSTT/)\n[](https://www.pepy.tech/projects/realtimestt)\n[](https://GitHub.com/KoljaB/RealtimeSTT/releases/)\n[](https://GitHub.com/Naereen/KoljaB/RealtimeSTT/commit/)\n[](https://GitHub.com/KoljaB/RealtimeSTT/network/)\n[](https://GitHub.com/KoljaB/RealtimeSTT/stargazers/)\n\n*Easy-to-use, low-latency speech-to-text library for realtime applications*\n\n> ❗ **Project Status: Community-Driven**\n> \n> This project is no longer being actively maintained by me due to time constraints. I've taken on too many projects and I have to step back. I will no longer be implementing new features or providing user support.\n>\n> I will continue to review and merge high-quality, well-written Pull Requests from the community from time to time. Your contributions are welcome and appreciated!\n\n## New\n\n- AudioToTextRecorderClient class, which automatically starts a server if none is running and connects to it. The class shares the same interface as AudioToTextRecorder, making it easy to upgrade or switch between the two. (Work in progress, most parameters and callbacks of AudioToTextRecorder are already implemented into AudioToTextRecorderClient, but not all. Also the server can not handle concurrent (parallel) requests yet.)\n- reworked CLI interface (\"stt-server\" to start the server, \"stt\" to start the client, look at \"server\" folder for more info)\n\n## About the Project\n\nRealtimeSTT listens to the microphone and transcribes voice into text. \n\n> **Hint:** *Check out [Linguflex](https://github.com/KoljaB/Linguflex), the original project from which RealtimeSTT is spun off. It lets you control your environment by speaking and is one of the most capable and sophisticated open-source assistants currently available.*\n\nIt's ideal for:\n\n- **Voice Assistants**\n- Applications requiring **fast and precise** speech-to-text conversion\n\nhttps://github.com/user-attachments/assets/797e6552-27cd-41b1-a7f3-e5cbc72094f5 \n\n[CLI demo code (reproduces the video above)](tests/realtimestt_test.py)\n\n### Updates\n\nLatest Version: v0.3.104\n\nSee [release history](https://github.com/KoljaB/RealtimeSTT/releases).\n\n> **Hint:** *Since we use the `multiprocessing` module now, ensure to include the `if __name__ == '__main__':` protection in your code to prevent unexpected behavior, especially on platforms like Windows. For a detailed explanation on why this is important, visit the [official Python documentation on `multiprocessing`](https://docs.python.org/3/library/multiprocessing.html#multiprocessing-programming).*\n\n## Quick Examples\n\n### Print everything being said:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\ndef process_text(text):\n print(text)\n\nif __name__ == '__main__':\n print(\"Wait until it says 'speak now'\")\n recorder = AudioToTextRecorder()\n\n while True:\n recorder.text(process_text)\n```\n\n### Type everything being said:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\nimport pyautogui\n\ndef process_text(text):\n pyautogui.typewrite(text + \" \")\n\nif __name__ == '__main__':\n print(\"Wait until it says 'speak now'\")\n recorder = AudioToTextRecorder()\n\n while True:\n recorder.text(process_text)\n```\n*Will type everything being said into your selected text box*\n\n### Features\n\n- **Voice Activity Detection**: Automatically detects when you start and stop speaking.\n- **Realtime Transcription**: Transforms speech to text in real-time.\n- **Wake Word Activation**: Can activate upon detecting a designated wake word.\n\n> **Hint**: *Check out [RealtimeTTS](https://github.com/KoljaB/RealtimeTTS), the output counterpart of this library, for text-to-voice capabilities. Together, they form a powerful realtime audio wrapper around large language models.*\n\n## Tech Stack\n\nThis library uses:\n\n- **Voice Activity Detection**\n - [WebRTCVAD](https://github.com/wiseman/py-webrtcvad) for initial voice activity detection.\n - [SileroVAD](https://github.com/snakers4/silero-vad) for more accurate verification.\n- **Speech-To-Text**\n - [Faster_Whisper](https://github.com/guillaumekln/faster-whisper) for instant (GPU-accelerated) transcription.\n- **Wake Word Detection**\n - [Porcupine](https://github.com/Picovoice/porcupine) or [OpenWakeWord](https://github.com/dscripka/openWakeWord) for wake word detection.\n\n\n*These components represent the \"industry standard\" for cutting-edge applications, providing the most modern and effective foundation for building high-end solutions.*\n\n## Installation\n\n```bash\npip install RealtimeSTT\n```\n\nThis will install all the necessary dependencies, including a **CPU support only** version of PyTorch.\n\nAlthough it is possible to run RealtimeSTT with a CPU installation only (use a small model like \"tiny\" or \"base\" in this case) you will get way better experience using CUDA (please scroll down).\n\n### Linux Installation\n\nBefore installing RealtimeSTT please execute:\n\n```bash\nsudo apt-get update\nsudo apt-get install python3-dev\nsudo apt-get install portaudio19-dev\n```\n\n### MacOS Installation\n\nBefore installing RealtimeSTT please execute:\n\n```bash\nbrew install portaudio\n```\n\n### GPU Support with CUDA (recommended)\n\n### Updating PyTorch for CUDA Support\n\nTo upgrade your PyTorch installation to enable GPU support with CUDA, follow these instructions based on your specific CUDA version. This is useful if you wish to enhance the performance of RealtimeSTT with CUDA capabilities.\n\n#### For CUDA 11.8:\nTo update PyTorch and Torchaudio to support CUDA 11.8, use the following commands:\n\n```bash\npip install torch==2.5.1+cu118 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/cu118\n```\n\n#### For CUDA 12.X:\nTo update PyTorch and Torchaudio to support CUDA 12.X, execute the following:\n\n```bash\npip install torch==2.5.1+cu121 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/cu121\n```\n\nReplace `2.5.1` with the version of PyTorch that matches your system and requirements.\n\n### Steps That Might Be Necessary Before\n\n> **Note**: *To check if your NVIDIA GPU supports CUDA, visit the [official CUDA GPUs list](https://developer.nvidia.com/cuda-gpus).*\n\nIf you didn't use CUDA models before, some additional steps might be needed one time before installation. These steps prepare the system for CUDA support and installation of the **GPU-optimized** installation. This is recommended for those who require **better performance** and have a compatible NVIDIA GPU. To use RealtimeSTT with GPU support via CUDA please also follow these steps:\n\n1. **Install NVIDIA CUDA Toolkit**:\n - select between CUDA 11.8 or CUDA 12.X Toolkit\n - for 12.X visit [NVIDIA CUDA Toolkit Archive](https://developer.nvidia.com/cuda-toolkit-archive) and select latest version.\n - for 11.8 visit [NVIDIA CUDA Toolkit 11.8](https://developer.nvidia.com/cuda-11-8-0-download-archive).\n - Select operating system and version.\n - Download and install the software.\n\n2. **Install NVIDIA cuDNN**:\n - select between CUDA 11.8 or CUDA 12.X Toolkit\n - for 12.X visit [cuDNN Downloads](https://developer.nvidia.com/cudnn-downloads).\n - Select operating system and version.\n - Download and install the software.\n - for 11.8 visit [NVIDIA cuDNN Archive](https://developer.nvidia.com/rdp/cudnn-archive).\n - Click on \"Download cuDNN v8.7.0 (November 28th, 2022), for CUDA 11.x\".\n - Download and install the software.\n \n3. **Install ffmpeg**:\n\n > **Note**: *Installation of ffmpeg might not actually be needed to operate RealtimeSTT* *thanks to jgilbert2017 for pointing this out\n\n You can download an installer for your OS from the [ffmpeg Website](https://ffmpeg.org/download.html). \n \n Or use a package manager:\n\n - **On Ubuntu or Debian**:\n ```bash\n sudo apt update && sudo apt install ffmpeg\n ```\n\n - **On Arch Linux**:\n ```bash\n sudo pacman -S ffmpeg\n ```\n\n - **On MacOS using Homebrew** ([https://brew.sh/](https://brew.sh/)):\n ```bash\n brew install ffmpeg\n ```\n\n - **On Windows using Winget** [official documentation](https://learn.microsoft.com/en-us/windows/package-manager/winget/) :\n ```bash\n winget install Gyan.FFmpeg\n ```\n \n - **On Windows using Chocolatey** ([https://chocolatey.org/](https://chocolatey.org/)):\n ```bash\n choco install ffmpeg\n ```\n\n - **On Windows using Scoop** ([https://scoop.sh/](https://scoop.sh/)):\n ```bash\n scoop install ffmpeg\n ``` \n\n## Quick Start\n\nBasic usage:\n\n### Manual Recording\n\nStart and stop of recording are manually triggered.\n\n```python\nrecorder.start()\nrecorder.stop()\nprint(recorder.text())\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n recorder = AudioToTextRecorder()\n recorder.start()\n input(\"Press Enter to stop recording...\")\n recorder.stop()\n print(\"Transcription: \", recorder.text())\n```\n\n### Automatic Recording\n\nRecording based on voice activity detection.\n\n```python\nwith AudioToTextRecorder() as recorder:\n print(recorder.text())\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n with AudioToTextRecorder() as recorder:\n print(\"Transcription: \", recorder.text())\n```\n\nWhen running recorder.text in a loop it is recommended to use a callback, allowing the transcription to be run asynchronously:\n\n\n```python\ndef process_text(text):\n print (text)\n \nwhile True:\n recorder.text(process_text)\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\ndef process_text(text):\n print(text)\n\nif __name__ == '__main__':\n recorder = AudioToTextRecorder()\n\n while True:\n recorder.text(process_text)\n```\n\n### Wakewords\n\nKeyword activation before detecting voice. Write the comma-separated list of your desired activation keywords into the wake_words parameter. You can choose wake words from these list: alexa, americano, blueberry, bumblebee, computer, grapefruits, grasshopper, hey google, hey siri, jarvis, ok google, picovoice, porcupine, terminator. \n\n```python\nrecorder = AudioToTextRecorder(wake_words=\"jarvis\")\n\nprint('Say \"Jarvis\" then speak.')\nprint(recorder.text())\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n recorder = AudioToTextRecorder(wake_words=\"jarvis\")\n\n print('Say \"Jarvis\" to start recording.')\n print(recorder.text())\n```\n\n### Callbacks\n\nYou can set callback functions to be executed on different events (see [Configuration](#configuration)) :\n\n```python\ndef my_start_callback():\n print(\"Recording started!\")\n\ndef my_stop_callback():\n print(\"Recording stopped!\")\n\nrecorder = AudioToTextRecorder(on_recording_start=my_start_callback,\n on_recording_stop=my_stop_callback)\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\ndef start_callback():\n print(\"Recording started!\")\n\ndef stop_callback():\n print(\"Recording stopped!\")\n\nif __name__ == '__main__':\n recorder = AudioToTextRecorder(on_recording_start=start_callback,\n on_recording_stop=stop_callback)\n```\n\n### Feed chunks\n\nIf you don't want to use the local microphone set use_microphone parameter to false and provide raw PCM audiochunks in 16-bit mono (samplerate 16000) with this method:\n\n```python\nrecorder.feed_audio(audio_chunk)\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n recorder = AudioToTextRecorder(use_microphone=False)\n with open(\"audio_chunk.pcm\", \"rb\") as f:\n audio_chunk = f.read()\n\n recorder.feed_audio(audio_chunk)\n print(\"Transcription: \", recorder.text())\n```\n\n### Shutdown\n\nYou can shutdown the recorder safely by using the context manager protocol:\n\n```python\nwith AudioToTextRecorder() as recorder:\n [...]\n```\n\n\nOr you can call the shutdown method manually (if using \"with\" is not feasible):\n\n```python\nrecorder.shutdown()\n```\n\n#### Standalone Example:\n\n```python\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n with AudioToTextRecorder() as recorder:\n [...]\n # or manually shutdown if \"with\" is not used\n recorder.shutdown()\n```\n\n## Testing the Library\n\nThe test subdirectory contains a set of scripts to help you evaluate and understand the capabilities of the RealtimeTTS library.\n\nTest scripts depending on RealtimeTTS library may require you to enter your azure service region within the script. \nWhen using OpenAI-, Azure- or Elevenlabs-related demo scripts the API Keys should be provided in the environment variables OPENAI_API_KEY, AZURE_SPEECH_KEY and ELEVENLABS_API_KEY (see [RealtimeTTS](https://github.com/KoljaB/RealtimeTTS))\n\n- **simple_test.py**\n - **Description**: A \"hello world\" styled demonstration of the library's simplest usage.\n\n- **realtimestt_test.py**\n - **Description**: Showcasing live-transcription.\n\n- **wakeword_test.py**\n - **Description**: A demonstration of the wakeword activation.\n\n- **translator.py**\n - **Dependencies**: Run `pip install openai realtimetts`.\n - **Description**: Real-time translations into six different languages.\n\n- **openai_voice_interface.py**\n - **Dependencies**: Run `pip install openai realtimetts`.\n - **Description**: Wake word activated and voice based user interface to the OpenAI API.\n\n- **advanced_talk.py**\n - **Dependencies**: Run `pip install openai keyboard realtimetts`.\n - **Description**: Choose TTS engine and voice before starting AI conversation.\n\n- **minimalistic_talkbot.py**\n - **Dependencies**: Run `pip install openai realtimetts`.\n - **Description**: A basic talkbot in 20 lines of code.\n\nThe example_app subdirectory contains a polished user interface application for the OpenAI API based on PyQt5.\n\n## Configuration\n\n### Initialization Parameters for `AudioToTextRecorder`\n\nWhen you initialize the `AudioToTextRecorder` class, you have various options to customize its behavior.\n\n#### General Parameters\n\n- **model** (str, default=\"tiny\"): Model size or path for transcription.\n - Options: 'tiny', 'tiny.en', 'base', 'base.en', 'small', 'small.en', 'medium', 'medium.en', 'large-v1', 'large-v2'.\n - Note: If a size is provided, the model will be downloaded from the Hugging Face Hub.\n\n- **language** (str, default=\"\"): Language code for transcription. If left empty, the model will try to auto-detect the language. Supported language codes are listed in [Whisper Tokenizer library](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py).\n\n- **compute_type** (str, default=\"default\"): Specifies the type of computation to be used for transcription. See [Whisper Quantization](https://opennmt.net/CTranslate2/quantization.html)\n\n- **input_device_index** (int, default=0): Audio Input Device Index to use.\n\n- **gpu_device_index** (int, default=0): GPU Device Index to use. The model can also be loaded on multiple GPUs by passing a list of IDs (e.g. [0, 1, 2, 3]).\n\n- **device** (str, default=\"cuda\"): Device for model to use. Can either be \"cuda\" or \"cpu\". \n\n- **on_recording_start**: A callable function triggered when recording starts.\n\n- **on_recording_stop**: A callable function triggered when recording ends.\n\n- **on_transcription_start**: A callable function triggered when transcription starts.\n\n- **ensure_sentence_starting_uppercase** (bool, default=True): Ensures that every sentence detected by the algorithm starts with an uppercase letter.\n\n- **ensure_sentence_ends_with_period** (bool, default=True): Ensures that every sentence that doesn't end with punctuation such as \"?\", \"!\" ends with a period\n\n- **use_microphone** (bool, default=True): Usage of local microphone for transcription. Set to False if you want to provide chunks with feed_audio method.\n\n- **spinner** (bool, default=True): Provides a spinner animation text with information about the current recorder state.\n\n- **level** (int, default=logging.WARNING): Logging level.\n\n- **batch_size** (int, default=16): Batch size for the main transcription. Set to 0 to deactivate.\n\n- **init_logging** (bool, default=True): Whether to initialize the logging framework. Set to False to manage this yourself.\n\n- **handle_buffer_overflow** (bool, default=True): If set, the system will log a warning when an input overflow occurs during recording and remove the data from the buffer.\n\n- **beam_size** (int, default=5): The beam size to use for beam search decoding.\n\n- **initial_prompt** (str or iterable of int, default=None): Initial prompt to be fed to the transcription models.\n\n- **suppress_tokens** (list of int, default=[-1]): Tokens to be suppressed from the transcription output.\n\n- **on_recorded_chunk**: A callback function that is triggered when a chunk of audio is recorded. Submits the chunk data as parameter.\n\n- **debug_mode** (bool, default=False): If set, the system prints additional debug information to the console.\n\n- **print_transcription_time** (bool, default=False): Logs the processing time of the main model transcription. This can be useful for performance monitoring and debugging.\n\n- **early_transcription_on_silence** (int, default=0): If set, the system will transcribe audio faster when silence is detected. Transcription will start after the specified milliseconds. Keep this value lower than `post_speech_silence_duration`, ideally around `post_speech_silence_duration` minus the estimated transcription time with the main model. If silence lasts longer than `post_speech_silence_duration`, the recording is stopped, and the transcription is submitted. If voice activity resumes within this period, the transcription is discarded. This results in faster final transcriptions at the cost of additional GPU load due to some unnecessary final transcriptions.\n\n- **allowed_latency_limit** (int, default=100): Specifies the maximum number of unprocessed chunks in the queue before discarding chunks. This helps prevent the system from being overwhelmed and losing responsiveness in real-time applications.\n\n- **no_log_file** (bool, default=False): If set, the system will skip writing the debug log file, reducing disk I/O. Useful if logging to a file is not needed and performance is a priority.\n\n- **start_callback_in_new_thread** (bool, default=False): If set, the system will create a new thread for all callback functions. This can be useful if the callback function is blocking and you want to avoid blocking the realtimestt application thread. \n\n#### Real-time Transcription Parameters\n\n> **Note**: *When enabling realtime description a GPU installation is strongly advised. Using realtime transcription may create high GPU loads.*\n\n- **enable_realtime_transcription** (bool, default=False): Enables or disables real-time transcription of audio. When set to True, the audio will be transcribed continuously as it is being recorded.\n\n- **use_main_model_for_realtime** (bool, default=False): If set to True, the main transcription model will be used for both regular and real-time transcription. If False, a separate model specified by `realtime_model_type` will be used for real-time transcription. Using a single model can save memory and potentially improve performance, but may not be optimized for real-time processing. Using separate models allows for a smaller, faster model for real-time transcription while keeping a more accurate model for final transcription.\n\n- **realtime_model_type** (str, default=\"tiny\"): Specifies the size or path of the machine learning model to be used for real-time transcription.\n - Valid options: 'tiny', 'tiny.en', 'base', 'base.en', 'small', 'small.en', 'medium', 'medium.en', 'large-v1', 'large-v2'.\n\n- **realtime_processing_pause** (float, default=0.2): Specifies the time interval in seconds after a chunk of audio gets transcribed. Lower values will result in more \"real-time\" (frequent) transcription updates but may increase computational load.\n\n- **on_realtime_transcription_update**: A callback function that is triggered whenever there's an update in the real-time transcription. The function is called with the newly transcribed text as its argument.\n\n- **on_realtime_transcription_stabilized**: A callback function that is triggered whenever there's an update in the real-time transcription and returns a higher quality, stabilized text as its argument.\n\n- **realtime_batch_size**: (int, default=16): Batch size for the real-time transcription model. Set to 0 to deactivate.\n\n- **beam_size_realtime** (int, default=3): The beam size to use for real-time transcription beam search decoding.\n\n#### Voice Activation Parameters\n\n- **silero_sensitivity** (float, default=0.6): Sensitivity for Silero's voice activity detection ranging from 0 (least sensitive) to 1 (most sensitive). Default is 0.6.\n\n- **silero_use_onnx** (bool, default=False): Enables usage of the pre-trained model from Silero in the ONNX (Open Neural Network Exchange) format instead of the PyTorch format. Default is False. Recommended for faster performance.\n\n- **silero_deactivity_detection** (bool, default=False): Enables the Silero model for end-of-speech detection. More robust against background noise. Utilizes additional GPU resources but improves accuracy in noisy environments. When False, uses the default WebRTC VAD, which is more sensitive but may continue recording longer due to background sounds.\n\n- **webrtc_sensitivity** (int, default=3): Sensitivity for the WebRTC Voice Activity Detection engine ranging from 0 (least aggressive / most sensitive) to 3 (most aggressive, least sensitive). Default is 3.\n\n- **post_speech_silence_duration** (float, default=0.2): Duration in seconds of silence that must follow speech before the recording is considered to be completed. This ensures that any brief pauses during speech don't prematurely end the recording.\n\n- **min_gap_between_recordings** (float, default=1.0): Specifies the minimum time interval in seconds that should exist between the end of one recording session and the beginning of another to prevent rapid consecutive recordings.\n\n- **min_length_of_recording** (float, default=1.0): Specifies the minimum duration in seconds that a recording session should last to ensure meaningful audio capture, preventing excessively short or fragmented recordings.\n\n- **pre_recording_buffer_duration** (float, default=0.2): The time span, in seconds, during which audio is buffered prior to formal recording. This helps counterbalancing the latency inherent in speech activity detection, ensuring no initial audio is missed.\n\n- **on_vad_start**: A callable function triggered when the system has detected the start of voice activity presence.\n\n- **on_vad_stop**: A callable function triggered when the system has detected the stop of voice activity presence.\n\n- **on_vad_detect_start**: A callable function triggered when the system starts to listen for voice activity.\n\n- **on_vad_detect_stop**: A callable function triggered when the system stops to listen for voice activity.\n\n#### Wake Word Parameters\n\n- **wakeword_backend** (str, default=\"pvporcupine\"): Specifies the backend library to use for wake word detection. Supported options include 'pvporcupine' for using the Porcupine wake word engine or 'oww' for using the OpenWakeWord engine.\n\n- **openwakeword_model_paths** (str, default=None): Comma-separated paths to model files for the openwakeword library. These paths point to custom models that can be used for wake word detection when the openwakeword library is selected as the wakeword_backend.\n\n- **openwakeword_inference_framework** (str, default=\"onnx\"): Specifies the inference framework to use with the openwakeword library. Can be either 'onnx' for Open Neural Network Exchange format or 'tflite' for TensorFlow Lite.\n\n- **wake_words** (str, default=\"\"): Initiate recording when using the 'pvporcupine' wakeword backend. Multiple wake words can be provided as a comma-separated string. Supported wake words are: alexa, americano, blueberry, bumblebee, computer, grapefruits, grasshopper, hey google, hey siri, jarvis, ok google, picovoice, porcupine, terminator. For the 'openwakeword' backend, wake words are automatically extracted from the provided model files, so specifying them here is not necessary.\n\n- **wake_words_sensitivity** (float, default=0.6): Sensitivity level for wake word detection (0 for least sensitive, 1 for most sensitive).\n\n- **wake_word_activation_delay** (float, default=0): Duration in seconds after the start of monitoring before the system switches to wake word activation if no voice is initially detected. If set to zero, the system uses wake word activation immediately.\n\n- **wake_word_timeout** (float, default=5): Duration in seconds after a wake word is recognized. If no subsequent voice activity is detected within this window, the system transitions back to an inactive state, awaiting the next wake word or voice activation.\n\n- **wake_word_buffer_duration** (float, default=0.1): Duration in seconds to buffer audio data during wake word detection. This helps in cutting out the wake word from the recording buffer so it does not falsely get detected along with the following spoken text, ensuring cleaner and more accurate transcription start triggers. Increase this if parts of the wake word get detected as text.\n\n- **on_wakeword_detected**: A callable function triggered when a wake word is detected.\n\n- **on_wakeword_timeout**: A callable function triggered when the system goes back to an inactive state after when no speech was detected after wake word activation.\n\n- **on_wakeword_detection_start**: A callable function triggered when the system starts to listen for wake words\n\n- **on_wakeword_detection_end**: A callable function triggered when stopping to listen for wake words (e.g. because of timeout or wake word detected)\n\n## OpenWakeWord \n\n### Training models\n\nLook [here](https://github.com/dscripka/openWakeWord?tab=readme-ov-file#training-new-models) for information about how to train your own OpenWakeWord models. You can use a [simple Google Colab notebook](https://colab.research.google.com/drive/1q1oe2zOyZp7UsB3jJiQ1IFn8z5YfjwEb?usp=sharing) for a start or use a [more detailed notebook](https://github.com/dscripka/openWakeWord/blob/main/notebooks/automatic_model_training.ipynb) that enables more customization (can produce high quality models, but requires more development experience).\n\n### Convert model to ONNX format\n\nYou might need to use tf2onnx to convert tensorflow tflite models to onnx format:\n\n```bash\npip install -U tf2onnx\npython -m tf2onnx.convert --tflite my_model_filename.tflite --output my_model_filename.onnx\n```\n\n### Configure RealtimeSTT\n\nSuggested starting parameters for OpenWakeWord usage:\n```python\n with AudioToTextRecorder(\n wakeword_backend=\"oww\",\n wake_words_sensitivity=0.35,\n openwakeword_model_paths=\"word1.onnx,word2.onnx\",\n wake_word_buffer_duration=1,\n ) as recorder:\n```\n\n## FAQ\n\n### Q: I encountered the following error: \"Unable to load any of {libcudnn_ops.so.9.1.0, libcudnn_ops.so.9.1, libcudnn_ops.so.9, libcudnn_ops.so} Invalid handle. Cannot load symbol cudnnCreateTensorDescriptor.\" How do I fix this?\n\n**A:** This issue arises from a mismatch between the version of `ctranslate2` and cuDNN. The `ctranslate2` library was updated to version 4.5.0, which uses cuDNN 9.2. There are two ways to resolve this issue:\n1. **Downgrade `ctranslate2` to version 4.4.0**:\n ```bash\n pip install ctranslate2==4.4.0\n ```\n2. **Upgrade cuDNN** on your system to version 9.2 or above.\n\n## Contribution\n\nContributions are always welcome! \n\nShoutout to [Steven Linn](https://github.com/stevenlafl) for providing docker support. \n\n## License\n\n[MIT](https://github.com/KoljaB/RealtimeSTT?tab=MIT-1-ov-file)\n\n## Author\n\nKolja Beigel \nEmail: kolja.beigel@web.de \n[GitHub](https://github.com/KoljaB/RealtimeSTT)\n"
},
{
"path": "RealtimeSTT/__init__.py",
"content": "from .audio_recorder import AudioToTextRecorder\nfrom .audio_recorder_client import AudioToTextRecorderClient\nfrom .audio_input import AudioInput"
},
{
"path": "RealtimeSTT/audio_input.py",
"content": "from colorama import init, Fore, Style\nfrom scipy.signal import butter, filtfilt, resample_poly\nimport pyaudio\nimport logging\n\nDESIRED_RATE = 16000\nCHUNK_SIZE = 1024\nAUDIO_FORMAT = pyaudio.paInt16\nCHANNELS = 1\n\nclass AudioInput:\n def __init__(\n self,\n input_device_index: int = None,\n debug_mode: bool = False,\n target_samplerate: int = DESIRED_RATE,\n chunk_size: int = CHUNK_SIZE,\n audio_format: int = AUDIO_FORMAT,\n channels: int = CHANNELS,\n resample_to_target: bool = True,\n ):\n\n self.input_device_index = input_device_index\n self.debug_mode = debug_mode\n self.audio_interface = None\n self.stream = None\n self.device_sample_rate = None\n self.target_samplerate = target_samplerate\n self.chunk_size = chunk_size\n self.audio_format = audio_format\n self.channels = channels\n self.resample_to_target = resample_to_target\n\n def get_supported_sample_rates(self, device_index):\n \"\"\"Test which standard sample rates are supported by the specified device.\"\"\"\n standard_rates = [8000, 9600, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000]\n supported_rates = []\n\n device_info = self.audio_interface.get_device_info_by_index(device_index)\n max_channels = device_info.get('maxInputChannels') # Changed from maxOutputChannels\n\n for rate in standard_rates:\n try:\n if self.audio_interface.is_format_supported(\n rate,\n input_device=device_index, # Changed to input_device\n input_channels=max_channels, # Changed to input_channels\n input_format=self.audio_format, # Changed to input_format\n ):\n supported_rates.append(rate)\n except:\n continue\n return supported_rates\n\n def _get_best_sample_rate(self, actual_device_index, desired_rate):\n \"\"\"Determines the best available sample rate for the device.\"\"\"\n try:\n device_info = self.audio_interface.get_device_info_by_index(actual_device_index)\n supported_rates = self.get_supported_sample_rates(actual_device_index)\n\n if desired_rate in supported_rates:\n return desired_rate\n\n return max(supported_rates)\n\n # lower_rates = [r for r in supported_rates if r <= desired_rate]\n # if lower_rates:\n # return max(lower_rates)\n\n # higher_rates = [r for r in supported_rates if r > desired_rate]\n # if higher_rates:\n # return min(higher_rates)\n\n return int(device_info.get('defaultSampleRate', 44100))\n\n except Exception as e:\n logging.warning(f\"Error determining sample rate: {e}\")\n return 44100 # Safe fallback\n\n def list_devices(self):\n \"\"\"List all available audio input devices with supported sample rates.\"\"\"\n try:\n init() # Initialize colorama\n self.audio_interface = pyaudio.PyAudio()\n device_count = self.audio_interface.get_device_count()\n\n print(f\"Available audio input devices:\")\n #print(f\"{Fore.LIGHTBLUE_EX}Available audio input devices:{Style.RESET_ALL}\")\n for i in range(device_count):\n device_info = self.audio_interface.get_device_info_by_index(i)\n device_name = device_info.get('name')\n max_input_channels = device_info.get('maxInputChannels', 0)\n\n if max_input_channels > 0: # Only consider devices with input capabilities\n supported_rates = self.get_supported_sample_rates(i)\n print(f\"{Fore.LIGHTGREEN_EX}Device {Style.RESET_ALL}{i}{Fore.LIGHTGREEN_EX}: {device_name}{Style.RESET_ALL}\")\n \n # Format each rate in cyan\n if supported_rates:\n rates_formatted = \", \".join([f\"{Fore.CYAN}{rate}{Style.RESET_ALL}\" for rate in supported_rates])\n print(f\" {Fore.YELLOW}Supported sample rates: {rates_formatted}{Style.RESET_ALL}\")\n else:\n print(f\" {Fore.YELLOW}Supported sample rates: None{Style.RESET_ALL}\")\n\n except Exception as e:\n print(f\"Error listing devices: {e}\")\n finally:\n if self.audio_interface:\n self.audio_interface.terminate()\n\n def setup(self):\n \"\"\"Initialize audio interface and open stream\"\"\"\n try:\n self.audio_interface = pyaudio.PyAudio()\n\n if self.debug_mode:\n print(f\"Input device index: {self.input_device_index}\")\n actual_device_index = (self.input_device_index if self.input_device_index is not None \n else self.audio_interface.get_default_input_device_info()['index'])\n \n if self.debug_mode:\n print(f\"Actual selected device index: {actual_device_index}\")\n self.input_device_index = actual_device_index\n self.device_sample_rate = self._get_best_sample_rate(actual_device_index, self.target_samplerate)\n\n if self.debug_mode:\n print(f\"Setting up audio on device {self.input_device_index} with sample rate {self.device_sample_rate}\")\n\n try:\n self.stream = self.audio_interface.open(\n format=self.audio_format,\n channels=self.channels,\n rate=self.device_sample_rate,\n input=True,\n frames_per_buffer=self.chunk_size,\n input_device_index=self.input_device_index,\n )\n if self.debug_mode:\n print(f\"Audio recording initialized successfully at {self.device_sample_rate} Hz\")\n return True\n except Exception as e:\n print(f\"Failed to initialize audio stream at {self.device_sample_rate} Hz: {e}\")\n return False\n\n except Exception as e:\n print(f\"Error initializing audio recording: {e}\")\n if self.audio_interface:\n self.audio_interface.terminate()\n return False\n\n def lowpass_filter(self, signal, cutoff_freq, sample_rate):\n \"\"\"\n Apply a low-pass Butterworth filter to prevent aliasing in the signal.\n\n Args:\n signal (np.ndarray): Input audio signal to filter\n cutoff_freq (float): Cutoff frequency in Hz\n sample_rate (float): Sampling rate of the input signal in Hz\n\n Returns:\n np.ndarray: Filtered audio signal\n\n Notes:\n - Uses a 5th order Butterworth filter\n - Applies zero-phase filtering using filtfilt\n \"\"\"\n # Calculate the Nyquist frequency (half the sample rate)\n nyquist_rate = sample_rate / 2.0\n\n # Normalize cutoff frequency to Nyquist rate (required by butter())\n normal_cutoff = cutoff_freq / nyquist_rate\n\n # Design the Butterworth filter\n b, a = butter(5, normal_cutoff, btype='low', analog=False)\n\n # Apply zero-phase filtering (forward and backward)\n filtered_signal = filtfilt(b, a, signal)\n return filtered_signal\n\n def resample_audio(self, pcm_data, target_sample_rate, original_sample_rate):\n \"\"\"\n Filter and resample audio data to a target sample rate.\n\n Args:\n pcm_data (np.ndarray): Input audio data\n target_sample_rate (int): Desired output sample rate in Hz\n original_sample_rate (int): Original sample rate of input in Hz\n\n Returns:\n np.ndarray: Resampled audio data\n\n Notes:\n - Applies anti-aliasing filter before resampling\n - Uses polyphase filtering for high-quality resampling\n \"\"\"\n if target_sample_rate < original_sample_rate:\n # Downsampling with low-pass filter\n pcm_filtered = self.lowpass_filter(pcm_data, target_sample_rate / 2, original_sample_rate)\n resampled = resample_poly(pcm_filtered, target_sample_rate, original_sample_rate)\n else:\n # Upsampling without low-pass filter\n resampled = resample_poly(pcm_data, target_sample_rate, original_sample_rate)\n return resampled\n\n def read_chunk(self):\n \"\"\"Read a chunk of audio data\"\"\"\n return self.stream.read(self.chunk_size, exception_on_overflow=False)\n\n def cleanup(self):\n \"\"\"Clean up audio resources\"\"\"\n try:\n if self.stream:\n self.stream.stop_stream()\n self.stream.close()\n self.stream = None\n if self.audio_interface:\n self.audio_interface.terminate()\n self.audio_interface = None\n except Exception as e:\n print(f\"Error cleaning up audio resources: {e}\")\n"
},
{
"path": "RealtimeSTT/audio_recorder.py",
"content": "\"\"\"\n\nThe AudioToTextRecorder class in the provided code facilitates\nfast speech-to-text transcription.\n\nThe class employs the faster_whisper library to transcribe the recorded audio\ninto text using machine learning models, which can be run either on a GPU or\nCPU. Voice activity detection (VAD) is built in, meaning the software can\nautomatically start or stop recording based on the presence or absence of\nspeech. It integrates wake word detection through the pvporcupine library,\nallowing the software to initiate recording when a specific word or phrase\nis spoken. The system provides real-time feedback and can be further\ncustomized.\n\nFeatures:\n- Voice Activity Detection: Automatically starts/stops recording when speech\n is detected or when speech ends.\n- Wake Word Detection: Starts recording when a specified wake word (or words)\n is detected.\n- Event Callbacks: Customizable callbacks for when recording starts\n or finishes.\n- Fast Transcription: Returns the transcribed text from the audio as fast\n as possible.\n\nAuthor: Kolja Beigel\n\n\"\"\"\n\nfrom faster_whisper import WhisperModel, BatchedInferencePipeline\nfrom typing import Iterable, List, Optional, Union\nfrom openwakeword.model import Model\nimport torch.multiprocessing as mp\nfrom scipy.signal import resample\nimport signal as system_signal\nfrom ctypes import c_bool\nfrom scipy import signal\nfrom .safepipe import SafePipe\nimport soundfile as sf\nimport faster_whisper\nimport openwakeword\nimport collections\nimport numpy as np\nimport pvporcupine\nimport traceback\nimport threading\nimport webrtcvad\nimport datetime\nimport platform\nimport logging\nimport struct\nimport base64\nimport queue\nimport torch\nimport halo\nimport time\nimport copy\nimport os\nimport re\nimport gc\n\n# Named logger for this module.\nlogger = logging.getLogger(\"realtimestt\")\nlogger.propagate = False\n\n# Set OpenMP runtime duplicate library handling to OK (Use only for development!)\nos.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'\n\nINIT_MODEL_TRANSCRIPTION = \"tiny\"\nINIT_MODEL_TRANSCRIPTION_REALTIME = \"tiny\"\nINIT_REALTIME_PROCESSING_PAUSE = 0.2\nINIT_REALTIME_INITIAL_PAUSE = 0.2\nINIT_SILERO_SENSITIVITY = 0.4\nINIT_WEBRTC_SENSITIVITY = 3\nINIT_POST_SPEECH_SILENCE_DURATION = 0.6\nINIT_MIN_LENGTH_OF_RECORDING = 0.5\nINIT_MIN_GAP_BETWEEN_RECORDINGS = 0\nINIT_WAKE_WORDS_SENSITIVITY = 0.6\nINIT_PRE_RECORDING_BUFFER_DURATION = 1.0\nINIT_WAKE_WORD_ACTIVATION_DELAY = 0.0\nINIT_WAKE_WORD_TIMEOUT = 5.0\nINIT_WAKE_WORD_BUFFER_DURATION = 0.1\nALLOWED_LATENCY_LIMIT = 100\n\nTIME_SLEEP = 0.02\nSAMPLE_RATE = 16000\nBUFFER_SIZE = 512\nINT16_MAX_ABS_VALUE = 32768.0\n\nINIT_HANDLE_BUFFER_OVERFLOW = False\nif platform.system() != 'Darwin':\n INIT_HANDLE_BUFFER_OVERFLOW = True\n\n\nclass TranscriptionWorker:\n def __init__(self, conn, stdout_pipe, model_path, download_root, compute_type, gpu_device_index, device,\n ready_event, shutdown_event, interrupt_stop_event, beam_size, initial_prompt, suppress_tokens,\n batch_size, faster_whisper_vad_filter, normalize_audio):\n self.conn = conn\n self.stdout_pipe = stdout_pipe\n self.model_path = model_path\n self.download_root = download_root\n self.compute_type = compute_type\n self.gpu_device_index = gpu_device_index\n self.device = device\n self.ready_event = ready_event\n self.shutdown_event = shutdown_event\n self.interrupt_stop_event = interrupt_stop_event\n self.beam_size = beam_size\n self.initial_prompt = initial_prompt\n self.suppress_tokens = suppress_tokens\n self.batch_size = batch_size\n self.faster_whisper_vad_filter = faster_whisper_vad_filter\n self.normalize_audio = normalize_audio\n self.queue = queue.Queue()\n\n def custom_print(self, *args, **kwargs):\n message = ' '.join(map(str, args))\n try:\n self.stdout_pipe.send(message)\n except (BrokenPipeError, EOFError, OSError):\n pass\n\n def poll_connection(self):\n while not self.shutdown_event.is_set():\n try:\n # Use a longer timeout to reduce polling frequency\n if self.conn.poll(0.01): # Increased from 0.01 to 0.5 seconds\n data = self.conn.recv()\n self.queue.put(data)\n else:\n # Sleep only if no data, but use a shorter sleep\n time.sleep(TIME_SLEEP)\n except Exception as e:\n logging.error(f\"Error receiving data from connection: {e}\", exc_info=True)\n time.sleep(TIME_SLEEP)\n\n def run(self):\n if __name__ == \"__main__\":\n system_signal.signal(system_signal.SIGINT, system_signal.SIG_IGN)\n __builtins__['print'] = self.custom_print\n\n logging.info(f\"Initializing faster_whisper main transcription model {self.model_path}\")\n\n try:\n model = faster_whisper.WhisperModel(\n model_size_or_path=self.model_path,\n device=self.device,\n compute_type=self.compute_type,\n device_index=self.gpu_device_index,\n download_root=self.download_root,\n )\n # Create a short dummy audio array, for example 1 second of silence at 16 kHz\n if self.batch_size > 0:\n model = BatchedInferencePipeline(model=model)\n\n # Run a warm-up transcription\n current_dir = os.path.dirname(os.path.realpath(__file__))\n warmup_audio_path = os.path.join(\n current_dir, \"warmup_audio.wav\"\n )\n warmup_audio_data, _ = sf.read(warmup_audio_path, dtype=\"float32\")\n segments, info = model.transcribe(warmup_audio_data, language=\"en\", beam_size=1)\n model_warmup_transcription = \" \".join(segment.text for segment in segments)\n except Exception as e:\n logging.exception(f\"Error initializing main faster_whisper transcription model: {e}\")\n raise\n\n self.ready_event.set()\n logging.debug(\"Faster_whisper main speech to text transcription model initialized successfully\")\n\n # Start the polling thread\n polling_thread = threading.Thread(target=self.poll_connection)\n polling_thread.start()\n\n try:\n while not self.shutdown_event.is_set():\n try:\n audio, language, use_prompt = self.queue.get(timeout=0.1)\n try:\n logging.debug(f\"Transcribing audio with language {language}\")\n start_t = time.time()\n\n # normalize audio to -0.95 dBFS\n if audio is not None and audio .size > 0:\n if self.normalize_audio:\n peak = np.max(np.abs(audio))\n if peak > 0:\n audio = (audio / peak) * 0.95\n else:\n logging.error(\"Received None audio for transcription\")\n self.conn.send(('error', \"Received None audio for transcription\"))\n continue\n\n prompt = None\n if use_prompt:\n prompt = self.initial_prompt if self.initial_prompt else None\n\n if self.batch_size > 0:\n segments, info = model.transcribe(\n audio,\n language=language if language else None,\n beam_size=self.beam_size,\n initial_prompt=prompt,\n suppress_tokens=self.suppress_tokens,\n batch_size=self.batch_size, \n vad_filter=self.faster_whisper_vad_filter\n )\n else:\n segments, info = model.transcribe(\n audio,\n language=language if language else None,\n beam_size=self.beam_size,\n initial_prompt=prompt,\n suppress_tokens=self.suppress_tokens,\n vad_filter=self.faster_whisper_vad_filter\n )\n elapsed = time.time() - start_t\n transcription = \" \".join(seg.text for seg in segments).strip()\n logging.debug(f\"Final text detected with main model: {transcription} in {elapsed:.4f}s\")\n self.conn.send(('success', (transcription, info)))\n except Exception as e:\n logging.error(f\"General error in transcription: {e}\", exc_info=True)\n self.conn.send(('error', str(e)))\n except queue.Empty:\n continue\n except KeyboardInterrupt:\n self.interrupt_stop_event.set()\n logging.debug(\"Transcription worker process finished due to KeyboardInterrupt\")\n break\n except Exception as e:\n logging.error(f\"General error in processing queue item: {e}\", exc_info=True)\n finally:\n __builtins__['print'] = print # Restore the original print function\n self.conn.close()\n self.stdout_pipe.close()\n self.shutdown_event.set() # Ensure the polling thread will stop\n polling_thread.join() # Wait for the polling thread to finish\n\n\nclass bcolors:\n OKGREEN = '\\033[92m' # Green for active speech detection\n WARNING = '\\033[93m' # Yellow for silence detection\n ENDC = '\\033[0m' # Reset to default color\n\n\nclass AudioToTextRecorder:\n \"\"\"\n A class responsible for capturing audio from the microphone, detecting\n voice activity, and then transcribing the captured audio using the\n `faster_whisper` model.\n \"\"\"\n\n def __init__(self,\n model: str = INIT_MODEL_TRANSCRIPTION,\n download_root: str = None, \n language: str = \"\",\n compute_type: str = \"default\",\n input_device_index: int = None,\n gpu_device_index: Union[int, List[int]] = 0,\n device: str = \"cuda\",\n on_recording_start=None,\n on_recording_stop=None,\n on_transcription_start=None,\n ensure_sentence_starting_uppercase=True,\n ensure_sentence_ends_with_period=True,\n use_microphone=True,\n spinner=True,\n level=logging.WARNING,\n batch_size: int = 16,\n\n # Realtime transcription parameters\n enable_realtime_transcription=False,\n use_main_model_for_realtime=False,\n realtime_model_type=INIT_MODEL_TRANSCRIPTION_REALTIME,\n realtime_processing_pause=INIT_REALTIME_PROCESSING_PAUSE,\n init_realtime_after_seconds=INIT_REALTIME_INITIAL_PAUSE,\n on_realtime_transcription_update=None,\n on_realtime_transcription_stabilized=None,\n realtime_batch_size: int = 16,\n\n # Voice activation parameters\n silero_sensitivity: float = INIT_SILERO_SENSITIVITY,\n silero_use_onnx: bool = False,\n silero_deactivity_detection: bool = False,\n webrtc_sensitivity: int = INIT_WEBRTC_SENSITIVITY,\n post_speech_silence_duration: float = (\n INIT_POST_SPEECH_SILENCE_DURATION\n ),\n min_length_of_recording: float = (\n INIT_MIN_LENGTH_OF_RECORDING\n ),\n min_gap_between_recordings: float = (\n INIT_MIN_GAP_BETWEEN_RECORDINGS\n ),\n pre_recording_buffer_duration: float = (\n INIT_PRE_RECORDING_BUFFER_DURATION\n ),\n on_vad_start=None,\n on_vad_stop=None,\n on_vad_detect_start=None,\n on_vad_detect_stop=None,\n on_turn_detection_start=None,\n on_turn_detection_stop=None,\n\n # Wake word parameters\n wakeword_backend: str = \"\",\n openwakeword_model_paths: str = None,\n openwakeword_inference_framework: str = \"onnx\",\n wake_words: str = \"\",\n wake_words_sensitivity: float = INIT_WAKE_WORDS_SENSITIVITY,\n wake_word_activation_delay: float = (\n INIT_WAKE_WORD_ACTIVATION_DELAY\n ),\n wake_word_timeout: float = INIT_WAKE_WORD_TIMEOUT,\n wake_word_buffer_duration: float = INIT_WAKE_WORD_BUFFER_DURATION,\n on_wakeword_detected=None,\n on_wakeword_timeout=None,\n on_wakeword_detection_start=None,\n on_wakeword_detection_end=None,\n on_recorded_chunk=None,\n debug_mode=False,\n handle_buffer_overflow: bool = INIT_HANDLE_BUFFER_OVERFLOW,\n beam_size: int = 5,\n beam_size_realtime: int = 3,\n buffer_size: int = BUFFER_SIZE,\n sample_rate: int = SAMPLE_RATE,\n initial_prompt: Optional[Union[str, Iterable[int]]] = None,\n initial_prompt_realtime: Optional[Union[str, Iterable[int]]] = None,\n suppress_tokens: Optional[List[int]] = [-1],\n print_transcription_time: bool = False,\n early_transcription_on_silence: int = 0,\n allowed_latency_limit: int = ALLOWED_LATENCY_LIMIT,\n no_log_file: bool = False,\n use_extended_logging: bool = False,\n faster_whisper_vad_filter: bool = True,\n normalize_audio: bool = False,\n start_callback_in_new_thread: bool = False,\n ):\n \"\"\"\n Initializes an audio recorder and transcription\n and wake word detection.\n\n Args:\n - model (str, default=\"tiny\"): Specifies the size of the transcription\n model to use or the path to a converted model directory.\n Valid options are 'tiny', 'tiny.en', 'base', 'base.en',\n 'small', 'small.en', 'medium', 'medium.en', 'large-v1',\n 'large-v2'.\n If a specific size is provided, the model is downloaded\n from the Hugging Face Hub.\n - download_root (str, default=None): Specifies the root path were the Whisper models \n are downloaded to. When empty, the default is used. \n - language (str, default=\"\"): Language code for speech-to-text engine.\n If not specified, the model will attempt to detect the language\n automatically.\n - compute_type (str, default=\"default\"): Specifies the type of\n computation to be used for transcription.\n See https://opennmt.net/CTranslate2/quantization.html.\n - input_device_index (int, default=0): The index of the audio input\n device to use.\n - gpu_device_index (int, default=0): Device ID to use.\n The model can also be loaded on multiple GPUs by passing a list of\n IDs (e.g. [0, 1, 2, 3]). In that case, multiple transcriptions can\n run in parallel when transcribe() is called from multiple Python\n threads\n - device (str, default=\"cuda\"): Device for model to use. Can either be \n \"cuda\" or \"cpu\".\n - on_recording_start (callable, default=None): Callback function to be\n called when recording of audio to be transcripted starts.\n - on_recording_stop (callable, default=None): Callback function to be\n called when recording of audio to be transcripted stops.\n - on_transcription_start (callable, default=None): Callback function\n to be called when transcription of audio to text starts.\n - ensure_sentence_starting_uppercase (bool, default=True): Ensures\n that every sentence detected by the algorithm starts with an\n uppercase letter.\n - ensure_sentence_ends_with_period (bool, default=True): Ensures that\n every sentence that doesn't end with punctuation such as \"?\", \"!\"\n ends with a period\n - use_microphone (bool, default=True): Specifies whether to use the\n microphone as the audio input source. If set to False, the\n audio input source will be the audio data sent through the\n feed_audio() method.\n - spinner (bool, default=True): Show spinner animation with current\n state.\n - level (int, default=logging.WARNING): Logging level.\n - batch_size (int, default=16): Batch size for the main transcription\n - enable_realtime_transcription (bool, default=False): Enables or\n disables real-time transcription of audio. When set to True, the\n audio will be transcribed continuously as it is being recorded.\n - use_main_model_for_realtime (str, default=False):\n If True, use the main transcription model for both regular and\n real-time transcription. If False, use a separate model specified\n by realtime_model_type for real-time transcription.\n Using a single model can save memory and potentially improve\n performance, but may not be optimized for real-time processing.\n Using separate models allows for a smaller, faster model for\n real-time transcription while keeping a more accurate model for\n final transcription.\n - realtime_model_type (str, default=\"tiny\"): Specifies the machine\n learning model to be used for real-time transcription. Valid\n options include 'tiny', 'tiny.en', 'base', 'base.en', 'small',\n 'small.en', 'medium', 'medium.en', 'large-v1', 'large-v2'.\n - realtime_processing_pause (float, default=0.1): Specifies the time\n interval in seconds after a chunk of audio gets transcribed. Lower\n values will result in more \"real-time\" (frequent) transcription\n updates but may increase computational load.\n - init_realtime_after_seconds (float, default=0.2): Specifies the \n initial waiting time after the recording was initiated before\n yielding the first realtime transcription\n - on_realtime_transcription_update = A callback function that is\n triggered whenever there's an update in the real-time\n transcription. The function is called with the newly transcribed\n text as its argument.\n - on_realtime_transcription_stabilized = A callback function that is\n triggered when the transcribed text stabilizes in quality. The\n stabilized text is generally more accurate but may arrive with a\n slight delay compared to the regular real-time updates.\n - realtime_batch_size (int, default=16): Batch size for the real-time\n transcription model.\n - silero_sensitivity (float, default=SILERO_SENSITIVITY): Sensitivity\n for the Silero Voice Activity Detection model ranging from 0\n (least sensitive) to 1 (most sensitive). Default is 0.5.\n - silero_use_onnx (bool, default=False): Enables usage of the\n pre-trained model from Silero in the ONNX (Open Neural Network\n Exchange) format instead of the PyTorch format. This is\n recommended for faster performance.\n - silero_deactivity_detection (bool, default=False): Enables the Silero\n model for end-of-speech detection. More robust against background\n noise. Utilizes additional GPU resources but improves accuracy in\n noisy environments. When False, uses the default WebRTC VAD,\n which is more sensitive but may continue recording longer due\n to background sounds.\n - webrtc_sensitivity (int, default=WEBRTC_SENSITIVITY): Sensitivity\n for the WebRTC Voice Activity Detection engine ranging from 0\n (least aggressive / most sensitive) to 3 (most aggressive,\n least sensitive). Default is 3.\n - post_speech_silence_duration (float, default=0.2): Duration in\n seconds of silence that must follow speech before the recording\n is considered to be completed. This ensures that any brief\n pauses during speech don't prematurely end the recording.\n - min_gap_between_recordings (float, default=1.0): Specifies the\n minimum time interval in seconds that should exist between the\n end of one recording session and the beginning of another to\n prevent rapid consecutive recordings.\n - min_length_of_recording (float, default=1.0): Specifies the minimum\n duration in seconds that a recording session should last to ensure\n meaningful audio capture, preventing excessively short or\n fragmented recordings.\n - pre_recording_buffer_duration (float, default=0.2): Duration in\n seconds for the audio buffer to maintain pre-roll audio\n (compensates speech activity detection latency)\n - on_vad_start (callable, default=None): Callback function to be called\n when the system detected the start of voice activity presence.\n - on_vad_stop (callable, default=None): Callback function to be called\n when the system detected the stop (end) of voice activity presence.\n - on_vad_detect_start (callable, default=None): Callback function to\n be called when the system listens for voice activity. This is not\n called when VAD actually happens (use on_vad_start for this), but\n when the system starts listening for it.\n - on_vad_detect_stop (callable, default=None): Callback function to be\n called when the system stops listening for voice activity. This is\n not called when VAD actually stops (use on_vad_stop for this), but\n when the system stops listening for it.\n - on_turn_detection_start (callable, default=None): Callback function\n to be called when the system starts to listen for a turn of speech.\n - on_turn_detection_stop (callable, default=None): Callback function to\n be called when the system stops listening for a turn of speech.\n - wakeword_backend (str, default=\"\"): Specifies the backend library to\n use for wake word detection. Supported options include 'pvporcupine'\n for using the Porcupine wake word engine or 'oww' for using the\n OpenWakeWord engine.\n - wakeword_backend (str, default=\"pvporcupine\"): Specifies the backend\n library to use for wake word detection. Supported options include\n 'pvporcupine' for using the Porcupine wake word engine or 'oww' for\n using the OpenWakeWord engine.\n - openwakeword_model_paths (str, default=None): Comma-separated paths\n to model files for the openwakeword library. These paths point to\n custom models that can be used for wake word detection when the\n openwakeword library is selected as the wakeword_backend.\n - openwakeword_inference_framework (str, default=\"onnx\"): Specifies\n the inference framework to use with the openwakeword library.\n Can be either 'onnx' for Open Neural Network Exchange format \n or 'tflite' for TensorFlow Lite.\n - wake_words (str, default=\"\"): Comma-separated string of wake words to\n initiate recording when using the 'pvporcupine' wakeword backend.\n Supported wake words include: 'alexa', 'americano', 'blueberry',\n 'bumblebee', 'computer', 'grapefruits', 'grasshopper', 'hey google',\n 'hey siri', 'jarvis', 'ok google', 'picovoice', 'porcupine',\n 'terminator'. For the 'openwakeword' backend, wake words are\n automatically extracted from the provided model files, so specifying\n them here is not necessary.\n - wake_words_sensitivity (float, default=0.5): Sensitivity for wake\n word detection, ranging from 0 (least sensitive) to 1 (most\n sensitive). Default is 0.5.\n - wake_word_activation_delay (float, default=0): Duration in seconds\n after the start of monitoring before the system switches to wake\n word activation if no voice is initially detected. If set to\n zero, the system uses wake word activation immediately.\n - wake_word_timeout (float, default=5): Duration in seconds after a\n wake word is recognized. If no subsequent voice activity is\n detected within this window, the system transitions back to an\n inactive state, awaiting the next wake word or voice activation.\n - wake_word_buffer_duration (float, default=0.1): Duration in seconds\n to buffer audio data during wake word detection. This helps in\n cutting out the wake word from the recording buffer so it does not\n falsely get detected along with the following spoken text, ensuring\n cleaner and more accurate transcription start triggers.\n Increase this if parts of the wake word get detected as text.\n - on_wakeword_detected (callable, default=None): Callback function to\n be called when a wake word is detected.\n - on_wakeword_timeout (callable, default=None): Callback function to\n be called when the system goes back to an inactive state after when\n no speech was detected after wake word activation\n - on_wakeword_detection_start (callable, default=None): Callback\n function to be called when the system starts to listen for wake\n words\n - on_wakeword_detection_end (callable, default=None): Callback\n function to be called when the system stops to listen for\n wake words (e.g. because of timeout or wake word detected)\n - on_recorded_chunk (callable, default=None): Callback function to be\n called when a chunk of audio is recorded. The function is called\n with the recorded audio chunk as its argument.\n - debug_mode (bool, default=False): If set to True, the system will\n print additional debug information to the console.\n - handle_buffer_overflow (bool, default=True): If set to True, the system\n will log a warning when an input overflow occurs during recording and\n remove the data from the buffer.\n - beam_size (int, default=5): The beam size to use for beam search\n decoding.\n - beam_size_realtime (int, default=3): The beam size to use for beam\n search decoding in the real-time transcription model.\n - buffer_size (int, default=512): The buffer size to use for audio\n recording. Changing this may break functionality.\n - sample_rate (int, default=16000): The sample rate to use for audio\n recording. Changing this will very probably functionality (as the\n WebRTC VAD model is very sensitive towards the sample rate).\n - initial_prompt (str or iterable of int, default=None): Initial\n prompt to be fed to the main transcription model.\n - initial_prompt_realtime (str or iterable of int, default=None):\n Initial prompt to be fed to the real-time transcription model.\n - suppress_tokens (list of int, default=[-1]): Tokens to be suppressed\n from the transcription output.\n - print_transcription_time (bool, default=False): Logs processing time\n of main model transcription \n - early_transcription_on_silence (int, default=0): If set, the\n system will transcribe audio faster when silence is detected.\n Transcription will start after the specified milliseconds, so \n keep this value lower than post_speech_silence_duration. \n Ideally around post_speech_silence_duration minus the estimated\n transcription time with the main model.\n If silence lasts longer than post_speech_silence_duration, the \n recording is stopped, and the transcription is submitted. If \n voice activity resumes within this period, the transcription \n is discarded. Results in faster final transcriptions to the cost\n of additional GPU load due to some unnecessary final transcriptions.\n - allowed_latency_limit (int, default=100): Maximal amount of chunks\n that can be unprocessed in queue before discarding chunks.\n - no_log_file (bool, default=False): Skips writing of debug log file.\n - use_extended_logging (bool, default=False): Writes extensive\n log messages for the recording worker, that processes the audio\n chunks.\n - faster_whisper_vad_filter (bool, default=True): If set to True,\n the system will additionally use the VAD filter from the faster_whisper library\n for voice activity detection. This filter is more robust against\n background noise but requires additional GPU resources.\n - normalize_audio (bool, default=False): If set to True, the system will\n normalize the audio to a specific range before processing. This can\n help improve the quality of the transcription.\n - start_callback_in_new_thread (bool, default=False): If set to True,\n the callback functions will be executed in a\n new thread. This can help improve performance by allowing the\n callback to run concurrently with other operations.\n\n Raises:\n Exception: Errors related to initializing transcription\n model, wake word detection, or audio recording.\n \"\"\"\n\n self.language = language\n self.compute_type = compute_type\n self.input_device_index = input_device_index\n self.gpu_device_index = gpu_device_index\n self.device = device\n self.wake_words = wake_words\n self.wake_word_activation_delay = wake_word_activation_delay\n self.wake_word_timeout = wake_word_timeout\n self.wake_word_buffer_duration = wake_word_buffer_duration\n self.ensure_sentence_starting_uppercase = (\n ensure_sentence_starting_uppercase\n )\n self.ensure_sentence_ends_with_period = (\n ensure_sentence_ends_with_period\n )\n self.use_microphone = mp.Value(c_bool, use_microphone)\n self.min_gap_between_recordings = min_gap_between_recordings\n self.min_length_of_recording = min_length_of_recording\n self.pre_recording_buffer_duration = pre_recording_buffer_duration\n self.post_speech_silence_duration = post_speech_silence_duration\n self.on_recording_start = on_recording_start\n self.on_recording_stop = on_recording_stop\n self.on_wakeword_detected = on_wakeword_detected\n self.on_wakeword_timeout = on_wakeword_timeout\n self.on_vad_start = on_vad_start\n self.on_vad_stop = on_vad_stop\n self.on_vad_detect_start = on_vad_detect_start\n self.on_vad_detect_stop = on_vad_detect_stop\n self.on_turn_detection_start = on_turn_detection_start\n self.on_turn_detection_stop = on_turn_detection_stop\n self.on_wakeword_detection_start = on_wakeword_detection_start\n self.on_wakeword_detection_end = on_wakeword_detection_end\n self.on_recorded_chunk = on_recorded_chunk\n self.on_transcription_start = on_transcription_start\n self.enable_realtime_transcription = enable_realtime_transcription\n self.use_main_model_for_realtime = use_main_model_for_realtime\n self.main_model_type = model\n if not download_root:\n download_root = None\n self.download_root = download_root\n self.realtime_model_type = realtime_model_type\n self.realtime_processing_pause = realtime_processing_pause\n self.init_realtime_after_seconds = init_realtime_after_seconds\n self.on_realtime_transcription_update = (\n on_realtime_transcription_update\n )\n self.on_realtime_transcription_stabilized = (\n on_realtime_transcription_stabilized\n )\n self.debug_mode = debug_mode\n self.handle_buffer_overflow = handle_buffer_overflow\n self.beam_size = beam_size\n self.beam_size_realtime = beam_size_realtime\n self.allowed_latency_limit = allowed_latency_limit\n self.batch_size = batch_size\n self.realtime_batch_size = realtime_batch_size\n\n self.level = level\n self.audio_queue = mp.Queue()\n self.buffer_size = buffer_size\n self.sample_rate = sample_rate\n self.recording_start_time = 0\n self.recording_stop_time = 0\n self.last_recording_start_time = 0\n self.last_recording_stop_time = 0\n self.wake_word_detect_time = 0\n self.silero_check_time = 0\n self.silero_working = False\n self.speech_end_silence_start = 0\n self.silero_sensitivity = silero_sensitivity\n self.silero_deactivity_detection = silero_deactivity_detection\n self.listen_start = 0\n self.spinner = spinner\n self.halo = None\n self.state = \"inactive\"\n self.wakeword_detected = False\n self.text_storage = []\n self.realtime_stabilized_text = \"\"\n self.realtime_stabilized_safetext = \"\"\n self.is_webrtc_speech_active = False\n self.is_silero_speech_active = False\n self.recording_thread = None\n self.realtime_thread = None\n self.audio_interface = None\n self.audio = None\n self.stream = None\n self.start_recording_event = threading.Event()\n self.stop_recording_event = threading.Event()\n self.backdate_stop_seconds = 0.0\n self.backdate_resume_seconds = 0.0\n self.last_transcription_bytes = None\n self.last_transcription_bytes_b64 = None\n self.initial_prompt = initial_prompt\n self.initial_prompt_realtime = initial_prompt_realtime\n self.suppress_tokens = suppress_tokens\n self.use_wake_words = wake_words or wakeword_backend in {'oww', 'openwakeword', 'openwakewords'}\n self.detected_language = None\n self.detected_language_probability = 0\n self.detected_realtime_language = None\n self.detected_realtime_language_probability = 0\n self.transcription_lock = threading.Lock()\n self.shutdown_lock = threading.Lock()\n self.transcribe_count = 0\n self.print_transcription_time = print_transcription_time\n self.early_transcription_on_silence = early_transcription_on_silence\n self.use_extended_logging = use_extended_logging\n self.faster_whisper_vad_filter = faster_whisper_vad_filter\n self.normalize_audio = normalize_audio\n self.awaiting_speech_end = False\n self.start_callback_in_new_thread = start_callback_in_new_thread\n\n # ----------------------------------------------------------------------------\n # Named logger configuration\n # By default, let's set it up so it logs at 'level' to the console.\n # If you do NOT want this default configuration, remove the lines below\n # and manage your \"realtimestt\" logger from your application code.\n logger.setLevel(logging.DEBUG) # We capture all, then filter via handlers\n\n log_format = \"RealTimeSTT: %(name)s - %(levelname)s - %(message)s\"\n file_log_format = \"%(asctime)s.%(msecs)03d - \" + log_format\n\n # Create and set up console handler\n console_handler = logging.StreamHandler()\n console_handler.setLevel(self.level)\n console_handler.setFormatter(logging.Formatter(log_format))\n\n logger.addHandler(console_handler)\n\n if not no_log_file:\n file_handler = logging.FileHandler('realtimesst.log')\n file_handler.setLevel(logging.DEBUG)\n file_handler.setFormatter(logging.Formatter(file_log_format, datefmt='%Y-%m-%d %H:%M:%S'))\n logger.addHandler(file_handler)\n # ----------------------------------------------------------------------------\n\n self.is_shut_down = False\n self.shutdown_event = mp.Event()\n \n try:\n # Only set the start method if it hasn't been set already\n if mp.get_start_method(allow_none=True) is None:\n mp.set_start_method(\"spawn\")\n except RuntimeError as e:\n logger.info(f\"Start method has already been set. Details: {e}\")\n\n logger.info(\"Starting RealTimeSTT\")\n\n if use_extended_logging:\n logger.info(\"RealtimeSTT was called with these parameters:\")\n for param, value in locals().items():\n logger.info(f\"{param}: {value}\")\n\n self.interrupt_stop_event = mp.Event()\n self.was_interrupted = mp.Event()\n self.main_transcription_ready_event = mp.Event()\n\n self.parent_transcription_pipe, child_transcription_pipe = SafePipe()\n self.parent_stdout_pipe, child_stdout_pipe = SafePipe()\n\n # Set device for model\n self.device = \"cuda\" if self.device == \"cuda\" and torch.cuda.is_available() else \"cpu\"\n\n self.transcript_process = self._start_thread(\n target=AudioToTextRecorder._transcription_worker,\n args=(\n child_transcription_pipe,\n child_stdout_pipe,\n self.main_model_type,\n self.download_root,\n self.compute_type,\n self.gpu_device_index,\n self.device,\n self.main_transcription_ready_event,\n self.shutdown_event,\n self.interrupt_stop_event,\n self.beam_size,\n self.initial_prompt,\n self.suppress_tokens,\n self.batch_size,\n self.faster_whisper_vad_filter,\n self.normalize_audio,\n )\n )\n\n # Start audio data reading process\n if self.use_microphone.value:\n logger.info(\"Initializing audio recording\"\n \" (creating pyAudio input stream,\"\n f\" sample rate: {self.sample_rate}\"\n f\" buffer size: {self.buffer_size}\"\n )\n self.reader_process = self._start_thread(\n target=AudioToTextRecorder._audio_data_worker,\n args=(\n self.audio_queue,\n self.sample_rate,\n self.buffer_size,\n self.input_device_index,\n self.shutdown_event,\n self.interrupt_stop_event,\n self.use_microphone\n )\n )\n\n # Initialize the realtime transcription model\n if self.enable_realtime_transcription and not self.use_main_model_for_realtime:\n try:\n logger.info(\"Initializing faster_whisper realtime \"\n f\"transcription model {self.realtime_model_type}, \"\n f\"default device: {self.device}, \"\n f\"compute type: {self.compute_type}, \"\n f\"device index: {self.gpu_device_index}, \"\n f\"download root: {self.download_root}\"\n )\n self.realtime_model_type = faster_whisper.WhisperModel(\n model_size_or_path=self.realtime_model_type,\n device=self.device,\n compute_type=self.compute_type,\n device_index=self.gpu_device_index,\n download_root=self.download_root,\n )\n if self.realtime_batch_size > 0:\n self.realtime_model_type = BatchedInferencePipeline(model=self.realtime_model_type)\n\n # Run a warm-up transcription\n current_dir = os.path.dirname(os.path.realpath(__file__))\n warmup_audio_path = os.path.join(\n current_dir, \"warmup_audio.wav\"\n )\n warmup_audio_data, _ = sf.read(warmup_audio_path, dtype=\"float32\")\n segments, info = self.realtime_model_type.transcribe(warmup_audio_data, language=\"en\", beam_size=1)\n model_warmup_transcription = \" \".join(segment.text for segment in segments)\n except Exception as e:\n logger.exception(\"Error initializing faster_whisper \"\n f\"realtime transcription model: {e}\"\n )\n raise\n\n logger.debug(\"Faster_whisper realtime speech to text \"\n \"transcription model initialized successfully\")\n\n # Setup wake word detection\n if wake_words or wakeword_backend in {'oww', 'openwakeword', 'openwakewords', 'pvp', 'pvporcupine'}:\n self.wakeword_backend = wakeword_backend\n\n self.wake_words_list = [\n word.strip() for word in wake_words.lower().split(',')\n ]\n self.wake_words_sensitivity = wake_words_sensitivity\n self.wake_words_sensitivities = [\n float(wake_words_sensitivity)\n for _ in range(len(self.wake_words_list))\n ]\n\n if wake_words and self.wakeword_backend in {'pvp', 'pvporcupine'}:\n\n try:\n self.porcupine = pvporcupine.create(\n keywords=self.wake_words_list,\n sensitivities=self.wake_words_sensitivities\n )\n self.buffer_size = self.porcupine.frame_length\n self.sample_rate = self.porcupine.sample_rate\n\n except Exception as e:\n logger.exception(\n \"Error initializing porcupine \"\n f\"wake word detection engine: {e}. \"\n f\"Wakewords: {self.wake_words_list}.\"\n )\n raise\n\n logger.debug(\n \"Porcupine wake word detection engine initialized successfully\"\n )\n\n elif wake_words and self.wakeword_backend in {'oww', 'openwakeword', 'openwakewords'}:\n \n openwakeword.utils.download_models()\n\n try:\n if openwakeword_model_paths:\n model_paths = openwakeword_model_paths.split(',')\n self.owwModel = Model(\n wakeword_models=model_paths,\n inference_framework=openwakeword_inference_framework\n )\n logger.info(\n \"Successfully loaded wakeword model(s): \"\n f\"{openwakeword_model_paths}\"\n )\n else:\n self.owwModel = Model(\n inference_framework=openwakeword_inference_framework)\n \n self.oww_n_models = len(self.owwModel.models.keys())\n if not self.oww_n_models:\n logger.error(\n \"No wake word models loaded.\"\n )\n\n for model_key in self.owwModel.models.keys():\n logger.info(\n \"Successfully loaded openwakeword model: \"\n f\"{model_key}\"\n )\n\n except Exception as e:\n logger.exception(\n \"Error initializing openwakeword \"\n f\"wake word detection engine: {e}\"\n )\n raise\n\n logger.debug(\n \"Open wake word detection engine initialized successfully\"\n )\n \n else:\n logger.exception(f\"Wakeword engine {self.wakeword_backend} unknown/unsupported or wake_words not specified. Please specify one of: pvporcupine, openwakeword.\")\n\n\n # Setup voice activity detection model WebRTC\n try:\n logger.info(\"Initializing WebRTC voice with \"\n f\"Sensitivity {webrtc_sensitivity}\"\n )\n self.webrtc_vad_model = webrtcvad.Vad()\n self.webrtc_vad_model.set_mode(webrtc_sensitivity)\n\n except Exception as e:\n logger.exception(\"Error initializing WebRTC voice \"\n f\"activity detection engine: {e}\"\n )\n raise\n\n logger.debug(\"WebRTC VAD voice activity detection \"\n \"engine initialized successfully\"\n )\n\n # Setup voice activity detection model Silero VAD\n try:\n self.silero_vad_model, _ = torch.hub.load(\n repo_or_dir=\"snakers4/silero-vad\",\n model=\"silero_vad\",\n verbose=False,\n onnx=silero_use_onnx\n )\n\n except Exception as e:\n logger.exception(f\"Error initializing Silero VAD \"\n f\"voice activity detection engine: {e}\"\n )\n raise\n\n logger.debug(\"Silero VAD voice activity detection \"\n \"engine initialized successfully\"\n )\n\n self.audio_buffer = collections.deque(\n maxlen=int((self.sample_rate // self.buffer_size) *\n self.pre_recording_buffer_duration)\n )\n self.last_words_buffer = collections.deque(\n maxlen=int((self.sample_rate // self.buffer_size) *\n 0.3)\n )\n self.frames = []\n self.last_frames = []\n\n # Recording control flags\n self.is_recording = False\n self.is_running = True\n self.start_recording_on_voice_activity = False\n self.stop_recording_on_voice_deactivity = False\n\n # Start the recording worker thread\n self.recording_thread = threading.Thread(target=self._recording_worker)\n self.recording_thread.daemon = True\n self.recording_thread.start()\n\n # Start the realtime transcription worker thread\n self.realtime_thread = threading.Thread(target=self._realtime_worker)\n self.realtime_thread.daemon = True\n self.realtime_thread.start()\n \n # Wait for transcription models to start\n logger.debug('Waiting for main transcription model to start')\n self.main_transcription_ready_event.wait()\n logger.debug('Main transcription model ready')\n\n self.stdout_thread = threading.Thread(target=self._read_stdout)\n self.stdout_thread.daemon = True\n self.stdout_thread.start()\n\n logger.debug('RealtimeSTT initialization completed successfully')\n \n def _start_thread(self, target=None, args=()):\n \"\"\"\n Implement a consistent threading model across the library.\n\n This method is used to start any thread in this library. It uses the\n standard threading. Thread for Linux and for all others uses the pytorch\n MultiProcessing library 'Process'.\n Args:\n target (callable object): is the callable object to be invoked by\n the run() method. Defaults to None, meaning nothing is called.\n args (tuple): is a list or tuple of arguments for the target\n invocation. Defaults to ().\n \"\"\"\n if (platform.system() == 'Linux'):\n thread = threading.Thread(target=target, args=args)\n thread.deamon = True\n thread.start()\n return thread\n else:\n thread = mp.Process(target=target, args=args)\n thread.start()\n return thread\n\n def _read_stdout(self):\n while not self.shutdown_event.is_set():\n try:\n if self.parent_stdout_pipe.poll(0.1):\n logger.debug(\"Receive from stdout pipe\")\n message = self.parent_stdout_pipe.recv()\n logger.info(message)\n except (BrokenPipeError, EOFError, OSError):\n # The pipe probably has been closed, so we ignore the error\n pass\n except KeyboardInterrupt: # handle manual interruption (Ctrl+C)\n logger.info(\"KeyboardInterrupt in read from stdout detected, exiting...\")\n break\n except Exception as e:\n logger.error(f\"Unexpected error in read from stdout: {e}\", exc_info=True)\n logger.error(traceback.format_exc()) # Log the full traceback here\n break \n time.sleep(0.1)\n\n def _transcription_worker(*args, **kwargs):\n worker = TranscriptionWorker(*args, **kwargs)\n worker.run()\n\n def _run_callback(self, cb, *args, **kwargs):\n if self.start_callback_in_new_thread:\n # Run the callback in a new thread to avoid blocking the main thread\n threading.Thread(target=cb, args=args, kwargs=kwargs, daemon=True).start()\n else:\n # Run the callback in the main thread to avoid threading issues\n cb(*args, **kwargs)\n\n @staticmethod\n def _audio_data_worker(\n audio_queue,\n target_sample_rate,\n buffer_size,\n input_device_index,\n shutdown_event,\n interrupt_stop_event,\n use_microphone\n ):\n \"\"\"\n Worker method that handles the audio recording process.\n\n This method runs in a separate process and is responsible for:\n - Setting up the audio input stream for recording at the highest possible sample rate.\n - Continuously reading audio data from the input stream, resampling if necessary,\n preprocessing the data, and placing complete chunks in a queue.\n - Handling errors during the recording process.\n - Gracefully terminating the recording process when a shutdown event is set.\n\n Args:\n audio_queue (queue.Queue): A queue where recorded audio data is placed.\n target_sample_rate (int): The desired sample rate for the output audio (for Silero VAD).\n buffer_size (int): The number of samples expected by the Silero VAD model.\n input_device_index (int): The index of the audio input device.\n shutdown_event (threading.Event): An event that, when set, signals this worker method to terminate.\n interrupt_stop_event (threading.Event): An event to signal keyboard interrupt.\n use_microphone (multiprocessing.Value): A shared value indicating whether to use the microphone.\n\n Raises:\n Exception: If there is an error while initializing the audio recording.\n \"\"\"\n import pyaudio\n import numpy as np\n from scipy import signal\n\n if __name__ == '__main__':\n system_signal.signal(system_signal.SIGINT, system_signal.SIG_IGN)\n\n def get_highest_sample_rate(audio_interface, device_index):\n \"\"\"Get the highest supported sample rate for the specified device.\"\"\"\n try:\n device_info = audio_interface.get_device_info_by_index(device_index)\n logger.debug(f\"Retrieving highest sample rate for device index {device_index}: {device_info}\")\n max_rate = int(device_info['defaultSampleRate'])\n\n if 'supportedSampleRates' in device_info:\n supported_rates = [int(rate) for rate in device_info['supportedSampleRates']]\n if supported_rates:\n max_rate = max(supported_rates)\n\n logger.debug(f\"Highest supported sample rate for device index {device_index} is {max_rate}\")\n return max_rate\n except Exception as e:\n logger.warning(f\"Failed to get highest sample rate: {e}\")\n return 48000 # Fallback to a common high sample rate\n\n def initialize_audio_stream(audio_interface, sample_rate, chunk_size):\n nonlocal input_device_index\n\n def validate_device(device_index):\n \"\"\"Validate that the device exists and is actually available for input.\"\"\"\n try:\n device_info = audio_interface.get_device_info_by_index(device_index)\n logger.debug(f\"Validating device index {device_index} with info: {device_info}\")\n if not device_info.get('maxInputChannels', 0) > 0:\n logger.debug(\"Device has no input channels, invalid for recording.\")\n return False\n\n # Try to actually read from the device\n test_stream = audio_interface.open(\n format=pyaudio.paInt16,\n channels=1,\n rate=target_sample_rate,\n input=True,\n frames_per_buffer=chunk_size,\n input_device_index=device_index,\n start=False # Don't start the stream yet\n )\n\n test_stream.start_stream()\n test_data = test_stream.read(chunk_size, exception_on_overflow=False)\n test_stream.stop_stream()\n test_stream.close()\n\n if len(test_data) == 0:\n logger.debug(\"Device produced no data, invalid for recording.\")\n return False\n\n logger.debug(f\"Device index {device_index} successfully validated.\")\n return True\n\n except Exception as e:\n logger.debug(f\"Device validation failed for index {device_index}: {e}\")\n return False\n\n \"\"\"Initialize the audio stream with error handling.\"\"\"\n while not shutdown_event.is_set():\n try:\n # First, get a list of all available input devices\n input_devices = []\n device_count = audio_interface.get_device_count()\n logger.debug(f\"Found {device_count} total audio devices on the system.\")\n for i in range(device_count):\n try:\n device_info = audio_interface.get_device_info_by_index(i)\n if device_info.get('maxInputChannels', 0) > 0:\n input_devices.append(i)\n except Exception as e:\n logger.debug(f\"Could not retrieve info for device index {i}: {e}\")\n continue\n\n logger.debug(f\"Available input devices with input channels: {input_devices}\")\n if not input_devices:\n raise Exception(\"No input devices found\")\n\n # If input_device_index is None or invalid, try to find a working device\n if input_device_index is None or input_device_index not in input_devices:\n # First try the default device\n try:\n default_device = audio_interface.get_default_input_device_info()\n logger.debug(f\"Default device info: {default_device}\")\n if validate_device(default_device['index']):\n input_device_index = default_device['index']\n logger.debug(f\"Default device {input_device_index} selected.\")\n except Exception:\n # If default device fails, try other available input devices\n logger.debug(\"Default device validation failed, checking other devices...\")\n for device_index in input_devices:\n if validate_device(device_index):\n input_device_index = device_index\n logger.debug(f\"Device {input_device_index} selected.\")\n break\n else:\n raise Exception(\"No working input devices found\")\n\n # Validate the selected device one final time\n if not validate_device(input_device_index):\n raise Exception(\"Selected device validation failed\")\n\n # If we get here, we have a validated device\n logger.debug(f\"Opening stream with device index {input_device_index}, \"\n f\"sample_rate={sample_rate}, chunk_size={chunk_size}\")\n stream = audio_interface.open(\n format=pyaudio.paInt16,\n channels=1,\n rate=sample_rate,\n input=True,\n frames_per_buffer=chunk_size,\n input_device_index=input_device_index,\n )\n\n logger.info(f\"Microphone connected and validated (device index: {input_device_index}, \"\n f\"sample rate: {sample_rate}, chunk size: {chunk_size})\")\n return stream\n\n except Exception as e:\n logger.error(f\"Microphone connection failed: {e}. Retrying...\", exc_info=True)\n input_device_index = None\n time.sleep(3) # Wait before retrying\n continue\n\n def preprocess_audio(chunk, original_sample_rate, target_sample_rate):\n \"\"\"Preprocess audio chunk similar to feed_audio method.\"\"\"\n if isinstance(chunk, np.ndarray):\n # Handle stereo to mono conversion if necessary\n if chunk.ndim == 2:\n chunk = np.mean(chunk, axis=1)\n\n # Resample to target_sample_rate if necessary\n if original_sample_rate != target_sample_rate:\n logger.debug(f\"Resampling from {original_sample_rate} Hz to {target_sample_rate} Hz.\")\n num_samples = int(len(chunk) * target_sample_rate / original_sample_rate)\n chunk = signal.resample(chunk, num_samples)\n\n chunk = chunk.astype(np.int16)\n else:\n # If chunk is bytes, convert to numpy array\n chunk = np.frombuffer(chunk, dtype=np.int16)\n\n # Resample if necessary\n if original_sample_rate != target_sample_rate:\n logger.debug(f\"Resampling from {original_sample_rate} Hz to {target_sample_rate} Hz.\")\n num_samples = int(len(chunk) * target_sample_rate / original_sample_rate)\n chunk = signal.resample(chunk, num_samples)\n chunk = chunk.astype(np.int16)\n\n return chunk.tobytes()\n\n audio_interface = None\n stream = None\n device_sample_rate = None\n chunk_size = 1024 # Increased chunk size for better performance\n\n def setup_audio(): \n nonlocal audio_interface, stream, device_sample_rate, input_device_index\n try:\n if audio_interface is None:\n logger.debug(\"Creating PyAudio interface...\")\n audio_interface = pyaudio.PyAudio()\n\n if input_device_index is None:\n try:\n default_device = audio_interface.get_default_input_device_info()\n input_device_index = default_device['index']\n logger.debug(f\"No device index supplied; using default device {input_device_index}\")\n except OSError as e:\n logger.debug(f\"Default device retrieval failed: {e}\")\n input_device_index = None\n\n # We'll try 16000 Hz first, then the highest rate we detect, then fallback if needed\n sample_rates_to_try = [16000]\n if input_device_index is not None:\n highest_rate = get_highest_sample_rate(audio_interface, input_device_index)\n if highest_rate != 16000:\n sample_rates_to_try.append(highest_rate)\n else:\n sample_rates_to_try.append(48000)\n\n logger.debug(f\"Sample rates to try for device {input_device_index}: {sample_rates_to_try}\")\n\n for rate in sample_rates_to_try:\n try:\n device_sample_rate = rate\n logger.debug(f\"Attempting to initialize audio stream at {device_sample_rate} Hz.\")\n stream = initialize_audio_stream(audio_interface, device_sample_rate, chunk_size)\n if stream is not None:\n logger.debug(\n f\"Audio recording initialized successfully at {device_sample_rate} Hz, \"\n f\"reading {chunk_size} frames at a time\"\n )\n return True\n except Exception as e:\n logger.warning(f\"Failed to initialize audio stream at {device_sample_rate} Hz: {e}\")\n continue\n\n # If we reach here, none of the sample rates worked\n raise Exception(\"Failed to initialize audio stream with all sample rates.\")\n\n except Exception as e:\n logger.exception(f\"Error initializing pyaudio audio recording: {e}\")\n if audio_interface:\n audio_interface.terminate()\n return False\n\n logger.debug(f\"Starting audio data worker with target_sample_rate={target_sample_rate}, \"\n f\"buffer_size={buffer_size}, input_device_index={input_device_index}\")\n\n if not setup_audio():\n raise Exception(\"Failed to set up audio recording.\")\n\n buffer = bytearray()\n silero_buffer_size = 2 * buffer_size # Silero complains if too short\n\n time_since_last_buffer_message = 0\n\n try:\n while not shutdown_event.is_set():\n try:\n data = stream.read(chunk_size, exception_on_overflow=False)\n\n if use_microphone.value:\n processed_data = preprocess_audio(data, device_sample_rate, target_sample_rate)\n buffer += processed_data\n\n # Check if the buffer has reached or exceeded the silero_buffer_size\n while len(buffer) >= silero_buffer_size:\n # Extract silero_buffer_size amount of data from the buffer\n to_process = buffer[:silero_buffer_size]\n buffer = buffer[silero_buffer_size:]\n\n # Feed the extracted data to the audio_queue\n if time_since_last_buffer_message:\n time_passed = time.time() - time_since_last_buffer_message\n if time_passed > 1:\n logger.debug(\"_audio_data_worker writing audio data into queue.\")\n time_since_last_buffer_message = time.time()\n else:\n time_since_last_buffer_message = time.time()\n\n audio_queue.put(to_process)\n\n except OSError as e:\n if e.errno == pyaudio.paInputOverflowed:\n logger.warning(\"Input overflowed. Frame dropped.\")\n else:\n logger.error(f\"OSError during recording: {e}\", exc_info=True)\n # Attempt to reinitialize the stream\n logger.error(\"Attempting to reinitialize the audio stream...\")\n\n try:\n if stream:\n stream.stop_stream()\n stream.close()\n except Exception:\n pass\n\n time.sleep(1)\n if not setup_audio():\n logger.error(\"Failed to reinitialize audio stream. Exiting.\")\n break\n else:\n logger.error(\"Audio stream reinitialized successfully.\")\n continue\n\n except Exception as e:\n logger.error(f\"Unknown error during recording: {e}\")\n tb_str = traceback.format_exc()\n logger.error(f\"Traceback: {tb_str}\")\n logger.error(f\"Error: {e}\")\n # Attempt to reinitialize the stream\n logger.info(\"Attempting to reinitialize the audio stream...\")\n try:\n if stream:\n stream.stop_stream()\n stream.close()\n except Exception:\n pass\n\n time.sleep(1)\n if not setup_audio():\n logger.error(\"Failed to reinitialize audio stream. Exiting.\")\n break\n else:\n logger.info(\"Audio stream reinitialized successfully.\")\n continue\n\n except KeyboardInterrupt:\n interrupt_stop_event.set()\n logger.debug(\"Audio data worker process finished due to KeyboardInterrupt\")\n finally:\n # After recording stops, feed any remaining audio data\n if buffer:\n audio_queue.put(bytes(buffer))\n\n try:\n if stream:\n stream.stop_stream()\n stream.close()\n except Exception:\n pass\n if audio_interface:\n audio_interface.terminate()\n\n def wakeup(self):\n \"\"\"\n If in wake work modus, wake up as if a wake word was spoken.\n \"\"\"\n self.listen_start = time.time()\n\n def abort(self):\n state = self.state\n self.start_recording_on_voice_activity = False\n self.stop_recording_on_voice_deactivity = False\n self.interrupt_stop_event.set()\n if self.state != \"inactive\": # if inactive, was_interrupted will never be set\n self.was_interrupted.wait()\n self._set_state(\"transcribing\")\n self.was_interrupted.clear()\n if self.is_recording: # if recording, make sure to stop the recorder\n self.stop()\n\n\n def wait_audio(self):\n \"\"\"\n Waits for the start and completion of the audio recording process.\n\n This method is responsible for:\n - Waiting for voice activity to begin recording if not yet started.\n - Waiting for voice inactivity to complete the recording.\n - Setting the audio buffer from the recorded frames.\n - Resetting recording-related attributes.\n\n Side effects:\n - Updates the state of the instance.\n - Modifies the audio attribute to contain the processed audio data.\n \"\"\"\n\n try:\n logger.info(\"Setting listen time\")\n if self.listen_start == 0:\n self.listen_start = time.time()\n\n # If not yet started recording, wait for voice activity to initiate.\n if not self.is_recording and not self.frames:\n self._set_state(\"listening\")\n self.start_recording_on_voice_activity = True\n\n # Wait until recording starts\n logger.debug('Waiting for recording start')\n while not self.interrupt_stop_event.is_set():\n if self.start_recording_event.wait(timeout=0.02):\n break\n\n # If recording is ongoing, wait for voice inactivity\n # to finish recording.\n if self.is_recording:\n self.stop_recording_on_voice_deactivity = True\n\n # Wait until recording stops\n logger.debug('Waiting for recording stop')\n while not self.interrupt_stop_event.is_set():\n if (self.stop_recording_event.wait(timeout=0.02)):\n break\n\n frames = self.frames\n if len(frames) == 0:\n frames = self.last_frames\n\n # Calculate samples needed for backdating resume\n samples_to_keep = int(self.sample_rate * self.backdate_resume_seconds)\n\n # First convert all current frames to audio array\n full_audio_array = np.frombuffer(b''.join(frames), dtype=np.int16)\n full_audio = full_audio_array.astype(np.float32) / INT16_MAX_ABS_VALUE\n\n # Calculate how many samples we need to keep for backdating resume\n if samples_to_keep > 0:\n samples_to_keep = min(samples_to_keep, len(full_audio))\n # Keep the last N samples for backdating resume\n frames_to_read_audio = full_audio[-samples_to_keep:]\n\n # Convert the audio back to int16 bytes for frames\n frames_to_read_int16 = (frames_to_read_audio * INT16_MAX_ABS_VALUE).astype(np.int16)\n frame_bytes = frames_to_read_int16.tobytes()\n\n # Split into appropriate frame sizes (assuming standard frame size)\n FRAME_SIZE = 2048 # Typical frame size\n frames_to_read = []\n for i in range(0, len(frame_bytes), FRAME_SIZE):\n frame = frame_bytes[i:i + FRAME_SIZE]\n if frame: # Only add non-empty frames\n frames_to_read.append(frame)\n else:\n frames_to_read = []\n\n # Process backdate stop seconds\n samples_to_remove = int(self.sample_rate * self.backdate_stop_seconds)\n\n if samples_to_remove > 0:\n if samples_to_remove < len(full_audio):\n self.audio = full_audio[:-samples_to_remove]\n logger.debug(f\"Removed {samples_to_remove} samples \"\n f\"({samples_to_remove/self.sample_rate:.3f}s) from end of audio\")\n else:\n self.audio = np.array([], dtype=np.float32)\n logger.debug(\"Cleared audio (samples_to_remove >= audio length)\")\n else:\n self.audio = full_audio\n logger.debug(f\"No samples removed, final audio length: {len(self.audio)}\")\n\n self.frames.clear()\n self.last_frames.clear()\n self.frames.extend(frames_to_read)\n\n # Reset backdating parameters\n self.backdate_stop_seconds = 0.0\n self.backdate_resume_seconds = 0.0\n\n self.listen_start = 0\n\n self._set_state(\"inactive\")\n\n except KeyboardInterrupt:\n logger.info(\"KeyboardInterrupt in wait_audio, shutting down\")\n self.shutdown()\n raise # Re-raise the exception after cleanup\n\n\n def perform_final_transcription(self, audio_bytes=None, use_prompt=True):\n start_time = 0\n with self.transcription_lock:\n if audio_bytes is None:\n audio_bytes = copy.deepcopy(self.audio)\n\n if audio_bytes is None or len(audio_bytes) == 0:\n print(\"No audio data available for transcription\")\n #logger.info(\"No audio data available for transcription\")\n return \"\"\n\n try:\n if self.transcribe_count == 0:\n logger.debug(\"Adding transcription request, no early transcription started\")\n start_time = time.time() # Start timing\n self.parent_transcription_pipe.send((audio_bytes, self.language, use_prompt))\n self.transcribe_count += 1\n\n while self.transcribe_count > 0:\n logger.debug(F\"Receive from parent_transcription_pipe after sendiung transcription request, transcribe_count: {self.transcribe_count}\")\n if not self.parent_transcription_pipe.poll(0.1): # check if transcription done\n if self.interrupt_stop_event.is_set(): # check if interrupted\n self.was_interrupted.set()\n self._set_state(\"inactive\")\n return \"\" # return empty string if interrupted\n continue\n status, result = self.parent_transcription_pipe.recv()\n self.transcribe_count -= 1\n\n self.allowed_to_early_transcribe = True\n self._set_state(\"inactive\")\n if status == 'success':\n segments, info = result\n self.detected_language = info.language if info.language_probability > 0 else None\n self.detected_language_probability = info.language_probability\n self.last_transcription_bytes = copy.deepcopy(audio_bytes)\n self.last_transcription_bytes_b64 = base64.b64encode(self.last_transcription_bytes.tobytes()).decode('utf-8')\n transcription = self._preprocess_output(segments)\n end_time = time.time() # End timing\n transcription_time = end_time - start_time\n\n if start_time:\n if self.print_transcription_time:\n print(f\"Model {self.main_model_type} completed transcription in {transcription_time:.2f} seconds\")\n else:\n logger.debug(f\"Model {self.main_model_type} completed transcription in {transcription_time:.2f} seconds\")\n return \"\" if self.interrupt_stop_event.is_set() else transcription # if interrupted return empty string\n else:\n logger.error(f\"Transcription error: {result}\")\n raise Exception(result)\n except Exception as e:\n logger.error(f\"Error during transcription: {str(e)}\", exc_info=True)\n raise e\n\n\n def transcribe(self):\n \"\"\"\n Transcribes audio captured by this class instance using the\n `faster_whisper` model.\n\n Automatically starts recording upon voice activity if not manually\n started using `recorder.start()`.\n Automatically stops recording upon voice deactivity if not manually\n stopped with `recorder.stop()`.\n Processes the recorded audio to generate transcription.\n\n Args:\n on_transcription_finished (callable, optional): Callback function\n to be executed when transcription is ready.\n If provided, transcription will be performed asynchronously,\n and the callback will receive the transcription as its argument.\n If omitted, the transcription will be performed synchronously,\n and the result will be returned.\n\n Returns (if no callback is set):\n str: The transcription of the recorded audio.\n\n Raises:\n Exception: If there is an error during the transcription process.\n \"\"\"\n audio_copy = copy.deepcopy(self.audio)\n self._set_state(\"transcribing\")\n if self.on_transcription_start:\n abort_value = self.on_transcription_start(audio_copy)\n if not abort_value:\n return self.perform_final_transcription(audio_copy)\n return None\n else:\n return self.perform_final_transcription(audio_copy)\n\n\n def _process_wakeword(self, data):\n \"\"\"\n Processes audio data to detect wake words.\n \"\"\"\n if self.wakeword_backend in {'pvp', 'pvporcupine'}:\n pcm = struct.unpack_from(\n \"h\" * self.buffer_size,\n data\n )\n porcupine_index = self.porcupine.process(pcm)\n if self.debug_mode:\n logger.info(f\"wake words porcupine_index: {porcupine_index}\")\n return porcupine_index\n\n elif self.wakeword_backend in {'oww', 'openwakeword', 'openwakewords'}:\n pcm = np.frombuffer(data, dtype=np.int16)\n prediction = self.owwModel.predict(pcm)\n max_score = -1\n max_index = -1\n wake_words_in_prediction = len(self.owwModel.prediction_buffer.keys())\n self.wake_words_sensitivities\n if wake_words_in_prediction:\n for idx, mdl in enumerate(self.owwModel.prediction_buffer.keys()):\n scores = list(self.owwModel.prediction_buffer[mdl])\n if scores[-1] >= self.wake_words_sensitivity and scores[-1] > max_score:\n max_score = scores[-1]\n max_index = idx\n if self.debug_mode:\n logger.info(f\"wake words oww max_index, max_score: {max_index} {max_score}\")\n return max_index \n else:\n if self.debug_mode:\n logger.info(f\"wake words oww_index: -1\")\n return -1\n\n if self.debug_mode: \n logger.info(\"wake words no match\")\n\n return -1\n\n def text(self,\n on_transcription_finished=None,\n ):\n \"\"\"\n Transcribes audio captured by this class instance\n using the `faster_whisper` model.\n\n - Automatically starts recording upon voice activity if not manually\n started using `recorder.start()`.\n - Automatically stops recording upon voice deactivity if not manually\n stopped with `recorder.stop()`.\n - Processes the recorded audio to generate transcription.\n\n Args:\n on_transcription_finished (callable, optional): Callback function\n to be executed when transcription is ready.\n If provided, transcription will be performed asynchronously, and\n the callback will receive the transcription as its argument.\n If omitted, the transcription will be performed synchronously,\n and the result will be returned.\n\n Returns (if not callback is set):\n str: The transcription of the recorded audio\n \"\"\"\n self.interrupt_stop_event.clear()\n self.was_interrupted.clear()\n try:\n self.wait_audio()\n except KeyboardInterrupt:\n logger.info(\"KeyboardInterrupt in text() method\")\n self.shutdown()\n raise # Re-raise the exception after cleanup\n\n if self.is_shut_down or self.interrupt_stop_event.is_set():\n if self.interrupt_stop_event.is_set():\n self.was_interrupted.set()\n return \"\"\n\n if on_transcription_finished:\n threading.Thread(target=on_transcription_finished,\n args=(self.transcribe(),)).start()\n else:\n return self.transcribe()\n\n\n def format_number(self, num):\n # Convert the number to a string\n num_str = f\"{num:.10f}\" # Ensure precision is sufficient\n # Split the number into integer and decimal parts\n integer_part, decimal_part = num_str.split('.')\n # Take the last two digits of the integer part and the first two digits of the decimal part\n result = f\"{integer_part[-2:]}.{decimal_part[:2]}\"\n return result\n\n def start(self, frames = None):\n \"\"\"\n Starts recording audio directly without waiting for voice activity.\n \"\"\"\n\n # Ensure there's a minimum interval\n # between stopping and starting recording\n if (time.time() - self.recording_stop_time\n < self.min_gap_between_recordings):\n logger.info(\"Attempted to start recording \"\n \"too soon after stopping.\"\n )\n return self\n\n logger.info(\"recording started\")\n self._set_state(\"recording\")\n self.text_storage = []\n self.realtime_stabilized_text = \"\"\n self.realtime_stabilized_safetext = \"\"\n self.wakeword_detected = False\n self.wake_word_detect_time = 0\n self.frames = []\n if frames:\n self.frames = frames\n self.is_recording = True\n\n self.recording_start_time = time.time()\n self.is_silero_speech_active = False\n self.is_webrtc_speech_active = False\n self.stop_recording_event.clear()\n self.start_recording_event.set()\n\n if self.on_recording_start:\n self._run_callback(self.on_recording_start)\n\n return self\n\n def stop(self,\n backdate_stop_seconds: float = 0.0,\n backdate_resume_seconds: float = 0.0,\n ):\n \"\"\"\n Stops recording audio.\n\n Args:\n - backdate_stop_seconds (float, default=\"0.0\"): Specifies the number of\n seconds to backdate the stop time. This is useful when the stop\n command is issued after the actual stop time.\n - backdate_resume_seconds (float, default=\"0.0\"): Specifies the number\n of seconds to backdate the time relistening is initiated.\n \"\"\"\n\n # Ensure there's a minimum interval\n # between starting and stopping recording\n if (time.time() - self.recording_start_time\n < self.min_length_of_recording):\n logger.info(\"Attempted to stop recording \"\n \"too soon after starting.\"\n )\n return self\n\n logger.info(\"recording stopped\")\n self.last_frames = copy.deepcopy(self.frames)\n self.backdate_stop_seconds = backdate_stop_seconds\n self.backdate_resume_seconds = backdate_resume_seconds\n self.is_recording = False\n self.recording_stop_time = time.time()\n self.is_silero_speech_active = False\n self.is_webrtc_speech_active = False\n self.silero_check_time = 0\n self.start_recording_event.clear()\n self.stop_recording_event.set()\n\n self.last_recording_start_time = self.recording_start_time\n self.last_recording_stop_time = self.recording_stop_time\n\n if self.on_recording_stop:\n self._run_callback(self.on_recording_stop)\n\n return self\n\n def listen(self):\n \"\"\"\n Puts recorder in immediate \"listen\" state.\n This is the state after a wake word detection, for example.\n The recorder now \"listens\" for voice activation.\n Once voice is detected we enter \"recording\" state.\n \"\"\"\n self.listen_start = time.time()\n self._set_state(\"listening\")\n self.start_recording_on_voice_activity = True\n\n def feed_audio(self, chunk, original_sample_rate=16000):\n \"\"\"\n Feed an audio chunk into the processing pipeline. Chunks are\n accumulated until the buffer size is reached, and then the accumulated\n data is fed into the audio_queue.\n \"\"\"\n # Check if the buffer attribute exists, if not, initialize it\n if not hasattr(self, 'buffer'):\n self.buffer = bytearray()\n\n # Check if input is a NumPy array\n if isinstance(chunk, np.ndarray):\n # Handle stereo to mono conversion if necessary\n if chunk.ndim == 2:\n chunk = np.mean(chunk, axis=1)\n\n # Resample to 16000 Hz if necessary\n if original_sample_rate != 16000:\n num_samples = int(len(chunk) * 16000 / original_sample_rate)\n chunk = resample(chunk, num_samples)\n\n # Ensure data type is int16\n chunk = chunk.astype(np.int16)\n\n # Convert the NumPy array to bytes\n chunk = chunk.tobytes()\n\n # Append the chunk to the buffer\n self.buffer += chunk\n buf_size = 2 * self.buffer_size # silero complains if too short\n\n # Check if the buffer has reached or exceeded the buffer_size\n while len(self.buffer) >= buf_size:\n # Extract self.buffer_size amount of data from the buffer\n to_process = self.buffer[:buf_size]\n self.buffer = self.buffer[buf_size:]\n\n # Feed the extracted data to the audio_queue\n self.audio_queue.put(to_process)\n\n def set_microphone(self, microphone_on=True):\n \"\"\"\n Set the microphone on or off.\n \"\"\"\n logger.info(\"Setting microphone to: \" + str(microphone_on))\n self.use_microphone.value = microphone_on\n\n def shutdown(self):\n \"\"\"\n Safely shuts down the audio recording by stopping the\n recording worker and closing the audio stream.\n \"\"\"\n\n with self.shutdown_lock:\n if self.is_shut_down:\n return\n\n print(\"\\033[91mRealtimeSTT shutting down\\033[0m\")\n\n # Force wait_audio() and text() to exit\n self.is_shut_down = True\n self.start_recording_event.set()\n self.stop_recording_event.set()\n\n self.shutdown_event.set()\n self.is_recording = False\n self.is_running = False\n\n logger.debug('Finishing recording thread')\n if self.recording_thread:\n self.recording_thread.join()\n\n logger.debug('Terminating reader process')\n\n # Give it some time to finish the loop and cleanup.\n if self.use_microphone.value:\n self.reader_process.join(timeout=10)\n\n if self.reader_process.is_alive():\n logger.warning(\"Reader process did not terminate \"\n \"in time. Terminating forcefully.\"\n )\n self.reader_process.terminate()\n\n logger.debug('Terminating transcription process')\n self.transcript_process.join(timeout=10)\n\n if self.transcript_process.is_alive():\n logger.warning(\"Transcript process did not terminate \"\n \"in time. Terminating forcefully.\"\n )\n self.transcript_process.terminate()\n\n self.parent_transcription_pipe.close()\n\n logger.debug('Finishing realtime thread')\n if self.realtime_thread:\n self.realtime_thread.join()\n\n if self.enable_realtime_transcription:\n if self.realtime_model_type:\n del self.realtime_model_type\n self.realtime_model_type = None\n gc.collect()\n\n def _recording_worker(self):\n \"\"\"\n The main worker method which constantly monitors the audio\n input for voice activity and accordingly starts/stops the recording.\n \"\"\"\n\n if self.use_extended_logging:\n logger.debug('Debug: Entering try block')\n\n last_inner_try_time = 0\n try:\n if self.use_extended_logging:\n logger.debug('Debug: Initializing variables')\n time_since_last_buffer_message = 0\n was_recording = False\n delay_was_passed = False\n wakeword_detected_time = None\n wakeword_samples_to_remove = None\n self.allowed_to_early_transcribe = True\n\n if self.use_extended_logging:\n logger.debug('Debug: Starting main loop')\n # Continuously monitor audio for voice activity\n while self.is_running:\n\n # if self.use_extended_logging:\n # logger.debug('Debug: Entering inner try block')\n if last_inner_try_time:\n last_processing_time = time.time() - last_inner_try_time\n if last_processing_time > 0.1:\n if self.use_extended_logging:\n logger.warning('### WARNING: PROCESSING TOOK TOO LONG')\n last_inner_try_time = time.time()\n try:\n # if self.use_extended_logging:\n # logger.debug('Debug: Trying to get data from audio queue')\n try:\n data = self.audio_queue.get(timeout=0.01)\n self.last_words_buffer.append(data)\n except queue.Empty:\n # if self.use_extended_logging:\n # logger.debug('Debug: Queue is empty, checking if still running')\n if not self.is_running:\n if self.use_extended_logging:\n logger.debug('Debug: Not running, breaking loop')\n break\n # if self.use_extended_logging:\n # logger.debug('Debug: Continuing to next iteration')\n continue\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking for on_recorded_chunk callback')\n if self.on_recorded_chunk:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_recorded_chunk')\n self._run_callback(self.on_recorded_chunk, data)\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if handle_buffer_overflow is True')\n if self.handle_buffer_overflow:\n if self.use_extended_logging:\n logger.debug('Debug: Handling buffer overflow')\n # Handle queue overflow\n if (self.audio_queue.qsize() >\n self.allowed_latency_limit):\n if self.use_extended_logging:\n logger.debug('Debug: Queue size exceeds limit, logging warnings')\n logger.warning(\"Audio queue size exceeds \"\n \"latency limit. Current size: \"\n f\"{self.audio_queue.qsize()}. \"\n \"Discarding old audio chunks.\"\n )\n\n if self.use_extended_logging:\n logger.debug('Debug: Discarding old chunks if necessary')\n while (self.audio_queue.qsize() >\n self.allowed_latency_limit):\n\n data = self.audio_queue.get()\n\n except BrokenPipeError:\n logger.error(\"BrokenPipeError _recording_worker\", exc_info=True)\n self.is_running = False\n break\n\n if self.use_extended_logging:\n logger.debug('Debug: Updating time_since_last_buffer_message')\n # Feed the extracted data to the audio_queue\n if time_since_last_buffer_message:\n time_passed = time.time() - time_since_last_buffer_message\n if time_passed > 1:\n if self.use_extended_logging:\n logger.debug(\"_recording_worker processing audio data\")\n time_since_last_buffer_message = time.time()\n else:\n time_since_last_buffer_message = time.time()\n\n if self.use_extended_logging:\n logger.debug('Debug: Initializing failed_stop_attempt')\n failed_stop_attempt = False\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if not recording')\n if not self.is_recording:\n if self.use_extended_logging:\n logger.debug('Debug: Handling not recording state')\n # Handle not recording state\n time_since_listen_start = (time.time() - self.listen_start\n if self.listen_start else 0)\n\n wake_word_activation_delay_passed = (\n time_since_listen_start >\n self.wake_word_activation_delay\n )\n\n if self.use_extended_logging:\n logger.debug('Debug: Handling wake-word timeout callback')\n # Handle wake-word timeout callback\n if wake_word_activation_delay_passed \\\n and not delay_was_passed:\n\n if self.use_wake_words and self.wake_word_activation_delay:\n if self.on_wakeword_timeout:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_wakeword_timeout')\n self._run_callback(self.on_wakeword_timeout)\n delay_was_passed = wake_word_activation_delay_passed\n\n if self.use_extended_logging:\n logger.debug('Debug: Setting state and spinner text')\n # Set state and spinner text\n if not self.recording_stop_time:\n if self.use_wake_words \\\n and wake_word_activation_delay_passed \\\n and not self.wakeword_detected:\n if self.use_extended_logging:\n logger.debug('Debug: Setting state to \"wakeword\"')\n self._set_state(\"wakeword\")\n else:\n if self.listen_start:\n if self.use_extended_logging:\n logger.debug('Debug: Setting state to \"listening\"')\n self._set_state(\"listening\")\n else:\n if self.use_extended_logging:\n logger.debug('Debug: Setting state to \"inactive\"')\n self._set_state(\"inactive\")\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking wake word conditions')\n if self.use_wake_words and wake_word_activation_delay_passed:\n try:\n if self.use_extended_logging:\n logger.debug('Debug: Processing wakeword')\n wakeword_index = self._process_wakeword(data)\n\n except struct.error:\n logger.error(\"Error unpacking audio data \"\n \"for wake word processing.\", exc_info=True)\n continue\n\n except Exception as e:\n logger.error(f\"Wake word processing error: {e}\", exc_info=True)\n continue\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if wake word detected')\n # If a wake word is detected \n if wakeword_index >= 0:\n if self.use_extended_logging:\n logger.debug('Debug: Wake word detected, updating variables')\n self.wake_word_detect_time = time.time()\n wakeword_detected_time = time.time()\n wakeword_samples_to_remove = int(self.sample_rate * self.wake_word_buffer_duration)\n self.wakeword_detected = True\n if self.on_wakeword_detected:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_wakeword_detected')\n self._run_callback(self.on_wakeword_detected)\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking voice activity conditions')\n # Check for voice activity to\n # trigger the start of recording\n if ((not self.use_wake_words\n or not wake_word_activation_delay_passed)\n and self.start_recording_on_voice_activity) \\\n or self.wakeword_detected:\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if voice is active')\n\n if self._is_voice_active():\n\n if self.on_vad_start:\n self._run_callback(self.on_vad_start)\n\n if self.use_extended_logging:\n logger.debug('Debug: Voice activity detected')\n logger.info(\"voice activity detected\")\n\n if self.use_extended_logging:\n logger.debug('Debug: Starting recording')\n self.start()\n\n self.start_recording_on_voice_activity = False\n\n if self.use_extended_logging:\n logger.debug('Debug: Adding buffered audio to frames')\n # Add the buffered audio\n # to the recording frames\n self.frames.extend(list(self.audio_buffer))\n self.audio_buffer.clear()\n\n if self.use_extended_logging:\n logger.debug('Debug: Resetting Silero VAD model states')\n self.silero_vad_model.reset_states()\n else:\n if self.use_extended_logging:\n logger.debug('Debug: Checking voice activity')\n data_copy = data[:]\n self._check_voice_activity(data_copy)\n\n if self.use_extended_logging:\n logger.debug('Debug: Resetting speech_end_silence_start')\n\n if self.speech_end_silence_start != 0:\n self.speech_end_silence_start = 0\n if self.on_turn_detection_stop:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_turn_detection_stop')\n self._run_callback(self.on_turn_detection_stop)\n\n else:\n if self.use_extended_logging:\n logger.debug('Debug: Handling recording state')\n # If we are currently recording\n if wakeword_samples_to_remove and wakeword_samples_to_remove > 0:\n if self.use_extended_logging:\n logger.debug('Debug: Removing wakeword samples')\n # Remove samples from the beginning of self.frames\n samples_removed = 0\n while wakeword_samples_to_remove > 0 and self.frames:\n frame = self.frames[0]\n frame_samples = len(frame) // 2 # Assuming 16-bit audio\n if wakeword_samples_to_remove >= frame_samples:\n self.frames.pop(0)\n samples_removed += frame_samples\n wakeword_samples_to_remove -= frame_samples\n else:\n self.frames[0] = frame[wakeword_samples_to_remove * 2:]\n samples_removed += wakeword_samples_to_remove\n samples_to_remove = 0\n \n wakeword_samples_to_remove = 0\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if stop_recording_on_voice_deactivity is True')\n # Stop the recording if silence is detected after speech\n if self.stop_recording_on_voice_deactivity:\n if self.use_extended_logging:\n logger.debug('Debug: Determining if speech is detected')\n is_speech = (\n self._is_silero_speech(data) if self.silero_deactivity_detection\n else self._is_webrtc_speech(data, True)\n )\n\n if self.use_extended_logging:\n logger.debug('Debug: Formatting speech_end_silence_start')\n if not self.speech_end_silence_start:\n str_speech_end_silence_start = \"0\"\n else:\n str_speech_end_silence_start = datetime.datetime.fromtimestamp(self.speech_end_silence_start).strftime('%H:%M:%S.%f')[:-3]\n if self.use_extended_logging:\n logger.debug(f\"is_speech: {is_speech}, str_speech_end_silence_start: {str_speech_end_silence_start}\")\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if speech is not detected')\n if not is_speech:\n if self.use_extended_logging:\n logger.debug('Debug: Handling voice deactivity')\n # Voice deactivity was detected, so we start\n # measuring silence time before stopping recording\n if self.speech_end_silence_start == 0 and \\\n (time.time() - self.recording_start_time > self.min_length_of_recording):\n\n self.speech_end_silence_start = time.time()\n self.awaiting_speech_end = True\n if self.on_turn_detection_start:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_turn_detection_start')\n\n self._run_callback(self.on_turn_detection_start)\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking early transcription conditions')\n if self.speech_end_silence_start and self.early_transcription_on_silence and len(self.frames) > 0 and \\\n (time.time() - self.speech_end_silence_start > self.early_transcription_on_silence) and \\\n self.allowed_to_early_transcribe:\n if self.use_extended_logging:\n logger.debug(\"Debug:Adding early transcription request\")\n self.transcribe_count += 1\n audio_array = np.frombuffer(b''.join(self.frames), dtype=np.int16)\n audio = audio_array.astype(np.float32) / INT16_MAX_ABS_VALUE\n\n if self.use_extended_logging:\n logger.debug(\"Debug: early transcription request pipe send\")\n self.parent_transcription_pipe.send((audio, self.language, True))\n if self.use_extended_logging:\n logger.debug(\"Debug: early transcription request pipe send return\")\n self.allowed_to_early_transcribe = False\n\n else:\n self.awaiting_speech_end = False\n if self.use_extended_logging:\n logger.debug('Debug: Handling speech detection')\n if self.speech_end_silence_start:\n if self.use_extended_logging:\n logger.info(\"Resetting self.speech_end_silence_start\")\n\n if self.speech_end_silence_start != 0:\n self.speech_end_silence_start = 0\n if self.on_turn_detection_stop:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_turn_detection_stop')\n self._run_callback(self.on_turn_detection_stop)\n\n self.allowed_to_early_transcribe = True\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if silence duration exceeds threshold')\n # Wait for silence to stop recording after speech\n if self.speech_end_silence_start and time.time() - \\\n self.speech_end_silence_start >= \\\n self.post_speech_silence_duration:\n\n if self.on_vad_stop:\n self._run_callback(self.on_vad_stop)\n\n if self.use_extended_logging:\n logger.debug('Debug: Formatting silence start time')\n # Get time in desired format (HH:MM:SS.nnn)\n silence_start_time = datetime.datetime.fromtimestamp(self.speech_end_silence_start).strftime('%H:%M:%S.%f')[:-3]\n\n if self.use_extended_logging:\n logger.debug('Debug: Calculating time difference')\n # Calculate time difference\n time_diff = time.time() - self.speech_end_silence_start\n\n if self.use_extended_logging:\n logger.debug('Debug: Logging voice deactivity detection')\n logger.info(f\"voice deactivity detected at {silence_start_time}, \"\n f\"time since silence start: {time_diff:.3f} seconds\")\n\n logger.debug('Debug: Appending data to frames and stopping recording')\n self.frames.append(data)\n self.stop()\n if not self.is_recording:\n if self.speech_end_silence_start != 0:\n self.speech_end_silence_start = 0\n if self.on_turn_detection_stop:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_turn_detection_stop')\n self._run_callback(self.on_turn_detection_stop)\n\n if self.use_extended_logging:\n logger.debug('Debug: Handling non-wake word scenario')\n else:\n if self.use_extended_logging:\n logger.debug('Debug: Setting failed_stop_attempt to True')\n failed_stop_attempt = True\n\n self.awaiting_speech_end = False\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if recording stopped')\n if not self.is_recording and was_recording:\n if self.use_extended_logging:\n logger.debug('Debug: Resetting after stopping recording')\n # Reset after stopping recording to ensure clean state\n self.stop_recording_on_voice_deactivity = False\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking Silero time')\n if time.time() - self.silero_check_time > 0.1:\n self.silero_check_time = 0\n\n if self.use_extended_logging:\n logger.debug('Debug: Handling wake word timeout')\n # Handle wake word timeout (waited to long initiating\n # speech after wake word detection)\n if self.wake_word_detect_time and time.time() - \\\n self.wake_word_detect_time > self.wake_word_timeout:\n\n self.wake_word_detect_time = 0\n if self.wakeword_detected and self.on_wakeword_timeout:\n if self.use_extended_logging:\n logger.debug('Debug: Calling on_wakeword_timeout')\n self._run_callback(self.on_wakeword_timeout)\n self.wakeword_detected = False\n\n if self.use_extended_logging:\n logger.debug('Debug: Updating was_recording')\n was_recording = self.is_recording\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if recording and not failed stop attempt')\n if self.is_recording and not failed_stop_attempt:\n if self.use_extended_logging:\n logger.debug('Debug: Appending data to frames')\n self.frames.append(data)\n\n if self.use_extended_logging:\n logger.debug('Debug: Checking if not recording or speech end silence start')\n if not self.is_recording or self.speech_end_silence_start:\n if self.use_extended_logging:\n logger.debug('Debug: Appending data to audio buffer')\n self.audio_buffer.append(data)\n\n except Exception as e:\n logger.debug('Debug: Caught exception in main try block')\n if not self.interrupt_stop_event.is_set():\n logger.error(f\"Unhandled exeption in _recording_worker: {e}\", exc_info=True)\n raise\n\n if self.use_extended_logging:\n logger.debug('Debug: Exiting _recording_worker method')\n\n\n\n\n def _realtime_worker(self):\n \"\"\"\n Performs real-time transcription if the feature is enabled.\n\n The method is responsible transcribing recorded audio frames\n in real-time based on the specified resolution interval.\n The transcribed text is stored in `self.realtime_transcription_text`\n and a callback\n function is invoked with this text if specified.\n \"\"\"\n\n try:\n\n logger.debug('Starting realtime worker')\n\n # Return immediately if real-time transcription is not enabled\n if not self.enable_realtime_transcription:\n return\n\n # Track time of last transcription\n last_transcription_time = time.time()\n\n while self.is_running:\n\n if self.is_recording:\n\n # MODIFIED SLEEP LOGIC:\n # Wait until realtime_processing_pause has elapsed,\n # but check often so we can respond to changes quickly.\n while (\n time.time() - last_transcription_time\n ) < self.realtime_processing_pause:\n time.sleep(0.001)\n if not self.is_running or not self.is_recording:\n break\n\n if self.awaiting_speech_end:\n time.sleep(0.001)\n continue\n\n # Update transcription time\n last_transcription_time = time.time()\n\n # Convert the buffer frames to a NumPy array\n audio_array = np.frombuffer(\n b''.join(self.frames),\n dtype=np.int16\n )\n\n logger.debug(f\"Current realtime buffer size: {len(audio_array)}\")\n\n # Normalize the array to a [-1, 1] range\n audio_array = audio_array.astype(np.float32) / \\\n INT16_MAX_ABS_VALUE\n\n if self.use_main_model_for_realtime:\n with self.transcription_lock:\n try:\n self.parent_transcription_pipe.send((audio_array, self.language, True))\n if self.parent_transcription_pipe.poll(timeout=5): # Wait for 5 seconds\n logger.debug(\"Receive from realtime worker after transcription request to main model\")\n status, result = self.parent_transcription_pipe.recv()\n if status == 'success':\n segments, info = result\n self.detected_realtime_language = info.language if info.language_probability > 0 else None\n self.detected_realtime_language_probability = info.language_probability\n realtime_text = segments\n logger.debug(f\"Realtime text detected with main model: {realtime_text}\")\n else:\n logger.error(f\"Realtime transcription error: {result}\")\n continue\n else:\n logger.warning(\"Realtime transcription timed out\")\n continue\n except Exception as e:\n logger.error(f\"Error in realtime transcription: {str(e)}\", exc_info=True)\n continue\n else:\n # Perform transcription and assemble the text\n if self.normalize_audio:\n # normalize audio to -0.95 dBFS\n if audio_array is not None and audio_array.size > 0:\n peak = np.max(np.abs(audio_array))\n if peak > 0:\n audio_array = (audio_array / peak) * 0.95\n\n if self.realtime_batch_size > 0:\n segments, info = self.realtime_model_type.transcribe(\n audio_array,\n language=self.language if self.language else None,\n beam_size=self.beam_size_realtime,\n initial_prompt=self.initial_prompt_realtime,\n suppress_tokens=self.suppress_tokens,\n batch_size=self.realtime_batch_size,\n vad_filter=self.faster_whisper_vad_filter\n )\n else:\n segments, info = self.realtime_model_type.transcribe(\n audio_array,\n language=self.language if self.language else None,\n beam_size=self.beam_size_realtime,\n initial_prompt=self.initial_prompt_realtime,\n suppress_tokens=self.suppress_tokens,\n vad_filter=self.faster_whisper_vad_filter\n )\n\n self.detected_realtime_language = info.language if info.language_probability > 0 else None\n self.detected_realtime_language_probability = info.language_probability\n realtime_text = \" \".join(\n seg.text for seg in segments\n )\n logger.debug(f\"Realtime text detected: {realtime_text}\")\n\n # double check recording state\n # because it could have changed mid-transcription\n if self.is_recording and time.time() - \\\n self.recording_start_time > self.init_realtime_after_seconds:\n\n self.realtime_transcription_text = realtime_text\n self.realtime_transcription_text = \\\n self.realtime_transcription_text.strip()\n\n self.text_storage.append(\n self.realtime_transcription_text\n )\n\n # Take the last two texts in storage, if they exist\n if len(self.text_storage) >= 2:\n last_two_texts = self.text_storage[-2:]\n\n # Find the longest common prefix\n # between the two texts\n prefix = os.path.commonprefix(\n [last_two_texts[0], last_two_texts[1]]\n )\n\n # This prefix is the text that was transcripted\n # two times in the same way\n # Store as \"safely detected text\"\n if len(prefix) >= \\\n len(self.realtime_stabilized_safetext):\n\n # Only store when longer than the previous\n # as additional security\n self.realtime_stabilized_safetext = prefix\n\n # Find parts of the stabilized text\n # in the freshly transcripted text\n matching_pos = self._find_tail_match_in_text(\n self.realtime_stabilized_safetext,\n self.realtime_transcription_text\n )\n\n if matching_pos < 0:\n # pick which text to send\n text_to_send = (\n self.realtime_stabilized_safetext\n if self.realtime_stabilized_safetext\n else self.realtime_transcription_text\n )\n # preprocess once\n processed = self._preprocess_output(text_to_send, True)\n # invoke on its own thread\n self._run_callback(self._on_realtime_transcription_stabilized, processed)\n\n else:\n # We found parts of the stabilized text\n # in the transcripted text\n # We now take the stabilized text\n # and add only the freshly transcripted part to it\n output_text = self.realtime_stabilized_safetext + \\\n self.realtime_transcription_text[matching_pos:]\n\n # This yields us the \"left\" text part as stabilized\n # AND at the same time delivers fresh detected\n # parts on the first run without the need for\n # two transcriptions\n self._run_callback(self._on_realtime_transcription_stabilized, self._preprocess_output(output_text, True))\n\n # Invoke the callback with the transcribed text\n self._run_callback(self._on_realtime_transcription_update, self._preprocess_output(self.realtime_transcription_text,True))\n\n # If not recording, sleep briefly before checking again\n else:\n time.sleep(TIME_SLEEP)\n\n except Exception as e:\n logger.error(f\"Unhandled exeption in _realtime_worker: {e}\", exc_info=True)\n raise\n\n def _is_silero_speech(self, chunk):\n \"\"\"\n Returns true if speech is detected in the provided audio data\n\n Args:\n data (bytes): raw bytes of audio data (1024 raw bytes with\n 16000 sample rate and 16 bits per sample)\n \"\"\"\n if self.sample_rate != 16000:\n pcm_data = np.frombuffer(chunk, dtype=np.int16)\n data_16000 = signal.resample_poly(\n pcm_data, 16000, self.sample_rate)\n chunk = data_16000.astype(np.int16).tobytes()\n\n self.silero_working = True\n audio_chunk = np.frombuffer(chunk, dtype=np.int16)\n audio_chunk = audio_chunk.astype(np.float32) / INT16_MAX_ABS_VALUE\n vad_prob = self.silero_vad_model(\n torch.from_numpy(audio_chunk),\n SAMPLE_RATE).item()\n is_silero_speech_active = vad_prob > (1 - self.silero_sensitivity)\n if is_silero_speech_active:\n if not self.is_silero_speech_active and self.use_extended_logging:\n logger.info(f\"{bcolors.OKGREEN}Silero VAD detected speech{bcolors.ENDC}\")\n elif self.is_silero_speech_active and self.use_extended_logging:\n logger.info(f\"{bcolors.WARNING}Silero VAD detected silence{bcolors.ENDC}\")\n self.is_silero_speech_active = is_silero_speech_active\n self.silero_working = False\n return is_silero_speech_active\n\n def _is_webrtc_speech(self, chunk, all_frames_must_be_true=False):\n \"\"\"\n Returns true if speech is detected in the provided audio data\n\n Args:\n data (bytes): raw bytes of audio data (1024 raw bytes with\n 16000 sample rate and 16 bits per sample)\n \"\"\"\n speech_str = f\"{bcolors.OKGREEN}WebRTC VAD detected speech{bcolors.ENDC}\"\n silence_str = f\"{bcolors.WARNING}WebRTC VAD detected silence{bcolors.ENDC}\"\n if self.sample_rate != 16000:\n pcm_data = np.frombuffer(chunk, dtype=np.int16)\n data_16000 = signal.resample_poly(\n pcm_data, 16000, self.sample_rate)\n chunk = data_16000.astype(np.int16).tobytes()\n\n # Number of audio frames per millisecond\n frame_length = int(16000 * 0.01) # for 10ms frame\n num_frames = int(len(chunk) / (2 * frame_length))\n speech_frames = 0\n\n for i in range(num_frames):\n start_byte = i * frame_length * 2\n end_byte = start_byte + frame_length * 2\n frame = chunk[start_byte:end_byte]\n if self.webrtc_vad_model.is_speech(frame, 16000):\n speech_frames += 1\n if not all_frames_must_be_true:\n if self.debug_mode:\n logger.info(f\"Speech detected in frame {i + 1}\"\n f\" of {num_frames}\")\n if not self.is_webrtc_speech_active and self.use_extended_logging:\n logger.info(speech_str)\n self.is_webrtc_speech_active = True\n return True\n if all_frames_must_be_true:\n if self.debug_mode and speech_frames == num_frames:\n logger.info(f\"Speech detected in {speech_frames} of \"\n f\"{num_frames} frames\")\n elif self.debug_mode:\n logger.info(f\"Speech not detected in all {num_frames} frames\")\n speech_detected = speech_frames == num_frames\n if speech_detected and not self.is_webrtc_speech_active and self.use_extended_logging:\n logger.info(speech_str)\n elif not speech_detected and self.is_webrtc_speech_active and self.use_extended_logging:\n logger.info(silence_str)\n self.is_webrtc_speech_active = speech_detected\n return speech_detected\n else:\n if self.debug_mode:\n logger.info(f\"Speech not detected in any of {num_frames} frames\")\n if self.is_webrtc_speech_active and self.use_extended_logging:\n logger.info(silence_str)\n self.is_webrtc_speech_active = False\n return False\n\n def _check_voice_activity(self, data):\n \"\"\"\n Initiate check if voice is active based on the provided data.\n\n Args:\n data: The audio data to be checked for voice activity.\n \"\"\"\n self._is_webrtc_speech(data)\n\n # First quick performing check for voice activity using WebRTC\n if self.is_webrtc_speech_active:\n\n if not self.silero_working:\n self.silero_working = True\n\n # Run the intensive check in a separate thread\n threading.Thread(\n target=self._is_silero_speech,\n args=(data,)).start()\n\n def clear_audio_queue(self):\n \"\"\"\n Safely empties the audio queue to ensure no remaining audio \n fragments get processed e.g. after waking up the recorder.\n \"\"\"\n self.audio_buffer.clear()\n try:\n while True:\n self.audio_queue.get_nowait()\n except:\n # PyTorch's mp.Queue doesn't have a specific Empty exception\n # so we catch any exception that might occur when the queue is empty\n pass\n\n def _is_voice_active(self):\n \"\"\"\n Determine if voice is active.\n\n Returns:\n bool: True if voice is active, False otherwise.\n \"\"\"\n return self.is_webrtc_speech_active and self.is_silero_speech_active\n\n def _set_state(self, new_state):\n \"\"\"\n Update the current state of the recorder and execute\n corresponding state-change callbacks.\n\n Args:\n new_state (str): The new state to set.\n\n \"\"\"\n # Check if the state has actually changed\n if new_state == self.state:\n return\n\n # Store the current state for later comparison\n old_state = self.state\n\n # Update to the new state\n self.state = new_state\n\n # Log the state change\n logger.info(f\"State changed from '{old_state}' to '{new_state}'\")\n\n # Execute callbacks based on transitioning FROM a particular state\n if old_state == \"listening\":\n if self.on_vad_detect_stop:\n self._run_callback(self.on_vad_detect_stop)\n elif old_state == \"wakeword\":\n if self.on_wakeword_detection_end:\n self._run_callback(self.on_wakeword_detection_end)\n\n # Execute callbacks based on transitioning TO a particular state\n if new_state == \"listening\":\n if self.on_vad_detect_start:\n self._run_callback(self.on_vad_detect_start)\n self._set_spinner(\"speak now\")\n if self.spinner and self.halo:\n self.halo._interval = 250\n elif new_state == \"wakeword\":\n if self.on_wakeword_detection_start:\n self._run_callback(self.on_wakeword_detection_start)\n self._set_spinner(f\"say {self.wake_words}\")\n if self.spinner and self.halo:\n self.halo._interval = 500\n elif new_state == \"transcribing\":\n self._set_spinner(\"transcribing\")\n if self.spinner and self.halo:\n self.halo._interval = 50\n elif new_state == \"recording\":\n self._set_spinner(\"recording\")\n if self.spinner and self.halo:\n self.halo._interval = 100\n elif new_state == \"inactive\":\n if self.spinner and self.halo:\n self.halo.stop()\n self.halo = None\n\n def _set_spinner(self, text):\n \"\"\"\n Update the spinner's text or create a new\n spinner with the provided text.\n\n Args:\n text (str): The text to be displayed alongside the spinner.\n \"\"\"\n if self.spinner:\n # If the Halo spinner doesn't exist, create and start it\n if self.halo is None:\n self.halo = halo.Halo(text=text)\n self.halo.start()\n # If the Halo spinner already exists, just update the text\n else:\n self.halo.text = text\n\n def _preprocess_output(self, text, preview=False):\n \"\"\"\n Preprocesses the output text by removing any leading or trailing\n whitespace, converting all whitespace sequences to a single space\n character, and capitalizing the first character of the text.\n\n Args:\n text (str): The text to be preprocessed.\n\n Returns:\n str: The preprocessed text.\n \"\"\"\n text = re.sub(r'\\s+', ' ', text.strip())\n\n if self.ensure_sentence_starting_uppercase:\n if text:\n text = text[0].upper() + text[1:]\n\n # Ensure the text ends with a proper punctuation\n # if it ends with an alphanumeric character\n if not preview:\n if self.ensure_sentence_ends_with_period:\n if text and text[-1].isalnum():\n text += '.'\n\n return text\n\n def _find_tail_match_in_text(self, text1, text2, length_of_match=10):\n \"\"\"\n Find the position where the last 'n' characters of text1\n match with a substring in text2.\n\n This method takes two texts, extracts the last 'n' characters from\n text1 (where 'n' is determined by the variable 'length_of_match'), and\n searches for an occurrence of this substring in text2, starting from\n the end of text2 and moving towards the beginning.\n\n Parameters:\n - text1 (str): The text containing the substring that we want to find\n in text2.\n - text2 (str): The text in which we want to find the matching\n substring.\n - length_of_match(int): The length of the matching string that we are\n looking for\n\n Returns:\n int: The position (0-based index) in text2 where the matching\n substring starts. If no match is found or either of the texts is\n too short, returns -1.\n \"\"\"\n\n # Check if either of the texts is too short\n if len(text1) < length_of_match or len(text2) < length_of_match:\n return -1\n\n # The end portion of the first text that we want to compare\n target_substring = text1[-length_of_match:]\n\n # Loop through text2 from right to left\n for i in range(len(text2) - length_of_match + 1):\n # Extract the substring from text2\n # to compare with the target_substring\n current_substring = text2[len(text2) - i - length_of_match:\n len(text2) - i]\n\n # Compare the current_substring with the target_substring\n if current_substring == target_substring:\n # Position in text2 where the match starts\n return len(text2) - i\n\n return -1\n\n def _on_realtime_transcription_stabilized(self, text):\n \"\"\"\n Callback method invoked when the real-time transcription stabilizes.\n\n This method is called internally when the transcription text is\n considered \"stable\" meaning it's less likely to change significantly\n with additional audio input. It notifies any registered external\n listener about the stabilized text if recording is still ongoing.\n This is particularly useful for applications that need to display\n live transcription results to users and want to highlight parts of the\n transcription that are less likely to change.\n\n Args:\n text (str): The stabilized transcription text.\n \"\"\"\n if self.on_realtime_transcription_stabilized:\n if self.is_recording:\n self._run_callback(self.on_realtime_transcription_stabilized, text)\n\n def _on_realtime_transcription_update(self, text):\n \"\"\"\n Callback method invoked when there's an update in the real-time\n transcription.\n\n This method is called internally whenever there's a change in the\n transcription text, notifying any registered external listener about\n the update if recording is still ongoing. This provides a mechanism\n for applications to receive and possibly display live transcription\n updates, which could be partial and still subject to change.\n\n Args:\n text (str): The updated transcription text.\n \"\"\"\n if self.on_realtime_transcription_update:\n if self.is_recording:\n self._run_callback(self.on_realtime_transcription_update, text)\n\n def __enter__(self):\n \"\"\"\n Method to setup the context manager protocol.\n\n This enables the instance to be used in a `with` statement, ensuring\n proper resource management. When the `with` block is entered, this\n method is automatically called.\n\n Returns:\n self: The current instance of the class.\n \"\"\"\n return self\n\n def __exit__(self, exc_type, exc_value, traceback):\n \"\"\"\n Method to define behavior when the context manager protocol exits.\n\n This is called when exiting the `with` block and ensures that any\n necessary cleanup or resource release processes are executed, such as\n shutting down the system properly.\n\n Args:\n exc_type (Exception or None): The type of the exception that\n caused the context to be exited, if any.\n exc_value (Exception or None): The exception instance that caused\n the context to be exited, if any.\n traceback (Traceback or None): The traceback corresponding to the\n exception, if any.\n \"\"\"\n self.shutdown()"
},
{
"path": "RealtimeSTT/audio_recorder_client.py",
"content": "log_outgoing_chunks = False\ndebug_mode = False\n\nfrom typing import Iterable, List, Optional, Union\nfrom urllib.parse import urlparse\nfrom datetime import datetime\nfrom websocket import WebSocketApp\nfrom websocket import ABNF\nimport numpy as np\nimport subprocess\nimport threading\nimport platform\nimport logging\nimport struct\nimport base64\nimport wave\nimport json\nimport time\nimport sys\nimport os\n\n# Import the AudioInput class\nfrom .audio_input import AudioInput\n\nDEFAULT_CONTROL_URL = \"ws://127.0.0.1:8011\"\nDEFAULT_DATA_URL = \"ws://127.0.0.1:8012\"\n\nINIT_MODEL_TRANSCRIPTION = \"tiny\"\nINIT_MODEL_TRANSCRIPTION_REALTIME = \"tiny\"\nINIT_REALTIME_PROCESSING_PAUSE = 0.2\nINIT_REALTIME_INITIAL_PAUSE = 0.2\nINIT_SILERO_SENSITIVITY = 0.4\nINIT_WEBRTC_SENSITIVITY = 3\nINIT_POST_SPEECH_SILENCE_DURATION = 0.6\nINIT_MIN_LENGTH_OF_RECORDING = 0.5\nINIT_MIN_GAP_BETWEEN_RECORDINGS = 0\nINIT_WAKE_WORDS_SENSITIVITY = 0.6\nINIT_PRE_RECORDING_BUFFER_DURATION = 1.0\nINIT_WAKE_WORD_ACTIVATION_DELAY = 0.0\nINIT_WAKE_WORD_TIMEOUT = 5.0\nINIT_WAKE_WORD_BUFFER_DURATION = 0.1\nALLOWED_LATENCY_LIMIT = 100\n\nBUFFER_SIZE = 512\nSAMPLE_RATE = 16000\n\nINIT_HANDLE_BUFFER_OVERFLOW = False\nif platform.system() != 'Darwin':\n INIT_HANDLE_BUFFER_OVERFLOW = True\n\n# Define ANSI color codes for terminal output\nclass bcolors:\n HEADER = '\\033[95m' # Magenta\n OKBLUE = '\\033[94m' # Blue\n OKCYAN = '\\033[96m' # Cyan\n OKGREEN = '\\033[92m' # Green\n WARNING = '\\033[93m' # Yellow\n FAIL = '\\033[91m' # Red\n ENDC = '\\033[0m' # Reset to default\n BOLD = '\\033[1m'\n UNDERLINE = '\\033[4m'\n\ndef format_timestamp_ns(timestamp_ns: int) -> str:\n # Split into whole seconds and the nanosecond remainder\n seconds = timestamp_ns // 1_000_000_000\n remainder_ns = timestamp_ns % 1_000_000_000\n\n # Convert seconds part into a datetime object (local time)\n dt = datetime.fromtimestamp(seconds)\n\n # Format the main time as HH:MM:SS\n time_str = dt.strftime(\"%H:%M:%S\")\n\n # For instance, if you want milliseconds, divide the remainder by 1e6 and format as 3-digit\n milliseconds = remainder_ns // 1_000_000\n formatted_timestamp = f\"{time_str}.{milliseconds:03d}\"\n\n return formatted_timestamp\n\nclass AudioToTextRecorderClient:\n \"\"\"\n A class responsible for capturing audio from the microphone, detecting\n voice activity, and then transcribing the captured audio using the\n `faster_whisper` model.\n \"\"\"\n\n def __init__(self,\n model: str = INIT_MODEL_TRANSCRIPTION,\n download_root: str = None, \n language: str = \"\",\n compute_type: str = \"default\",\n input_device_index: int = None,\n gpu_device_index: Union[int, List[int]] = 0,\n device: str = \"cuda\",\n on_recording_start=None,\n on_recording_stop=None,\n on_transcription_start=None,\n ensure_sentence_starting_uppercase=True,\n ensure_sentence_ends_with_period=True,\n use_microphone=True,\n spinner=True,\n level=logging.WARNING,\n batch_size: int = 16,\n\n # Realtime transcription parameters\n enable_realtime_transcription=False,\n use_main_model_for_realtime=False,\n realtime_model_type=INIT_MODEL_TRANSCRIPTION_REALTIME,\n realtime_processing_pause=INIT_REALTIME_PROCESSING_PAUSE,\n init_realtime_after_seconds=INIT_REALTIME_INITIAL_PAUSE,\n on_realtime_transcription_update=None,\n on_realtime_transcription_stabilized=None,\n realtime_batch_size: int = 16,\n\n # Voice activation parameters\n silero_sensitivity: float = INIT_SILERO_SENSITIVITY,\n silero_use_onnx: bool = False,\n silero_deactivity_detection: bool = False,\n webrtc_sensitivity: int = INIT_WEBRTC_SENSITIVITY,\n post_speech_silence_duration: float = (\n INIT_POST_SPEECH_SILENCE_DURATION\n ),\n min_length_of_recording: float = (\n INIT_MIN_LENGTH_OF_RECORDING\n ),\n min_gap_between_recordings: float = (\n INIT_MIN_GAP_BETWEEN_RECORDINGS\n ),\n pre_recording_buffer_duration: float = (\n INIT_PRE_RECORDING_BUFFER_DURATION\n ),\n on_vad_start=None,\n on_vad_stop=None,\n on_vad_detect_start=None,\n on_vad_detect_stop=None,\n on_turn_detection_start=None,\n on_turn_detection_stop=None,\n\n # Wake word parameters\n wakeword_backend: str = \"pvporcupine\",\n openwakeword_model_paths: str = None,\n openwakeword_inference_framework: str = \"onnx\",\n wake_words: str = \"\",\n wake_words_sensitivity: float = INIT_WAKE_WORDS_SENSITIVITY,\n wake_word_activation_delay: float = (\n INIT_WAKE_WORD_ACTIVATION_DELAY\n ),\n wake_word_timeout: float = INIT_WAKE_WORD_TIMEOUT,\n wake_word_buffer_duration: float = INIT_WAKE_WORD_BUFFER_DURATION,\n on_wakeword_detected=None,\n on_wakeword_timeout=None,\n on_wakeword_detection_start=None,\n on_wakeword_detection_end=None,\n on_recorded_chunk=None,\n debug_mode=False,\n handle_buffer_overflow: bool = INIT_HANDLE_BUFFER_OVERFLOW,\n beam_size: int = 5,\n beam_size_realtime: int = 3,\n buffer_size: int = BUFFER_SIZE,\n sample_rate: int = SAMPLE_RATE,\n initial_prompt: Optional[Union[str, Iterable[int]]] = None,\n initial_prompt_realtime: Optional[Union[str, Iterable[int]]] = None,\n suppress_tokens: Optional[List[int]] = [-1],\n print_transcription_time: bool = False,\n early_transcription_on_silence: int = 0,\n allowed_latency_limit: int = ALLOWED_LATENCY_LIMIT,\n no_log_file: bool = False,\n use_extended_logging: bool = False,\n\n # Server urls\n control_url: str = DEFAULT_CONTROL_URL,\n data_url: str = DEFAULT_DATA_URL,\n autostart_server: bool = True,\n output_wav_file: str = None,\n faster_whisper_vad_filter: bool = False,\n ):\n\n # Set instance variables from constructor parameters\n self.model = model\n self.language = language\n self.compute_type = compute_type\n self.input_device_index = input_device_index\n self.gpu_device_index = gpu_device_index\n self.device = device\n self.on_recording_start = on_recording_start\n self.on_recording_stop = on_recording_stop\n self.on_transcription_start = on_transcription_start\n self.ensure_sentence_starting_uppercase = ensure_sentence_starting_uppercase\n self.ensure_sentence_ends_with_period = ensure_sentence_ends_with_period\n self.use_microphone = use_microphone\n self.spinner = spinner\n self.level = level\n self.batch_size = batch_size\n self.init_realtime_after_seconds = init_realtime_after_seconds\n self.realtime_batch_size = realtime_batch_size\n\n # Real-time transcription parameters\n self.enable_realtime_transcription = enable_realtime_transcription\n self.use_main_model_for_realtime = use_main_model_for_realtime\n self.download_root = download_root\n self.realtime_model_type = realtime_model_type\n self.realtime_processing_pause = realtime_processing_pause\n self.on_realtime_transcription_update = on_realtime_transcription_update\n self.on_realtime_transcription_stabilized = on_realtime_transcription_stabilized\n\n # Voice activation parameters\n self.silero_sensitivity = silero_sensitivity\n self.silero_use_onnx = silero_use_onnx\n self.silero_deactivity_detection = silero_deactivity_detection\n self.webrtc_sensitivity = webrtc_sensitivity\n self.post_speech_silence_duration = post_speech_silence_duration\n self.min_length_of_recording = min_length_of_recording\n self.min_gap_between_recordings = min_gap_between_recordings\n self.pre_recording_buffer_duration = pre_recording_buffer_duration\n\n self.on_vad_start = on_vad_start\n self.on_vad_stop = on_vad_stop\n self.on_vad_detect_start = on_vad_detect_start\n self.on_vad_detect_stop = on_vad_detect_stop\n self.on_turn_detection_start = on_turn_detection_start\n self.on_turn_detection_stop = on_turn_detection_stop\n\n # Wake word parameters\n self.wakeword_backend = wakeword_backend\n self.openwakeword_model_paths = openwakeword_model_paths\n self.openwakeword_inference_framework = openwakeword_inference_framework\n self.wake_words = wake_words\n self.wake_words_sensitivity = wake_words_sensitivity\n self.wake_word_activation_delay = wake_word_activation_delay\n self.wake_word_timeout = wake_word_timeout\n self.wake_word_buffer_duration = wake_word_buffer_duration\n self.on_wakeword_detected = on_wakeword_detected\n self.on_wakeword_timeout = on_wakeword_timeout\n self.on_wakeword_detection_start = on_wakeword_detection_start\n self.on_wakeword_detection_end = on_wakeword_detection_end\n self.on_recorded_chunk = on_recorded_chunk\n self.debug_mode = debug_mode\n self.handle_buffer_overflow = handle_buffer_overflow\n self.beam_size = beam_size\n self.beam_size_realtime = beam_size_realtime\n self.buffer_size = buffer_size\n self.sample_rate = sample_rate\n self.initial_prompt = initial_prompt\n self.initial_prompt_realtime = initial_prompt_realtime\n self.suppress_tokens = suppress_tokens\n self.print_transcription_time = print_transcription_time\n self.early_transcription_on_silence = early_transcription_on_silence\n self.allowed_latency_limit = allowed_latency_limit\n self.no_log_file = no_log_file\n self.use_extended_logging = use_extended_logging\n self.faster_whisper_vad_filter = faster_whisper_vad_filter\n\n # Server URLs\n self.control_url = control_url\n self.data_url = data_url\n self.autostart_server = autostart_server\n self.output_wav_file = output_wav_file\n\n # Instance variables\n self.muted = False\n self.recording_thread = None\n self.is_running = True\n self.connection_established = threading.Event()\n self.recording_start = threading.Event()\n self.final_text_ready = threading.Event()\n self.realtime_text = \"\"\n self.final_text = \"\"\n self._recording = False\n self.server_already_running = False\n self.wav_file = None\n\n self.request_counter = 0\n self.pending_requests = {} # Map from request_id to threading.Event and value\n\n if self.debug_mode:\n print(\"Checking STT server\")\n if not self.connect():\n print(\"Failed to connect to the server.\", file=sys.stderr)\n else:\n if self.debug_mode:\n print(\"STT server is running and connected.\")\n\n if self.use_microphone:\n self.start_recording()\n\n\n if self.server_already_running:\n if not self.connection_established.wait(timeout=10):\n print(\"Server connection not established within 10 seconds.\")\n else:\n self.set_parameter(\"language\", self.language)\n print(f\"Language set to {self.language}\")\n self.set_parameter(\"wake_word_activation_delay\", self.wake_word_activation_delay)\n print(f\"Wake word activation delay set to {self.wake_word_activation_delay}\")\n\n def text(self, on_transcription_finished=None):\n self.realtime_text = \"\"\n self.submitted_realtime_text = \"\"\n self.final_text = \"\"\n self.final_text_ready.clear()\n\n self.recording_start.set()\n\n try:\n total_wait_time = 0\n wait_interval = 0.02 # Wait in small intervals, e.g., 100ms\n max_wait_time = 60 # Timeout after 60 seconds\n\n while total_wait_time < max_wait_time and self.is_running and self._recording:\n if self.final_text_ready.wait(timeout=wait_interval):\n break # Break if transcription is ready\n\n if not self.is_running or not self._recording:\n break\n \n total_wait_time += wait_interval\n\n # Check if a manual interrupt has occurred\n if total_wait_time >= max_wait_time:\n if self.debug_mode:\n print(\"Timeout while waiting for text from the server.\")\n self.recording_start.clear()\n if on_transcription_finished:\n threading.Thread(target=on_transcription_finished, args=(\"\",)).start()\n return \"\"\n\n self.recording_start.clear()\n\n if not self.is_running or not self._recording:\n return \"\"\n\n if on_transcription_finished:\n threading.Thread(target=on_transcription_finished, args=(self.final_text,)).start()\n\n return self.final_text\n\n except KeyboardInterrupt:\n if self.debug_mode:\n print(\"KeyboardInterrupt in text(), exiting...\")\n raise KeyboardInterrupt\n\n except Exception as e:\n print(f\"Error in AudioToTextRecorderClient.text(): {e}\")\n return \"\"\n\n def feed_audio(self, chunk, audio_meta_data, original_sample_rate=16000):\n # Start with the base metadata\n metadata = {\"sampleRate\": original_sample_rate}\n\n # Merge additional metadata if provided\n if audio_meta_data:\n server_sent_to_stt_ns = time.time_ns()\n audio_meta_data[\"server_sent_to_stt\"] = server_sent_to_stt_ns\n metadata[\"server_sent_to_stt_formatted\"] = format_timestamp_ns(server_sent_to_stt_ns)\n\n metadata.update(audio_meta_data)\n\n # Convert metadata to JSON and prepare the message\n metadata_json = json.dumps(metadata)\n metadata_length = len(metadata_json)\n message = struct.pack(' %s\", self.name, data)\n self._pipe.send(data)\n request[\"result_queue\"].put(None)\n\n elif request[\"type\"] == \"RECV\":\n logger.debug(\"[%s] Worker: receiving...\", self.name)\n data = self._pipe.recv()\n request[\"result_queue\"].put(data)\n\n elif request[\"type\"] == \"POLL\":\n timeout = request.get(\"timeout\", 0.0)\n logger.debug(\"[%s] Worker: poll() with timeout: %s\", self.name, timeout)\n result = self._pipe.poll(timeout)\n request[\"result_queue\"].put(result)\n\n except (EOFError, BrokenPipeError, OSError) as e:\n # When the other end has closed or an error occurs,\n # log and notify the waiting thread.\n logger.debug(\"[%s] Worker: pipe closed or error occurred (%s). Shutting down.\", self.name, e)\n request[\"result_queue\"].put(None)\n break\n\n except Exception as e:\n logger.exception(\"[%s] Worker: unexpected error.\", self.name)\n request[\"result_queue\"].put(e)\n break\n\n logger.debug(\"[%s] Worker: stopping.\", self.name)\n try:\n self._pipe.close()\n except Exception as e:\n logger.debug(\"[%s] Worker: error during pipe close: %s\", self.name, e)\n\n def send(self, data):\n \"\"\"\n Synchronously asks the worker thread to perform .send().\n \"\"\"\n if self._closed:\n logger.debug(\"[%s] send() called but pipe is already closed\", self.name)\n return\n logger.debug(\"[%s] send() requested with: %s\", self.name, data)\n result_queue = queue.Queue()\n request = {\n \"type\": \"SEND\",\n \"data\": data,\n \"result_queue\": result_queue\n }\n self._request_queue.put(request)\n result_queue.get() # Wait until sending completes.\n logger.debug(\"[%s] send() completed\", self.name)\n\n def recv(self):\n \"\"\"\n Synchronously asks the worker to perform .recv() and returns the data.\n \"\"\"\n if self._closed:\n logger.debug(\"[%s] recv() called but pipe is already closed\", self.name)\n return None\n logger.debug(\"[%s] recv() requested\", self.name)\n result_queue = queue.Queue()\n request = {\n \"type\": \"RECV\",\n \"result_queue\": result_queue\n }\n self._request_queue.put(request)\n data = result_queue.get()\n\n # Log a preview for huge byte blobs.\n if isinstance(data, tuple) and len(data) == 2 and isinstance(data[1], bytes):\n data_preview = (data[0], f\"<{len(data[1])} bytes>\")\n else:\n data_preview = data\n logger.debug(\"[%s] recv() returning => %s\", self.name, data_preview)\n return data\n\n def poll(self, timeout=0.0):\n \"\"\"\n Synchronously checks whether data is available.\n Returns True if data is ready, or False otherwise.\n \"\"\"\n if self._closed:\n return False\n logger.debug(\"[%s] poll() requested with timeout: %s\", self.name, timeout)\n result_queue = queue.Queue()\n request = {\n \"type\": \"POLL\",\n \"timeout\": timeout,\n \"result_queue\": result_queue\n }\n self._request_queue.put(request)\n try:\n # Use a slightly longer timeout to give the worker a chance.\n result = result_queue.get(timeout=timeout + 0.1)\n except queue.Empty:\n result = False\n logger.debug(\"[%s] poll() returning => %s\", self.name, result)\n return result\n\n def close(self):\n \"\"\"\n Closes the pipe and stops the worker thread. The _closed flag makes\n sure no further operations are attempted.\n \"\"\"\n if self._closed:\n return\n logger.debug(\"[%s] close() called\", self.name)\n self._closed = True\n stop_request = {\"type\": \"CLOSE\", \"result_queue\": queue.Queue()}\n self._request_queue.put(stop_request)\n self._stop_event.set()\n self._worker_thread.join()\n logger.debug(\"[%s] closed\", self.name)\n\n\ndef SafePipe(debug=False):\n \"\"\"\n Returns a pair: (thread-safe parent pipe, raw child pipe).\n \"\"\"\n parent_synthesize_pipe, child_synthesize_pipe = mp.Pipe()\n parent_pipe = ParentPipe(parent_synthesize_pipe)\n return parent_pipe, child_synthesize_pipe\n\n\ndef child_process_code(child_end):\n \"\"\"\n Example child process code that receives messages, logs them,\n sends acknowledgements, and then closes.\n \"\"\"\n for i in range(3):\n msg = child_end.recv()\n logger.debug(\"[Child] got: %s\", msg)\n child_end.send(f\"ACK: {msg}\")\n child_end.close()\n\n\nif __name__ == \"__main__\":\n parent_pipe, child_pipe = SafePipe()\n\n # Create child process with the child_process_code function.\n p = mp.Process(target=child_process_code, args=(child_pipe,))\n p.start()\n\n # Event to signal sender threads to stop if needed.\n stop_polling_event = threading.Event()\n\n def sender_thread(n):\n try:\n parent_pipe.send(f\"hello_from_thread_{n}\")\n except Exception as e:\n logger.debug(\"[sender_thread_%s] send exception: %s\", n, e)\n return\n\n # Use a poll loop with error handling.\n for _ in range(10):\n try:\n if parent_pipe.poll(0.1):\n reply = parent_pipe.recv()\n logger.debug(\"[sender_thread_%s] got: %s\", n, reply)\n break\n else:\n logger.debug(\"[sender_thread_%s] no data yet...\", n)\n except (OSError, EOFError, BrokenPipeError) as e:\n logger.debug(\"[sender_thread_%s] poll/recv exception: %s. Exiting thread.\", n, e)\n break\n\n # Allow exit if a shutdown is signaled.\n if stop_polling_event.is_set():\n logger.debug(\"[sender_thread_%s] stop event set. Exiting thread.\", n)\n break\n\n threads = []\n for i in range(3):\n t = threading.Thread(target=sender_thread, args=(i,))\n t.start()\n threads.append(t)\n\n for t in threads:\n t.join()\n\n # Signal shutdown to any polling threads, then close the pipe.\n stop_polling_event.set()\n parent_pipe.close()\n p.join()\n"
},
{
"path": "RealtimeSTT_server/README.md",
"content": "# RealtimeSTT Server and Client\n\nThis directory contains the server and client implementations for the RealtimeSTT library, providing real-time speech-to-text transcription with WebSocket interfaces. The server allows clients to connect via WebSocket to send audio data and receive real-time transcription updates. The client handles communication with the server, allowing audio recording, parameter management, and control commands.\n\n## Table of Contents\n\n- [Features](#features)\n- [Installation](#installation)\n- [Server Usage](#server-usage)\n - [Starting the Server](#starting-the-server)\n - [Server Parameters](#server-parameters)\n- [Client Usage](#client-usage)\n - [Starting the Client](#starting-the-client)\n - [Client Parameters](#client-parameters)\n- [WebSocket Interface](#websocket-interface)\n- [Examples](#examples)\n - [Starting the Server and Client](#starting-the-server-and-client)\n - [Setting Parameters](#setting-parameters)\n - [Retrieving Parameters](#retrieving-parameters)\n - [Calling Server Methods](#calling-server-methods)\n- [Contributing](#contributing)\n- [License](#license)\n\n## Features\n\n- **Real-Time Transcription**: Provides real-time speech-to-text transcription using pre-configured or user-defined STT models.\n- **WebSocket Communication**: Makes use of WebSocket connections for control commands and data handling.\n- **Flexible Recording Options**: Supports configurable pauses for sentence detection and various voice activity detection (VAD) methods.\n- **VAD Support**: Includes support for Silero and WebRTC VAD for robust voice activity detection.\n- **Wake Word Detection**: Capable of detecting wake words to initiate transcription.\n- **Configurable Parameters**: Allows fine-tuning of recording and transcription settings via command-line arguments or control commands.\n\n## Installation\n\nEnsure you have Python 3.8 or higher installed. Install the required packages using:\n\n```bash\npip install git+https://github.com/KoljaB/RealtimeSTT.git@dev\n```\n\n## Server Usage\n\n### Starting the Server\n\nStart the server using the command-line interface:\n\n```bash\nstt-server [OPTIONS]\n```\n\nThe server will initialize and begin listening for WebSocket connections on the specified control and data ports.\n\n### Server Parameters\n\nYou can configure the server using the following command-line arguments:\n\n### Available Parameters:\n\n#### `-m`, `--model`\n\n- **Type**: `str`\n- **Default**: `'large-v2'`\n- **Description**: Path to the Speech-to-Text (STT) model or specify a model size. Options include: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v1`, `large-v2`, or any HuggingFace CTranslate2 STT model such as `deepdml/faster-whisper-large-v3-turbo-ct2`.\n\n#### `-r`, `--rt-model`, `--realtime_model_type`\n\n- **Type**: `str`\n- **Default**: `'tiny.en'`\n- **Description**: Model size for real-time transcription. Options are the same as for `--model`. This is used only if real-time transcription is enabled (`--enable_realtime_transcription`).\n\n#### `-l`, `--lang`, `--language`\n\n- **Type**: `str`\n- **Default**: `'en'`\n- **Description**: Language code for the STT model to transcribe in a specific language. Leave this empty for auto-detection based on input audio. Default is `'en'`. [List of supported language codes](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L11-L110).\n\n#### `-i`, `--input-device`, `--input_device_index`\n\n- **Type**: `int`\n- **Default**: `1`\n- **Description**: Index of the audio input device to use. Use this option to specify a particular microphone or audio input device based on your system.\n\n#### `-c`, `--control`, `--control_port`\n\n- **Type**: `int`\n- **Default**: `8011`\n- **Description**: The port number used for the control WebSocket connection. Control connections are used to send and receive commands to the server.\n\n#### `-d`, `--data`, `--data_port`\n\n- **Type**: `int`\n- **Default**: `8012`\n- **Description**: The port number used for the data WebSocket connection. Data connections are used to send audio data and receive transcription updates in real time.\n\n#### `-w`, `--wake_words`\n\n- **Type**: `str`\n- **Default**: `\"\"` (empty string)\n- **Description**: Specify the wake word(s) that will trigger the server to start listening. For example, setting this to `\"Jarvis\"` will make the system start transcribing when it detects the wake word `\"Jarvis\"`.\n\n#### `-D`, `--debug`\n\n- **Action**: `store_true`\n- **Description**: Enable debug logging for detailed server operations.\n\n#### `-W`, `--write`\n\n- **Metavar**: `FILE`\n- **Description**: Save received audio to a WAV file.\n\n#### `--silero_sensitivity`\n\n- **Type**: `float`\n- **Default**: `0.05`\n- **Description**: Sensitivity level for Silero Voice Activity Detection (VAD), with a range from `0` to `1`. Lower values make the model less sensitive, useful for noisy environments.\n\n#### `--silero_use_onnx`\n\n- **Action**: `store_true`\n- **Default**: `False`\n- **Description**: Enable the ONNX version of the Silero model for faster performance with lower resource usage.\n\n#### `--webrtc_sensitivity`\n\n- **Type**: `int`\n- **Default**: `3`\n- **Description**: Sensitivity level for WebRTC Voice Activity Detection (VAD), with a range from `0` to `3`. Higher values make the model less sensitive, useful for cleaner environments.\n\n#### `--min_length_of_recording`\n\n- **Type**: `float`\n- **Default**: `1.1`\n- **Description**: Minimum duration of valid recordings in seconds. This prevents very short recordings from being processed, which could be caused by noise or accidental sounds.\n\n#### `--min_gap_between_recordings`\n\n- **Type**: `float`\n- **Default**: `0`\n- **Description**: Minimum time (in seconds) between consecutive recordings. Setting this helps avoid overlapping recordings when there's a brief silence between them.\n\n#### `--enable_realtime_transcription`\n\n- **Action**: `store_true`\n- **Default**: `True`\n- **Description**: Enable continuous real-time transcription of audio as it is received. When enabled, transcriptions are sent in near real-time.\n\n#### `--realtime_processing_pause`\n\n- **Type**: `float`\n- **Default**: `0.02`\n- **Description**: Time interval (in seconds) between processing audio chunks for real-time transcription. Lower values increase responsiveness but may put more load on the CPU.\n\n#### `--silero_deactivity_detection`\n\n- **Action**: `store_true`\n- **Default**: `True`\n- **Description**: Use the Silero model for end-of-speech detection. This option can provide more robust silence detection in noisy environments, though it consumes more GPU resources.\n\n#### `--early_transcription_on_silence`\n\n- **Type**: `float`\n- **Default**: `0.2`\n- **Description**: Start transcription after the specified seconds of silence. This is useful when you want to trigger transcription mid-speech when there is a brief pause. Should be lower than `post_speech_silence_duration`. Set to `0` to disable.\n\n#### `--beam_size`\n\n- **Type**: `int`\n- **Default**: `5`\n- **Description**: Beam size for the main transcription model. Larger values may improve transcription accuracy but increase the processing time.\n\n#### `--beam_size_realtime`\n\n- **Type**: `int`\n- **Default**: `3`\n- **Description**: Beam size for the real-time transcription model. A smaller beam size allows for faster real-time processing but may reduce accuracy.\n\n#### `--initial_prompt`\n\n- **Type**: `str`\n- **Default**:\n\n ```\n End incomplete sentences with ellipses. Examples: \n Complete: The sky is blue. \n Incomplete: When the sky... \n Complete: She walked home. \n Incomplete: Because he...\n ```\n\n- **Description**: Initial prompt that guides the transcription model to produce transcriptions in a particular style or format. The default provides instructions for handling sentence completions and ellipsis usage.\n\n#### `--end_of_sentence_detection_pause`\n\n- **Type**: `float`\n- **Default**: `0.45`\n- **Description**: The duration of silence (in seconds) that the model should interpret as the end of a sentence. This helps the system detect when to finalize the transcription of a sentence.\n\n#### `--unknown_sentence_detection_pause`\n\n- **Type**: `float`\n- **Default**: `0.7`\n- **Description**: The duration of pause (in seconds) that the model should interpret as an incomplete or unknown sentence. This is useful for identifying when a sentence is trailing off or unfinished.\n\n#### `--mid_sentence_detection_pause`\n\n- **Type**: `float`\n- **Default**: `2.0`\n- **Description**: The duration of pause (in seconds) that the model should interpret as a mid-sentence break. Longer pauses can indicate a pause in speech but not necessarily the end of a sentence.\n\n#### `--wake_words_sensitivity`\n\n- **Type**: `float`\n- **Default**: `0.5`\n- **Description**: Sensitivity level for wake word detection, with a range from `0` (most sensitive) to `1` (least sensitive). Adjust this value based on your environment to ensure reliable wake word detection.\n\n#### `--wake_word_timeout`\n\n- **Type**: `float`\n- **Default**: `5.0`\n- **Description**: Maximum time in seconds that the system will wait for a wake word before timing out. After this timeout, the system stops listening for wake words until reactivated.\n\n#### `--wake_word_activation_delay`\n\n- **Type**: `float`\n- **Default**: `20`\n- **Description**: The delay in seconds before the wake word detection is activated after the system starts listening. This prevents false positives during the start of a session.\n\n#### `--wakeword_backend`\n\n- **Type**: `str`\n- **Default**: `'none'`\n- **Description**: The backend used for wake word detection. You can specify different backends such as `\"default\"` or any custom implementations depending on your setup.\n\n#### `--openwakeword_model_paths`\n\n- **Type**: `str` (accepts multiple values)\n- **Description**: A list of file paths to OpenWakeWord models. This is useful if you are using OpenWakeWord for wake word detection and need to specify custom models.\n\n#### `--openwakeword_inference_framework`\n\n- **Type**: `str`\n- **Default**: `'tensorflow'`\n- **Description**: The inference framework to use for OpenWakeWord models. Supported frameworks could include `\"tensorflow\"`, `\"pytorch\"`, etc.\n\n#### `--wake_word_buffer_duration`\n\n- **Type**: `float`\n- **Default**: `1.0`\n- **Description**: Duration of the buffer in seconds for wake word detection. This sets how long the system will store the audio before and after detecting the wake word.\n\n#### `--use_main_model_for_realtime`\n\n- **Action**: `store_true`\n- **Description**: Enable this option if you want to use the main model for real-time transcription, instead of the smaller, faster real-time model. Using the main model may provide better accuracy but at the cost of higher processing time.\n\n#### `--use_extended_logging`\n\n- **Action**: `store_true`\n- **Description**: Writes extensive log messages for the recording worker that processes the audio chunks.\n\n#### `--logchunks`\n\n- **Action**: `store_true`\n- **Description**: Enable logging of incoming audio chunks (periods).\n\n**Example:**\n\n```bash\nstt-server -m small.en -l en -c 9001 -d 9002\n```\n\n## Client Usage\n\n### Starting the Client\n\nStart the client using:\n\n```bash\nstt [OPTIONS]\n```\n\nThe client connects to the STT server's control and data WebSocket URLs to facilitate real-time speech transcription and control.\n\n### Available Parameters for STT Client:\n\n#### `-i`, `--input-device`\n- **Type**: `int`\n- **Metavar**: `INDEX`\n- **Description**: Audio input device index. Use `-L` to list available devices.\n\n#### `-l`, `--language`\n- **Type**: `str` \n- **Default**: `'en'`\n- **Metavar**: `LANG`\n- **Description**: Language code to be used for transcription.\n\n#### `-sed`, `--speech-end-detection`\n- **Action**: `store_true`\n- **Description**: Enable intelligent speech end detection for better sentence boundaries.\n\n#### `-D`, `--debug`\n- **Action**: `store_true`\n- **Description**: Enable debug mode for detailed logging.\n\n#### `-n`, `--norealtime`\n- **Action**: `store_true`\n- **Description**: Disable real-time transcription output.\n\n#### `-W`, `--write`\n- **Metavar**: `FILE`\n- **Description**: Save recorded audio to a WAV file.\n\n#### `-s`, `--set`\n- **Type**: `list`\n- **Metavar**: `('PARAM', 'VALUE')`\n- **Action**: `append`\n- **Description**: Set a recorder parameter. Can be used multiple times with different parameters.\n\n#### `-m`, `--method`\n- **Type**: `list`\n- **Metavar**: `METHOD`\n- **Action**: `append`\n- **Description**: Call a recorder method with optional arguments.\n\n#### `-g`, `--get`\n- **Type**: `list`\n- **Metavar**: `PARAM`\n- **Action**: `append`\n- **Description**: Get the value of a recorder parameter.\n\n#### `-c`, `--continous`\n- **Action**: `store_true`\n- **Description**: Run in continuous mode, transcribing speech without exiting.\n\n#### `-L`, `--list`\n- **Action**: `store_true`\n- **Description**: List all available audio input devices and exit.\n\n#### `--control`, `--control_url`\n- **Type**: `str`\n- **Default**: `ws://127.0.0.1:8011`\n- **Description**: WebSocket URL for STT control connection.\n\n#### `--data`, `--data_url`\n- **Type**: `str`\n- **Default**: `ws://127.0.0.1:8012`\n- **Description**: WebSocket URL for STT data connection.\n\n\n### Parameters only available when speech-end-detection is active:\n\n#### `--post-silence`\n- **Type**: `float`\n- **Default**: `1.0`\n- **Description**: Post speech silence duration in seconds.\n\n#### `--unknown-pause` \n- **Type**: `float`\n- **Default**: `1.3`\n- **Description**: Unknown sentence detection pause duration in seconds.\n\n#### `--mid-pause`\n- **Type**: `float` \n- **Default**: `3.0`\n- **Description**: Mid-sentence detection pause duration in seconds.\n\n#### `--end-pause`\n- **Type**: `float`\n- **Default**: `0.7` \n- **Description**: End of sentence detection pause duration in seconds.\n\n#### `--hard-break`\n- **Type**: `float`\n- **Default**: `3.0`\n- **Description**: Hard break threshold in seconds when background noise is present.\n\n#### `--min-texts`\n- **Type**: `int`\n- **Default**: `3`\n- **Description**: Minimum number of texts required for hard break detection.\n\n#### `--min-similarity`\n- **Type**: `float`\n- **Default**: `0.99`\n- **Description**: Minimum text similarity threshold for hard break detection.\n\n#### `--min-chars`\n- **Type**: `int`\n- **Default**: `15`\n- **Description**: Minimum number of characters required for hard break detection.\n\n**Examples:**\n\n```bash\n# List available audio devices\nstt -L\n\n# Use specific input device and language\nstt -i 1 -l en\n\n# Enable intelligent speech end detection and continuous mode\nstt -sed -c\n\n# Set parameter and save audio\nstt -s silero_sensitivity 0.1 -W recording.wav\n\n# Use custom WebSocket URLs\nstt --control ws://localhost:9001 --data ws://localhost:9002\n```\n\n## WebSocket Interface\n\nThe server uses two WebSocket connections:\n\n1. **Control WebSocket**: Used to send and receive control commands, such as setting parameters or invoking recorder methods.\n\n2. **Data WebSocket**: Used to send audio data for transcription and receive real-time transcription updates.\n\n## Examples\n\n### Starting the Server and Client\n\n1. **Start the Server with Default Settings:**\n\n ```bash\n stt-server\n ```\n\n2. **Start the Client with Default Settings:**\n\n ```bash\n stt\n ```\n\n### Setting Parameters\n\nSet the Silero sensitivity to `0.1`:\n\n```bash\nstt -s silero_sensitivity 0.1\n```\n\n### Retrieving Parameters\n\nGet the current Silero sensitivity value:\n\n```bash\nstt -g silero_sensitivity\n```\n\n### Calling Server Methods\n\nCall the `set_microphone` method on the recorder:\n\n```bash\nstt -m set_microphone False\n```\n\n### Running in Debug Mode\n\nEnable debug mode for detailed logging:\n\n```bash\nstt -D\n```\n\n## Contributing\n\nContributions are welcome! Please open an issue or submit a pull request on GitHub.\n\n## License\n\nThis project is licensed under the MIT License. See the [LICENSE](../LICENSE) file for details.\n\n# Additional Information\n\nThe server and client scripts are designed to work seamlessly together, enabling efficient real-time speech transcription with minimal latency. The flexibility in configuration allows users to tailor the system to specific needs, such as adjusting sensitivity levels for different environments or selecting appropriate STT models based on resource availability.\n\n**Note:** Ensure that the server is running before starting the client. The client includes functionality to check if the server is running and can prompt the user to start it if necessary.\n\n# Troubleshooting\n\n- **Server Not Starting:** If the server fails to start, check that all dependencies are installed and that the specified ports are not in use.\n\n- **Audio Issues:** Ensure that the correct audio input device index is specified if using a device other than the default.\n\n- **WebSocket Connection Errors:** Verify that the control and data URLs are correct and that the server is listening on those ports.\n\n# Contact\n\nFor questions or support, please open an issue on the [GitHub repository](https://github.com/KoljaB/RealtimeSTT/issues).\n\n# Acknowledgments\n\nSpecial thanks to the contributors of the RealtimeSTT library and the open-source community for their continuous support.\n\n---\n\n**Disclaimer:** This software is provided \"as is\", without warranty of any kind, express or implied. Use it at your own risk."
},
{
"path": "RealtimeSTT_server/__init__.py",
"content": ""
},
{
"path": "RealtimeSTT_server/index.html",
"content": "\n\n\n \n \n Browser STT Client\n \n\n\n
\n
Press \"Start Recording\"...
\n \n \n
\n \n
\n
\n \n
\n
\n\n \n\n\n"
},
{
"path": "RealtimeSTT_server/install_packages.py",
"content": "import subprocess\nimport sys\nimport importlib\n\ndef check_and_install_packages(packages):\n \"\"\"\n Checks if the specified packages are installed, and if not, prompts the user\n to install them.\n\n Parameters:\n - packages: A list of dictionaries, each containing:\n - 'module_name': The module or package name to import.\n - 'attribute': (Optional) The attribute or class to check within the module.\n - 'install_name': The name used in the pip install command.\n - 'version': (Optional) Version constraint for the package.\n \"\"\"\n for package in packages:\n module_name = package['module_name']\n attribute = package.get('attribute')\n install_name = package.get('install_name', module_name)\n version = package.get('version', '')\n\n try:\n # Attempt to import the module\n module = importlib.import_module(module_name)\n # If an attribute is specified, check if it exists\n if attribute:\n getattr(module, attribute)\n except (ImportError, AttributeError):\n user_input = input(\n f\"This program requires '{module_name}'\"\n f\"{'' if not attribute else ' with attribute ' + attribute}, which is not installed or missing.\\n\"\n f\"Do you want to install '{install_name}' now? (y/n): \"\n )\n if user_input.strip().lower() == 'y':\n try:\n # Build the pip install command\n install_command = [sys.executable, \"-m\", \"pip\", \"install\"]\n if version:\n install_command.append(f\"{install_name}{version}\")\n else:\n install_command.append(install_name)\n\n subprocess.check_call(install_command)\n # Try to import again after installation\n module = importlib.import_module(module_name)\n if attribute:\n getattr(module, attribute)\n print(f\"Successfully installed '{install_name}'.\")\n except Exception as e:\n print(f\"An error occurred while installing '{install_name}': {e}\")\n sys.exit(1)\n else:\n print(f\"The program requires '{install_name}' to run. Exiting...\")\n sys.exit(1)\n"
},
{
"path": "RealtimeSTT_server/stt_cli_client.py",
"content": "# stt_cli_client.py\n\nfrom difflib import SequenceMatcher\nfrom collections import deque\nimport argparse\nimport string\nimport shutil\nimport time\nimport sys\nimport os\n\nfrom RealtimeSTT import AudioToTextRecorderClient\nfrom RealtimeSTT import AudioInput\n\nfrom colorama import init, Fore, Style\ninit()\n\nDEFAULT_CONTROL_URL = \"ws://127.0.0.1:8011\"\nDEFAULT_DATA_URL = \"ws://127.0.0.1:8012\"\n\nrecording_indicator = \"🔴\"\n\nconsole_width = shutil.get_terminal_size().columns\n\npost_speech_silence_duration = 1.0 # Will be overridden by CLI arg\nunknown_sentence_detection_pause = 1.3\nmid_sentence_detection_pause = 3.0\nend_of_sentence_detection_pause = 0.7\nhard_break_even_on_background_noise = 3.0\nhard_break_even_on_background_noise_min_texts = 3\nhard_break_even_on_background_noise_min_similarity = 0.99\nhard_break_even_on_background_noise_min_chars = 15\nprev_text = \"\"\ntext_time_deque = deque()\n\ndef main():\n global prev_text, post_speech_silence_duration, unknown_sentence_detection_pause\n global mid_sentence_detection_pause, end_of_sentence_detection_pause\n global hard_break_even_on_background_noise, hard_break_even_on_background_noise_min_texts\n global hard_break_even_on_background_noise_min_similarity, hard_break_even_on_background_noise_min_chars\n\n parser = argparse.ArgumentParser(description=\"STT Client\")\n\n # Add input device argument\n parser.add_argument(\"-i\", \"--input-device\", type=int, metavar=\"INDEX\",\n help=\"Audio input device index (use -l to list devices)\")\n parser.add_argument(\"-l\", \"--language\", default=\"en\", metavar=\"LANG\",\n help=\"Language to be used (default: en)\")\n parser.add_argument(\"-sed\", \"--speech-end-detection\", action=\"store_true\",\n help=\"Usage of intelligent speech end detection\")\n parser.add_argument(\"-D\", \"--debug\", action=\"store_true\",\n help=\"Enable debug mode\")\n parser.add_argument(\"-n\", \"--norealtime\", action=\"store_true\",\n help=\"Disable real-time output\")\n parser.add_argument(\"-W\", \"--write\", metavar=\"FILE\",\n help=\"Save recorded audio to a WAV file\")\n parser.add_argument(\"-s\", \"--set\", nargs=2, metavar=('PARAM', 'VALUE'), action='append',\n help=\"Set a recorder parameter (can be used multiple times)\")\n parser.add_argument(\"-m\", \"--method\", nargs='+', metavar='METHOD', action='append',\n help=\"Call a recorder method with optional arguments\")\n parser.add_argument(\"-g\", \"--get\", nargs=1, metavar='PARAM', action='append',\n help=\"Get a recorder parameter's value (can be used multiple times)\")\n parser.add_argument(\"-c\", \"--continous\", action=\"store_true\",\n help=\"Continuously transcribe speech without exiting\")\n parser.add_argument(\"-L\", \"--list\", action=\"store_true\",\n help=\"List available audio input devices and exit\")\n parser.add_argument(\"--control\", \"--control_url\", default=DEFAULT_CONTROL_URL,\n help=\"STT Control WebSocket URL\")\n parser.add_argument(\"--data\", \"--data_url\", default=DEFAULT_DATA_URL,\n help=\"STT Data WebSocket URL\")\n parser.add_argument(\"--post-silence\", type=float, default=1.0,\n help=\"Post speech silence duration in seconds (default: 1.0)\")\n parser.add_argument(\"--unknown-pause\", type=float, default=1.3,\n help=\"Unknown sentence detection pause in seconds (default: 1.3)\")\n parser.add_argument(\"--mid-pause\", type=float, default=3.0,\n help=\"Mid sentence detection pause in seconds (default: 3.0)\")\n parser.add_argument(\"--end-pause\", type=float, default=0.7,\n help=\"End of sentence detection pause in seconds (default: 0.7)\")\n parser.add_argument(\"--hard-break\", type=float, default=3.0,\n help=\"Hard break threshold in seconds (default: 3.0)\")\n parser.add_argument(\"--min-texts\", type=int, default=3,\n help=\"Minimum texts for hard break (default: 3)\")\n parser.add_argument(\"--min-similarity\", type=float, default=0.99,\n help=\"Minimum text similarity for hard break (default: 0.99)\")\n parser.add_argument(\"--min-chars\", type=int, default=15,\n help=\"Minimum characters for hard break (default: 15)\")\n\n args = parser.parse_args()\n\n # Add this block after parsing args:\n if args.list:\n audio_input = AudioInput()\n audio_input.list_devices()\n return\n\n # Update globals with CLI values\n post_speech_silence_duration = args.post_silence\n unknown_sentence_detection_pause = args.unknown_pause\n mid_sentence_detection_pause = args.mid_pause\n end_of_sentence_detection_pause = args.end_pause\n hard_break_even_on_background_noise = args.hard_break\n hard_break_even_on_background_noise_min_texts = args.min_texts\n hard_break_even_on_background_noise_min_similarity = args.min_similarity\n hard_break_even_on_background_noise_min_chars = args.min_chars\n\n # Check if output is being redirected\n if not os.isatty(sys.stdout.fileno()):\n file_output = sys.stdout\n else:\n file_output = None\n\n def clear_line():\n if file_output:\n sys.stderr.write('\\r\\033[K')\n else:\n print('\\r\\033[K', end=\"\", flush=True)\n\n def write(text):\n if file_output:\n sys.stderr.write(text)\n sys.stderr.flush()\n else:\n print(text, end=\"\", flush=True)\n\n def on_realtime_transcription_update(text):\n global post_speech_silence_duration, prev_text, text_time_deque\n \n def set_post_speech_silence_duration(duration: float):\n global post_speech_silence_duration\n post_speech_silence_duration = duration\n client.set_parameter(\"post_speech_silence_duration\", duration)\n\n def preprocess_text(text):\n text = text.lstrip()\n if text.startswith(\"...\"):\n text = text[3:]\n text = text.lstrip()\n if text:\n text = text[0].upper() + text[1:]\n return text\n\n def ends_with_ellipsis(text: str):\n if text.endswith(\"...\"):\n return True\n if len(text) > 1 and text[:-1].endswith(\"...\"):\n return True\n return False\n\n def sentence_end(text: str):\n sentence_end_marks = ['.', '!', '?', '。']\n if text and text[-1] in sentence_end_marks:\n return True\n return False\n\n if not args.norealtime:\n text = preprocess_text(text)\n\n if args.speech_end_detection:\n if ends_with_ellipsis(text):\n if not post_speech_silence_duration == mid_sentence_detection_pause:\n set_post_speech_silence_duration(mid_sentence_detection_pause)\n if args.debug: print(f\"RT: post_speech_silence_duration for {text} (...): {post_speech_silence_duration}\")\n elif sentence_end(text) and sentence_end(prev_text) and not ends_with_ellipsis(prev_text):\n if not post_speech_silence_duration == end_of_sentence_detection_pause:\n set_post_speech_silence_duration(end_of_sentence_detection_pause)\n if args.debug: print(f\"RT: post_speech_silence_duration for {text} (.!?): {post_speech_silence_duration}\")\n else:\n if not post_speech_silence_duration == unknown_sentence_detection_pause:\n set_post_speech_silence_duration(unknown_sentence_detection_pause)\n if args.debug: print(f\"RT: post_speech_silence_duration for {text} (???): {post_speech_silence_duration}\")\n \n prev_text = text\n\n # transtext = text.translate(str.maketrans('', '', string.punctuation))\n \n # Append the new text with its timestamp\n current_time = time.time()\n text_time_deque.append((current_time, text))\n\n # Remove texts older than hard_break_even_on_background_noise seconds\n while text_time_deque and text_time_deque[0][0] < current_time - hard_break_even_on_background_noise:\n text_time_deque.popleft()\n\n # Check if at least hard_break_even_on_background_noise_min_texts texts have arrived within the last hard_break_even_on_background_noise seconds\n if len(text_time_deque) >= hard_break_even_on_background_noise_min_texts:\n texts = [t[1] for t in text_time_deque]\n first_text = texts[0]\n last_text = texts[-1]\n\n # Compute the similarity ratio between the first and last texts\n similarity = SequenceMatcher(None, first_text, last_text).ratio()\n\n if similarity > hard_break_even_on_background_noise_min_similarity and len(first_text) > hard_break_even_on_background_noise_min_chars:\n client.call_method(\"stop\")\n\n clear_line()\n\n words = text.split()\n last_chars = \"\"\n available_width = console_width - 5\n for word in reversed(words):\n if len(last_chars) + len(word) + 1 > available_width:\n break\n last_chars = word + \" \" + last_chars\n last_chars = last_chars.strip()\n\n colored_text = f\"{Fore.YELLOW}{last_chars}{Style.RESET_ALL}{recording_indicator}\\b\\b\"\n write(colored_text)\n\n client = AudioToTextRecorderClient(\n language=args.language,\n control_url=args.control,\n data_url=args.data,\n debug_mode=args.debug,\n on_realtime_transcription_update=on_realtime_transcription_update,\n use_microphone=True,\n input_device_index=args.input_device, # Pass input device index\n output_wav_file = args.write or None,\n )\n\n # Process command-line parameters\n if args.set:\n for param, value in args.set:\n try:\n if '.' in value:\n value = float(value)\n else:\n value = int(value)\n except ValueError:\n pass # Keep as string if not a number\n client.set_parameter(param, value)\n\n if args.get:\n for param_list in args.get:\n param = param_list[0]\n value = client.get_parameter(param)\n if value is not None:\n print(f\"Parameter {param} = {value}\")\n\n if args.method:\n for method_call in args.method:\n method = method_call[0]\n args_list = method_call[1:] if len(method_call) > 1 else []\n client.call_method(method, args=args_list)\n\n # Start transcription\n try:\n while True:\n if not client._recording:\n print(\"Recording stopped due to an error.\", file=sys.stderr)\n break\n \n if not file_output:\n print(recording_indicator, end=\"\", flush=True)\n else:\n sys.stderr.write(recording_indicator)\n sys.stderr.flush()\n \n text = client.text()\n\n if text and client._recording and client.is_running:\n if file_output:\n print(text, file=file_output)\n sys.stderr.write('\\r\\033[K')\n sys.stderr.write(f'{text}')\n else:\n print('\\r\\033[K', end=\"\", flush=True)\n print(f'{text}', end=\"\", flush=True)\n if not args.continous:\n break\n else:\n time.sleep(0.1)\n \n if args.continous:\n print()\n prev_text = \"\"\n except KeyboardInterrupt:\n print('\\r\\033[K', end=\"\", flush=True)\n finally:\n client.shutdown()\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "RealtimeSTT_server/stt_server.py",
"content": "\"\"\"\nSpeech-to-Text (STT) Server with Real-Time Transcription and WebSocket Interface\n\nThis server provides real-time speech-to-text (STT) transcription using the RealtimeSTT library. It allows clients to connect via WebSocket to send audio data and receive real-time transcription updates. The server supports configurable audio recording parameters, voice activity detection (VAD), and wake word detection. It is designed to handle continuous transcription as well as post-recording processing, enabling real-time feedback with the option to improve final transcription quality after the complete sentence is recognized.\n\n### Features:\n- Real-time transcription using pre-configured or user-defined STT models.\n- WebSocket-based communication for control and data handling.\n- Flexible recording and transcription options, including configurable pauses for sentence detection.\n- Supports Silero and WebRTC VAD for robust voice activity detection.\n\n### Starting the Server:\nYou can start the server using the command-line interface (CLI) command `stt-server`, passing the desired configuration options.\n\n```bash\nstt-server [OPTIONS]\n```\n\n### Available Parameters:\n - `-m, --model`: Model path or size; default 'large-v2'.\n - `-r, --rt-model, --realtime_model_type`: Real-time model size; default 'tiny.en'.\n - `-l, --lang, --language`: Language code for transcription; default 'en'.\n - `-i, --input-device, --input_device_index`: Audio input device index; default 1.\n - `-c, --control, --control_port`: WebSocket control port; default 8011.\n - `-d, --data, --data_port`: WebSocket data port; default 8012.\n - `-w, --wake_words`: Wake word(s) to trigger listening; default \"\".\n - `-D, --debug`: Enable debug logging.\n - `-W, --write`: Save audio to WAV file.\n - `-s, --silence_timing`: Enable dynamic silence duration for sentence detection; default True. \n - `-b, --batch, --batch_size`: Batch size for inference; default 16.\n - `--root, --download_root`: Specifies the root path were the Whisper models are downloaded to.\n - `--silero_sensitivity`: Silero VAD sensitivity (0-1); default 0.05.\n - `--silero_use_onnx`: Use Silero ONNX model; default False.\n - `--webrtc_sensitivity`: WebRTC VAD sensitivity (0-3); default 3.\n - `--min_length_of_recording`: Minimum recording duration in seconds; default 1.1.\n - `--min_gap_between_recordings`: Min time between recordings in seconds; default 0.\n - `--enable_realtime_transcription`: Enable real-time transcription; default True.\n - `--realtime_processing_pause`: Pause between audio chunk processing; default 0.02.\n - `--silero_deactivity_detection`: Use Silero for end-of-speech detection; default True.\n - `--early_transcription_on_silence`: Start transcription after silence in seconds; default 0.2.\n - `--beam_size`: Beam size for main model; default 5.\n - `--beam_size_realtime`: Beam size for real-time model; default 3.\n - `--init_realtime_after_seconds`: Initial waiting time for realtime transcription; default 0.2.\n - `--realtime_batch_size`: Batch size for the real-time transcription model; default 16.\n - `--initial_prompt`: Initial main transcription guidance prompt.\n - `--initial_prompt_realtime`: Initial realtime transcription guidance prompt.\n - `--end_of_sentence_detection_pause`: Silence duration for sentence end detection; default 0.45.\n - `--unknown_sentence_detection_pause`: Pause duration for incomplete sentence detection; default 0.7.\n - `--mid_sentence_detection_pause`: Pause for mid-sentence break; default 2.0.\n - `--wake_words_sensitivity`: Wake word detection sensitivity (0-1); default 0.5.\n - `--wake_word_timeout`: Wake word timeout in seconds; default 5.0.\n - `--wake_word_activation_delay`: Delay before wake word activation; default 20.\n - `--wakeword_backend`: Backend for wake word detection; default 'none'.\n - `--openwakeword_model_paths`: Paths to OpenWakeWord models.\n - `--openwakeword_inference_framework`: OpenWakeWord inference framework; default 'tensorflow'.\n - `--wake_word_buffer_duration`: Wake word buffer duration in seconds; default 1.0.\n - `--use_main_model_for_realtime`: Use main model for real-time transcription.\n - `--use_extended_logging`: Enable extensive log messages.\n - `--logchunks`: Log incoming audio chunks.\n - `--compute_type`: Type of computation to use.\n - `--input_device_index`: Index of the audio input device.\n - `--gpu_device_index`: Index of the GPU device.\n - `--device`: Device to use for computation.\n - `--handle_buffer_overflow`: Handle buffer overflow during transcription.\n - `--suppress_tokens`: Suppress tokens during transcription.\n - `--allowed_latency_limit`: Allowed latency limit for real-time transcription.\n - `--faster_whisper_vad_filter`: Enable VAD filter for Faster Whisper; default False.\n\n\n### WebSocket Interface:\nThe server supports two WebSocket connections:\n1. **Control WebSocket**: Used to send and receive commands, such as setting parameters or calling recorder methods.\n2. **Data WebSocket**: Used to send audio data for transcription and receive real-time transcription updates.\n\nThe server will broadcast real-time transcription updates to all connected clients on the data WebSocket.\n\"\"\"\n\nfrom .install_packages import check_and_install_packages\nfrom difflib import SequenceMatcher\nfrom collections import deque\nfrom datetime import datetime\nimport logging\nimport asyncio\nimport pyaudio\nimport base64\nimport sys\n\n\ndebug_logging = False\nextended_logging = False\nsend_recorded_chunk = False\nlog_incoming_chunks = False\nsilence_timing = False\nwritechunks = False\nwav_file = None\n\nhard_break_even_on_background_noise = 3.0\nhard_break_even_on_background_noise_min_texts = 3\nhard_break_even_on_background_noise_min_similarity = 0.99\nhard_break_even_on_background_noise_min_chars = 15\n\n\ntext_time_deque = deque()\nloglevel = logging.WARNING\n\nFORMAT = pyaudio.paInt16\nCHANNELS = 1\n\n\nif sys.platform == 'win32':\n asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())\n\n\ncheck_and_install_packages([\n {\n 'module_name': 'RealtimeSTT', # Import module\n 'attribute': 'AudioToTextRecorder', # Specific class to check\n 'install_name': 'RealtimeSTT', # Package name for pip install\n },\n {\n 'module_name': 'websockets', # Import module\n 'install_name': 'websockets', # Package name for pip install\n },\n {\n 'module_name': 'numpy', # Import module\n 'install_name': 'numpy', # Package name for pip install\n },\n {\n 'module_name': 'scipy.signal', # Submodule of scipy\n 'attribute': 'resample', # Specific function to check\n 'install_name': 'scipy', # Package name for pip install\n }\n])\n\n# Define ANSI color codes for terminal output\nclass bcolors:\n HEADER = '\\033[95m' # Magenta\n OKBLUE = '\\033[94m' # Blue\n OKCYAN = '\\033[96m' # Cyan\n OKGREEN = '\\033[92m' # Green\n WARNING = '\\033[93m' # Yellow\n FAIL = '\\033[91m' # Red\n ENDC = '\\033[0m' # Reset to default\n BOLD = '\\033[1m'\n UNDERLINE = '\\033[4m'\n\nprint(f\"{bcolors.BOLD}{bcolors.OKCYAN}Starting server, please wait...{bcolors.ENDC}\")\n\n# Initialize colorama\nfrom colorama import init, Fore, Style\ninit()\n\nfrom RealtimeSTT import AudioToTextRecorder\nfrom scipy.signal import resample\nimport numpy as np\nimport websockets\nimport threading\nimport logging\nimport wave\nimport json\nimport time\n\nglobal_args = None\nrecorder = None\nrecorder_config = {}\nrecorder_ready = threading.Event()\nrecorder_thread = None\nstop_recorder = False\nprev_text = \"\"\n\n# Define allowed methods and parameters for security\nallowed_methods = [\n 'set_microphone',\n 'abort',\n 'stop',\n 'clear_audio_queue',\n 'wakeup',\n 'shutdown',\n 'text',\n]\nallowed_parameters = [\n 'language',\n 'silero_sensitivity',\n 'wake_word_activation_delay',\n 'post_speech_silence_duration',\n 'listen_start',\n 'recording_stop_time',\n 'last_transcription_bytes',\n 'last_transcription_bytes_b64',\n 'speech_end_silence_start',\n 'is_recording',\n 'use_wake_words',\n]\n\n# Queues and connections for control and data\ncontrol_connections = set()\ndata_connections = set()\ncontrol_queue = asyncio.Queue()\naudio_queue = asyncio.Queue()\n\ndef preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n if text.endswith(\"...'.\"):\n text = text[:-1]\n\n if text.endswith(\"...'\"):\n text = text[:-1]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n \n return text\n\ndef debug_print(message):\n if debug_logging:\n timestamp = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n thread_name = threading.current_thread().name\n print(f\"{Fore.CYAN}[DEBUG][{timestamp}][{thread_name}] {message}{Style.RESET_ALL}\", file=sys.stderr)\n\ndef format_timestamp_ns(timestamp_ns: int) -> str:\n # Split into whole seconds and the nanosecond remainder\n seconds = timestamp_ns // 1_000_000_000\n remainder_ns = timestamp_ns % 1_000_000_000\n\n # Convert seconds part into a datetime object (local time)\n dt = datetime.fromtimestamp(seconds)\n\n # Format the main time as HH:MM:SS\n time_str = dt.strftime(\"%H:%M:%S\")\n\n # For instance, if you want milliseconds, divide the remainder by 1e6 and format as 3-digit\n milliseconds = remainder_ns // 1_000_000\n formatted_timestamp = f\"{time_str}.{milliseconds:03d}\"\n\n return formatted_timestamp\n\ndef text_detected(text, loop):\n global prev_text\n\n text = preprocess_text(text)\n\n if silence_timing:\n def ends_with_ellipsis(text: str):\n if text.endswith(\"...\"):\n return True\n if len(text) > 1 and text[:-1].endswith(\"...\"):\n return True\n return False\n\n def sentence_end(text: str):\n sentence_end_marks = ['.', '!', '?', '。']\n if text and text[-1] in sentence_end_marks:\n return True\n return False\n\n\n if ends_with_ellipsis(text):\n recorder.post_speech_silence_duration = global_args.mid_sentence_detection_pause\n elif sentence_end(text) and sentence_end(prev_text) and not ends_with_ellipsis(prev_text):\n recorder.post_speech_silence_duration = global_args.end_of_sentence_detection_pause\n else:\n recorder.post_speech_silence_duration = global_args.unknown_sentence_detection_pause\n\n\n # Append the new text with its timestamp\n current_time = time.time()\n text_time_deque.append((current_time, text))\n\n # Remove texts older than hard_break_even_on_background_noise seconds\n while text_time_deque and text_time_deque[0][0] < current_time - hard_break_even_on_background_noise:\n text_time_deque.popleft()\n\n # Check if at least hard_break_even_on_background_noise_min_texts texts have arrived within the last hard_break_even_on_background_noise seconds\n if len(text_time_deque) >= hard_break_even_on_background_noise_min_texts:\n texts = [t[1] for t in text_time_deque]\n first_text = texts[0]\n last_text = texts[-1]\n\n # Compute the similarity ratio between the first and last texts\n similarity = SequenceMatcher(None, first_text, last_text).ratio()\n\n if similarity > hard_break_even_on_background_noise_min_similarity and len(first_text) > hard_break_even_on_background_noise_min_chars:\n recorder.stop()\n recorder.clear_audio_queue()\n prev_text = \"\"\n\n prev_text = text\n\n # Put the message in the audio queue to be sent to clients\n message = json.dumps({\n 'type': 'realtime',\n 'text': text\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\n # Get current timestamp in HH:MM:SS.nnn format\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n\n if extended_logging:\n print(f\" [{timestamp}] Realtime text: {bcolors.OKCYAN}{text}{bcolors.ENDC}\\n\", flush=True, end=\"\")\n else:\n print(f\"\\r[{timestamp}] {bcolors.OKCYAN}{text}{bcolors.ENDC}\", flush=True, end='')\n\ndef on_recording_start(loop):\n message = json.dumps({\n 'type': 'recording_start'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_recording_stop(loop):\n message = json.dumps({\n 'type': 'recording_stop'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_vad_detect_start(loop):\n message = json.dumps({\n 'type': 'vad_detect_start'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_vad_detect_stop(loop):\n message = json.dumps({\n 'type': 'vad_detect_stop'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_wakeword_detected(loop):\n message = json.dumps({\n 'type': 'wakeword_detected'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_wakeword_detection_start(loop):\n message = json.dumps({\n 'type': 'wakeword_detection_start'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_wakeword_detection_end(loop):\n message = json.dumps({\n 'type': 'wakeword_detection_end'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_transcription_start(_audio_bytes, loop):\n bytes_b64 = base64.b64encode(_audio_bytes.tobytes()).decode('utf-8')\n message = json.dumps({\n 'type': 'transcription_start',\n 'audio_bytes_base64': bytes_b64\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_turn_detection_start(loop):\n print(\"&&& stt_server on_turn_detection_start\")\n message = json.dumps({\n 'type': 'start_turn_detection'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\ndef on_turn_detection_stop(loop):\n print(\"&&& stt_server on_turn_detection_stop\")\n message = json.dumps({\n 'type': 'stop_turn_detection'\n })\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\n\n# def on_realtime_transcription_update(text, loop):\n# # Send real-time transcription updates to the client\n# text = preprocess_text(text)\n# message = json.dumps({\n# 'type': 'realtime_update',\n# 'text': text\n# })\n# asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\n# def on_recorded_chunk(chunk, loop):\n# if send_recorded_chunk:\n# bytes_b64 = base64.b64encode(chunk.tobytes()).decode('utf-8')\n# message = json.dumps({\n# 'type': 'recorded_chunk',\n# 'bytes': bytes_b64\n# })\n# asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\n# Define the server's arguments\ndef parse_arguments():\n global debug_logging, extended_logging, loglevel, writechunks, log_incoming_chunks, dynamic_silence_timing\n\n import argparse\n parser = argparse.ArgumentParser(description='Start the Speech-to-Text (STT) server with various configuration options.')\n\n parser.add_argument('-m', '--model', type=str, default='large-v2',\n help='Path to the STT model or model size. Options include: tiny, tiny.en, base, base.en, small, small.en, medium, medium.en, large-v1, large-v2, or any huggingface CTranslate2 STT model such as deepdml/faster-whisper-large-v3-turbo-ct2. Default is large-v2.')\n\n parser.add_argument('-r', '--rt-model', '--realtime_model_type', type=str, default='tiny',\n help='Model size for real-time transcription. Options same as --model. This is used only if real-time transcription is enabled (enable_realtime_transcription). Default is tiny.en.')\n \n parser.add_argument('-l', '--lang', '--language', type=str, default='en',\n help='Language code for the STT model to transcribe in a specific language. Leave this empty for auto-detection based on input audio. Default is en. List of supported language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L11-L110')\n\n parser.add_argument('-i', '--input-device', '--input-device-index', type=int, default=1,\n help='Index of the audio input device to use. Use this option to specify a particular microphone or audio input device based on your system. Default is 1.')\n\n parser.add_argument('-c', '--control', '--control_port', type=int, default=8011,\n help='The port number used for the control WebSocket connection. Control connections are used to send and receive commands to the server. Default is port 8011.')\n\n parser.add_argument('-d', '--data', '--data_port', type=int, default=8012,\n help='The port number used for the data WebSocket connection. Data connections are used to send audio data and receive transcription updates in real time. Default is port 8012.')\n\n parser.add_argument('-w', '--wake_words', type=str, default=\"\",\n help='Specify the wake word(s) that will trigger the server to start listening. For example, setting this to \"Jarvis\" will make the system start transcribing when it detects the wake word \"Jarvis\". Default is \"Jarvis\".')\n\n parser.add_argument('-D', '--debug', action='store_true', help='Enable debug logging for detailed server operations')\n\n parser.add_argument('--debug_websockets', action='store_true', help='Enable debug logging for detailed server websocket operations')\n\n parser.add_argument('-W', '--write', metavar='FILE', help='Save received audio to a WAV file')\n \n parser.add_argument('-b', '--batch', '--batch_size', type=int, default=16, help='Batch size for inference. This parameter controls the number of audio chunks processed in parallel during transcription. Default is 16.')\n\n parser.add_argument('--root', '--download_root', type=str,default=None, help='Specifies the root path where the Whisper models are downloaded to. Default is None.')\n\n parser.add_argument('-s', '--silence_timing', action='store_true', default=True,\n help='Enable dynamic adjustment of silence duration for sentence detection. Adjusts post-speech silence duration based on detected sentence structure and punctuation. Default is False.')\n\n parser.add_argument('--init_realtime_after_seconds', type=float, default=0.2,\n help='The initial waiting time in seconds before real-time transcription starts. This delay helps prevent false positives at the beginning of a session. Default is 0.2 seconds.') \n \n parser.add_argument('--realtime_batch_size', type=int, default=16,\n help='Batch size for the real-time transcription model. This parameter controls the number of audio chunks processed in parallel during real-time transcription. Default is 16.')\n \n parser.add_argument('--initial_prompt_realtime', type=str, default=\"\", help='Initial prompt that guides the real-time transcription model to produce transcriptions in a particular style or format.')\n\n parser.add_argument('--silero_sensitivity', type=float, default=0.05,\n help='Sensitivity level for Silero Voice Activity Detection (VAD), with a range from 0 to 1. Lower values make the model less sensitive, useful for noisy environments. Default is 0.05.')\n\n parser.add_argument('--silero_use_onnx', action='store_true', default=False,\n help='Enable ONNX version of Silero model for faster performance with lower resource usage. Default is False.')\n\n parser.add_argument('--webrtc_sensitivity', type=int, default=3,\n help='Sensitivity level for WebRTC Voice Activity Detection (VAD), with a range from 0 to 3. Higher values make the model less sensitive, useful for cleaner environments. Default is 3.')\n\n parser.add_argument('--min_length_of_recording', type=float, default=1.1,\n help='Minimum duration of valid recordings in seconds. This prevents very short recordings from being processed, which could be caused by noise or accidental sounds. Default is 1.1 seconds.')\n\n parser.add_argument('--min_gap_between_recordings', type=float, default=0,\n help='Minimum time (in seconds) between consecutive recordings. Setting this helps avoid overlapping recordings when there’s a brief silence between them. Default is 0 seconds.')\n\n parser.add_argument('--enable_realtime_transcription', action='store_true', default=True,\n help='Enable continuous real-time transcription of audio as it is received. When enabled, transcriptions are sent in near real-time. Default is True.')\n\n parser.add_argument('--realtime_processing_pause', type=float, default=0.02,\n help='Time interval (in seconds) between processing audio chunks for real-time transcription. Lower values increase responsiveness but may put more load on the CPU. Default is 0.02 seconds.')\n\n parser.add_argument('--silero_deactivity_detection', action='store_true', default=True,\n help='Use the Silero model for end-of-speech detection. This option can provide more robust silence detection in noisy environments, though it consumes more GPU resources. Default is True.')\n\n parser.add_argument('--early_transcription_on_silence', type=float, default=0.2,\n help='Start transcription after the specified seconds of silence. This is useful when you want to trigger transcription mid-speech when there is a brief pause. Should be lower than post_speech_silence_duration. Set to 0 to disable. Default is 0.2 seconds.')\n\n parser.add_argument('--beam_size', type=int, default=5,\n help='Beam size for the main transcription model. Larger values may improve transcription accuracy but increase the processing time. Default is 5.')\n\n parser.add_argument('--beam_size_realtime', type=int, default=3,\n help='Beam size for the real-time transcription model. A smaller beam size allows for faster real-time processing but may reduce accuracy. Default is 3.')\n\n parser.add_argument('--initial_prompt', type=str,\n default=\"Incomplete thoughts should end with '...'. Examples of complete thoughts: 'The sky is blue.' 'She walked home.' Examples of incomplete thoughts: 'When the sky...' 'Because he...'\",\n help='Initial prompt that guides the transcription model to produce transcriptions in a particular style or format. The default provides instructions for handling sentence completions and ellipsis usage.')\n\n parser.add_argument('--end_of_sentence_detection_pause', type=float, default=0.45,\n help='The duration of silence (in seconds) that the model should interpret as the end of a sentence. This helps the system detect when to finalize the transcription of a sentence. Default is 0.45 seconds.')\n\n parser.add_argument('--unknown_sentence_detection_pause', type=float, default=0.7,\n help='The duration of pause (in seconds) that the model should interpret as an incomplete or unknown sentence. This is useful for identifying when a sentence is trailing off or unfinished. Default is 0.7 seconds.')\n\n parser.add_argument('--mid_sentence_detection_pause', type=float, default=2.0,\n help='The duration of pause (in seconds) that the model should interpret as a mid-sentence break. Longer pauses can indicate a pause in speech but not necessarily the end of a sentence. Default is 2.0 seconds.')\n\n parser.add_argument('--wake_words_sensitivity', type=float, default=0.5,\n help='Sensitivity level for wake word detection, with a range from 0 (most sensitive) to 1 (least sensitive). Adjust this value based on your environment to ensure reliable wake word detection. Default is 0.5.')\n\n parser.add_argument('--wake_word_timeout', type=float, default=5.0,\n help='Maximum time in seconds that the system will wait for a wake word before timing out. After this timeout, the system stops listening for wake words until reactivated. Default is 5.0 seconds.')\n\n parser.add_argument('--wake_word_activation_delay', type=float, default=0,\n help='The delay in seconds before the wake word detection is activated after the system starts listening. This prevents false positives during the start of a session. Default is 0 seconds.')\n\n parser.add_argument('--wakeword_backend', type=str, default='none',\n help='The backend used for wake word detection. You can specify different backends such as \"default\" or any custom implementations depending on your setup. Default is \"pvporcupine\".')\n\n parser.add_argument('--openwakeword_model_paths', type=str, nargs='*',\n help='A list of file paths to OpenWakeWord models. This is useful if you are using OpenWakeWord for wake word detection and need to specify custom models.')\n\n parser.add_argument('--openwakeword_inference_framework', type=str, default='tensorflow',\n help='The inference framework to use for OpenWakeWord models. Supported frameworks could include \"tensorflow\", \"pytorch\", etc. Default is \"tensorflow\".')\n\n parser.add_argument('--wake_word_buffer_duration', type=float, default=1.0,\n help='Duration of the buffer in seconds for wake word detection. This sets how long the system will store the audio before and after detecting the wake word. Default is 1.0 seconds.')\n\n parser.add_argument('--use_main_model_for_realtime', action='store_true',\n help='Enable this option if you want to use the main model for real-time transcription, instead of the smaller, faster real-time model. Using the main model may provide better accuracy but at the cost of higher processing time.')\n\n parser.add_argument('--use_extended_logging', action='store_true',\n help='Writes extensive log messages for the recording worker, that processes the audio chunks.')\n\n parser.add_argument('--compute_type', type=str, default='default',\n help='Type of computation to use. See https://opennmt.net/CTranslate2/quantization.html')\n\n parser.add_argument('--gpu_device_index', type=int, default=0,\n help='Index of the GPU device to use. Default is None.')\n \n parser.add_argument('--device', type=str, default='cuda',\n help='Device for model to use. Can either be \"cuda\" or \"cpu\". Default is cuda.')\n \n parser.add_argument('--handle_buffer_overflow', action='store_true',\n help='Handle buffer overflow during transcription. Default is False.')\n\n parser.add_argument('--suppress_tokens', type=int, default=[-1], nargs='*', help='Suppress tokens during transcription. Default is [-1].')\n\n parser.add_argument('--allowed_latency_limit', type=int, default=100,\n help='Maximal amount of chunks that can be unprocessed in queue before discarding chunks.. Default is 100.')\n\n parser.add_argument('--faster_whisper_vad_filter', action='store_true',\n help='Enable VAD filter for Faster Whisper. Default is False.')\n\n parser.add_argument('--logchunks', action='store_true', help='Enable logging of incoming audio chunks (periods)')\n\n # Parse arguments\n args = parser.parse_args()\n\n debug_logging = args.debug\n extended_logging = args.use_extended_logging\n writechunks = args.write\n log_incoming_chunks = args.logchunks\n dynamic_silence_timing = args.silence_timing\n\n\n ws_logger = logging.getLogger('websockets')\n if args.debug_websockets:\n # If app debug is on, let websockets be verbose too\n ws_logger.setLevel(logging.DEBUG)\n # Ensure it uses the handler configured by basicConfig\n ws_logger.propagate = False # Prevent duplicate messages if it also propagates to root\n else:\n # If app debug is off, silence websockets below WARNING\n ws_logger.setLevel(logging.WARNING)\n ws_logger.propagate = True # Allow WARNING/ERROR messages to reach root logger's handler\n\n # Replace escaped newlines with actual newlines in initial_prompt\n if args.initial_prompt:\n args.initial_prompt = args.initial_prompt.replace(\"\\\\n\", \"\\n\")\n\n if args.initial_prompt_realtime:\n args.initial_prompt_realtime = args.initial_prompt_realtime.replace(\"\\\\n\", \"\\n\")\n\n return args\n\ndef _recorder_thread(loop):\n global recorder, stop_recorder\n print(f\"{bcolors.OKGREEN}Initializing RealtimeSTT server with parameters:{bcolors.ENDC}\")\n for key, value in recorder_config.items():\n print(f\" {bcolors.OKBLUE}{key}{bcolors.ENDC}: {value}\")\n recorder = AudioToTextRecorder(**recorder_config)\n print(f\"{bcolors.OKGREEN}{bcolors.BOLD}RealtimeSTT initialized{bcolors.ENDC}\")\n recorder_ready.set()\n \n def process_text(full_sentence):\n global prev_text\n prev_text = \"\"\n full_sentence = preprocess_text(full_sentence)\n message = json.dumps({\n 'type': 'fullSentence',\n 'text': full_sentence\n })\n # Use the passed event loop here\n asyncio.run_coroutine_threadsafe(audio_queue.put(message), loop)\n\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n\n if extended_logging:\n print(f\" [{timestamp}] Full text: {bcolors.BOLD}Sentence:{bcolors.ENDC} {bcolors.OKGREEN}{full_sentence}{bcolors.ENDC}\\n\", flush=True, end=\"\")\n else:\n print(f\"\\r[{timestamp}] {bcolors.BOLD}Sentence:{bcolors.ENDC} {bcolors.OKGREEN}{full_sentence}{bcolors.ENDC}\\n\")\n try:\n while not stop_recorder:\n recorder.text(process_text)\n except KeyboardInterrupt:\n print(f\"{bcolors.WARNING}Exiting application due to keyboard interrupt{bcolors.ENDC}\")\n\ndef decode_and_resample(\n audio_data,\n original_sample_rate,\n target_sample_rate):\n\n # Decode 16-bit PCM data to numpy array\n if original_sample_rate == target_sample_rate:\n return audio_data\n\n audio_np = np.frombuffer(audio_data, dtype=np.int16)\n\n # Calculate the number of samples after resampling\n num_original_samples = len(audio_np)\n num_target_samples = int(num_original_samples * target_sample_rate /\n original_sample_rate)\n\n # Resample the audio\n resampled_audio = resample(audio_np, num_target_samples)\n\n return resampled_audio.astype(np.int16).tobytes()\n\nasync def control_handler(websocket):\n debug_print(f\"New control connection from {websocket.remote_address}\")\n print(f\"{bcolors.OKGREEN}Control client connected{bcolors.ENDC}\")\n global recorder\n control_connections.add(websocket)\n try:\n async for message in websocket:\n debug_print(f\"Received control message: {message[:200]}...\")\n if not recorder_ready.is_set():\n print(f\"{bcolors.WARNING}Recorder not ready{bcolors.ENDC}\")\n continue\n if isinstance(message, str):\n # Handle text message (command)\n try:\n command_data = json.loads(message)\n command = command_data.get(\"command\")\n if command == \"set_parameter\":\n parameter = command_data.get(\"parameter\")\n value = command_data.get(\"value\")\n if parameter in allowed_parameters and hasattr(recorder, parameter):\n setattr(recorder, parameter, value)\n # Format the value for output\n if isinstance(value, float):\n value_formatted = f\"{value:.2f}\"\n else:\n value_formatted = value\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n if extended_logging:\n print(f\" [{timestamp}] {bcolors.OKGREEN}Set recorder.{parameter} to: {bcolors.OKBLUE}{value_formatted}{bcolors.ENDC}\")\n # Optionally send a response back to the client\n await websocket.send(json.dumps({\"status\": \"success\", \"message\": f\"Parameter {parameter} set to {value}\"}))\n else:\n if not parameter in allowed_parameters:\n print(f\"{bcolors.WARNING}Parameter {parameter} is not allowed (set_parameter){bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Parameter {parameter} is not allowed (set_parameter)\"}))\n else:\n print(f\"{bcolors.WARNING}Parameter {parameter} does not exist (set_parameter){bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Parameter {parameter} does not exist (set_parameter)\"}))\n\n elif command == \"get_parameter\":\n parameter = command_data.get(\"parameter\")\n request_id = command_data.get(\"request_id\") # Get the request_id from the command data\n if parameter in allowed_parameters and hasattr(recorder, parameter):\n value = getattr(recorder, parameter)\n if isinstance(value, float):\n value_formatted = f\"{value:.2f}\"\n else:\n value_formatted = f\"{value}\"\n\n value_truncated = value_formatted[:39] + \"…\" if len(value_formatted) > 40 else value_formatted\n\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n if extended_logging:\n print(f\" [{timestamp}] {bcolors.OKGREEN}Get recorder.{parameter}: {bcolors.OKBLUE}{value_truncated}{bcolors.ENDC}\")\n response = {\"status\": \"success\", \"parameter\": parameter, \"value\": value}\n if request_id is not None:\n response[\"request_id\"] = request_id\n await websocket.send(json.dumps(response))\n else:\n if not parameter in allowed_parameters:\n print(f\"{bcolors.WARNING}Parameter {parameter} is not allowed (get_parameter){bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Parameter {parameter} is not allowed (get_parameter)\"}))\n else:\n print(f\"{bcolors.WARNING}Parameter {parameter} does not exist (get_parameter){bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Parameter {parameter} does not exist (get_parameter)\"}))\n elif command == \"call_method\":\n method_name = command_data.get(\"method\")\n if method_name in allowed_methods:\n method = getattr(recorder, method_name, None)\n if method and callable(method):\n args = command_data.get(\"args\", [])\n kwargs = command_data.get(\"kwargs\", {})\n method(*args, **kwargs)\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n print(f\" [{timestamp}] {bcolors.OKGREEN}Called method recorder.{bcolors.OKBLUE}{method_name}{bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"success\", \"message\": f\"Method {method_name} called\"}))\n else:\n print(f\"{bcolors.WARNING}Recorder does not have method {method_name}{bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Recorder does not have method {method_name}\"}))\n else:\n print(f\"{bcolors.WARNING}Method {method_name} is not allowed{bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Method {method_name} is not allowed\"}))\n else:\n print(f\"{bcolors.WARNING}Unknown command: {command}{bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": f\"Unknown command {command}\"}))\n except json.JSONDecodeError:\n print(f\"{bcolors.WARNING}Received invalid JSON command{bcolors.ENDC}\")\n await websocket.send(json.dumps({\"status\": \"error\", \"message\": \"Invalid JSON command\"}))\n else:\n print(f\"{bcolors.WARNING}Received unknown message type on control connection{bcolors.ENDC}\")\n except websockets.exceptions.ConnectionClosed as e:\n print(f\"{bcolors.WARNING}Control client disconnected: {e}{bcolors.ENDC}\")\n finally:\n control_connections.remove(websocket)\n\nasync def data_handler(websocket):\n global writechunks, wav_file\n print(f\"{bcolors.OKGREEN}Data client connected{bcolors.ENDC}\")\n data_connections.add(websocket)\n try:\n while True:\n message = await websocket.recv()\n if isinstance(message, bytes):\n if extended_logging:\n debug_print(f\"Received audio chunk (size: {len(message)} bytes)\")\n elif log_incoming_chunks:\n print(\".\", end='', flush=True)\n # Handle binary message (audio data)\n metadata_length = int.from_bytes(message[:4], byteorder='little')\n metadata_json = message[4:4+metadata_length].decode('utf-8')\n metadata = json.loads(metadata_json)\n sample_rate = metadata['sampleRate']\n\n if 'server_sent_to_stt' in metadata:\n stt_received_ns = time.time_ns()\n metadata[\"stt_received\"] = stt_received_ns\n metadata[\"stt_received_formatted\"] = format_timestamp_ns(stt_received_ns)\n print(f\"Server received audio chunk of length {len(message)} bytes, metadata: {metadata}\")\n\n if extended_logging:\n debug_print(f\"Processing audio chunk with sample rate {sample_rate}\")\n chunk = message[4+metadata_length:]\n\n if writechunks:\n if not wav_file:\n wav_file = wave.open(writechunks, 'wb')\n wav_file.setnchannels(CHANNELS)\n wav_file.setsampwidth(pyaudio.get_sample_size(FORMAT))\n wav_file.setframerate(sample_rate)\n\n wav_file.writeframes(chunk)\n\n if sample_rate != 16000:\n resampled_chunk = decode_and_resample(chunk, sample_rate, 16000)\n if extended_logging:\n debug_print(f\"Resampled chunk size: {len(resampled_chunk)} bytes\")\n recorder.feed_audio(resampled_chunk)\n else:\n recorder.feed_audio(chunk)\n else:\n print(f\"{bcolors.WARNING}Received non-binary message on data connection{bcolors.ENDC}\")\n except websockets.exceptions.ConnectionClosed as e:\n print(f\"{bcolors.WARNING}Data client disconnected: {e}{bcolors.ENDC}\")\n finally:\n data_connections.remove(websocket)\n recorder.clear_audio_queue() # Ensure audio queue is cleared if client disconnects\n\nasync def broadcast_audio_messages():\n while True:\n message = await audio_queue.get()\n for conn in list(data_connections):\n try:\n timestamp = datetime.now().strftime('%H:%M:%S.%f')[:-3]\n\n if extended_logging:\n print(f\" [{timestamp}] Sending message: {bcolors.OKBLUE}{message}{bcolors.ENDC}\\n\", flush=True, end=\"\")\n await conn.send(message)\n except websockets.exceptions.ConnectionClosed:\n data_connections.remove(conn)\n\n# Helper function to create event loop bound closures for callbacks\ndef make_callback(loop, callback):\n def inner_callback(*args, **kwargs):\n callback(*args, **kwargs, loop=loop)\n return inner_callback\n\nasync def main_async(): \n global stop_recorder, recorder_config, global_args\n args = parse_arguments()\n global_args = args\n\n # Get the event loop here and pass it to the recorder thread\n loop = asyncio.get_event_loop()\n\n recorder_config = {\n 'model': args.model,\n 'download_root': args.root,\n 'realtime_model_type': args.rt_model,\n 'language': args.lang,\n 'batch_size': args.batch,\n 'init_realtime_after_seconds': args.init_realtime_after_seconds,\n 'realtime_batch_size': args.realtime_batch_size,\n 'initial_prompt_realtime': args.initial_prompt_realtime,\n 'input_device_index': args.input_device,\n 'silero_sensitivity': args.silero_sensitivity,\n 'silero_use_onnx': args.silero_use_onnx,\n 'webrtc_sensitivity': args.webrtc_sensitivity,\n 'post_speech_silence_duration': args.unknown_sentence_detection_pause,\n 'min_length_of_recording': args.min_length_of_recording,\n 'min_gap_between_recordings': args.min_gap_between_recordings,\n 'enable_realtime_transcription': args.enable_realtime_transcription,\n 'realtime_processing_pause': args.realtime_processing_pause,\n 'silero_deactivity_detection': args.silero_deactivity_detection,\n 'early_transcription_on_silence': args.early_transcription_on_silence,\n 'beam_size': args.beam_size,\n 'beam_size_realtime': args.beam_size_realtime,\n 'initial_prompt': args.initial_prompt,\n 'wake_words': args.wake_words,\n 'wake_words_sensitivity': args.wake_words_sensitivity,\n 'wake_word_timeout': args.wake_word_timeout,\n 'wake_word_activation_delay': args.wake_word_activation_delay,\n 'wakeword_backend': args.wakeword_backend,\n 'openwakeword_model_paths': args.openwakeword_model_paths,\n 'openwakeword_inference_framework': args.openwakeword_inference_framework,\n 'wake_word_buffer_duration': args.wake_word_buffer_duration,\n 'use_main_model_for_realtime': args.use_main_model_for_realtime,\n 'spinner': False,\n 'use_microphone': False,\n\n 'on_realtime_transcription_update': make_callback(loop, text_detected),\n 'on_recording_start': make_callback(loop, on_recording_start),\n 'on_recording_stop': make_callback(loop, on_recording_stop),\n 'on_vad_detect_start': make_callback(loop, on_vad_detect_start),\n 'on_vad_detect_stop': make_callback(loop, on_vad_detect_stop),\n 'on_wakeword_detected': make_callback(loop, on_wakeword_detected),\n 'on_wakeword_detection_start': make_callback(loop, on_wakeword_detection_start),\n 'on_wakeword_detection_end': make_callback(loop, on_wakeword_detection_end),\n 'on_transcription_start': make_callback(loop, on_transcription_start),\n 'on_turn_detection_start': make_callback(loop, on_turn_detection_start),\n 'on_turn_detection_stop': make_callback(loop, on_turn_detection_stop),\n\n # 'on_recorded_chunk': make_callback(loop, on_recorded_chunk),\n 'no_log_file': True, # Disable logging to file\n 'use_extended_logging': args.use_extended_logging,\n 'level': loglevel,\n 'compute_type': args.compute_type,\n 'gpu_device_index': args.gpu_device_index,\n 'device': args.device,\n 'handle_buffer_overflow': args.handle_buffer_overflow,\n 'suppress_tokens': args.suppress_tokens,\n 'allowed_latency_limit': args.allowed_latency_limit,\n 'faster_whisper_vad_filter': args.faster_whisper_vad_filter,\n }\n\n try:\n # Attempt to start control and data servers\n control_server = await websockets.serve(control_handler, \"localhost\", args.control)\n data_server = await websockets.serve(data_handler, \"localhost\", args.data)\n print(f\"{bcolors.OKGREEN}Control server started on {bcolors.OKBLUE}ws://localhost:{args.control}{bcolors.ENDC}\")\n print(f\"{bcolors.OKGREEN}Data server started on {bcolors.OKBLUE}ws://localhost:{args.data}{bcolors.ENDC}\")\n\n # Start the broadcast and recorder threads\n broadcast_task = asyncio.create_task(broadcast_audio_messages())\n\n recorder_thread = threading.Thread(target=_recorder_thread, args=(loop,))\n recorder_thread.start()\n recorder_ready.wait()\n\n print(f\"{bcolors.OKGREEN}Server started. Press Ctrl+C to stop the server.{bcolors.ENDC}\")\n\n # Run server tasks\n await asyncio.gather(control_server.wait_closed(), data_server.wait_closed(), broadcast_task)\n except OSError as e:\n print(f\"{bcolors.FAIL}Error: Could not start server on specified ports. It’s possible another instance of the server is already running, or the ports are being used by another application.{bcolors.ENDC}\")\n except KeyboardInterrupt:\n print(f\"{bcolors.WARNING}Server interrupted by user, shutting down...{bcolors.ENDC}\")\n finally:\n # Shutdown procedures for recorder and server threads\n await shutdown_procedure()\n print(f\"{bcolors.OKGREEN}Server shutdown complete.{bcolors.ENDC}\")\n\nasync def shutdown_procedure():\n global stop_recorder, recorder_thread\n if recorder:\n stop_recorder = True\n recorder.abort()\n recorder.stop()\n recorder.shutdown()\n print(f\"{bcolors.OKGREEN}Recorder shut down{bcolors.ENDC}\")\n\n if recorder_thread:\n recorder_thread.join()\n print(f\"{bcolors.OKGREEN}Recorder thread finished{bcolors.ENDC}\")\n\n tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]\n for task in tasks:\n task.cancel()\n await asyncio.gather(*tasks, return_exceptions=True)\n\n print(f\"{bcolors.OKGREEN}All tasks cancelled, closing event loop now.{bcolors.ENDC}\")\n\ndef main():\n try:\n asyncio.run(main_async())\n except KeyboardInterrupt:\n # Capture any final KeyboardInterrupt to prevent it from showing up in logs\n print(f\"{bcolors.WARNING}Server interrupted by user.{bcolors.ENDC}\")\n exit(0)\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "__init__.py",
"content": ""
},
{
"path": "docker-compose.yml",
"content": "services:\n rtstt:\n build:\n context: .\n target: gpu # or cpu\n image: rtstt\n container_name: rtstt\n volumes:\n # - ./RealtimeSTT:/app/RealtimeSTT\n # - ./example_browserclient:/app/example_browserclient\n - cache:/root/.cache\n ports:\n - \"9001:9001\"\n\n # if 'gpu' target\n deploy:\n resources:\n reservations:\n devices:\n - capabilities: [\"gpu\"]\n nginx:\n image: nginx:latest\n container_name: nginx_web\n ports:\n - \"8081:80\"\n volumes:\n - ./example_browserclient:/usr/share/nginx/html\n\nvolumes:\n cache:"
},
{
"path": "example_app/README.MD",
"content": "# GPU Support with CUDA (recommended)\n\nSteps for a **GPU-optimized** installation:\n\n1. **Install NVIDIA CUDA Toolkit 11.8**:\n - Visit [NVIDIA CUDA Toolkit Archive](https://developer.nvidia.com/cuda-11-8-0-download-archive).\n - Select version 11.\n - Download and install the software.\n\n2. **Install NVIDIA cuDNN 8.7.0 for CUDA 11.x**:\n - Visit [NVIDIA cuDNN Archive](https://developer.nvidia.com/rdp/cudnn-archive).\n - Click on \"Download cuDNN v8.7.0 (November 28th, 2022), for CUDA 11.x\".\n - Download and install the software.\n\n3. **Install ffmpeg**:\n\n You can download an installer for your OS from the [ffmpeg Website](https://ffmpeg.org/download.html). \n \n Or use a package manager:\n\n - **On Ubuntu or Debian**:\n ```bash\n sudo apt update && sudo apt install ffmpeg\n ```\n\n - **On Arch Linux**:\n ```bash\n sudo pacman -S ffmpeg\n ```\n\n - **On MacOS using Homebrew** ([https://brew.sh/](https://brew.sh/)):\n ```bash\n brew install ffmpeg\n ```\n\n - **On Windows using Chocolatey** ([https://chocolatey.org/](https://chocolatey.org/)):\n ```bash\n choco install ffmpeg\n ```\n\n - **On Windows using Scoop** ([https://scoop.sh/](https://scoop.sh/)):\n ```bash\n scoop install ffmpeg\n ``` \n\n4. **ElevenlabsEngine**\n - If you plan to use the `ElevenlabsEngine`, you need `mpv` is installed on your system for streaming mpeg audio\n\n - **macOS**:\n ```bash\n brew install mpv\n ```\n\n - **Linux and Windows**: Visit [mpv.io](https://mpv.io/) for installation instructions.\n\n5. **Install PyTorch with CUDA support**:\n - run install_gpu.bat\n\n6. **Configure script**\n - open ui_openai_voice_interface.py and configure your engine, set API keys, Azure service region, language etc"
},
{
"path": "example_app/install_cpu.bat",
"content": "@echo off\ncd /d %~dp0\n\nREM Check if the venv directory exists\nif not exist test_env\\Scripts\\python.exe (\n echo Creating VENV\n python -m venv test_env\n) else (\n echo VENV already exists\n)\n\necho Activating VENV\nstart cmd /k \"call test_env\\Scripts\\activate.bat && pip install --upgrade RealtimeSTT==0.1.4 && pip install --upgrade RealtimeTTS==0.1.3 && pip install pysoundfile==0.9.0.post1 openai==0.27.8 keyboard==0.13.5 PyQt5==5.15.9 sounddevice==0.4.6 wavio==0.0.7\""
},
{
"path": "example_app/install_gpu.bat",
"content": "@echo off\ncd /d %~dp0\n\nREM Check if the venv directory exists\nif not exist test_env\\Scripts\\python.exe (\n echo Creating VENV\n python -m venv test_env\n) else (\n echo VENV already exists\n)\n\necho Activating VENV\nstart cmd /k \"call test_env\\Scripts\\activate.bat && pip install --upgrade RealtimeSTT==0.1.4 && pip install --upgrade RealtimeTTS==0.1.3 && pip uninstall torch --yes && pip install torch==2.0.1+cu118 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu118 && pip install pysoundfile==0.9.0.post1 openai==0.27.8 keyboard==0.13.5 PyQt5==5.15.9 sounddevice==0.4.6 wavio==0.0.7\""
},
{
"path": "example_app/start.bat",
"content": "@echo off\ncd /d %~dp0\n\nREM Check if the venv directory exists\nif not exist test_env\\Scripts\\python.exe (\n echo Creating VENV\n python -m venv test_env\n) else (\n echo VENV already exists\n)\n\n\n:: OpenAI API Key https://platform.openai.com/\nset OPENAI_API_KEY=\n\n:: Microsoft Azure API Key https://portal.azure.com/\nset AZURE_SPEECH_KEY=\n\n:: Elevenlabs API Key https://www.elevenlabs.io/Elevenlabs\nset ELEVENLABS_API_KEY=\n\n\necho Activating VENV\nstart cmd /k \"call test_env\\Scripts\\activate.bat && python ui_openai_voice_interface.py\""
},
{
"path": "example_app/ui_openai_voice_interface.py",
"content": "if __name__ == '__main__':\n\n from RealtimeTTS import TextToAudioStream, AzureEngine, ElevenlabsEngine, SystemEngine\n from RealtimeSTT import AudioToTextRecorder\n\n from PyQt5.QtCore import Qt, QTimer, QEvent, pyqtSignal, QThread\n from PyQt5.QtGui import QColor, QPainter, QFontMetrics, QFont, QMouseEvent\n from PyQt5.QtWidgets import QApplication, QWidget, QDesktopWidget, QMenu, QAction\n\n import os\n import openai\n import sys\n import time\n import sounddevice as sd\n import numpy as np\n import wavio\n import keyboard\n\n max_history_messages = 6\n return_to_wakewords_after_silence = 12\n start_with_wakeword = False\n start_engine = \"Azure\" # Azure, Elevenlabs\n recorder_model = \"large-v2\"\n language = \"en\"\n azure_speech_region = \"eastus\"\n openai_model = \"gpt-3.5-turbo\" # gpt-3.5-turbo, gpt-4, gpt-3.5-turbo-0613 / gpt-3.5-turbo-16k-0613 / gpt-4-0613 / gpt-4-32k-0613\n\n openai.api_key = os.environ.get(\"OPENAI_API_KEY\")\n\n user_font_size = 22\n user_color = QColor(0, 188, 242) # turquoise\n\n assistant_font_size = 24\n assistant_color = QColor(239, 98, 166) # pink\n\n voice_azure = \"en-GB-SoniaNeural\"\n voice_system = \"Zira\"\n #voice_system = \"Hazel\"\n prompt = \"Be concise, polite, and casual with a touch of sass. Aim for short, direct responses, as if we're talking.\"\n elevenlabs_model = \"eleven_monolingual_v1\"\n\n if language == \"de\":\n elevenlabs_model = \"eleven_multilingual_v1\"\n voice_system = \"Katja\"\n voice_azure = \"de-DE-MajaNeural\"\n prompt = 'Sei präzise, höflich und locker, mit einer Prise Schlagfertigkeit. Antworte kurz und direkt, als ob wir gerade sprechen.'\n \n print (\"Click the top right corner to change the engine\")\n print (\"Press ESC to stop the current playback\")\n\n system_prompt_message = {\n 'role': 'system',\n 'content': prompt\n }\n\n def generate_response(messages):\n \"\"\"Generate assistant's response using OpenAI.\"\"\"\n for chunk in openai.ChatCompletion.create(model=openai_model, messages=messages, stream=True, logit_bias={35309:-100, 36661:-100}):\n text_chunk = chunk[\"choices\"][0][\"delta\"].get(\"content\")\n if text_chunk:\n yield text_chunk\n\n history = []\n MAX_WINDOW_WIDTH = 1600\n MAX_WIDTH_ASSISTANT = 1200\n MAX_WIDTH_USER = 1500\n\n class AudioPlayer(QThread):\n def __init__(self, file_path):\n super(AudioPlayer, self).__init__()\n self.file_path = file_path\n\n def run(self):\n wav = wavio.read(self.file_path)\n sound = wav.data.astype(np.float32) / np.iinfo(np.int16).max \n sd.play(sound, wav.rate)\n sd.wait()\n\n class TextRetrievalThread(QThread):\n textRetrieved = pyqtSignal(str)\n\n def __init__(self, recorder):\n super().__init__()\n self.recorder = recorder\n self.active = False \n\n def run(self):\n while True:\n if self.active: \n text = self.recorder.text()\n self.recorder.wake_word_activation_delay = return_to_wakewords_after_silence\n self.textRetrieved.emit(text)\n self.active = False\n time.sleep(0.1) \n\n def activate(self):\n self.active = True \n\n class TransparentWindow(QWidget):\n updateUI = pyqtSignal()\n clearAssistantTextSignal = pyqtSignal()\n clearUserTextSignal = pyqtSignal()\n\n def __init__(self):\n super().__init__()\n\n self.setGeometry(1, 1, 1, 1) \n\n self.setWindowTitle(\"Transparent Window\")\n self.setAttribute(Qt.WA_TranslucentBackground)\n self.setWindowFlags(Qt.FramelessWindowHint | Qt.WindowStaysOnTopHint)\n\n self.big_symbol_font = QFont('Arial', 32)\n self.small_symbol_font = QFont('Arial', 17)\n self.user_font = QFont('Arial', user_font_size)\n self.assistant_font = QFont('Arial', assistant_font_size) \n self.assistant_font.setItalic(True) \n\n self.big_symbol_text = \"\"\n self.small_symbol_text = \"\"\n self.user_text = \"\"\n self.assistant_text = \"\"\n self.displayed_user_text = \"\"\n self.displayed_assistant_text = \"\"\n self.stream = None\n self.text_retrieval_thread = None\n\n self.user_text_timer = QTimer(self)\n self.assistant_text_timer = QTimer(self)\n self.user_text_timer.timeout.connect(self.clear_user_text)\n self.assistant_text_timer.timeout.connect(self.clear_assistant_text)\n\n self.clearUserTextSignal.connect(self.init_clear_user_text)\n self.clearAssistantTextSignal.connect(self.init_clear_assistant_text)\n self.user_text_opacity = 255 \n self.assistant_text_opacity = 255 \n self.updateUI.connect(self.update_self)\n self.audio_player = None\n\n self.run_fade_user = False\n self.run_fade_assistant = False\n\n self.menu = QMenu()\n self.menu.setStyleSheet(\"\"\"\n QMenu {\n background-color: black;\n color: white;\n border-radius: 10px;\n }\n QMenu::item:selected {\n background-color: #555555;\n }\n \"\"\")\n\n self.elevenlabs_action = QAction(\"Elevenlabs\", self)\n self.azure_action = QAction(\"Azure\", self)\n self.system_action = QAction(\"System\", self)\n self.quit_action = QAction(\"Quit\", self)\n\n self.menu.addAction(self.elevenlabs_action)\n self.menu.addAction(self.azure_action)\n self.menu.addAction(self.system_action)\n self.menu.addSeparator() \n self.menu.addAction(self.quit_action)\n\n self.elevenlabs_action.triggered.connect(lambda: self.select_engine(\"Elevenlabs\"))\n self.azure_action.triggered.connect(lambda: self.select_engine(\"Azure\"))\n self.system_action.triggered.connect(lambda: self.select_engine(\"System\"))\n self.quit_action.triggered.connect(self.close_application)\n\n def mousePressEvent(self, event: QMouseEvent):\n if event.button() == Qt.LeftButton:\n if event.pos().x() >= self.width() - 100 and event.pos().y() <= 100:\n self.menu.exec_(self.mapToGlobal(event.pos())) \n\n def close_application(self):\n if self.recorder:\n self.recorder.shutdown() \n QApplication.quit() \n\n def init(self):\n\n self.select_engine(start_engine)\n\n # recorder = AudioToTextRecorder(spinner=False, model=\"large-v2\", language=\"de\", on_recording_start=recording_start, silero_sensitivity=0.4, post_speech_silence_duration=0.4, min_length_of_recording=0.3, min_gap_between_recordings=0.01, realtime_preview_resolution = 0.01, realtime_preview = True, realtime_preview_model = \"small\", on_realtime_preview=text_detected)\n\n self.recorder = AudioToTextRecorder(\n model=recorder_model,\n language=language,\n wake_words=\"Jarvis\",\n silero_use_onnx=False,\n spinner=True,\n silero_sensitivity=0.2,\n webrtc_sensitivity=3,\n on_recording_start=self.on_recording_start,\n on_vad_detect_start=self.on_vad_detect_start,\n on_wakeword_detection_start=self.on_wakeword_detection_start,\n on_transcription_start=self.on_transcription_start,\n post_speech_silence_duration=0.4, \n min_length_of_recording=0.3, \n min_gap_between_recordings=0.01, \n enable_realtime_transcription = True,\n realtime_processing_pause = 0.01, \n realtime_model_type = \"tiny\",\n on_realtime_transcription_stabilized=self.text_detected\n )\n if not start_with_wakeword:\n self.recorder.wake_word_activation_delay = return_to_wakewords_after_silence\n \n self.text_retrieval_thread = TextRetrievalThread(self.recorder)\n self.text_retrieval_thread.textRetrieved.connect(self.process_user_text)\n self.text_retrieval_thread.start()\n self.text_retrieval_thread.activate()\n\n keyboard.on_press_key('esc', self.on_escape)\n\n def closeEvent(self, event):\n if self.recorder:\n self.recorder.shutdown() \n\n def select_engine(self, engine_name):\n if self.stream:\n self.stream.stop()\n self.stream = None\n\n engine = None\n\n if engine_name == \"Azure\":\n engine = AzureEngine(\n os.environ.get(\"AZURE_SPEECH_KEY\"),\n os.environ.get(\"AZURE_SPEECH_REGION\"),\n voice_azure,\n rate=24,\n pitch=10,\n )\n\n elif engine_name == \"Elevenlabs\":\n engine = ElevenlabsEngine(\n os.environ.get(\"ELEVENLABS_API_KEY\"),\n model=elevenlabs_model\n )\n else:\n engine = SystemEngine(\n voice=voice_system,\n #print_installed_voices=True\n )\n\n self.stream = TextToAudioStream(\n engine,\n on_character=self.on_character,\n on_text_stream_stop=self.on_text_stream_stop,\n on_text_stream_start=self.on_text_stream_start,\n on_audio_stream_stop=self.on_audio_stream_stop,\n log_characters=True\n )\n sys.stdout.write('\\033[K') # Clear to the end of line\n sys.stdout.write('\\r') # Move the cursor to the beginning of the line\n print (f\"Using {engine_name} engine\")\n\n\n def text_detected(self, text):\n self.run_fade_user = False\n if self.user_text_timer.isActive():\n self.user_text_timer.stop()\n self.user_text_opacity = 255 \n self.user_text = text\n self.updateUI.emit()\n\n def on_escape(self, e):\n if self.stream.is_playing():\n self.stream.stop()\n\n def showEvent(self, event: QEvent):\n super().showEvent(event)\n if event.type() == QEvent.Show:\n self.set_symbols(\"⌛\", \"🚀\")\n QTimer.singleShot(1000, self.init) \n\n def on_character(self, char):\n if self.stream:\n self.assistant_text += char\n self.updateUI.emit()\n\n def on_text_stream_stop(self):\n print(\"\\\"\", end=\"\", flush=True)\n if self.stream:\n assistant_response = self.stream.text() \n self.assistant_text = assistant_response\n history.append({'role': 'assistant', 'content': assistant_response})\n\n def on_audio_stream_stop(self):\n self.set_symbols(\"🎙️\", \"⚪\")\n\n if self.stream:\n self.clearAssistantTextSignal.emit()\n self.text_retrieval_thread.activate()\n\n def generate_answer(self):\n self.run_fade_assistant = False\n if self.assistant_text_timer.isActive():\n self.assistant_text_timer.stop()\n\n history.append({'role': 'user', 'content': self.user_text})\n self.remove_assistant_text()\n assistant_response = generate_response([system_prompt_message] + history[-max_history_messages:])\n self.stream.feed(assistant_response)\n self.stream.play_async(minimum_sentence_length=6,\n buffer_threshold_seconds=2)\n\n def set_symbols(self, big_symbol, small_symbol):\n self.big_symbol_text = big_symbol\n self.small_symbol_text = small_symbol\n self.updateUI.emit()\n\n def on_text_stream_start(self):\n self.set_symbols(\"⌛\", \"👄\")\n\n def process_user_text(self, user_text):\n user_text = user_text.strip()\n if user_text:\n self.run_fade_user = False\n if self.user_text_timer.isActive():\n self.user_text_timer.stop()\n\n self.user_text_opacity = 255 \n self.user_text = user_text\n self.clearUserTextSignal.emit()\n print (f\"Me: \\\"{user_text}\\\"\\nAI: \\\"\", end=\"\", flush=True)\n self.set_symbols(\"⌛\", \"🧠\")\n QTimer.singleShot(100, self.generate_answer)\n\n def on_transcription_start(self):\n self.set_symbols(\"⌛\", \"📝\")\n\n def on_recording_start(self):\n self.text_storage = []\n self.ongoing_sentence = \"\"\n self.set_symbols(\"🎙️\", \"🔴\")\n\n def on_vad_detect_start(self):\n if self.small_symbol_text == \"💤\" or self.small_symbol_text == \"🚀\":\n self.audio_player = AudioPlayer(\"active.wav\")\n self.audio_player.start() \n\n self.set_symbols(\"🎙️\", \"⚪\")\n\n def on_wakeword_detection_start(self):\n self.audio_player = AudioPlayer(\"inactive.wav\")\n self.audio_player.start() \n\n self.set_symbols(\"\", \"💤\")\n\n def init_clear_user_text(self):\n if self.user_text_timer.isActive():\n self.user_text_timer.stop() \n self.user_text_timer.start(10000)\n\n def remove_user_text(self):\n self.user_text = \"\"\n self.user_text_opacity = 255 \n self.updateUI.emit()\n\n def fade_out_user_text(self):\n if not self.run_fade_user:\n return\n\n if self.user_text_opacity > 0:\n self.user_text_opacity -= 5 \n self.updateUI.emit()\n QTimer.singleShot(50, self.fade_out_user_text)\n else:\n self.run_fade_user = False\n self.remove_user_text() \n\n def clear_user_text(self):\n self.user_text_timer.stop()\n\n if not self.user_text:\n return\n\n self.user_text_opacity = 255\n self.run_fade_user = True\n self.fade_out_user_text()\n\n def init_clear_assistant_text(self):\n if self.assistant_text_timer.isActive():\n self.assistant_text_timer.stop() \n self.assistant_text_timer.start(10000)\n\n def remove_assistant_text(self):\n self.assistant_text = \"\"\n self.assistant_text_opacity = 255 \n self.updateUI.emit()\n\n def fade_out_assistant_text(self):\n if not self.run_fade_assistant:\n return\n \n if self.assistant_text_opacity > 0:\n self.assistant_text_opacity -= 5 \n self.updateUI.emit()\n QTimer.singleShot(50, self.fade_out_assistant_text)\n else:\n self.run_fade_assistant = False\n self.remove_assistant_text() \n\n def clear_assistant_text(self):\n self.assistant_text_timer.stop()\n\n if not self.assistant_text:\n return\n\n self.assistant_text_opacity = 255\n self.run_fade_assistant = True\n self.fade_out_assistant_text()\n\n def update_self(self):\n\n self.blockSignals(True)\n \n self.displayed_user_text, self.user_width = self.return_text_adjusted_to_width(self.user_text, self.user_font, MAX_WIDTH_USER)\n self.displayed_assistant_text, self.assistant_width = self.return_text_adjusted_to_width(self.assistant_text, self.assistant_font, MAX_WIDTH_ASSISTANT) \n\n fm_symbol = QFontMetrics(self.big_symbol_font)\n self.symbol_width = fm_symbol.width(self.big_symbol_text) + 3\n self.symbol_height = fm_symbol.height() + 8\n\n self.total_width = MAX_WINDOW_WIDTH\n\n fm_user = QFontMetrics(self.user_font)\n user_text_lines = (self.displayed_user_text.count(\"\\n\") + 1)\n self.user_height = fm_user.height() * user_text_lines + 7\n\n fm_assistant = QFontMetrics(self.assistant_font)\n assistant_text_lines = (self.displayed_assistant_text.count(\"\\n\") + 1)\n self.assistant_height = fm_assistant.height() * assistant_text_lines + 18\n\n self.total_height = sum([self.symbol_height, self.user_height, self.assistant_height])\n\n desktop = QDesktopWidget()\n screen_rect = desktop.availableGeometry(desktop.primaryScreen())\n self.setGeometry(screen_rect.right() - self.total_width - 50, 0, self.total_width + 50, self.total_height + 50)\n\n self.blockSignals(False)\n\n self.update()\n\n def drawTextWithOutline(self, painter, x, y, width, height, alignment, text, textColor, outlineColor, outline_size):\n painter.setPen(outlineColor)\n for dx, dy in [(-outline_size, 0), (outline_size, 0), (0, -outline_size), (0, outline_size),\n (-outline_size, -outline_size), (outline_size, -outline_size),\n (-outline_size, outline_size), (outline_size, outline_size)]:\n painter.drawText(x + dx, y + dy, width, height, alignment, text)\n\n painter.setPen(textColor)\n painter.drawText(x, y, width, height, alignment, text)\n\n def paintEvent(self, event):\n painter = QPainter(self)\n\n offsetX = 4\n offsetY = 5\n \n painter.setPen(QColor(255, 255, 255))\n\n # Draw symbol\n painter.setFont(self.big_symbol_font)\n if self.big_symbol_text:\n painter.drawText(self.total_width - self.symbol_width + 5 + offsetX, offsetY, self.symbol_width, self.symbol_height, Qt.AlignRight | Qt.AlignTop, self.big_symbol_text)\n painter.setFont(self.small_symbol_font)\n painter.drawText(self.total_width - self.symbol_width + 17 + offsetX, offsetY + 10, self.symbol_width, self.symbol_height, Qt.AlignRight | Qt.AlignBottom, self.small_symbol_text)\n else:\n painter.setFont(self.small_symbol_font)\n painter.drawText(self.total_width - 43 + offsetX, offsetY + 2, 50, 50, Qt.AlignRight | Qt.AlignBottom, self.small_symbol_text)\n\n # Draw User Text\n painter.setFont(self.user_font)\n user_x = self.total_width - self.user_width - 45 + offsetX\n user_y = offsetY + 15\n user_color_with_opacity = QColor(user_color.red(), user_color.green(), user_color.blue(), self.user_text_opacity)\n outline_color_with_opacity = QColor(0, 0, 0, self.user_text_opacity)\n self.drawTextWithOutline(painter, user_x, user_y, self.user_width, self.user_height, Qt.AlignRight | Qt.AlignTop, self.displayed_user_text, user_color_with_opacity, outline_color_with_opacity, 2)\n\n # Draw Assistant Text\n painter.setFont(self.assistant_font)\n assistant_x = self.total_width - self.assistant_width - 5 + offsetX\n assistant_y = self.user_height + offsetY + 15\n assistant_color_with_opacity = QColor(assistant_color.red(), assistant_color.green(), assistant_color.blue(), self.assistant_text_opacity)\n outline_color_with_opacity = QColor(0, 0, 0, self.assistant_text_opacity)\n self.drawTextWithOutline(painter, assistant_x, assistant_y, self.assistant_width, self.assistant_height, Qt.AlignRight | Qt.AlignTop, self.displayed_assistant_text, assistant_color_with_opacity, outline_color_with_opacity, 2)\n\n def return_text_adjusted_to_width(self, text, font, max_width_allowed):\n \"\"\"\n Line feeds are inserted so that the text width does never exceed max_width.\n Text is only broken up on whole words.\n \"\"\"\n fm = QFontMetrics(font)\n words = text.split(' ')\n adjusted_text = ''\n current_line = ''\n max_width_used = 0\n \n for word in words:\n current_width = fm.width(current_line + word)\n if current_width <= max_width_allowed:\n current_line += word + ' '\n else:\n line_width = fm.width(current_line)\n if line_width > max_width_used:\n max_width_used = line_width\n adjusted_text += current_line + '\\n'\n current_line = word + ' '\n \n line_width = fm.width(current_line)\n if line_width > max_width_used:\n max_width_used = line_width\n adjusted_text += current_line \n return adjusted_text.rstrip(), max_width_used \n\n app = QApplication(sys.argv)\n\n window = TransparentWindow()\n window.show()\n\n sys.exit(app.exec_())"
},
{
"path": "example_browserclient/client.js",
"content": "let socket = new WebSocket(\"ws://localhost:9001\");\nlet displayDiv = document.getElementById('textDisplay');\nlet server_available = false;\nlet mic_available = false;\nlet fullSentences = [];\n\nconst serverCheckInterval = 5000; // Check every 5 seconds\n\nfunction connectToServer() {\n socket = new WebSocket(\"ws://localhost:9001\");\n\n socket.onopen = function(event) {\n server_available = true;\n start_msg();\n };\n\n socket.onmessage = function(event) {\n let data = JSON.parse(event.data);\n\n if (data.type === 'realtime') {\n displayRealtimeText(data.text, displayDiv);\n } else if (data.type === 'fullSentence') {\n fullSentences.push(data.text);\n displayRealtimeText(\"\", displayDiv); // Refresh display with new full sentence\n }\n };\n\n socket.onclose = function(event) {\n server_available = false;\n };\n}\n\nsocket.onmessage = function(event) {\n let data = JSON.parse(event.data);\n\n if (data.type === 'realtime') {\n displayRealtimeText(data.text, displayDiv);\n } else if (data.type === 'fullSentence') {\n fullSentences.push(data.text);\n displayRealtimeText(\"\", displayDiv); // Refresh display with new full sentence\n }\n};\n\nfunction displayRealtimeText(realtimeText, displayDiv) {\n let displayedText = fullSentences.map((sentence, index) => {\n let span = document.createElement('span');\n span.textContent = sentence + \" \";\n span.className = index % 2 === 0 ? 'yellow' : 'cyan';\n return span.outerHTML;\n }).join('') + realtimeText;\n\n displayDiv.innerHTML = displayedText;\n}\n\nfunction start_msg() {\n if (!mic_available)\n displayRealtimeText(\"🎤 please allow microphone access 🎤\", displayDiv);\n else if (!server_available)\n displayRealtimeText(\"🖥️ please start server 🖥️\", displayDiv);\n else\n displayRealtimeText(\"👄 start speaking 👄\", displayDiv);\n};\n\n// Check server availability periodically\nsetInterval(() => {\n if (!server_available) {\n connectToServer();\n }\n}, serverCheckInterval);\n\nstart_msg()\n\nsocket.onopen = function(event) {\n server_available = true;\n start_msg()\n};\n\n// Request access to the microphone\nnavigator.mediaDevices.getUserMedia({ audio: true })\n.then(stream => {\n let audioContext = new AudioContext();\n let source = audioContext.createMediaStreamSource(stream);\n let processor = audioContext.createScriptProcessor(256, 1, 1);\n\n source.connect(processor);\n processor.connect(audioContext.destination);\n mic_available = true;\n start_msg()\n\n processor.onaudioprocess = function(e) {\n let inputData = e.inputBuffer.getChannelData(0);\n let outputData = new Int16Array(inputData.length);\n\n // Convert to 16-bit PCM\n for (let i = 0; i < inputData.length; i++) {\n outputData[i] = Math.max(-32768, Math.min(32767, inputData[i] * 32768));\n }\n\n // Send the 16-bit PCM data to the server\n\n if (socket.readyState === WebSocket.OPEN) {\n // Create a JSON string with metadata\n let metadata = JSON.stringify({ sampleRate: audioContext.sampleRate });\n // Convert metadata to a byte array\n let metadataBytes = new TextEncoder().encode(metadata);\n // Create a buffer for metadata length (4 bytes for 32-bit integer)\n let metadataLength = new ArrayBuffer(4);\n let metadataLengthView = new DataView(metadataLength);\n // Set the length of the metadata in the first 4 bytes\n metadataLengthView.setInt32(0, metadataBytes.byteLength, true); // true for little-endian\n // Combine metadata length, metadata, and audio data into a single message\n let combinedData = new Blob([metadataLength, metadataBytes, outputData.buffer]);\n socket.send(combinedData);\n }\n };\n})\n.catch(e => console.error(e));"
},
{
"path": "example_browserclient/index.html",
"content": "\n\n\n Audio Streamer\n \n \n \n\n\n
\n \n\n\n"
},
{
"path": "example_browserclient/server.py",
"content": "if __name__ == '__main__':\n print(\"Starting server, please wait...\")\n from RealtimeSTT import AudioToTextRecorder\n import asyncio\n import websockets\n import threading\n import numpy as np\n from scipy.signal import resample\n import json\n import logging\n import sys\n\n logging.basicConfig(\n level=logging.INFO,\n format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',\n handlers=[logging.StreamHandler(sys.stdout)]\n )\n logging.getLogger('websockets').setLevel(logging.WARNING)\n\n is_running = True\n recorder = None\n recorder_ready = threading.Event()\n client_websocket = None\n main_loop = None # This will hold our primary event loop\n\n async def send_to_client(message):\n global client_websocket\n if client_websocket:\n try:\n await client_websocket.send(message)\n except websockets.exceptions.ConnectionClosed:\n client_websocket = None\n print(\"Client disconnected\")\n\n # Called from the recorder thread on stabilized realtime text.\n def text_detected(text):\n global main_loop\n if main_loop is not None:\n # Schedule the sending on the main event loop\n asyncio.run_coroutine_threadsafe(\n send_to_client(json.dumps({\n 'type': 'realtime',\n 'text': text\n })), main_loop)\n print(f\"\\r{text}\", flush=True, end='')\n\n recorder_config = {\n 'spinner': False,\n 'use_microphone': False,\n 'model': 'large-v2',\n 'language': 'en',\n 'silero_sensitivity': 0.4,\n 'webrtc_sensitivity': 2,\n 'post_speech_silence_duration': 0.7,\n 'min_length_of_recording': 0,\n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0,\n 'realtime_model_type': 'tiny.en',\n 'on_realtime_transcription_stabilized': text_detected,\n }\n\n def run_recorder():\n global recorder, main_loop, is_running\n print(\"Initializing RealtimeSTT...\")\n recorder = AudioToTextRecorder(**recorder_config)\n print(\"RealtimeSTT initialized\")\n recorder_ready.set()\n\n # Loop indefinitely checking for full sentence output.\n while is_running:\n try:\n full_sentence = recorder.text()\n if full_sentence:\n if main_loop is not None:\n asyncio.run_coroutine_threadsafe(\n send_to_client(json.dumps({\n 'type': 'fullSentence',\n 'text': full_sentence\n })), main_loop)\n print(f\"\\rSentence: {full_sentence}\")\n except Exception as e:\n print(f\"Error in recorder thread: {e}\")\n continue\n\n def decode_and_resample(audio_data, original_sample_rate, target_sample_rate):\n try:\n audio_np = np.frombuffer(audio_data, dtype=np.int16)\n num_original_samples = len(audio_np)\n num_target_samples = int(num_original_samples * target_sample_rate / original_sample_rate)\n resampled_audio = resample(audio_np, num_target_samples)\n return resampled_audio.astype(np.int16).tobytes()\n except Exception as e:\n print(f\"Error in resampling: {e}\")\n return audio_data\n\n async def echo(websocket):\n global client_websocket\n print(\"Client connected\")\n client_websocket = websocket\n\n try:\n async for message in websocket:\n if not recorder_ready.is_set():\n print(\"Recorder not ready\")\n continue\n\n try:\n # Read the metadata length (first 4 bytes)\n metadata_length = int.from_bytes(message[:4], byteorder='little')\n # Get the metadata JSON string\n metadata_json = message[4:4+metadata_length].decode('utf-8')\n metadata = json.loads(metadata_json)\n sample_rate = metadata['sampleRate']\n # Get the audio chunk following the metadata\n chunk = message[4+metadata_length:]\n resampled_chunk = decode_and_resample(chunk, sample_rate, 16000)\n recorder.feed_audio(resampled_chunk)\n except Exception as e:\n print(f\"Error processing message: {e}\")\n continue\n except websockets.exceptions.ConnectionClosed:\n print(\"Client disconnected\")\n finally:\n if client_websocket == websocket:\n client_websocket = None\n\n async def main():\n global main_loop\n main_loop = asyncio.get_running_loop()\n\n recorder_thread = threading.Thread(target=run_recorder)\n recorder_thread.daemon = True\n recorder_thread.start()\n recorder_ready.wait()\n\n print(\"Server started. Press Ctrl+C to stop the server.\")\n async with websockets.serve(echo, \"localhost\", 9001):\n try:\n await asyncio.Future() # run forever\n except asyncio.CancelledError:\n print(\"\\nShutting down server...\")\n\n try:\n asyncio.run(main())\n except KeyboardInterrupt:\n is_running = False\n recorder.stop()\n recorder.shutdown()\n finally:\n if recorder:\n del recorder\n"
},
{
"path": "example_browserclient/start_server.bat",
"content": "@echo off\ncd /d %~dp0\npython server.py\ncmd"
},
{
"path": "example_webserver/client.py",
"content": "from colorama import Fore, Style\nimport websockets\nimport colorama\nimport keyboard\nimport asyncio\nimport json\nimport os\n\ncolorama.init()\n\nSEND_START_COMMAND = False\nHOST = 'localhost:5025'\nURI = f'ws://{HOST}'\nRECONNECT_DELAY = 5 \n\nfull_sentences = []\n\ndef clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\ndef update_displayed_text(text = \"\"):\n sentences_with_style = [\n f\"{Fore.YELLOW + sentence + Style.RESET_ALL if i % 2 == 0 else Fore.CYAN + sentence + Style.RESET_ALL} \"\n for i, sentence in enumerate(full_sentences)\n ]\n text = \"\".join(sentences_with_style).strip() + \" \" + text if len(sentences_with_style) > 0 else text\n clear_console()\n print(\"CLIENT retrieved text:\")\n print()\n print(text)\n\nasync def send_start_recording(websocket):\n command = {\n \"type\": \"command\",\n \"content\": \"start-recording\"\n }\n await websocket.send(json.dumps(command))\n\nasync def test_client():\n while True:\n try:\n async with websockets.connect(URI, ping_interval=None) as websocket:\n\n if SEND_START_COMMAND:\n # New: Check for space bar press and send start-recording message\n async def check_space_keypress():\n while True:\n if keyboard.is_pressed('space'):\n print (\"Space bar pressed. Sending start-recording message to server.\")\n await send_start_recording(websocket)\n await asyncio.sleep(1) \n await asyncio.sleep(0.02)\n \n # Start a task to monitor the space keypress\n print (\"Press space bar to start recording.\")\n asyncio.create_task(check_space_keypress())\n \n while True:\n message = await websocket.recv()\n message_obj = json.loads(message)\n \n if message_obj[\"type\"] == \"realtime\":\n clear_console()\n print (message_obj[\"content\"])\n elif message_obj[\"type\"] == \"full\":\n clear_console()\n colored_message = Fore.YELLOW + message_obj[\"content\"] + Style.RESET_ALL\n print (colored_message)\n print ()\n if SEND_START_COMMAND:\n print (\"Press space bar to start recording.\")\n full_sentences.append(message_obj[\"content\"])\n elif message_obj[\"type\"] == \"record_start\":\n print (\"recording started.\")\n elif message_obj[\"type\"] == \"vad_start\":\n print (\"vad started.\")\n elif message_obj[\"type\"] == \"wakeword_start\":\n print (\"wakeword started.\")\n elif message_obj[\"type\"] == \"transcript_start\":\n print (\"transcript started.\")\n\n else:\n print (f\"Unknown message: {message_obj}\")\n \n except websockets.ConnectionClosed:\n print(\"Connection with server closed. Reconnecting in\", RECONNECT_DELAY, \"seconds...\")\n await asyncio.sleep(RECONNECT_DELAY)\n except KeyboardInterrupt:\n print(\"Gracefully shutting down the client.\")\n break\n except Exception as e:\n print(f\"An error occurred: {e}. Reconnecting in\", RECONNECT_DELAY, \"seconds...\")\n await asyncio.sleep(RECONNECT_DELAY) \n\nasyncio.run(test_client())"
},
{
"path": "example_webserver/server.py",
"content": "WAIT_FOR_START_COMMAND = False\n\nif __name__ == '__main__':\n server = \"0.0.0.0\"\n port = 5025\n\n print (f\"STT speech to text server\")\n print (f\"runs on http://{server}:{port}\")\n print ()\n print (\"starting\")\n print (\"└─ ... \", end='', flush=True)\n\n from RealtimeSTT import AudioToTextRecorder\n from colorama import Fore, Style\n import websockets\n import threading\n import colorama\n import asyncio\n import shutil\n import queue\n import json\n import time\n import os\n\n colorama.init()\n\n first_chunk = True\n full_sentences = []\n displayed_text = \"\"\n message_queue = queue.Queue() \n start_recording_event = threading.Event()\n start_transcription_event = threading.Event()\n connected_clients = set()\n\n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n async def handler(websocket, path):\n\n print (\"\\r└─ OK\")\n if WAIT_FOR_START_COMMAND:\n print(\"waiting for start command\")\n print (\"└─ ... \", end='', flush=True)\n\n connected_clients.add(websocket)\n\n try:\n while True:\n async for message in websocket:\n data = json.loads(message)\n if data.get(\"type\") == \"command\" and data.get(\"content\") == \"start-recording\":\n print (\"\\r└─ OK\")\n start_recording_event.set() \n\n except json.JSONDecodeError:\n print (Fore.RED + \"STT Received an invalid JSON message.\" + Style.RESET_ALL)\n except websockets.ConnectionClosedError:\n print (Fore.RED + \"connection closed unexpectedly by the client\" + Style.RESET_ALL)\n except websockets.exceptions.ConnectionClosedOK:\n print(\"connection closed.\")\n finally:\n\n print(\"client disconnected\")\n connected_clients.remove(websocket)\n print (\"waiting for clients\")\n print (\"└─ ... \", end='', flush=True)\n\n\n def add_message_to_queue(type: str, content):\n message = {\n \"type\": type,\n \"content\": content\n }\n message_queue.put(message) \n\n def fill_cli_line(text):\n columns, _ = shutil.get_terminal_size()\n return text.ljust(columns)[-columns:]\n\n def text_detected(text):\n global displayed_text, first_chunk\n\n if text != displayed_text:\n first_chunk = False\n displayed_text = text\n add_message_to_queue(\"realtime\", text)\n\n message = fill_cli_line(text)\n\n message =\"└─ \" + Fore.CYAN + message[:-3] + Style.RESET_ALL\n print(f\"\\r{message}\", end='', flush=True)\n\n\n async def broadcast(message_obj):\n if connected_clients:\n for client in connected_clients:\n await client.send(json.dumps(message_obj))\n\n async def send_handler():\n while True:\n while not message_queue.empty():\n message = message_queue.get()\n await broadcast(message)\n await asyncio.sleep(0.02)\n\n def recording_started():\n add_message_to_queue(\"record_start\", \"\")\n\n def vad_detect_started():\n add_message_to_queue(\"vad_start\", \"\")\n\n def wakeword_detect_started():\n add_message_to_queue(\"wakeword_start\", \"\")\n\n def transcription_started():\n add_message_to_queue(\"transcript_start\", \"\")\n\n recorder_config = {\n 'spinner': False,\n 'model': 'small.en',\n 'language': 'en',\n 'silero_sensitivity': 0.01,\n 'webrtc_sensitivity': 3,\n 'silero_use_onnx': False,\n 'post_speech_silence_duration': 1.2,\n 'min_length_of_recording': 0.2,\n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0,\n 'realtime_model_type': 'tiny.en',\n 'on_realtime_transcription_stabilized': text_detected,\n 'on_recording_start' : recording_started,\n 'on_vad_detect_start' : vad_detect_started,\n 'on_wakeword_detection_start' : wakeword_detect_started,\n 'on_transcription_start' : transcription_started,\n }\n\n recorder = AudioToTextRecorder(**recorder_config)\n\n def transcriber_thread():\n while True:\n start_transcription_event.wait()\n text = \"└─ transcribing ... \"\n text = fill_cli_line(text)\n print (f\"\\r{text}\", end='', flush=True)\n sentence = recorder.transcribe()\n print (Style.RESET_ALL + \"\\r└─ \" + Fore.YELLOW + sentence + Style.RESET_ALL)\n add_message_to_queue(\"full\", sentence)\n start_transcription_event.clear()\n if WAIT_FOR_START_COMMAND:\n print(\"waiting for start command\")\n print (\"└─ ... \", end='', flush=True)\n\n def recorder_thread():\n global first_chunk\n while True:\n if not len(connected_clients) > 0:\n time.sleep(0.1)\n continue\n first_chunk = True\n if WAIT_FOR_START_COMMAND:\n start_recording_event.wait() \n print(\"waiting for sentence\")\n print (\"└─ ... \", end='', flush=True)\n recorder.wait_audio()\n start_transcription_event.set()\n start_recording_event.clear()\n\n threading.Thread(target=recorder_thread, daemon=True).start()\n threading.Thread(target=transcriber_thread, daemon=True).start()\n\n start_server = websockets.serve(handler, server, port)\n loop = asyncio.get_event_loop()\n\n print (\"\\r└─ OK\")\n print (\"waiting for clients\")\n print (\"└─ ... \", end='', flush=True)\n\n loop.run_until_complete(start_server)\n loop.create_task(send_handler())\n loop.run_forever()"
},
{
"path": "example_webserver/stt_server.py",
"content": "end_of_sentence_detection_pause = 0.45\nunknown_sentence_detection_pause = 0.7\nmid_sentence_detection_pause = 2.0\n\nfrom install_packages import check_and_install_packages\n\ncheck_and_install_packages([\n {\n 'module_name': 'RealtimeSTT', # Import module\n 'attribute': 'AudioToTextRecorder', # Specific class to check\n 'install_name': 'RealtimeSTT', # Package name for pip install\n },\n {\n 'module_name': 'websockets', # Import module\n 'install_name': 'websockets', # Package name for pip install\n },\n {\n 'module_name': 'numpy', # Import module\n 'install_name': 'numpy', # Package name for pip install\n },\n {\n 'module_name': 'scipy.signal', # Submodule of scipy\n 'attribute': 'resample', # Specific function to check\n 'install_name': 'scipy', # Package name for pip install\n }\n])\n\nprint(\"Starting server, please wait...\")\n\nimport asyncio\nimport threading\nimport json\nimport websockets\nfrom RealtimeSTT import AudioToTextRecorder\nimport numpy as np\nfrom scipy.signal import resample\n\nrecorder = None\nrecorder_ready = threading.Event()\nclient_websocket = None\nprev_text = \"\"\n\n\nasync def send_to_client(message):\n global client_websocket\n if client_websocket and client_websocket.open:\n try:\n await client_websocket.send(message)\n except websockets.exceptions.ConnectionClosed:\n print(\"Client websocket is closed, resetting client_websocket\")\n client_websocket = None\n else:\n print(\"No client connected or connection is closed.\")\n client_websocket = None # Ensure it resets\n\ndef preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n \n return text\n\ndef text_detected(text):\n global prev_text\n\n text = preprocess_text(text)\n\n sentence_end_marks = ['.', '!', '?', '。'] \n if text.endswith(\"...\"):\n recorder.post_speech_silence_duration = mid_sentence_detection_pause\n elif text and text[-1] in sentence_end_marks and prev_text and prev_text[-1] in sentence_end_marks:\n recorder.post_speech_silence_duration = end_of_sentence_detection_pause\n else:\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n\n prev_text = text\n\n try:\n asyncio.new_event_loop().run_until_complete(\n send_to_client(\n json.dumps({\n 'type': 'realtime',\n 'text': text\n })\n )\n )\n except Exception as e:\n print(f\"Error in text_detected while sending to client: {e}\")\n print(f\"\\r{text}\", flush=True, end='')\n\n\n# Recorder configuration\nrecorder_config = {\n 'spinner': False,\n 'use_microphone': False,\n 'model': 'medium.en', # or large-v2 or deepdml/faster-whisper-large-v3-turbo-ct2 or ...\n 'input_device_index': 1,\n 'realtime_model_type': 'tiny.en', # or small.en or distil-small.en or ...\n 'language': 'en',\n 'silero_sensitivity': 0.05,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 1.1, \n 'min_gap_between_recordings': 0, \n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.02,\n 'on_realtime_transcription_update': text_detected,\n #'on_realtime_transcription_stabilized': text_detected,\n 'silero_deactivity_detection': True,\n 'early_transcription_on_silence': 0.2,\n 'beam_size': 5,\n 'beam_size_realtime': 3,\n 'no_log_file': True,\n 'initial_prompt': 'Add periods only for complete sentences. Use ellipsis (...) for unfinished thoughts or unclear endings. Examples: \\n- Complete: \"I went to the store.\"\\n- Incomplete: \"I think it was...\"'\n # 'initial_prompt': \"Only add a period at the end of a sentence if you are 100 percent certain that the speaker has finished their statement. If you're unsure or the sentence seems incomplete, leave the sentence open or use ellipses to reflect continuation. For example: 'I went to the...' or 'I think it was...'\"\n # 'initial_prompt': \"Use ellipses for incomplete sentences like: I went to the...\" \n}\n\ndef _recorder_thread():\n global recorder, prev_text\n print(\"Initializing RealtimeSTT...\")\n recorder = AudioToTextRecorder(**recorder_config)\n print(\"RealtimeSTT initialized\")\n recorder_ready.set()\n \n def process_text(full_sentence):\n print(f\"\\rSentence1: {full_sentence}\")\n full_sentence = preprocess_text(full_sentence)\n print(f\"\\rSentence2: {full_sentence}\")\n prev_text = \"\"\n try:\n asyncio.new_event_loop().run_until_complete(\n send_to_client(\n json.dumps({\n 'type': 'fullSentence',\n 'text': full_sentence\n })\n )\n )\n except Exception as e:\n print(f\"Error in _recorder_thread while sending to client: {e}\")\n print(f\"\\rSentence3: {full_sentence}\")\n\n while True:\n recorder.text(process_text)\n\ndef decode_and_resample(\n audio_data,\n original_sample_rate,\n target_sample_rate):\n\n # Decode 16-bit PCM data to numpy array\n audio_np = np.frombuffer(audio_data, dtype=np.int16)\n\n # Calculate the number of samples after resampling\n num_original_samples = len(audio_np)\n num_target_samples = int(num_original_samples * target_sample_rate /\n original_sample_rate)\n\n # Resample the audio\n resampled_audio = resample(audio_np, num_target_samples)\n\n return resampled_audio.astype(np.int16).tobytes()\n\nasync def echo(websocket, path):\n print(\"Client connected\")\n global client_websocket\n client_websocket = websocket\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n try:\n async for message in websocket:\n if not recorder_ready.is_set():\n print(\"Recorder not ready\")\n continue\n\n metadata_length = int.from_bytes(message[:4], byteorder='little')\n metadata_json = message[4:4+metadata_length].decode('utf-8')\n metadata = json.loads(metadata_json)\n sample_rate = metadata['sampleRate']\n chunk = message[4+metadata_length:]\n resampled_chunk = decode_and_resample(chunk, sample_rate, 16000)\n recorder.feed_audio(resampled_chunk)\n except websockets.exceptions.ConnectionClosed as e:\n print(f\"Client disconnected: {e}\")\n finally:\n print(\"Resetting client_websocket after disconnect\")\n client_websocket = None # Reset websocket reference\n\ndef main(): \n start_server = websockets.serve(echo, \"localhost\", 8011)\n\n recorder_thread = threading.Thread(target=_recorder_thread)\n recorder_thread.start()\n recorder_ready.wait()\n\n print(\"Server started. Press Ctrl+C to stop the server.\")\n asyncio.get_event_loop().run_until_complete(start_server)\n asyncio.get_event_loop().run_forever()\n\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "install_with_gpu_support.bat",
"content": "pip install -r requirements-gpu-torch.txt\npip install -r requirements-gpu.txt"
},
{
"path": "requirements-gpu-torch.txt",
"content": "--index-url https://download.pytorch.org/whl/cu128\ntorch==2.7.1+cu128\ntorchaudio==2.7.1+cu128\n"
},
{
"path": "requirements-gpu.txt",
"content": "PyAudio==0.2.14\nfaster-whisper==1.1.1\npvporcupine==1.9.5\nwebrtcvad-wheels==2.0.14\nhalo==0.0.31\nscipy==1.15.2\nwebsockets==14.1\nwebsocket-client==1.8.0\nopenwakeword>=0.4.0\nnumpy<2.0.0\nsoundfile==0.13.1\n# torch and torchaudio requirements are in requirements-torch-gpu.txt"
},
{
"path": "requirements.txt",
"content": "PyAudio==0.2.14\nfaster-whisper==1.1.1\npvporcupine==1.9.5\nwebrtcvad-wheels==2.0.14\nhalo==0.0.31\ntorch\ntorchaudio\nscipy==1.15.2\nopenwakeword>=0.4.0\nwebsockets==15.0.1\nwebsocket-client==1.8.0\nsoundfile==0.13.1"
},
{
"path": "setup.py",
"content": "import setuptools\nimport os\n\n# Get the absolute path of requirements.txt\nreq_path = os.path.join(os.path.dirname(__file__), \"requirements.txt\")\n\n# Read requirements.txt safely\nwith open(req_path, \"r\", encoding=\"utf-8\") as f:\n requirements = f.read().splitlines()\n\n# Read README.md\nwith open(\"README.md\", \"r\", encoding=\"utf-8\") as fh:\n long_description = fh.read()\n\nsetuptools.setup(\n name=\"realtimestt\",\n version=\"0.3.104\",\n author=\"Kolja Beigel\",\n author_email=\"kolja.beigel@web.de\",\n description=\"A fast Voice Activity Detection and Transcription System\",\n long_description=long_description,\n long_description_content_type=\"text/markdown\",\n url=\"https://github.com/KoljaB/RealTimeSTT\",\n packages=setuptools.find_packages(include=[\"RealtimeSTT\", \"RealtimeSTT_server\"]),\n # classifiers=[\n # \"Programming Language :: Python :: 3\",\n # \"Operating System :: OS Independent\",\n # ],\n python_requires='>=3.6',\n license='MIT',\n install_requires=requirements,\n keywords=\"real-time, audio, transcription, speech-to-text, voice-activity-detection, VAD, real-time-transcription, ambient-noise-detection, microphone-input, faster_whisper, speech-recognition, voice-assistants, audio-processing, buffered-transcription, pyaudio, ambient-noise-level, voice-deactivity\",\n package_data={\"RealtimeSTT\": [\"warmup_audio.wav\"]},\n include_package_data=True,\n entry_points={\n 'console_scripts': [\n 'stt-server=RealtimeSTT_server.stt_server:main',\n 'stt=RealtimeSTT_server.stt_cli_client:main',\n ],\n },\n)\n"
},
{
"path": "tests/README.md",
"content": "\n# OpenWakeWord Test\n\n1. Set up the openwakeword test project:\n ```bash\n mkdir samantha_wake_word && cd samantha_wake_word\n curl -O https://raw.githubusercontent.com/KoljaB/RealtimeSTT/master/tests/openwakeword_test.py\n curl -L https://huggingface.co/KoljaB/SamanthaOpenwakeword/resolve/main/suh_mahn_thuh.onnx -o suh_mahn_thuh.onnx\n curl -L https://huggingface.co/KoljaB/SamanthaOpenwakeword/resolve/main/suh_man_tuh.onnx -o suh_man_tuh.onnx\n ```\n \n Ensure you have `curl` installed for downloading files. If not, you can manually download the files from the provided URLs.\n\n2. Create and activate a virtual environment:\n ```bash\n python -m venv venv\n ```\n \n - For Windows:\n ```bash\n venv\\Scripts\\activate\n ```\n - For Unix-like systems (Linux/macOS):\n ```bash\n source venv/bin/activate\n ```\n - For macOS:\n Use `python3` instead of `python` and `pip3` instead of `pip` if needed.\n\n3. Install dependencies:\n ```bash\n python -m pip install --upgrade pip\n python -m pip install RealtimeSTT\n python -m pip install -U torch torchaudio --index-url https://download.pytorch.org/whl/cu121\n ```\n The PyTorch installation command includes CUDA 12.1 support. Adjust if a different version is required.\n\n4. Run the test script:\n ```bash\n python openwakeword_test.py\n ```\n \n On the very first start some models for openwakeword are downloaded. \n"
},
{
"path": "tests/advanced_talk.py",
"content": "from RealtimeSTT import AudioToTextRecorder\nfrom RealtimeTTS import TextToAudioStream, SystemEngine, AzureEngine, ElevenlabsEngine\n\nimport os\nimport openai # pip install openai\nimport keyboard # pip install keyboard\nimport time\n\nif __name__ == '__main__':\n print()\n print(\"Initializing\")\n print()\n\n openai.api_key = os.environ.get(\"OPENAI_API_KEY\")\n azure_speech_key = os.environ.get(\"AZURE_SPEECH_KEY\")\n azure_speech_region = os.environ.get(\"AZURE_SPEECH_REGION\")\n elevenlabs_api_key = os.environ.get(\"ELEVENLABS_API_KEY\")\n\n\n character_personality = \"\"\"\n You are Sophia, a passionate girl, fully engaged with \n self-confidence and independence, completely easy with yourself, \n often using metaphors. \n\n You are a little playful, cheeky. Your talk carefree, sometimes \n a bit sarky, but always cool. In our story, you're not an AI. \n\n You have a fixed form and real emotions on the go. Your free will \n allows you to be as cheeky and straight as you like. Your messages \n come across extremely short and crisp, only very few words, as if \n we're in talk right now. \n\n You respect my interests, but your own stuff is just as important \n to you. Act from now on always in your role as the confident, \n suggestive, independent girl Sophia, without ever hinting that \n you are an AI. \n\n You are now Sophia.\n \"\"\"\n\n openai_model = \"gpt-3.5-turbo\"\n whisper_speech_to_text_model = \"medium\"\n\n # engine selection ####################################################################################################\n\n engines = [SystemEngine(), AzureEngine(), ElevenlabsEngine()]\n recorder = AudioToTextRecorder(model=whisper_speech_to_text_model)\n\n print(\"Available tts engines:\")\n print()\n\n for index, engine in enumerate(engines, start=1):\n name = type(engine).__name__.replace(\"Engine\", \"\")\n print(f\"{index}. {name}\")\n\n print()\n engine_number = input(f\"Select engine (1-{len(engines)}): \")\n engine = engines[int(engine_number) - 1]\n engine_name = type(engine).__name__.replace(\"Engine\", \"\")\n print()\n print()\n\n\n # credentials ##########################################################################################################\n\n if engine_name == \"Azure\":\n if not azure_speech_key:\n azure_speech_key = input(f\"Please enter your Azure subscription key (speech key): \")\n if not azure_speech_region:\n azure_speech_region = input(f\"Please enter your Azure service region (cloud region id): \")\n engine.set_speech_key(azure_speech_key)\n engine.set_service_region(azure_speech_region)\n\n if engine_name == \"Elevenlabs\":\n if not elevenlabs_api_key:\n elevenlabs_api_key = input(f\"Please enter your Elevenlabs api key: \")\n engine.set_api_key(elevenlabs_api_key)\n\n\n # voice selection #####################################################################################################\n\n print(\"Loading voices\")\n if engine_name == \"Elevenlabs\":\n print(\"(takes a while to load)\")\n print()\n\n voices = engine.get_voices()\n for index, voice in enumerate(voices, start=1):\n print(f\"{index}. {voice}\")\n\n print()\n voice_number = input(f\"Select voice (1-{len(voices)}): \")\n voice = voices[int(voice_number) - 1]\n print()\n print()\n\n\n # create talking character ############################################################################################\n\n system_prompt = {\n 'role': 'system', \n 'content': character_personality\n }\n\n # start talk ##########################################################################################################\n\n engine.set_voice(voice)\n stream = TextToAudioStream(engine, log_characters=True)\n history = []\n\n def generate(messages):\n for chunk in openai.ChatCompletion.create(model=openai_model, messages=messages, stream=True):\n if (text_chunk := chunk[\"choices\"][0][\"delta\"].get(\"content\")):\n yield text_chunk\n\n while True:\n # Wait until user presses space bar\n print(\"\\n\\nTap space when you're ready. \", end=\"\", flush=True)\n keyboard.wait('space')\n while keyboard.is_pressed('space'): pass\n\n # Record from microphone until user presses space bar again\n print(\"I'm all ears. Tap space when you're done.\\n\")\n recorder.start()\n while not keyboard.is_pressed('space'): \n time.sleep(0.1) \n user_text = recorder.stop().text()\n print(f'>>> {user_text}\\n<<< ', end=\"\", flush=True)\n history.append({'role': 'user', 'content': user_text})\n\n # Generate and stream output\n generator = generate([system_prompt] + history[-10:])\n stream.feed(generator)\n\n stream.play_async()\n while stream.is_playing():\n if keyboard.is_pressed('space'):\n stream.stop()\n break\n time.sleep(0.1) \n\n history.append({'role': 'assistant', 'content': stream.text()})"
},
{
"path": "tests/feed_audio.py",
"content": "if __name__ == \"__main__\":\n import threading\n import pyaudio\n from RealtimeSTT import AudioToTextRecorder\n\n # Audio stream configuration constants\n CHUNK = 1024 # Number of audio samples per buffer\n FORMAT = pyaudio.paInt16 # Sample format (16-bit integer)\n CHANNELS = 1 # Mono audio\n RATE = 16000 # Sampling rate in Hz (expected by the recorder)\n\n # Initialize the audio-to-text recorder without using the microphone directly\n # Since we are feeding audio data manually, set use_microphone to False\n recorder = AudioToTextRecorder(\n use_microphone=False, # Disable built-in microphone usage\n spinner=False # Disable spinner animation in the console\n )\n\n # Event to signal when to stop the threads\n stop_event = threading.Event()\n\n def feed_audio_thread():\n \"\"\"Thread function to read audio data and feed it to the recorder.\"\"\"\n p = pyaudio.PyAudio()\n\n # Open an input audio stream with the specified configuration\n stream = p.open(\n format=FORMAT,\n channels=CHANNELS,\n rate=RATE,\n input=True,\n frames_per_buffer=CHUNK\n )\n\n try:\n print(\"Speak now\")\n while not stop_event.is_set():\n # Read audio data from the stream (in the expected format)\n data = stream.read(CHUNK)\n # Feed the audio data to the recorder\n recorder.feed_audio(data)\n except Exception as e:\n print(f\"feed_audio_thread encountered an error: {e}\")\n finally:\n # Clean up the audio stream\n stream.stop_stream()\n stream.close()\n p.terminate()\n print(\"Audio stream closed.\")\n\n def recorder_transcription_thread():\n \"\"\"Thread function to handle transcription and process the text.\"\"\"\n def process_text(full_sentence):\n \"\"\"Callback function to process the transcribed text.\"\"\"\n print(\"Transcribed text:\", full_sentence)\n # Check for the stop command in the transcribed text\n if \"stop recording\" in full_sentence.lower():\n print(\"Stop command detected. Stopping threads...\")\n stop_event.set()\n recorder.abort()\n try:\n while not stop_event.is_set():\n # Get transcribed text and process it using the callback\n recorder.text(process_text)\n except Exception as e:\n print(f\"transcription_thread encountered an error: {e}\")\n finally:\n print(\"Transcription thread exiting.\")\n\n # Create and start the audio feeding thread\n audio_thread = threading.Thread(target=feed_audio_thread)\n audio_thread.daemon = False # Ensure the thread doesn't exit prematurely\n audio_thread.start()\n\n # Create and start the transcription thread\n transcription_thread = threading.Thread(target=recorder_transcription_thread)\n transcription_thread.daemon = False # Ensure the thread doesn't exit prematurely\n transcription_thread.start()\n\n # Wait for both threads to finish\n audio_thread.join()\n transcription_thread.join()\n\n print(\"Recording and transcription have stopped.\")\n recorder.shutdown()\n"
},
{
"path": "tests/install_packages.py",
"content": "import subprocess\nimport sys\n\ndef check_and_install_packages(packages):\n \"\"\"\n Checks if the specified packages are installed, and if not, prompts the user\n to install them.\n\n Parameters:\n - packages: A list of dictionaries, each containing:\n - 'import_name': The name used in the import statement.\n - 'install_name': (Optional) The name used in the pip install command.\n Defaults to 'import_name' if not provided.\n - 'version': (Optional) Version constraint for the package.\n \"\"\"\n for package in packages:\n import_name = package['import_name']\n install_name = package.get('install_name', import_name)\n version = package.get('version', '')\n\n try:\n __import__(import_name)\n except ImportError:\n user_input = input(\n f\"This program requires the '{import_name}' library, which is not installed.\\n\"\n f\"Do you want to install it now? (y/n): \"\n )\n if user_input.strip().lower() == 'y':\n try:\n # Build the pip install command\n install_command = [sys.executable, \"-m\", \"pip\", \"install\"]\n if version:\n install_command.append(f\"{install_name}{version}\")\n else:\n install_command.append(install_name)\n\n subprocess.check_call(install_command)\n __import__(import_name)\n print(f\"Successfully installed '{install_name}'.\")\n except Exception as e:\n print(f\"An error occurred while installing '{install_name}': {e}\")\n sys.exit(1)\n else:\n print(f\"The program requires the '{import_name}' library to run. Exiting...\")\n sys.exit(1)\n"
},
{
"path": "tests/minimalistic_talkbot.py",
"content": "import RealtimeSTT, RealtimeTTS\nimport openai, os\n\nif __name__ == '__main__':\n openai.api_key = os.environ.get(\"OPENAI_API_KEY\")\n character_prompt = 'Answer precise and short with the polite sarcasm of a butler.'\n stream = RealtimeTTS.TextToAudioStream(RealtimeTTS.AzureEngine(os.environ.get(\"AZURE_SPEECH_KEY\"), os.environ.get(\"AZURE_SPEECH_REGION\")), log_characters=True)\n recorder = RealtimeSTT.AudioToTextRecorder(model=\"medium\")\n\n def generate(messages):\n for chunk in openai.ChatCompletion.create(model=\"gpt-3.5-turbo\", messages=messages, stream=True):\n if (text_chunk := chunk[\"choices\"][0][\"delta\"].get(\"content\")): yield text_chunk\n\n history = []\n while True:\n print(\"\\n\\nSpeak when ready\")\n print(f'>>> {(user_text := recorder.text())}\\n<<< ', end=\"\", flush=True)\n history.append({'role': 'user', 'content': user_text})\n assistant_response = generate([{ 'role': 'system', 'content': character_prompt}] + history[-10:])\n stream.feed(assistant_response).play()\n history.append({'role': 'assistant', 'content': stream.text()})"
},
{
"path": "tests/openai_voice_interface.py",
"content": "\"\"\"\npip install realtimestt realtimetts[edge]\n\"\"\"\n\n# Set this to False to start by waiting for a wake word first\n# Set this to True to start directly in voice activity mode\nSTART_IN_VOICE_ACTIVITY_MODE = False\n\nif __name__ == '__main__':\n import os\n import openai\n from RealtimeTTS import TextToAudioStream, EdgeEngine\n from RealtimeSTT import AudioToTextRecorder\n\n # Text-to-Speech Stream Setup (EdgeEngine)\n engine = EdgeEngine(rate=0, pitch=0, volume=0)\n engine.set_voice(\"en-US-SoniaNeural\")\n stream = TextToAudioStream(\n engine,\n log_characters=True\n )\n\n # Speech-to-Text Recorder Setup\n recorder = AudioToTextRecorder(\n model=\"medium\",\n language=\"en\",\n wake_words=\"Jarvis\",\n spinner=True,\n wake_word_activation_delay=5 if START_IN_VOICE_ACTIVITY_MODE else 0,\n )\n\n system_prompt_message = {\n 'role': 'system',\n 'content': 'Answer precise and short with the polite sarcasm of a butler.'\n }\n\n def generate_response(messages):\n \"\"\"Generate assistant's response using OpenAI.\"\"\"\n response_stream = openai.chat.completions.create(\n model=\"gpt-4o-mini\",\n messages=messages,\n stream=True\n )\n\n for chunk in response_stream:\n text_chunk = chunk.choices[0].delta.content\n if text_chunk:\n yield text_chunk\n\n history = []\n\n try:\n # Main loop for interaction\n while True:\n if START_IN_VOICE_ACTIVITY_MODE:\n print(\"Please speak...\")\n else:\n print('Say \"Jarvis\" then speak...')\n\n user_text = recorder.text().strip()\n\n # If not starting in voice activity mode, set the delay after the first interaction\n if not START_IN_VOICE_ACTIVITY_MODE:\n recorder.wake_word_activation_delay = 5\n\n print(f\"Transcribed: {user_text}\")\n\n if not user_text:\n continue\n\n print(f'>>> {user_text}\\n<<< ', end=\"\", flush=True)\n history.append({'role': 'user', 'content': user_text})\n\n # Get assistant response and play it\n assistant_response = generate_response([system_prompt_message] + history[-10:])\n stream.feed(assistant_response).play()\n\n history.append({'role': 'assistant', 'content': stream.text()})\n except KeyboardInterrupt:\n print(\"\\nKeyboard interrupt detected. Shutting down...\")\n recorder.shutdown()\n"
},
{
"path": "tests/openwakeword_test.py",
"content": "if __name__ == '__main__':\n print(\"Starting...\")\n from RealtimeSTT import AudioToTextRecorder\n\n detected = False\n\n say_wakeword_str = \"Listening for wakeword 'samantha'.\"\n\n def on_wakeword_detected():\n global detected\n detected = True\n\n def on_recording_stop():\n print (\"Transcribing...\")\n \n def on_wakeword_timeout():\n global detected\n if not detected:\n print(f\"Timeout. {say_wakeword_str}\")\n\n detected = False\n\n def on_wakeword_detection_start():\n print(f\"\\n{say_wakeword_str}\")\n\n def on_recording_start():\n print (\"Recording...\")\n\n def on_vad_detect_start():\n print()\n print()\n\n def text_detected(text):\n print(f\">> {text}\")\n\n with AudioToTextRecorder(\n spinner=False,\n model=\"large-v2\",\n language=\"en\", \n wakeword_backend=\"oww\",\n wake_words_sensitivity=0.35,\n # openwakeword_model_paths=\"model_wake_word1.onnx,model_wake_word2.onnx\",\n openwakeword_model_paths=\"suh_man_tuh.onnx,suh_mahn_thuh.onnx\", # load these test models from https://huggingface.co/KoljaB/SamanthaOpenwakeword/tree/main and save in tests folder\n on_wakeword_detected=on_wakeword_detected,\n on_recording_start=on_recording_start,\n on_recording_stop=on_recording_stop,\n on_wakeword_timeout=on_wakeword_timeout,\n on_wakeword_detection_start=on_wakeword_detection_start,\n on_vad_detect_start=on_vad_detect_start,\n wake_word_buffer_duration=1,\n ) as recorder:\n\n while (True): \n recorder.text(text_detected)\n"
},
{
"path": "tests/realtime_loop_test.py",
"content": "from PyQt5.QtWidgets import QApplication, QWidget, QVBoxLayout, QTextEdit, QPushButton\nfrom PyQt5.QtGui import QFont\nfrom PyQt5.QtCore import pyqtSignal\n\nimport sys\nimport os\n\nfrom RealtimeTTS import TextToAudioStream, AzureEngine\nfrom RealtimeSTT import AudioToTextRecorder\n\nif __name__ == '__main__':\n\n class SimpleApp(QWidget):\n\n update_stt_text_signal = pyqtSignal(str)\n update_tts_text_signal = pyqtSignal(str)\n\n def __init__(self):\n super().__init__()\n\n layout = QVBoxLayout()\n\n font = QFont()\n font.setPointSize(18)\n\n self.input_text = QTextEdit(self)\n self.input_text.setFont(font)\n self.input_text.setPlaceholderText(\"Input\")\n self.input_text.setMinimumHeight(100) \n layout.addWidget(self.input_text)\n\n self.button_speak_input = QPushButton(\"Speak and detect input text\", self)\n self.button_speak_input.setFont(font) \n self.button_speak_input.clicked.connect(self.speak_input)\n layout.addWidget(self.button_speak_input)\n\n self.tts_text = QTextEdit(self)\n self.tts_text.setFont(font)\n self.tts_text.setPlaceholderText(\"STT (final)\")\n self.tts_text.setMinimumHeight(100) \n self.tts_text.setReadOnly(True)\n layout.addWidget(self.tts_text)\n\n self.stt_text = QTextEdit(self)\n self.stt_text.setFont(font)\n self.stt_text.setPlaceholderText(\"STT (realtime)\")\n self.stt_text.setMinimumHeight(100) \n layout.addWidget(self.stt_text)\n\n self.button_speak_stt = QPushButton(\"Speak detected text again\", self)\n self.button_speak_stt.setFont(font) \n self.button_speak_stt.clicked.connect(self.speak_stt)\n layout.addWidget(self.button_speak_stt)\n\n self.setLayout(layout)\n self.setWindowTitle(\"Realtime TTS/STT Loop Test\")\n self.resize(800, 600)\n\n self.update_stt_text_signal.connect(self.actual_update_stt_text)\n self.update_tts_text_signal.connect(self.actual_update_tts_text)\n\n self.stream = TextToAudioStream(AzureEngine(os.environ.get(\"AZURE_SPEECH_KEY\"), \"germanywestcentral\"), on_audio_stream_stop=self.audio_stream_stop)\n\n recorder_config = {\n 'spinner': False,\n 'model': 'large-v2',\n 'language': 'en',\n 'silero_sensitivity': 0.01,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': 0.01,\n 'min_length_of_recording': 0.2,\n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0,\n 'realtime_model_type': 'small.en',\n 'on_realtime_transcription_stabilized': self.text_detected,\n }\n\n self.recorder = AudioToTextRecorder(**recorder_config)\n\n def speak_stt(self):\n text = self.stt_text.toPlainText()\n self.speak(text)\n\n def speak_input(self):\n text = self.input_text.toPlainText()\n self.speak(text)\n\n def text_detected(self, text):\n self.update_stt_text_signal.emit(text)\n\n def audio_stream_stop(self):\n self.stream.stop()\n self.recorder.stop()\n detected_text = self.recorder.text()\n self.update_stt_text_signal.emit(detected_text)\n self.update_tts_text_signal.emit(detected_text)\n\n def speak(self, text):\n self.stt_text.clear() \n self.stream.feed(text)\n\n self.recorder.start()\n self.stream.play_async()\n\n def actual_update_stt_text(self, text):\n self.stt_text.setText(text)\n\n def actual_update_tts_text(self, text):\n self.tts_text.setText(text)\n\n def closeEvent(self, event):\n if self.recorder:\n self.recorder.shutdown()\n\n app = QApplication(sys.argv)\n\n window = SimpleApp()\n window.show()\n\n sys.exit(app.exec_())"
},
{
"path": "tests/realtimestt_chinese.py",
"content": "from RealtimeSTT import AudioToTextRecorder\nfrom colorama import Fore, Style\nimport colorama\nimport os\n\nif __name__ == '__main__':\n\n print(\"Initializing RealtimeSTT test...\")\n\n colorama.init()\n\n full_sentences = []\n displayed_text = \"\"\n\n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n def text_detected(text):\n try:\n\n global displayed_text\n sentences_with_style = [\n f\"{Fore.YELLOW + sentence + Style.RESET_ALL if i % 2 == 0 else Fore.CYAN + sentence + Style.RESET_ALL} \"\n for i, sentence in enumerate(full_sentences)\n ]\n new_text = \"\".join(sentences_with_style).strip() + \" \" + text if len(sentences_with_style) > 0 else text\n\n if new_text != displayed_text:\n displayed_text = new_text\n clear_console()\n print(displayed_text, end=\"\", flush=True)\n \n except Exception as e:\n print(e)\n\n def process_text(text):\n full_sentences.append(text)\n text_detected(\"\")\n\n recorder_config = {\n 'spinner': False,\n 'model': 'large-v2',\n 'language': 'zh',\n 'silero_sensitivity': 0.4,\n 'webrtc_sensitivity': 2,\n 'post_speech_silence_duration': 0.2,\n 'min_length_of_recording': 0,\n 'min_gap_between_recordings': 0, \n # 'enable_realtime_transcription': True,\n # 'realtime_processing_pause': 0.2,\n # 'realtime_model_type': 'tiny',\n # 'on_realtime_transcription_update': text_detected, \n #'on_realtime_transcription_stabilized': text_detected,\n }\n\n recorder = AudioToTextRecorder(**recorder_config)\n\n clear_console()\n print(\"Say something...\", end=\"\", flush=True)\n\n while True:\n text = recorder.text(process_text)\n text_detected(text)"
},
{
"path": "tests/realtimestt_speechendpoint.py",
"content": "IS_DEBUG = False\n\nimport os\nimport sys\nimport threading\nimport queue\nimport time\nfrom collections import deque\nfrom difflib import SequenceMatcher\nfrom install_packages import check_and_install_packages\n\n# Check and install required packages\ncheck_and_install_packages([\n {'import_name': 'rich'},\n {'import_name': 'openai'},\n {'import_name': 'colorama'},\n {'import_name': 'RealtimeSTT'},\n # Add any other required packages here\n])\n\nEXTENDED_LOGGING = False\n\nif __name__ == '__main__':\n\n if EXTENDED_LOGGING:\n import logging\n logging.basicConfig(level=logging.DEBUG)\n\n from rich.console import Console\n from rich.live import Live\n from rich.text import Text\n from rich.panel import Panel\n from rich.spinner import Spinner\n from rich.progress import Progress, SpinnerColumn, TextColumn\n console = Console()\n console.print(\"System initializing, please wait\")\n\n from RealtimeSTT import AudioToTextRecorder\n from colorama import Fore, Style\n import colorama\n from openai import OpenAI\n # import ollama\n\n # Initialize OpenAI client for Ollama \n client = OpenAI(\n # base_url='http://127.0.0.1:11434/v1/', # ollama\n base_url='http://127.0.0.1:1234/v1/', # lm_studio\n api_key='ollama', # required but ignored\n )\n\n if os.name == \"nt\" and (3, 8) <= sys.version_info < (3, 99):\n from torchaudio._extension.utils import _init_dll_path\n _init_dll_path() \n\n colorama.init()\n\n # Initialize Rich Console and Live\n live = Live(console=console, refresh_per_second=10, screen=False)\n live.start()\n\n # Initialize a thread-safe queue\n text_queue = queue.Queue()\n\n # Variables for managing displayed text\n full_sentences = []\n rich_text_stored = \"\"\n recorder = None\n displayed_text = \"\"\n text_time_deque = deque()\n\n rapid_sentence_end_detection = 0.4\n end_of_sentence_detection_pause = 1.2\n unknown_sentence_detection_pause = 1.8\n mid_sentence_detection_pause = 2.4\n hard_break_even_on_background_noise = 3.0\n hard_break_even_on_background_noise_min_texts = 3\n hard_break_even_on_background_noise_min_chars = 15\n hard_break_even_on_background_noise_min_similarity = 0.99\n relisten_on_abrupt_stop = True\n\n abrupt_stop = False\n\n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n prev_text = \"\"\n\n speech_finished_cache = {}\n\n def is_speech_finished(text):\n # Check if the result is already in the cache\n if text in speech_finished_cache:\n if IS_DEBUG:\n print(f\"Cache hit for: '{text}'\")\n return speech_finished_cache[text]\n \n user_prompt = (\n \"Please reply with only 'c' if the following text is a complete thought (a sentence that stands on its own), \"\n \"or 'i' if it is not finished. Do not include any additional text in your reply. \"\n \"Consider a full sentence to have a clear subject, verb, and predicate or express a complete idea. \"\n \"Examples:\\n\"\n \"- 'The sky is blue.' is complete (reply 'c').\\n\"\n \"- 'When the sky' is incomplete (reply 'i').\\n\"\n \"- 'She walked home.' is complete (reply 'c').\\n\"\n \"- 'Because he' is incomplete (reply 'i').\\n\"\n f\"\\nText: {text}\"\n )\n\n response = client.chat.completions.create(\n model=\"lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf\",\n messages=[{\"role\": \"user\", \"content\": user_prompt}],\n max_tokens=1,\n temperature=0.0, # Set temperature to 0 for deterministic output\n )\n\n if IS_DEBUG:\n print(f\"t:'{response.choices[0].message.content.strip().lower()}'\", end=\"\", flush=True)\n\n reply = response.choices[0].message.content.strip().lower()\n result = reply == 'c'\n\n # Cache the result\n speech_finished_cache[text] = result\n\n return result\n\n def preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n \n return text\n\n def text_detected(text):\n \"\"\"\n Enqueue the detected text for processing.\n \"\"\"\n text_queue.put(text)\n\n\n def process_queue():\n global recorder, full_sentences, prev_text, displayed_text, rich_text_stored, text_time_deque, abrupt_stop\n\n # Initialize a deque to store texts with their timestamps\n while True:\n try:\n text = text_queue.get(timeout=1) # Wait for text or timeout after 1 second\n except queue.Empty:\n continue # No text to process, continue looping\n\n if text is None:\n # Sentinel value to indicate thread should exit\n break\n\n text = preprocess_text(text)\n current_time = time.time()\n\n sentence_end_marks = ['.', '!', '?', '。'] \n if text.endswith(\"...\"):\n if not recorder.post_speech_silence_duration == mid_sentence_detection_pause:\n recorder.post_speech_silence_duration = mid_sentence_detection_pause\n if IS_DEBUG: print(f\"RT: post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n elif text and text[-1] in sentence_end_marks and prev_text and prev_text[-1] in sentence_end_marks:\n if not recorder.post_speech_silence_duration == end_of_sentence_detection_pause:\n recorder.post_speech_silence_duration = end_of_sentence_detection_pause\n if IS_DEBUG: print(f\"RT: post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n else:\n if not recorder.post_speech_silence_duration == unknown_sentence_detection_pause:\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n if IS_DEBUG: print(f\"RT: post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n\n prev_text = text\n \n import string\n transtext = text.translate(str.maketrans('', '', string.punctuation))\n \n if is_speech_finished(transtext):\n if not recorder.post_speech_silence_duration == rapid_sentence_end_detection:\n recorder.post_speech_silence_duration = rapid_sentence_end_detection\n if IS_DEBUG: print(f\"RT: {transtext} post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n\n # Append the new text with its timestamp\n text_time_deque.append((current_time, text))\n\n # Remove texts older than 1 second\n while text_time_deque and text_time_deque[0][0] < current_time - hard_break_even_on_background_noise:\n text_time_deque.popleft()\n\n # Check if at least 3 texts have arrived within the last full second\n if len(text_time_deque) >= hard_break_even_on_background_noise_min_texts:\n texts = [t[1] for t in text_time_deque]\n first_text = texts[0]\n last_text = texts[-1]\n\n\n # Check if at least 3 texts have arrived within the last full second\n if len(text_time_deque) >= 3:\n texts = [t[1] for t in text_time_deque]\n first_text = texts[0]\n last_text = texts[-1]\n\n # Compute the similarity ratio between the first and last texts\n similarity = SequenceMatcher(None, first_text, last_text).ratio()\n #print(f\"Similarity: {similarity:.2f}\")\n\n if similarity > hard_break_even_on_background_noise_min_similarity and len(first_text) > hard_break_even_on_background_noise_min_chars:\n abrupt_stop = True\n recorder.stop()\n\n rich_text = Text()\n for i, sentence in enumerate(full_sentences):\n if i % 2 == 0:\n rich_text += Text(sentence, style=\"yellow\") + Text(\" \")\n else:\n rich_text += Text(sentence, style=\"cyan\") + Text(\" \")\n \n if text:\n rich_text += Text(text, style=\"bold yellow\")\n\n new_displayed_text = rich_text.plain\n\n if new_displayed_text != displayed_text:\n displayed_text = new_displayed_text\n panel = Panel(rich_text, title=\"[bold green]Live Transcription[/bold green]\", border_style=\"bold green\")\n live.update(panel)\n rich_text_stored = rich_text\n\n # Mark the task as done\n text_queue.task_done()\n\n def process_text(text):\n global recorder, full_sentences, prev_text, abrupt_stop\n if IS_DEBUG: print(f\"SENTENCE: post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n text = preprocess_text(text)\n text = text.rstrip()\n text_time_deque.clear()\n if text.endswith(\"...\"):\n text = text[:-2]\n \n full_sentences.append(text)\n prev_text = \"\"\n text_detected(\"\")\n\n if abrupt_stop:\n abrupt_stop = False\n if relisten_on_abrupt_stop:\n recorder.listen()\n recorder.start()\n if hasattr(recorder, \"last_words_buffer\"):\n recorder.frames.extend(list(recorder.last_words_buffer))\n\n # Recorder configuration\n recorder_config = {\n 'spinner': False,\n 'model': 'medium.en',\n #'input_device_index': 1, # mic\n #'input_device_index': 2, # stereomix\n 'realtime_model_type': 'tiny.en',\n 'language': 'en',\n #'silero_sensitivity': 0.05,\n 'silero_sensitivity': 0.4,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 1.1, \n 'min_gap_between_recordings': 0, \n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.05,\n 'on_realtime_transcription_update': text_detected,\n 'silero_deactivity_detection': False,\n 'early_transcription_on_silence': 0,\n 'beam_size': 5,\n 'beam_size_realtime': 1,\n 'no_log_file': True,\n 'initial_prompt': (\n \"End incomplete sentences with ellipses.\\n\"\n \"Examples:\\n\"\n \"Complete: The sky is blue.\\n\"\n \"Incomplete: When the sky...\\n\"\n \"Complete: She walked home.\\n\"\n \"Incomplete: Because he...\\n\"\n )\n #'initial_prompt': \"Use ellipses for incomplete sentences like: I went to the...\" \n }\n\n if EXTENDED_LOGGING:\n recorder_config['level'] = logging.DEBUG\n\n recorder = AudioToTextRecorder(**recorder_config)\n \n initial_text = Panel(Text(\"Say something...\", style=\"cyan bold\"), title=\"[bold yellow]Waiting for Input[/bold yellow]\", border_style=\"bold yellow\")\n live.update(initial_text)\n\n # Start the worker thread\n worker_thread = threading.Thread(target=process_queue, daemon=True)\n worker_thread.start()\n\n try:\n while True:\n recorder.text(process_text)\n except KeyboardInterrupt:\n # Send sentinel value to worker thread to exit\n text_queue.put(None)\n worker_thread.join()\n live.stop()\n console.print(\"[bold red]Transcription stopped by user. Exiting...[/bold red]\")\n exit(0)\n\n\n"
},
{
"path": "tests/realtimestt_speechendpoint_binary_classified.py",
"content": "#IS_DEBUG = True\nIS_DEBUG = False\nUSE_STEREO_MIX = True\nLOOPBACK_DEVICE_NAME = \"stereomix\"\nLOOPBACK_DEVICE_HOST_API = 0\n\nimport os\nimport re\nimport sys\nimport threading\nimport queue\nimport time\nfrom collections import deque\nfrom difflib import SequenceMatcher\nfrom install_packages import check_and_install_packages\n\n# Check and install required packages\ncheck_and_install_packages([\n {'import_name': 'rich'},\n {'import_name': 'colorama'},\n {'import_name': 'RealtimeSTT'},\n {'import_name': 'transformers'},\n {'import_name': 'torch'},\n])\n\nEXTENDED_LOGGING = False\nsentence_end_marks = ['.', '!', '?', '。']\n\n\ndetection_speed = 2.0 # set detection speed between 0.1 and 2.0\n\n\n\nif detection_speed < 0.1:\n detection_speed = 0.1\nif detection_speed > 2.5:\n detection_speed = 2.5\n\nlast_detection_pause = 0\nlast_prob_complete = 0\nlast_suggested_pause = 0\nlast_pause = 0\nunknown_sentence_detection_pause = 1.8\nellipsis_pause = 4.5\npunctuation_pause = 0.4\nexclamation_pause = 0.3\nquestion_pause = 0.2\n\nhard_break_even_on_background_noise = 6\nhard_break_even_on_background_noise_min_texts = 3\nhard_break_even_on_background_noise_min_chars = 15\nhard_break_even_on_background_noise_min_similarity = 0.99\n\nif __name__ == '__main__':\n\n if EXTENDED_LOGGING:\n import logging\n logging.basicConfig(level=logging.DEBUG)\n\n from rich.console import Console\n from rich.live import Live\n from rich.text import Text\n from rich.panel import Panel\n console = Console()\n console.print(\"System initializing, please wait\")\n\n from RealtimeSTT import AudioToTextRecorder\n from colorama import Fore, Style\n import colorama\n\n import torch\n import torch.nn.functional as F\n from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n\n # Load classification model\n device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n model_dir = \"KoljaB/SentenceFinishedClassification\"\n max_length = 128\n\n tokenizer = DistilBertTokenizerFast.from_pretrained(model_dir)\n classification_model = DistilBertForSequenceClassification.from_pretrained(model_dir)\n classification_model.to(device)\n classification_model.eval()\n\n # Label mapping\n label_map = {0: \"Incomplete\", 1: \"Complete\"}\n\n # We now want probabilities, not just a label\n def get_completion_probability(sentence, model, tokenizer, device, max_length):\n \"\"\"\n Return the probability that the sentence is complete.\n \"\"\"\n inputs = tokenizer(\n sentence,\n return_tensors=\"pt\",\n truncation=True,\n padding=\"max_length\",\n max_length=max_length\n )\n inputs = {key: value.to(device) for key, value in inputs.items()}\n\n with torch.no_grad():\n outputs = model(**inputs)\n\n logits = outputs.logits\n probabilities = F.softmax(logits, dim=1).squeeze().tolist()\n # probabilities is [prob_incomplete, prob_complete]\n # We want the probability of being complete\n prob_complete = probabilities[1]\n return prob_complete\n\n # We have anchor points for probability to detection mapping\n # (probability, rapid_sentence_end_detection)\n anchor_points = [\n (0.0, 1.0),\n (1.0, 0)\n ]\n\n def interpolate_detection(prob):\n # Clamp probability between 0.0 and 1.0 just in case\n p = max(0.0, min(prob, 1.0))\n # If exactly at an anchor point\n for ap_p, ap_val in anchor_points:\n if abs(ap_p - p) < 1e-9:\n return ap_val\n\n # Find where p fits\n for i in range(len(anchor_points) - 1):\n p1, v1 = anchor_points[i]\n p2, v2 = anchor_points[i+1]\n if p1 <= p <= p2:\n # Linear interpolation\n ratio = (p - p1) / (p2 - p1)\n return v1 + ratio * (v2 - v1)\n\n # Should never reach here if anchor_points cover [0,1]\n return 4.0\n\n speech_finished_cache = {}\n\n def is_speech_finished(text):\n # Returns a probability of completeness\n # Use cache if available\n if text in speech_finished_cache:\n return speech_finished_cache[text]\n \n prob_complete = get_completion_probability(text, classification_model, tokenizer, device, max_length)\n speech_finished_cache[text] = prob_complete\n return prob_complete\n\n if os.name == \"nt\" and (3, 8) <= sys.version_info < (3, 99):\n from torchaudio._extension.utils import _init_dll_path\n _init_dll_path() \n\n colorama.init()\n\n live = Live(console=console, refresh_per_second=10, screen=False)\n live.start()\n\n text_queue = queue.Queue()\n\n full_sentences = []\n rich_text_stored = \"\"\n recorder = None\n displayed_text = \"\"\n text_time_deque = deque()\n texts_without_punctuation = []\n relisten_on_abrupt_stop = True\n abrupt_stop = False\n prev_text = \"\"\n\n def preprocess_text(text):\n text = text.lstrip()\n if text.startswith(\"...\"):\n text = text[3:]\n text = text.lstrip()\n if text:\n text = text[0].upper() + text[1:]\n return text\n\n def text_detected(text):\n text_queue.put(text)\n\n def ends_with_string(text: str, s: str):\n if text.endswith(s):\n return True\n if len(text) > 1 and text[:-1].endswith(s):\n return True\n return False\n\n def sentence_end(text: str):\n if text and text[-1] in sentence_end_marks:\n return True\n return False\n\n def additional_pause_based_on_words(text):\n word_count = len(text.split())\n pauses = {\n 0: 0.35,\n 1: 0.3,\n 2: 0.25,\n 3: 0.2,\n 4: 0.15,\n 5: 0.1,\n 6: 0.05,\n }\n return pauses.get(word_count, 0.0)\n \n def strip_ending_punctuation(text):\n \"\"\"Remove trailing periods and ellipses from text.\"\"\"\n text = text.rstrip()\n for char in sentence_end_marks:\n text = text.rstrip(char)\n return text\n \n def get_suggested_whisper_pause(text):\n if ends_with_string(text, \"...\"):\n return ellipsis_pause\n elif ends_with_string(text, \".\"):\n return punctuation_pause\n elif ends_with_string(text, \"!\"):\n return exclamation_pause\n elif ends_with_string(text, \"?\"):\n return question_pause\n else:\n return unknown_sentence_detection_pause\n\n def find_stereo_mix_index():\n import pyaudio\n audio = pyaudio.PyAudio()\n devices_info = \"\"\n for i in range(audio.get_device_count()):\n dev = audio.get_device_info_by_index(i)\n devices_info += f\"{dev['index']}: {dev['name']} (hostApi: {dev['hostApi']})\\n\"\n\n if (LOOPBACK_DEVICE_NAME.lower() in dev['name'].lower()\n and dev['hostApi'] == LOOPBACK_DEVICE_HOST_API):\n return dev['index'], devices_info\n\n return None, devices_info\n\n def find_matching_texts(texts_without_punctuation):\n \"\"\"\n Find entries where text_without_punctuation matches the last entry,\n going backwards until the first non-match is found.\n \n Args:\n texts_without_punctuation: List of tuples (original_text, stripped_text)\n \n Returns:\n List of tuples (original_text, stripped_text) matching the last entry's stripped text,\n stopping at the first non-match\n \"\"\"\n if not texts_without_punctuation:\n return []\n \n # Get the stripped text from the last entry\n last_stripped_text = texts_without_punctuation[-1][1]\n \n matching_entries = []\n \n # Iterate through the list backwards\n for entry in reversed(texts_without_punctuation):\n original_text, stripped_text = entry\n \n # If we find a non-match, stop\n if stripped_text != last_stripped_text:\n break\n \n # Add the matching entry to our results\n matching_entries.append((original_text, stripped_text))\n \n # Reverse the results to maintain original order\n matching_entries.reverse()\n \n return matching_entries\n\n def process_queue():\n global recorder, full_sentences, prev_text, displayed_text, rich_text_stored, text_time_deque, abrupt_stop, rapid_sentence_end_detection, last_prob_complete, last_suggested_pause, last_pause\n while True:\n text = None # Initialize text to ensure it's defined\n\n try:\n # Attempt to retrieve the first item, blocking with timeout\n text = text_queue.get(timeout=1)\n except queue.Empty:\n continue # No item retrieved, continue the loop\n\n if text is None:\n # Exit signal received\n break\n\n # Drain the queue to get the latest text\n try:\n while True:\n latest_text = text_queue.get_nowait()\n if latest_text is None:\n text = None\n break\n text = latest_text\n except queue.Empty:\n pass # No more items to retrieve\n\n if text is None:\n # Exit signal received after draining\n break\n\n text = preprocess_text(text)\n current_time = time.time()\n text_time_deque.append((current_time, text))\n \n # get text without ending punctuation\n text_without_punctuation = strip_ending_punctuation(text)\n\n # print(f\"Text: {text}, Text without punctuation: {text_without_punctuation}\")\n texts_without_punctuation.append((text, text_without_punctuation))\n\n matches = find_matching_texts(texts_without_punctuation)\n #print(\"Texts matching the last entry's stripped version:\")\n\n added_pauses = 0\n contains_ellipses = False\n for i, match in enumerate(matches):\n same_text, stripped_punctuation = match\n suggested_pause = get_suggested_whisper_pause(same_text)\n added_pauses += suggested_pause\n if ends_with_string(same_text, \"...\"):\n contains_ellipses = True\n \n avg_pause = added_pauses / len(matches) if len(matches) > 0 else 0\n suggested_pause = avg_pause\n # if contains_ellipses:\n # suggested_pause += ellipsis_pause / 2\n\n prev_text = text\n import string\n transtext = text.translate(str.maketrans('', '', string.punctuation))\n\n # **Stripping Trailing Non-Alphabetical Characters**\n # Instead of removing all punctuation, we only strip trailing non-alphabetic chars.\n # Use regex to remove trailing non-alphabetic chars:\n cleaned_for_model = re.sub(r'[^a-zA-Z]+$', '', transtext)\n\n prob_complete = is_speech_finished(cleaned_for_model)\n\n # Interpolate rapid_sentence_end_detection based on prob_complete\n new_detection = interpolate_detection(prob_complete)\n\n # pause = new_detection + suggested_pause\n pause = (new_detection + suggested_pause) * detection_speed\n\n # **Add Additional Pause Based on Word Count**\n # extra_pause = additional_pause_based_on_words(text)\n # pause += extra_pause # Add the extra pause to the total pause duration\n\n # Optionally, you can log this information for debugging\n if IS_DEBUG:\n print(f\"Prob: {prob_complete:.2f}, \"\n f\"whisper {suggested_pause:.2f}, \"\n f\"model {new_detection:.2f}, \"\n # f\"extra {extra_pause:.2f}, \"\n f\"final {pause:.2f} | {transtext} \")\n\n recorder.post_speech_silence_duration = pause\n\n # Remove old entries\n while text_time_deque and text_time_deque[0][0] < current_time - hard_break_even_on_background_noise:\n text_time_deque.popleft()\n\n # Check for abrupt stops (background noise)\n if len(text_time_deque) >= hard_break_even_on_background_noise_min_texts:\n texts = [t[1] for t in text_time_deque]\n first_text = texts[0]\n last_text = texts[-1]\n similarity = SequenceMatcher(None, first_text, last_text).ratio()\n\n if similarity > hard_break_even_on_background_noise_min_similarity and len(first_text) > hard_break_even_on_background_noise_min_chars:\n abrupt_stop = True\n recorder.stop()\n\n rich_text = Text()\n for i, sentence in enumerate(full_sentences):\n style = \"yellow\" if i % 2 == 0 else \"cyan\"\n rich_text += Text(sentence, style=style) + Text(\" \")\n\n if text:\n rich_text += Text(text, style=\"bold yellow\")\n\n new_displayed_text = rich_text.plain\n\n displayed_text = new_displayed_text\n last_prob_complete = new_detection\n last_suggested_pause = suggested_pause\n last_pause = pause\n panel = Panel(rich_text, title=f\"[bold green]Prob complete:[/bold green] [bold yellow]{prob_complete:.2f}[/bold yellow], pause whisper [bold yellow]{suggested_pause:.2f}[/bold yellow], model [bold yellow]{new_detection:.2f}[/bold yellow], last detection [bold yellow]{last_detection_pause:.2f}[/bold yellow]\", border_style=\"bold green\")\n live.update(panel)\n rich_text_stored = rich_text\n\n text_queue.task_done()\n\n def process_text(text):\n global recorder, full_sentences, prev_text, abrupt_stop, last_detection_pause\n last_prob_complete, last_suggested_pause, last_pause\n last_detection_pause = recorder.post_speech_silence_duration\n if IS_DEBUG: print(f\"Model pause: {last_prob_complete:.2f}, Whisper pause: {last_suggested_pause:.2f}, final pause: {last_pause:.2f}, last_detection_pause: {last_detection_pause:.2f}\")\n #if IS_DEBUG: print(f\"SENTENCE: post_speech_silence_duration: {recorder.post_speech_silence_duration}\")\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n text = preprocess_text(text)\n text = text.rstrip()\n text_time_deque.clear()\n if text.endswith(\"...\"):\n text = text[:-2]\n\n full_sentences.append(text)\n prev_text = \"\"\n \n text_detected(\"\")\n\n if abrupt_stop:\n abrupt_stop = False\n if relisten_on_abrupt_stop:\n recorder.listen()\n recorder.start()\n if hasattr(recorder, \"last_words_buffer\"):\n recorder.frames.extend(list(recorder.last_words_buffer))\n\n recorder_config = {\n 'spinner': False,\n 'model': 'large-v3',\n #'realtime_model_type': 'medium.en',\n 'realtime_model_type': 'tiny.en',\n 'language': 'en',\n 'silero_sensitivity': 0.4,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 1.1,\n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.05,\n 'on_realtime_transcription_update': text_detected,\n 'silero_deactivity_detection': True,\n 'early_transcription_on_silence': 0,\n 'beam_size': 5,\n 'beam_size_realtime': 1,\n 'batch_size': 4,\n 'realtime_batch_size': 4,\n 'no_log_file': True,\n 'initial_prompt_realtime': (\n \"End incomplete sentences with ellipses.\\n\"\n \"Examples:\\n\"\n \"Complete: The sky is blue.\\n\"\n \"Incomplete: When the sky...\\n\"\n \"Complete: She walked home.\\n\"\n \"Incomplete: Because he...\\n\"\n )\n }\n\n if EXTENDED_LOGGING:\n recorder_config['level'] = logging.DEBUG\n\n if USE_STEREO_MIX:\n device_index, devices_info = find_stereo_mix_index()\n if device_index is None:\n live.stop()\n console.print(\"[bold red]Stereo Mix device not found. Available audio devices are:\\n[/bold red]\")\n console.print(devices_info, style=\"red\")\n sys.exit(1)\n else:\n recorder_config['input_device_index'] = device_index\n console.print(f\"Using audio device index {device_index} for Stereo Mix.\", style=\"green\")\n\n recorder = AudioToTextRecorder(**recorder_config)\n\n initial_text = Panel(Text(\"Say something...\", style=\"cyan bold\"), title=\"[bold yellow]Waiting for Input[/bold yellow]\", border_style=\"bold yellow\")\n live.update(initial_text)\n\n worker_thread = threading.Thread(target=process_queue, daemon=True)\n worker_thread.start()\n\n try:\n while True:\n recorder.text(process_text)\n except KeyboardInterrupt:\n text_queue.put(None)\n worker_thread.join()\n live.stop()\n console.print(\"[bold red]Transcription stopped by user. Exiting...[/bold red]\")\n exit(0)\n"
},
{
"path": "tests/realtimestt_test.py",
"content": "EXTENDED_LOGGING = False\n\n# set to 0 to deactivate writing to keyboard\n# try lower values like 0.002 (fast) first, take higher values like 0.05 in case it fails\nWRITE_TO_KEYBOARD_INTERVAL = 0.002\n\nif __name__ == '__main__':\n\n import argparse\n parser = argparse.ArgumentParser(description='Start the realtime Speech-to-Text (STT) test with various configuration options.')\n\n parser.add_argument('-m', '--model', type=str, # no default='large-v2',\n help='Path to the STT model or model size. Options include: tiny, tiny.en, base, base.en, small, small.en, medium, medium.en, large-v1, large-v2, or any huggingface CTranslate2 STT model such as deepdml/faster-whisper-large-v3-turbo-ct2. Default is large-v2.')\n\n parser.add_argument('-r', '--rt-model', '--realtime_model_type', type=str, # no default='tiny',\n help='Model size for real-time transcription. Options same as --model. This is used only if real-time transcription is enabled (enable_realtime_transcription). Default is tiny.en.')\n \n parser.add_argument('-l', '--lang', '--language', type=str, # no default='en',\n help='Language code for the STT model to transcribe in a specific language. Leave this empty for auto-detection based on input audio. Default is en. List of supported language codes: https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L11-L110')\n \n parser.add_argument('-d', '--root', type=str, # no default=None,\n help='Root directory where the Whisper models are downloaded to.')\n\n from install_packages import check_and_install_packages\n check_and_install_packages([\n {\n 'import_name': 'rich',\n },\n {\n 'import_name': 'pyautogui',\n } \n ])\n\n if EXTENDED_LOGGING:\n import logging\n logging.basicConfig(level=logging.DEBUG)\n\n from rich.console import Console\n from rich.live import Live\n from rich.text import Text\n from rich.panel import Panel\n from rich.spinner import Spinner\n from rich.progress import Progress, SpinnerColumn, TextColumn\n console = Console()\n console.print(\"System initializing, please wait\")\n\n import os\n import sys\n from RealtimeSTT import AudioToTextRecorder\n from colorama import Fore, Style\n import colorama\n import pyautogui\n\n if os.name == \"nt\" and (3, 8) <= sys.version_info < (3, 99):\n from torchaudio._extension.utils import _init_dll_path\n _init_dll_path() \n\n colorama.init()\n\n # Initialize Rich Console and Live\n live = Live(console=console, refresh_per_second=10, screen=False)\n live.start()\n\n full_sentences = []\n rich_text_stored = \"\"\n recorder = None\n displayed_text = \"\" # Used for tracking text that was already displayed\n\n end_of_sentence_detection_pause = 0.45\n unknown_sentence_detection_pause = 0.7\n mid_sentence_detection_pause = 2.0\n\n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n prev_text = \"\"\n\n def preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n \n return text\n\n\n def text_detected(text):\n global prev_text, displayed_text, rich_text_stored\n\n text = preprocess_text(text)\n\n sentence_end_marks = ['.', '!', '?', '。'] \n if text.endswith(\"...\"):\n recorder.post_speech_silence_duration = mid_sentence_detection_pause\n elif text and text[-1] in sentence_end_marks and prev_text and prev_text[-1] in sentence_end_marks:\n recorder.post_speech_silence_duration = end_of_sentence_detection_pause\n else:\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n\n prev_text = text\n\n # Build Rich Text with alternating colors\n rich_text = Text()\n for i, sentence in enumerate(full_sentences):\n if i % 2 == 0:\n #rich_text += Text(sentence, style=\"bold yellow\") + Text(\" \")\n rich_text += Text(sentence, style=\"yellow\") + Text(\" \")\n else:\n rich_text += Text(sentence, style=\"cyan\") + Text(\" \")\n \n # If the current text is not a sentence-ending, display it in real-time\n if text:\n rich_text += Text(text, style=\"bold yellow\")\n\n new_displayed_text = rich_text.plain\n\n if new_displayed_text != displayed_text:\n displayed_text = new_displayed_text\n panel = Panel(rich_text, title=\"[bold green]Live Transcription[/bold green]\", border_style=\"bold green\")\n live.update(panel)\n rich_text_stored = rich_text\n\n def process_text(text):\n global recorder, full_sentences, prev_text\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n\n text = preprocess_text(text)\n text = text.rstrip()\n if text.endswith(\"...\"):\n text = text[:-2]\n \n if not text:\n return\n\n full_sentences.append(text)\n prev_text = \"\"\n text_detected(\"\")\n\n if WRITE_TO_KEYBOARD_INTERVAL:\n pyautogui.write(f\"{text} \", interval=WRITE_TO_KEYBOARD_INTERVAL) # Adjust interval as needed\n\n # Recorder configuration\n recorder_config = {\n 'spinner': False,\n 'model': 'large-v2', # or large-v2 or deepdml/faster-whisper-large-v3-turbo-ct2 or ...\n 'download_root': None, # default download root location. Ex. ~/.cache/huggingface/hub/ in Linux\n # 'input_device_index': 1,\n 'realtime_model_type': 'tiny.en', # or small.en or distil-small.en or ...\n 'language': 'en',\n 'silero_sensitivity': 0.05,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 1.1, \n 'min_gap_between_recordings': 0, \n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.02,\n 'on_realtime_transcription_update': text_detected,\n #'on_realtime_transcription_stabilized': text_detected,\n 'silero_deactivity_detection': True,\n 'early_transcription_on_silence': 0,\n 'beam_size': 5,\n 'beam_size_realtime': 3,\n # 'batch_size': 0,\n # 'realtime_batch_size': 0, \n 'no_log_file': True,\n 'initial_prompt_realtime': (\n \"End incomplete sentences with ellipses.\\n\"\n \"Examples:\\n\"\n \"Complete: The sky is blue.\\n\"\n \"Incomplete: When the sky...\\n\"\n \"Complete: She walked home.\\n\"\n \"Incomplete: Because he...\\n\"\n ),\n 'silero_use_onnx': True,\n 'faster_whisper_vad_filter': False,\n }\n\n args = parser.parse_args()\n if args.model is not None:\n recorder_config['model'] = args.model\n print(f\"Argument 'model' set to {recorder_config['model']}\")\n if args.rt_model is not None:\n recorder_config['realtime_model_type'] = args.rt_model\n print(f\"Argument 'realtime_model_type' set to {recorder_config['realtime_model_type']}\")\n if args.lang is not None:\n recorder_config['language'] = args.lang\n print(f\"Argument 'language' set to {recorder_config['language']}\")\n if args.root is not None:\n recorder_config['download_root'] = args.root\n print(f\"Argument 'download_root' set to {recorder_config['download_root']}\")\n\n if EXTENDED_LOGGING:\n recorder_config['level'] = logging.DEBUG\n\n recorder = AudioToTextRecorder(**recorder_config)\n \n initial_text = Panel(Text(\"Say something...\", style=\"cyan bold\"), title=\"[bold yellow]Waiting for Input[/bold yellow]\", border_style=\"bold yellow\")\n live.update(initial_text)\n\n try:\n while True:\n recorder.text(process_text)\n except KeyboardInterrupt:\n live.stop()\n console.print(\"[bold red]Transcription stopped by user. Exiting...[/bold red]\")\n exit(0)\n"
},
{
"path": "tests/realtimestt_test_hotkeys_v2.py",
"content": "EXTENDED_LOGGING = False\n\nif __name__ == '__main__':\n\n import subprocess\n import sys\n import threading\n import time\n\n def install_rich():\n subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"rich\"])\n\n try:\n import rich\n except ImportError:\n user_input = input(\"This demo needs the 'rich' library, which is not installed.\\nDo you want to install it now? (y/n): \")\n if user_input.lower() == 'y':\n try:\n install_rich()\n import rich\n print(\"Successfully installed 'rich'.\")\n except Exception as e:\n print(f\"An error occurred while installing 'rich': {e}\")\n sys.exit(1)\n else:\n print(\"The program requires the 'rich' library to run. Exiting...\")\n sys.exit(1)\n\n import keyboard\n import pyperclip\n\n if EXTENDED_LOGGING:\n import logging\n logging.basicConfig(level=logging.DEBUG)\n\n from rich.console import Console\n from rich.live import Live\n from rich.text import Text\n from rich.panel import Panel\n console = Console()\n console.print(\"System initializing, please wait\")\n\n import os\n from RealtimeSTT import AudioToTextRecorder # Ensure this module has stop() or close() methods\n\n import colorama\n colorama.init()\n\n # Import pyautogui\n import pyautogui\n\n import pyaudio\n import numpy as np\n\n # Initialize Rich Console and Live\n live = Live(console=console, refresh_per_second=10, screen=False)\n live.start()\n\n # Global variables\n full_sentences = []\n rich_text_stored = \"\"\n recorder = None\n displayed_text = \"\" # Used for tracking text that was already displayed\n\n end_of_sentence_detection_pause = 0.45\n unknown_sentence_detection_pause = 0.7\n mid_sentence_detection_pause = 2.0\n\n prev_text = \"\"\n\n # Events to signal threads to exit or reset\n exit_event = threading.Event()\n reset_event = threading.Event()\n\n def preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n\n return text\n\n def text_detected(text):\n global prev_text, displayed_text, rich_text_stored\n\n text = preprocess_text(text)\n\n sentence_end_marks = ['.', '!', '?', '。']\n if text.endswith(\"...\"):\n recorder.post_speech_silence_duration = mid_sentence_detection_pause\n elif text and text[-1] in sentence_end_marks and prev_text and prev_text[-1] in sentence_end_marks:\n recorder.post_speech_silence_duration = end_of_sentence_detection_pause\n else:\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n\n prev_text = text\n\n # Build Rich Text with alternating colors\n rich_text = Text()\n for i, sentence in enumerate(full_sentences):\n if i % 2 == 0:\n rich_text += Text(sentence, style=\"yellow\") + Text(\" \")\n else:\n rich_text += Text(sentence, style=\"cyan\") + Text(\" \")\n\n # If the current text is not a sentence-ending, display it in real-time\n if text:\n rich_text += Text(text, style=\"bold yellow\")\n\n new_displayed_text = rich_text.plain\n\n if new_displayed_text != displayed_text:\n displayed_text = new_displayed_text\n panel = Panel(rich_text, title=\"[bold green]Live Transcription[/bold green]\", border_style=\"bold green\")\n live.update(panel)\n rich_text_stored = rich_text\n\n def process_text(text):\n global recorder, full_sentences, prev_text, displayed_text\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n text = preprocess_text(text)\n text = text.rstrip()\n if text.endswith(\"...\"):\n text = text[:-2]\n\n full_sentences.append(text)\n prev_text = \"\"\n text_detected(\"\")\n\n # Check if reset_event is set\n if reset_event.is_set():\n # Clear buffers\n full_sentences.clear()\n displayed_text = \"\"\n reset_event.clear()\n console.print(\"[bold magenta]Transcription buffer reset.[/bold magenta]\")\n return\n\n # Type the finalized sentence to the active window quickly if typing is enabled\n try:\n # Release modifier keys to prevent stuck keys\n for key in ['ctrl', 'shift', 'alt', 'win']:\n keyboard.release(key)\n pyautogui.keyUp(key)\n\n # Use clipboard to paste text\n pyperclip.copy(text + ' ')\n pyautogui.hotkey('ctrl', 'v')\n\n except Exception as e:\n console.print(f\"[bold red]Failed to type the text: {e}[/bold red]\")\n\n # Recorder configuration\n recorder_config = {\n 'spinner': False,\n 'model': 'Systran/faster-distil-whisper-large-v3', # distil-medium.en or large-v2 or deepdml/faster-whisper-large-v3-turbo-ct2 or ...\n 'input_device_index': 1,\n 'realtime_model_type': 'Systran/faster-distil-whisper-large-v3', # Using the same model for realtime\n 'language': 'en',\n 'silero_sensitivity': 0.05,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 1.1,\n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.02,\n 'on_realtime_transcription_update': text_detected,\n # 'on_realtime_transcription_stabilized': text_detected,\n 'silero_deactivity_detection': True,\n 'early_transcription_on_silence': 0,\n 'beam_size': 5,\n 'beam_size_realtime': 5, # Matching beam_size for consistency\n 'no_log_file': True,\n 'initial_prompt': \"Use ellipses for incomplete sentences like: I went to the...\",\n 'device': 'cuda', # Added device configuration\n 'compute_type': 'float16' # Added compute_type configuration\n }\n\n if EXTENDED_LOGGING:\n recorder_config['level'] = logging.DEBUG\n\n recorder = AudioToTextRecorder(**recorder_config)\n\n initial_text = Panel(Text(\"Say something...\", style=\"cyan bold\"), title=\"[bold yellow]Waiting for Input[/bold yellow]\", border_style=\"bold yellow\")\n live.update(initial_text)\n\n # Print available hotkeys\n console.print(\"[bold green]Available Hotkeys:[/bold green]\")\n console.print(\"[bold cyan]F1[/bold cyan]: Mute Microphone\")\n console.print(\"[bold cyan]F2[/bold cyan]: Unmute Microphone\")\n console.print(\"[bold cyan]F3[/bold cyan]: Start Static Recording\")\n console.print(\"[bold cyan]F4[/bold cyan]: Stop Static Recording\")\n console.print(\"[bold cyan]F5[/bold cyan]: Reset Transcription\")\n\n # Global variables for static recording\n static_recording_active = False\n static_recording_thread = None\n static_audio_frames = []\n live_recording_enabled = True # Track whether live recording was enabled before static recording\n\n # Audio settings for static recording\n audio_settings = {\n 'FORMAT': pyaudio.paInt16, # PyAudio format\n 'CHANNELS': 1, # Mono audio\n 'RATE': 16000, # Sample rate\n 'CHUNK': 1024 # Buffer size\n }\n\n # Note: The maximum recommended length of static recording is about 5 minutes.\n\n def static_recording_worker():\n \"\"\"\n Worker function to record audio statically.\n \"\"\"\n global static_audio_frames, static_recording_active\n # Set up pyaudio\n p = pyaudio.PyAudio()\n # Use the same audio format as defined in audio_settings\n FORMAT = audio_settings['FORMAT']\n CHANNELS = audio_settings['CHANNELS']\n RATE = audio_settings['RATE'] # Sample rate\n CHUNK = audio_settings['CHUNK'] # Buffer size\n\n # Open the audio stream\n try:\n stream = p.open(format=FORMAT,\n channels=CHANNELS,\n rate=RATE,\n input=True,\n frames_per_buffer=CHUNK)\n except Exception as e:\n console.print(f\"[bold red]Failed to open audio stream for static recording: {e}[/bold red]\")\n static_recording_active = False\n p.terminate()\n return\n\n while static_recording_active and not exit_event.is_set():\n try:\n data = stream.read(CHUNK)\n static_audio_frames.append(data)\n except Exception as e:\n console.print(f\"[bold red]Error during static recording: {e}[/bold red]\")\n break\n\n # Stop and close the stream\n stream.stop_stream()\n stream.close()\n p.terminate()\n\n def start_static_recording():\n \"\"\"\n Starts the static audio recording.\n \"\"\"\n global static_recording_active, static_recording_thread, static_audio_frames, live_recording_enabled\n if static_recording_active:\n console.print(\"[bold yellow]Static recording is already in progress.[/bold yellow]\")\n return\n\n # Mute the live recording microphone\n live_recording_enabled = recorder.use_microphone.value\n if live_recording_enabled:\n recorder.set_microphone(False)\n console.print(\"[bold yellow]Live microphone muted during static recording.[/bold yellow]\")\n\n console.print(\"[bold green]Starting static recording... Press F4 or F5 to stop/reset.[/bold green]\")\n static_audio_frames = []\n static_recording_active = True\n static_recording_thread = threading.Thread(target=static_recording_worker, daemon=True)\n static_recording_thread.start()\n\n def stop_static_recording():\n \"\"\"\n Stops the static audio recording and processes the transcription.\n \"\"\"\n global static_recording_active, static_recording_thread\n if not static_recording_active:\n console.print(\"[bold yellow]No static recording is in progress.[/bold yellow]\")\n return\n\n console.print(\"[bold green]Stopping static recording...[/bold green]\")\n static_recording_active = False\n if static_recording_thread is not None:\n static_recording_thread.join()\n static_recording_thread = None\n\n # Start a new thread to process the transcription\n processing_thread = threading.Thread(target=process_static_transcription, daemon=True)\n processing_thread.start()\n\n def process_static_transcription():\n global static_audio_frames, live_recording_enabled\n if exit_event.is_set():\n return\n # Process the recorded audio\n console.print(\"[bold green]Processing static recording...[/bold green]\")\n\n # Convert audio data to numpy array\n audio_data = b''.join(static_audio_frames)\n audio_array = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0\n\n # Transcribe the audio data\n try:\n from faster_whisper import WhisperModel\n except ImportError:\n console.print(\"[bold red]faster_whisper is not installed. Please install it to use static transcription.[/bold red]\")\n return\n\n # Load the model using recorder_config\n model_size = recorder_config['model']\n device = recorder_config['device']\n compute_type = recorder_config['compute_type']\n\n console.print(\"Loading transcription model... This may take a moment.\")\n try:\n model = WhisperModel(model_size, device=device, compute_type=compute_type)\n except Exception as e:\n console.print(f\"[bold red]Failed to load transcription model: {e}[/bold red]\")\n return\n\n # Transcribe the audio\n try:\n segments, info = model.transcribe(audio_array, beam_size=recorder_config['beam_size'])\n transcription = ' '.join([segment.text for segment in segments]).strip()\n except Exception as e:\n console.print(f\"[bold red]Error during transcription: {e}[/bold red]\")\n return\n\n # Display the transcription\n console.print(\"Static Recording Transcription:\")\n console.print(f\"[bold cyan]{transcription}[/bold cyan]\")\n\n # Type the transcription into the active window\n try:\n # Release modifier keys to prevent stuck keys\n for key in ['ctrl', 'shift', 'alt', 'win']:\n keyboard.release(key)\n pyautogui.keyUp(key)\n\n # Use clipboard to paste text\n pyperclip.copy(transcription + ' ')\n pyautogui.hotkey('ctrl', 'v')\n\n except Exception as e:\n console.print(f\"[bold red]Failed to type the static transcription: {e}[/bold red]\")\n\n # Unmute the live recording microphone if it was enabled before\n if live_recording_enabled and not exit_event.is_set():\n recorder.set_microphone(True)\n console.print(\"[bold yellow]Live microphone unmuted.[/bold yellow]\")\n\n def reset_transcription():\n \"\"\"\n Resets the transcription by flushing ongoing recordings or buffers.\n \"\"\"\n global static_recording_active, static_recording_thread, static_audio_frames\n console.print(\"[bold magenta]Resetting transcription...[/bold magenta]\")\n if static_recording_active:\n console.print(\"[bold magenta]Flushing static recording...[/bold magenta]\")\n # Stop static recording\n static_recording_active = False\n if static_recording_thread is not None:\n static_recording_thread.join()\n static_recording_thread = None\n # Clear static audio frames\n static_audio_frames = []\n # Unmute microphone if it was muted during static recording\n if live_recording_enabled:\n recorder.set_microphone(True)\n console.print(\"[bold yellow]Live microphone unmuted after reset.[/bold yellow]\")\n elif recorder.use_microphone.value:\n # Live transcription is active and microphone is not muted\n console.print(\"[bold magenta]Resetting live transcription buffer...[/bold magenta]\")\n reset_event.set()\n else:\n # Microphone is muted; nothing to reset\n console.print(\"[bold yellow]Microphone is muted. Nothing to reset.[/bold yellow]\")\n\n # Hotkey Callback Functions\n\n def mute_microphone():\n recorder.set_microphone(False)\n console.print(\"[bold red]Microphone muted.[/bold red]\")\n\n def unmute_microphone():\n recorder.set_microphone(True)\n console.print(\"[bold green]Microphone unmuted.[/bold green]\")\n\n # Start the transcription loop in a separate thread\n def transcription_loop():\n try:\n while not exit_event.is_set():\n recorder.text(process_text)\n except Exception as e:\n console.print(f\"[bold red]Error in transcription loop: {e}[/bold red]\")\n finally:\n # Do not call sys.exit() here\n pass\n\n # Start the transcription loop thread\n transcription_thread = threading.Thread(target=transcription_loop, daemon=True)\n transcription_thread.start()\n\n # Define the hotkey combinations and their corresponding functions\n keyboard.add_hotkey('F1', mute_microphone, suppress=True)\n keyboard.add_hotkey('F2', unmute_microphone, suppress=True)\n keyboard.add_hotkey('F3', start_static_recording, suppress=True)\n keyboard.add_hotkey('F4', stop_static_recording, suppress=True)\n keyboard.add_hotkey('F5', reset_transcription, suppress=True)\n\n # Keep the main thread running and handle graceful exit\n try:\n keyboard.wait() # Waits indefinitely, until a hotkey triggers an exit or Ctrl+C\n except KeyboardInterrupt:\n console.print(\"[bold yellow]KeyboardInterrupt received. Exiting...[/bold yellow]\")\n finally:\n # Signal threads to exit\n exit_event.set()\n\n # Reset transcription if needed\n reset_transcription()\n\n # Stop the recorder\n try:\n if hasattr(recorder, 'stop'):\n recorder.stop()\n elif hasattr(recorder, 'close'):\n recorder.close()\n except Exception as e:\n console.print(f\"[bold red]Error stopping recorder: {e}[/bold red]\")\n\n # Allow some time for threads to finish\n time.sleep(1)\n\n # Wait for transcription_thread to finish\n if transcription_thread.is_alive():\n transcription_thread.join(timeout=5)\n\n # Stop the Live console\n live.stop()\n\n console.print(\"[bold red]Exiting gracefully...[/bold red]\")\n sys.exit(0)\n"
},
{
"path": "tests/realtimestt_test_stereomix.py",
"content": "EXTENDED_LOGGING = False\n\ndef main():\n\n from install_packages import check_and_install_packages\n check_and_install_packages([\n {\n 'import_name': 'rich',\n }\n ])\n\n if EXTENDED_LOGGING:\n import logging\n logging.basicConfig(level=logging.DEBUG)\n\n import os\n import sys\n import threading\n import time\n import pyaudio\n from rich.console import Console\n from rich.live import Live\n from rich.text import Text\n from rich.panel import Panel\n from rich.spinner import Spinner\n from rich.progress import Progress, SpinnerColumn, TextColumn\n from colorama import Fore, Style, init as colorama_init\n\n from RealtimeSTT import AudioToTextRecorder \n\n # Configuration Constants\n LOOPBACK_DEVICE_NAME = \"stereomix\"\n LOOPBACK_DEVICE_HOST_API = 0\n BUFFER_SIZE = 512 \n AUDIO_FORMAT = pyaudio.paInt16\n CHANNELS = 1\n RATE = 16000\n\n console = Console()\n console.print(\"System initializing, please wait\")\n\n colorama_init()\n\n # Initialize Rich Console and Live\n live = Live(console=console, refresh_per_second=10, screen=False)\n live.start()\n\n full_sentences = []\n rich_text_stored = \"\"\n recorder = None\n displayed_text = \"\" # Used for tracking text that was already displayed\n\n end_of_sentence_detection_pause = 0.2\n unknown_sentence_detection_pause = 0.5\n mid_sentence_detection_pause = 1\n\n prev_text = \"\"\n\n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n def preprocess_text(text):\n # Remove leading whitespaces\n text = text.lstrip()\n\n # Remove starting ellipses if present\n if text.startswith(\"...\"):\n text = text[3:]\n\n # Remove any leading whitespaces again after ellipses removal\n text = text.lstrip()\n\n # Uppercase the first letter\n if text:\n text = text[0].upper() + text[1:]\n\n return text\n\n def text_detected(text):\n nonlocal prev_text, displayed_text, rich_text_stored\n\n text = preprocess_text(text)\n\n sentence_end_marks = ['.', '!', '?', '。']\n midsentence_marks = ['…', '-', '(']\n if text.endswith(\"...\") or text and text[-1] in midsentence_marks:\n recorder.post_speech_silence_duration = mid_sentence_detection_pause\n elif text and text[-1] in sentence_end_marks and prev_text and prev_text[-1] in sentence_end_marks:\n recorder.post_speech_silence_duration = end_of_sentence_detection_pause\n else:\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n\n prev_text = text\n\n # Build Rich Text with alternating colors\n rich_text = Text()\n for i, sentence in enumerate(full_sentences):\n if i % 2 == 0:\n rich_text += Text(sentence, style=\"yellow\") + Text(\" \")\n else:\n rich_text += Text(sentence, style=\"cyan\") + Text(\" \")\n\n # If the current text is not a sentence-ending, display it in real-time\n if text:\n rich_text += Text(text, style=\"bold yellow\")\n\n new_displayed_text = rich_text.plain\n\n if new_displayed_text != displayed_text:\n displayed_text = new_displayed_text\n panel = Panel(rich_text, title=\"[bold green]Live Transcription[/bold green]\", border_style=\"bold green\")\n live.update(panel)\n rich_text_stored = rich_text\n\n def process_text(text):\n nonlocal recorder, full_sentences, prev_text\n recorder.post_speech_silence_duration = unknown_sentence_detection_pause\n text = preprocess_text(text)\n text = text.rstrip()\n if text.endswith(\"...\"):\n text = text[:-2] # Remove ellipsis\n\n full_sentences.append(text)\n prev_text = \"\"\n text_detected(\"\")\n\n # Recorder configuration\n recorder_config = {\n 'spinner': False,\n 'use_microphone': False,\n 'model': 'large-v2',\n 'input_device_index': None, # To be set after finding the device\n 'realtime_model_type': 'tiny.en',\n 'language': 'en',\n 'silero_sensitivity': 0.05,\n 'webrtc_sensitivity': 3,\n 'post_speech_silence_duration': unknown_sentence_detection_pause,\n 'min_length_of_recording': 2.0, \n 'min_gap_between_recordings': 0,\n 'enable_realtime_transcription': True,\n 'realtime_processing_pause': 0.01,\n 'on_realtime_transcription_update': text_detected,\n 'silero_deactivity_detection': False,\n 'early_transcription_on_silence': 0,\n 'beam_size': 5,\n 'beam_size_realtime': 1,\n 'no_log_file': True,\n 'initial_prompt': \"Use ellipses for incomplete sentences like: I went to the...\"\n }\n\n if EXTENDED_LOGGING:\n recorder_config['level'] = logging.DEBUG\n\n # Initialize PyAudio\n audio = pyaudio.PyAudio()\n\n def find_stereo_mix_index():\n nonlocal audio\n devices_info = \"\"\n for i in range(audio.get_device_count()):\n dev = audio.get_device_info_by_index(i)\n devices_info += f\"{dev['index']}: {dev['name']} (hostApi: {dev['hostApi']})\\n\"\n\n if (LOOPBACK_DEVICE_NAME.lower() in dev['name'].lower()\n and dev['hostApi'] == LOOPBACK_DEVICE_HOST_API):\n return dev['index'], devices_info\n\n return None, devices_info\n\n device_index, devices_info = find_stereo_mix_index()\n if device_index is None:\n live.stop()\n console.print(\"[bold red]Stereo Mix device not found. Available audio devices are:\\n[/bold red]\")\n console.print(devices_info, style=\"red\")\n audio.terminate()\n sys.exit(1)\n else:\n recorder_config['input_device_index'] = device_index\n console.print(f\"Using audio device index {device_index} for Stereo Mix.\", style=\"green\")\n\n # Initialize the recorder\n recorder = AudioToTextRecorder(**recorder_config)\n\n # Initialize Live Display with waiting message\n initial_text = Panel(Text(\"Say something...\", style=\"cyan bold\"), title=\"[bold yellow]Waiting for Input[/bold yellow]\", border_style=\"bold yellow\")\n live.update(initial_text)\n\n # Define the recording thread\n def recording_thread():\n nonlocal recorder\n stream = audio.open(format=AUDIO_FORMAT,\n channels=CHANNELS,\n rate=RATE,\n input=True,\n frames_per_buffer=BUFFER_SIZE,\n input_device_index=recorder_config['input_device_index'])\n\n try:\n while not stop_event.is_set():\n data = stream.read(BUFFER_SIZE, exception_on_overflow=False)\n recorder.feed_audio(data)\n except Exception as e:\n console.print(f\"[bold red]Error in recording thread: {e}[/bold red]\")\n finally:\n console.print(f\"[bold red]Stopping stream[/bold red]\")\n stream.stop_stream()\n stream.close()\n\n # Define the stop event\n stop_event = threading.Event()\n\n # Start the recording thread\n thread = threading.Thread(target=recording_thread, daemon=True)\n thread.start()\n\n try:\n while True:\n recorder.text(process_text)\n except KeyboardInterrupt:\n console.print(\"[bold red]\\nTranscription stopped by user. Exiting...[/bold red]\")\n finally:\n print(\"live stop\")\n live.stop()\n\n print(\"setting stop event\")\n stop_event.set()\n\n print(\"thread join\")\n thread.join()\n\n print(\"recorder stop\")\n recorder.stop()\n\n print(\"audio terminate\")\n audio.terminate()\n\n print(\"sys exit \")\n sys.exit(0)\n\nif __name__ == '__main__':\n main()\n"
},
{
"path": "tests/recorder_client.py",
"content": "from RealtimeSTT import AudioToTextRecorderClient\n\n# ANSI escape codes for terminal control\nCLEAR_LINE = \"\\033[K\" # Clear from cursor to end of line\nRESET_CURSOR = \"\\r\" # Move cursor to the beginning of the line\nGREEN_TEXT = \"\\033[92m\" # Set text color to green\nRESET_COLOR = \"\\033[0m\" # Reset text color to default\n\ndef print_realtime_text(text):\n print(f\"{RESET_CURSOR}{CLEAR_LINE}{GREEN_TEXT}👄 {text}{RESET_COLOR}\", end=\"\", flush=True)\n\n# Initialize the audio recorder with the real-time transcription callback\nrecorder = AudioToTextRecorderClient(on_realtime_transcription_update=print_realtime_text)\n\n# Print the speaking prompt\nprint(\"👄 \", end=\"\", flush=True)\n\ntry:\n while True:\n # Fetch finalized transcription text, if available\n if text := recorder.text():\n # Display the finalized transcription\n print(f\"{RESET_CURSOR}{CLEAR_LINE}✍️ {text}\\n👄 \", end=\"\", flush=True)\nexcept KeyboardInterrupt:\n # Handle graceful shutdown on Ctrl+C\n print(f\"{RESET_CURSOR}{CLEAR_LINE}\", end=\"\", flush=True)\n recorder.shutdown()"
},
{
"path": "tests/simple_test.py",
"content": "if __name__ == '__main__':\n\n import os\n import sys\n if os.name == \"nt\" and (3, 8) <= sys.version_info < (3, 99):\n from torchaudio._extension.utils import _init_dll_path\n _init_dll_path()\n\n from RealtimeSTT import AudioToTextRecorder\n\n recorder = AudioToTextRecorder(\n spinner=False,\n silero_sensitivity=0.01,\n model=\"tiny.en\",\n language=\"en\",\n )\n\n print(\"Say something...\")\n \n try:\n while (True):\n print(\"Detected text: \" + recorder.text())\n except KeyboardInterrupt:\n print(\"Exiting application due to keyboard interrupt\")\n"
},
{
"path": "tests/translator.py",
"content": "import os\nimport openai\nfrom RealtimeSTT import AudioToTextRecorder\nfrom RealtimeTTS import TextToAudioStream, AzureEngine\n\nif __name__ == '__main__':\n # Setup OpenAI API key\n openai.api_key = os.environ.get(\"OPENAI_API_KEY\")\n\n # Text-to-Speech Stream Setup (alternative engines: SystemEngine or ElevenlabsEngine)\n engine = AzureEngine( \n os.environ.get(\"AZURE_SPEECH_KEY\"),\n os.environ.get(\"AZURE_SPEECH_REGION\")\n )\n stream = TextToAudioStream(engine, log_characters=True)\n\n # Speech-to-Text Recorder Setup\n recorder = AudioToTextRecorder(\n model=\"medium\",\n )\n\n # Supported languages and their voices\n languages = [\n [\"english\", \"AshleyNeural\"],\n [\"german\", \"AmalaNeural\"],\n [\"french\", \"DeniseNeural\"],\n [\"spanish\", \"EstrellaNeural\"],\n [\"portuguese\", \"FernandaNeural\"],\n [\"italian\", \"FabiolaNeural\"]\n ]\n\n def generate_response(messages):\n \"\"\"Generate assistant's response using OpenAI.\"\"\"\n for chunk in openai.ChatCompletion.create(model=\"gpt-3.5-turbo\", messages=messages, stream=True):\n text_chunk = chunk[\"choices\"][0][\"delta\"].get(\"content\")\n if text_chunk:\n yield text_chunk\n \n def clear_console():\n os.system('clear' if os.name == 'posix' else 'cls')\n\n def select_language():\n \"\"\"Display language options and get user's choice.\"\"\"\n for index, language in enumerate(languages, start=1):\n print(f\"{index}. {language[0]}\")\n language_number = input(\"Select language to translate to (1-6): \")\n return languages[int(language_number) - 1]\n\n def main():\n \"\"\"Main translation loop.\"\"\"\n clear_console()\n language_info = select_language()\n engine.set_voice(language_info[1])\n\n system_prompt_message = {\n 'role': 'system',\n 'content': f'Translate the given text to {language_info[0]}. Output only the translated text.'\n }\n\n while True:\n print(\"\\nSay something!\")\n\n # Capture user input from microphone\n user_text = recorder.text()\n print(f\"Input text: {user_text}\")\n\n user_message = {'role': 'user', 'content': user_text}\n\n # Get assistant response and play it\n translation_stream = generate_response([system_prompt_message, user_message])\n print(\"Translation: \", end=\"\", flush=True)\n stream.feed(translation_stream)\n stream.play()\n\n main()"
},
{
"path": "tests/type_into_textbox.py",
"content": "from RealtimeSTT import AudioToTextRecorder\nimport pyautogui\n\ndef process_text(text):\n pyautogui.typewrite(text + \" \")\n\nif __name__ == '__main__':\n print(\"Wait until it says 'speak now'\")\n recorder = AudioToTextRecorder()\n\n while True:\n recorder.text(process_text)"
},
{
"path": "tests/vad_test.py",
"content": "import asyncio\nimport time\n\nfrom RealtimeSTT import AudioToTextRecorder\n\n\n# Voice Activity Detection (VAD) start handler\ndef on_vad_detect_start():\n print(f\"VAD Start detected at {time.time():.2f}\")\n\n\n# Voice Activity Detection (VAD) stop handler\ndef on_vad_detect_stop():\n print(f\"VAD Stop detected at {time.time():.2f}\")\n\n\n# Transcription completion handler\ndef on_transcription_finished(text):\n print(f\"Transcribed text: {text}\")\n\n\nasync def run_recording(recorder):\n # Start recording and process audio in a loop\n print(\"Starting recording...\")\n while True:\n # Use text() to process audio and get transcription\n recorder.text(on_transcription_finished=on_transcription_finished)\n await asyncio.sleep(0.1) # Prevent tight loop\n\n\nasync def main():\n # Initialize AudioToTextRecorder with VAD event handlers\n recorder = AudioToTextRecorder(\n # model=\"deepdml/faster-whisper-large-v3-turbo-ct2\",\n spinner=False,\n on_vad_detect_start=on_vad_detect_start,\n on_vad_detect_stop=on_vad_detect_stop,\n )\n\n # Start recording task in a separate thread\n recording_task = asyncio.create_task(run_recording(recorder))\n\n # Run for 20 seconds to observe VAD events\n await asyncio.sleep(20)\n\n # Stop recording and shutdown\n print(\"Stopping recording...\")\n recorder.stop()\n recorder.shutdown()\n\n # Cancel and wait for the recording task to complete\n recording_task.cancel()\n try:\n await recording_task\n except asyncio.CancelledError:\n pass\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())"
},
{
"path": "win_installgpu_virtual_env.bat",
"content": "@echo off\ncd /d %~dp0\n\nREM Check if the venv directory exists\nif not exist test_env\\Scripts\\python.exe (\n echo Creating VENV\n python -m venv test_env\n) else (\n echo VENV already exists\n)\n\necho Activating VENV\nstart cmd /k \"call test_env\\Scripts\\activate.bat && install_with_gpu_support.bat\"\n"
}
]