Repository: microsoft/ai-chat-protocol Branch: main Commit: 540122f753d5 Files: 74 Total size: 158.9 KB Directory structure: gitextract_8epire6f/ ├── .devcontainer/ │ └── devcontainer.json ├── .github/ │ ├── CODEOWNERS │ ├── dependabot.yml │ └── workflows/ │ ├── typescript-build.yml │ └── typescript-release.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── omnisharp.json ├── samples/ │ ├── README.md │ ├── backend/ │ │ ├── csharp/ │ │ │ ├── ChatProtocolBackend.csproj │ │ │ ├── Controllers/ │ │ │ │ └── ChatController.cs │ │ │ ├── Converters/ │ │ │ │ └── JsonCamelCaseEnumConverter.cs │ │ │ ├── Interfaces/ │ │ │ │ ├── ISecretStore.cs │ │ │ │ ├── ISemanticKernelApp.cs │ │ │ │ ├── ISemanticKernelSession.cs │ │ │ │ └── IStateStore.cs │ │ │ ├── Model/ │ │ │ │ ├── AIChatCompletion.cs │ │ │ │ ├── AIChatCompletionDelta.cs │ │ │ │ ├── AIChatFile.cs │ │ │ │ ├── AIChatMessage.cs │ │ │ │ ├── AIChatMessageDelta.cs │ │ │ │ ├── AIChatRequest.cs │ │ │ │ └── AIChatRole.cs │ │ │ ├── Program.cs │ │ │ ├── Properties/ │ │ │ │ └── launchSettings.json │ │ │ ├── Services/ │ │ │ │ ├── EnvVarSecretStore.cs │ │ │ │ ├── InMemoryStore.cs │ │ │ │ ├── KeyVaultSecretStore.cs │ │ │ │ └── SemanticKernelApp.cs │ │ │ ├── appsettings.Development.json │ │ │ └── appsettings.json │ │ ├── js/ │ │ │ └── expressjs/ │ │ │ ├── .gitignore │ │ │ ├── package.json │ │ │ ├── src/ │ │ │ │ ├── config.ts │ │ │ │ ├── index.ts │ │ │ │ ├── routes/ │ │ │ │ │ └── chat.ts │ │ │ │ └── state-store.ts │ │ │ └── tsconfig.json │ │ └── python/ │ │ └── quart/ │ │ ├── .gitignore │ │ ├── __init__.py │ │ ├── model/ │ │ │ ├── __init__.py │ │ │ └── model.py │ │ ├── pyproject.toml │ │ ├── requirements-dev.txt │ │ └── requirements.txt │ └── frontend/ │ └── js/ │ └── react/ │ ├── .eslintrc.cjs │ ├── .gitignore │ ├── README.md │ ├── index.html │ ├── package.json │ ├── src/ │ │ ├── App.module.css │ │ ├── App.tsx │ │ ├── Chat.module.css │ │ ├── Chat.tsx │ │ ├── Readme.tsx │ │ ├── globals.d.ts │ │ ├── main.tsx │ │ └── vite-env.d.ts │ ├── tsconfig.json │ ├── tsconfig.node.json │ └── vite.config.ts ├── sdk/ │ └── js/ │ └── .gitignore └── spec/ ├── .gitignore ├── README.md ├── legacy/ │ └── 2024-01-28.md ├── main.tsp ├── models.tsp ├── operations.tsp ├── package.json └── tspconfig.yaml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .devcontainer/devcontainer.json ================================================ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu { "name": "Ubuntu", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "ghcr.io/devcontainers/features/dotnet:2": {}, "ghcr.io/devcontainers/features/node:1": {}, "ghcr.io/devcontainers/features/python": "3.12", "ghcr.io/devcontainers/features/azure-cli": "latest" } // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. // "postCreateCommand": "uname -a", // Configure tool-specific properties. // "customizations": {}, // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" } ================================================ FILE: .github/CODEOWNERS ================================================ * @dargilco * @glecaros * @rohit-ganguly ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for more information: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates # https://containers.dev/guide/dependabot version: 2 updates: - package-ecosystem: "devcontainers" directory: "/" schedule: interval: weekly ================================================ FILE: .github/workflows/typescript-build.yml ================================================ name: TypeScript Build on: push: branches: - main paths: - 'sdk/js/packages/**' pull_request: branches: - main paths: - 'sdk/js/packages/**' jobs: build: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '20.11.1' - name: Install Dependencies run: npm ci working-directory: ./sdk/js/packages/client - name: Run Linter run: npm run lint working-directory: ./sdk/js/packages/client - name: Check Format run: npm run check-format working-directory: ./sdk/js/packages/client - name: Build run: npm run build working-directory: ./sdk/js/packages/client - name: Test run: npm test working-directory: ./sdk/js/packages/client - name: Update Version run: | version=$(npm pkg get version | tr -d '"') npm pkg set version="${version}-beta.${GITHUB_RUN_NUMBER}" working-directory: ./sdk/js/packages/client - name: Pack run: npm pack working-directory: ./sdk/js/packages/client - name: Upload Artifact uses: actions/upload-artifact@v4 with: name: npm-package path: ./sdk/js/packages/client/*.tgz ================================================ FILE: .github/workflows/typescript-release.yml ================================================ name: TypeScript Release on: push: tags: - 'release/js/*' jobs: build: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Setup Node.js uses: actions/setup-node@v4 with: node-version: '20.11.1' - name: Install Dependencies run: npm ci working-directory: ./sdk/js/packages/client - name: Run Linter run: npm run lint working-directory: ./sdk/js/packages/client - name: Check Format run: npm run check-format working-directory: ./sdk/js/packages/client - name: Build run: npm run build working-directory: ./sdk/js/packages/client - name: Test run: npm test working-directory: ./sdk/js/packages/client - name: Set Version run: | version=${GITHUB_REF#refs/tags/release/js/} npm pkg set version="${version}" working-directory: ./sdk/js/packages/client - name: Pack run: npm pack working-directory: ./sdk/js/packages/client - name: Upload Artifact uses: actions/upload-artifact@v4 with: name: npm-package path: ./sdk/js/packages/client/*.tgz release: needs: build runs-on: ubuntu-latest permissions: write-all steps: - name: Checkout code uses: actions/checkout@v4 - name: Download Artifact uses: actions/download-artifact@v4 with: name: npm-package path: ./packages/ - name: Create Release env: GH_TOKEN: ${{ github.token}} run: | tag_name=${GITHUB_REF#refs/tags/} version=${tag_name#release/js/} gh release create $tag_name -t "Typescript Release $version" ./packages/*.tgz ================================================ FILE: .gitignore ================================================ ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. ## ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore # User-specific files *.rsuser *.suo *.user *.userosscache *.sln.docstates # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs # Mono auto generated files mono_crash.* # Build results [Dd]ebug/ [Dd]ebugPublic/ [Rr]elease/ [Rr]eleases/ x64/ x86/ [Ww][Ii][Nn]32/ [Aa][Rr][Mm]/ [Aa][Rr][Mm]64/ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ [Ll]ogs/ # Visual Studio 2015/2017 cache/options directory .vs/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ # Visual Studio 2017 auto generated files Generated\ Files/ # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* # NUnit *.VisualState.xml TestResult.xml nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ [Rr]eleasePS/ dlldata.c # Benchmark Results BenchmarkDotNet.Artifacts/ # .NET Core project.lock.json project.fragment.lock.json artifacts/ # ASP.NET Scaffolding ScaffoldingReadMe.txt # StyleCop StyleCopReport.xml # Files built by Visual Studio *_i.c *_p.c *_h.h *.ilk *.meta *.obj *.iobj *.pch *.pdb *.ipdb *.pgc *.pgd *.rsp *.sbr *.tlb *.tli *.tlh *.tmp *.tmp_proj *_wpftmp.csproj *.log *.tlog *.vspscc *.vssscc .builds *.pidb *.svclog *.scc # Chutzpah Test files _Chutzpah* # Visual C++ cache files ipch/ *.aps *.ncb *.opendb *.opensdf *.sdf *.cachefile *.VC.db *.VC.VC.opendb # Visual Studio profiler *.psess *.vsp *.vspx *.sap # Visual Studio Trace Files *.e2e # TFS 2012 Local Workspace $tf/ # Guidance Automation Toolkit *.gpState # ReSharper is a .NET coding add-in _ReSharper*/ *.[Rr]e[Ss]harper *.DotSettings.user # TeamCity is a build add-in _TeamCity* # DotCover is a Code Coverage Tool *.dotCover # AxoCover is a Code Coverage Tool .axoCover/* !.axoCover/settings.json # Coverlet is a free, cross platform Code Coverage Tool coverage*.json coverage*.xml coverage*.info # Visual Studio code coverage results *.coverage *.coveragexml # NCrunch _NCrunch_* .*crunch*.local.xml nCrunchTemp_* # MightyMoose *.mm.* AutoTest.Net/ # Web workbench (sass) .sass-cache/ # Installshield output folder [Ee]xpress/ # DocProject is a documentation generator add-in DocProject/buildhelp/ DocProject/Help/*.HxT DocProject/Help/*.HxC DocProject/Help/*.hhc DocProject/Help/*.hhk DocProject/Help/*.hhp DocProject/Help/Html2 DocProject/Help/html # Click-Once directory publish/ # Publish Web Output *.[Pp]ublish.xml *.azurePubxml # Note: Comment the next line if you want to checkin your web deploy settings, # but database connection strings (with potential passwords) will be unencrypted *.pubxml *.publishproj # Microsoft Azure Web App publish settings. Comment the next line if you want to # checkin your Azure Web App publish settings, but sensitive information contained # in these scripts will be unencrypted PublishScripts/ # NuGet Packages *.nupkg # NuGet Symbol Packages *.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. !**/[Pp]ackages/build/ # Uncomment if necessary however generally it will be regenerated when needed #!**/[Pp]ackages/repositories.config # NuGet v3's project.json files produces more ignorable files *.nuget.props *.nuget.targets # Microsoft Azure Build Output csx/ *.build.csdef # Microsoft Azure Emulator ecf/ rcf/ # Windows Store app package directories and files AppPackages/ BundleArtifacts/ Package.StoreAssociation.xml _pkginfo.txt *.appx *.appxbundle *.appxupload # Visual Studio cache files # files ending in .cache can be ignored *.[Cc]ache # but keep track of directories ending in .cache !?*.[Cc]ache/ # Others ClientBin/ ~$* *~ *.dbmdl *.dbproj.schemaview *.jfm *.pfx *.publishsettings orleans.codegen.cs # Including strong name files can present a security risk # (https://github.com/github/gitignore/pull/2483#issue-259490424) #*.snk # Since there are multiple workflows, uncomment next line to ignore bower_components # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) #bower_components/ # RIA/Silverlight projects Generated_Code/ # Backup & report files from converting an old project file # to a newer Visual Studio version. Backup files are not needed, # because we have git ;-) _UpgradeReport_Files/ Backup*/ UpgradeLog*.XML UpgradeLog*.htm ServiceFabricBackup/ *.rptproj.bak # SQL Server files *.mdf *.ldf *.ndf # Business Intelligence projects *.rdl.data *.bim.layout *.bim_*.settings *.rptproj.rsuser *- [Bb]ackup.rdl *- [Bb]ackup ([0-9]).rdl *- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ # GhostDoc plugin setting file *.GhostDoc.xml # Node.js Tools for Visual Studio .ntvs_analysis.dat node_modules/ # Visual Studio 6 build log *.plg # Visual Studio 6 workspace options file *.opt # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) *.vbw # Visual Studio 6 auto-generated project file (contains which files were open etc.) *.vbp # Visual Studio 6 workspace and project file (working project files containing files to include in project) *.dsw *.dsp # Visual Studio 6 technical files *.ncb *.aps # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts **/*.DesktopClient/ModelManifest.xml **/*.Server/GeneratedArtifacts **/*.Server/ModelManifest.xml _Pvt_Extensions # Paket dependency manager .paket/paket.exe paket-files/ # FAKE - F# Make .fake/ # CodeRush personal settings .cr/personal # Python Tools for Visual Studio (PTVS) __pycache__/ *.pyc # Cake - Uncomment if you are using it # tools/** # !tools/packages.config # Tabs Studio *.tss # Telerik's JustMock configuration file *.jmconfig # BizTalk build output *.btp.cs *.btm.cs *.odx.cs *.xsd.cs # OpenCover UI analysis results OpenCover/ # Azure Stream Analytics local run output ASALocalRun/ # MSBuild Binary and Structured Log *.binlog # NVidia Nsight GPU debugger configuration file *.nvuser # MFractors (Xamarin productivity tool) working folder .mfractor/ # Local History for Visual Studio .localhistory/ # Visual Studio History (VSHistory) files .vshistory/ # BeatPulse healthcheck temp database healthchecksdb # Backup folder for Package Reference Convert tool in Visual Studio 2017 MigrationBackup/ # Ionide (cross platform F# VS Code tools) working folder .ionide/ # Fody - auto-generated XML schema FodyWeavers.xsd # VS Code files for those working on multiple tools .vscode/* !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json *.code-workspace # Local History for Visual Studio Code .history/ # Windows Installer files from build outputs *.cab *.msi *.msix *.msm *.msp # JetBrains Rider *.sln.iml # For any other files you don't want Git to track .hide/ ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Microsoft Open Source Code of Conduct This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). Resources: - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns ================================================ FILE: CONTRIBUTING.md ================================================ ## Contributing As an open source repository, the AI Chat Protocol SDK welcomes contributions in the form of issues and pull requests (PRs). Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA. This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) Microsoft Corporation. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE ================================================ FILE: README.md ================================================ # Microsoft AI Chat Protocol [![NPM Package](https://img.shields.io/npm/v/@microsoft/ai-chat-protocol)](https://www.npmjs.com/package/@microsoft/ai-chat-protocol) [![TypeScript Build](https://github.com/microsoft/ai-chat-protocol/actions/workflows/typescript-build.yml/badge.svg)](https://github.com/microsoft/ai-chat-protocol/actions/workflows/typescript-build.yml) The Microsoft [AI Chat Protocol SDK](/sdk) is a library for easily building AI Chat interfaces from services that follow the [AI Chat Protocol API Specification](https://aka.ms/chatprotocol), both of which are located in this repository. By agreeing on a standard API contract, AI backend consumption and evaluation can be performed easily and consistently across different services regardless of the models, orchestration tooling, or design patterns used. *Note: we are currently in public preview. Your feedback is greatly appreciated as we get ready to be generally available!* With the AI Chat Protocol, you will be able to: * Develop AI chat interfaces, components, and applications in JavaScript/TypeScript (more languages to follow!) * Consistently consume and evaluate AI inference backends and middle tiers with ease, either synchronously or by streaming * Easily incorporate HTTP middleware for logging, authentication, and more. **Please star the repo to show your support for this project!** ## Getting Started Our comprehensive getting started guide is coming soon! Be sure to check out the samples and API specification for more details. * [Samples](/samples) * [API Specification](/spec) * [Samples on Azure](#samples-on-azure) To take a look locally, install the library via npm: ```bash npm install @microsoft/ai-chat-protocol ``` Create the client object: ```javascript const client = new AIChatProtocolClient("/api/chat"); ``` Stream completions to your UI: ```javascript let sessionState = undefined; // add any logic to handle state here function setSessionState(value) { sessionState = value; } const message: AIChatMessage = { role: "user", content: "Hello World!", }; const result = await client.getStreamedCompletion([message], { sessionState: sessionState, }); for await (const response of result) { if (response.sessionState) { //do something with the session state returned } if (response.delta.role) { // do something with the information about the role } if (response.delta.content) { // do something with the content of the message } } ``` ## Samples on Azure If you're curious on samples hosted on Azure, the following samples utilize the AI Chat Protocol SDK on the frontend: * [Serverless AI Chat with RAG using LangChain.js](https://github.com/Azure-Samples/serverless-chat-langchainjs) * [Chat Application using Azure OpenAI (Python)](https://github.com/Azure-Samples/openai-chat-app-quickstart) * [OpenAI Chat Application with Microsoft Entra Authentication (Python) - Local](https://github.com/Azure-Samples//openai-chat-app-entra-auth-local) * [OpenAI Chat Application with Microsoft Entra Authentication (Python) - Builtin](https://github.com/Azure-Samples/openai-chat-app-entra-auth-builtin) * [OpenAI Chat App Frontend (Vanilla JS)](https://github.com/Azure-Samples/openai-chat-frontend-vanillajs) * [Chat Application using Azure OpenAI (Python)](https://github.com/Azure-Samples/openai-chat-app-quickstart) Additionally, many Azure AI sample projects utilize the AI Chat Protocol API spec without the SDK, either because they don't have a frontend, or because they were made before the library's release: * [ChatGPT + Enterprise data with Azure OpenAI and AI Search in Python](https://github.com/Azure-samples/azure-search-openai-demo) * [ChatGPT + Enterprise data with Azure OpenAI and Azure AI Search in JavaScript](https://github.com/Azure-samples/azure-search-openai-javascript) * [Chat with GPT Modes - FastAPI backend](https://github.com/Azure-Samples/openai-chat-backend-fastapi) * [Evaluating a RAG Chat App](https://github.com/Azure-Samples/ai-rag-chat-evaluator) ## Code of Conduct This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. ## License Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the [MIT](LICENSE) license. ================================================ FILE: SECURITY.md ================================================ ## Security Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. ## Reporting Security Issues **Please do not report security vulnerabilities through public GitHub issues.** Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) * Full paths of source file(s) related to the manifestation of the issue * The location of the affected source code (tag/branch/commit or direct URL) * Any special configuration required to reproduce the issue * Step-by-step instructions to reproduce the issue * Proof-of-concept or exploit code (if possible) * Impact of the issue, including how an attacker might exploit the issue This information will help us triage your report more quickly. If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. ## Preferred Languages We prefer all communications to be in English. ## Policy Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). ================================================ FILE: omnisharp.json ================================================ { "projectLoadTimeout": 100, "maxProjectResults": 1, "excludeSearchPatterns": ["samples/**/*"] } ================================================ FILE: samples/README.md ================================================ # Microsoft AI Chat Protocol Samples > This directory contains basic starters for using the AI Chat Protocol. If you're interested in more in-depth, end-to-end samples hosted on Azure, visit this [link](https://aka.ms/aichat/templates). If you'd like to run the samples, follow these steps: ## Frontend 1. Clone the repository to your machine. 1. In one terminal, navigate to the `frontend/js/react` directory. 1. In the `frontend/js/react` directory, run `npm install` to install your dependencies, including [`@microsoft/ai-chat-protocol`](https://www.npmjs.com/package/@microsoft/ai-chat-protocol). 1. Next, run `npm run dev` to start your web application. ## Backend The backend directory has both a .NET and a JavaScript (Express) backend sample. Follow the steps below for the sample you'd like to run. ### .NET (with Semantic Kernel) 1. In one terminal, navigate to the `backend/csharp` directory. 2. Set the following environment variables: 1. UseAzureOpenAI - either `true` or `false` 1. If using Azure OpenAI, set your `AzureDeployment` and `AzureEndpoint` according to this [guide](https://learn.microsoft.com/en-us/azure/ai-services/openai/quickstart?tabs=command-line%2Cpython-new&pivots=programming-language-python#retrieve-key-and-endpoint). 2. Sign into Azure using the Azure CLI (`az login`) or Azure Developer CLI (`azd auth login`). 2. If you're using OpenAI (*not* Azure OpenAI), set the environment variables `APIKey` and `Model`. 3. Next, run `dotnet restore` to restore your dependencies and `dotnet run` to run the backend. ### JavaScript (Express) 1. In one terminal, navigate to the `backend/js` directory. 2. In the `.env` file, update `AZURE_OPENAI_ENDPOINT` and `AZURE_OPENAI_DEPLOYMENT` according to this [guide](https://learn.microsoft.com/en-us/azure/ai-services/openai/quickstart?tabs=command-line%2Cpython-new&pivots=programming-language-python#retrieve-key-and-endpoint). 3. Run `npm install` to install your dependencies. 4. Run `npm run dev` to run the backend. ================================================ FILE: samples/backend/csharp/ChatProtocolBackend.csproj ================================================ net8.0 enable enable ================================================ FILE: samples/backend/csharp/Controllers/ChatController.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json; using Microsoft.AspNetCore.Mvc; using Backend.Interfaces; using Backend.Model; using System.Text.RegularExpressions; namespace Backend.Controllers; [ApiController, Route("api/[controller]")] public partial class ChatController : ControllerBase { private readonly ISemanticKernelApp _semanticKernelApp; public ChatController(ISemanticKernelApp semanticKernelApp) { _semanticKernelApp = semanticKernelApp; } [GeneratedRegex(@"messages\[(\d+)\]\.files\[(\d+)\]")] private static partial Regex MessageFilesRegex(); private (int MessageIndex, int FileIndex, IFormFile File) GetPosition(IFormFile formFile) { var match = MessageFilesRegex().Match(formFile.Name); if (match.Success && int.TryParse(match.Groups[1].ValueSpan, out var messageIndex) && int.TryParse(match.Groups[2].ValueSpan, out var fileIndex)) { return (messageIndex, fileIndex, formFile); } throw new ArgumentException("Malformed multipart request: Invalid file name."); } private async Task RequestFromMultipart(IFormFileCollection formFiles) { using var jsonFileStream = formFiles .First(f => f.Name == "json") .OpenReadStream(); if (jsonFileStream is null) { throw new Exception("Malformed multipart request: Missing json part."); } var request = await JsonSerializer.DeserializeAsync(jsonFileStream) ?? throw new Exception("Malformed multipart request: Invalid json part."); foreach (var (messageIndex, fileIndex, file) in formFiles.Where(f => f.Name != "json").Select(GetPosition).OrderBy(p => p.MessageIndex).ThenBy(p => p.FileIndex)) { if (request.Messages.Count <= messageIndex) { throw new Exception("Malformed multipart request: Invalid message index."); } var message = request.Messages[messageIndex]; message.Files ??= new List(); if (message.Files.Count != fileIndex) { throw new Exception("Malformed multipart request: Invalid file index."); } using var fileStream = file.OpenReadStream(); var fileData = await BinaryData.FromStreamAsync(fileStream); message.Files.Add(new AIChatFile { ContentType = file.ContentType, Data = fileData }); } return request; } [HttpPost] [Consumes("multipart/form-data")] public async Task ProcessMessage(IFormFileCollection files) { try { var request = await RequestFromMultipart(files); var session = request.SessionState switch { Guid sessionId => await _semanticKernelApp.GetSession(sessionId), _ => await _semanticKernelApp.CreateSession(Guid.NewGuid()) }; return Ok(await session.ProcessRequest(request)); } catch (Exception e) { return BadRequest(e.Message); } } [HttpPost] [Consumes("application/json")] public async Task ProcessMessage(AIChatRequest request) { var session = request.SessionState switch { Guid sessionId => await _semanticKernelApp.GetSession(sessionId), _ => await _semanticKernelApp.CreateSession(Guid.NewGuid()) }; var response = await session.ProcessRequest(request); return Ok(response); } [HttpPost("stream")] [Consumes("application/json")] public async Task ProcessStreamingMessage(AIChatRequest request) { var session = request.SessionState switch { Guid sessionId => await _semanticKernelApp.GetSession(sessionId), _ => await _semanticKernelApp.CreateSession(Guid.NewGuid()) }; var response = Response; response.Headers.Append("Content-Type", "application/jsonl"); await foreach (var delta in session.ProcessStreamingRequest(request)) { await response.WriteAsync($"{JsonSerializer.Serialize(delta)}\r\n"); await response.Body.FlushAsync(); } } } ================================================ FILE: samples/backend/csharp/Converters/JsonCamelCaseEnumConverter.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json; using System.Text.Json.Serialization; namespace Backend.Converters; public class JsonCamelCaseEnumConverter : JsonStringEnumConverter where T : struct, Enum { public JsonCamelCaseEnumConverter() : base(JsonNamingPolicy.CamelCase) { } } ================================================ FILE: samples/backend/csharp/Interfaces/ISecretStore.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. namespace Backend.Interfaces; public interface ISecretStore { Task GetSecretAsync(string secretName, CancellationToken cancellationToken = default); } ================================================ FILE: samples/backend/csharp/Interfaces/ISemanticKernelApp.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. namespace Backend.Interfaces; public interface ISemanticKernelApp { Task CreateSession(Guid sessionId); Task GetSession(Guid sessionId); } ================================================ FILE: samples/backend/csharp/Interfaces/ISemanticKernelSession.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Backend.Model; namespace Backend.Interfaces; public interface ISemanticKernelSession { Guid Id { get; } Task ProcessRequest(AIChatRequest request); IAsyncEnumerable ProcessStreamingRequest(AIChatRequest request); } ================================================ FILE: samples/backend/csharp/Interfaces/IStateStore.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. namespace Backend.Interfaces; public interface IStateStore { Task GetStateAsync(Guid sessionId); Task SetStateAsync(Guid sessionId, T state); Task RemoveStateAsync(Guid sessionId); } ================================================ FILE: samples/backend/csharp/Model/AIChatCompletion.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public record AIChatCompletion([property: JsonPropertyName("message")] AIChatMessage Message) { [JsonInclude, JsonPropertyName("sessionState"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Guid? SessionState; [JsonInclude, JsonPropertyName("context"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public BinaryData? Context; } ================================================ FILE: samples/backend/csharp/Model/AIChatCompletionDelta.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public record AIChatCompletionDelta([property: JsonPropertyName("delta")] AIChatMessageDelta Delta) { [JsonInclude, JsonPropertyName("sessionState"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public Guid? SessionState; [JsonInclude, JsonPropertyName("context"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public BinaryData? Context; } ================================================ FILE: samples/backend/csharp/Model/AIChatFile.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public struct AIChatFile { [JsonPropertyName("contentType")] public string ContentType { get; set; } [JsonPropertyName("data")] public BinaryData Data { get; set; } } ================================================ FILE: samples/backend/csharp/Model/AIChatMessage.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public struct AIChatMessage { [JsonPropertyName("content")] public string Content { get; set; } [JsonPropertyName("role")] public AIChatRole Role { get; set; } [JsonPropertyName("context")] public BinaryData? Context { get; set; } [JsonPropertyName("files")] public IList? Files { get; set; } } ================================================ FILE: samples/backend/csharp/Model/AIChatMessageDelta.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public struct AIChatMessageDelta { [JsonPropertyName("content")] public string? Content { get; set; } [JsonPropertyName("role")] public AIChatRole? Role { get; set; } [JsonPropertyName("context"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public BinaryData? Context { get; set; } } ================================================ FILE: samples/backend/csharp/Model/AIChatRequest.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; namespace Backend.Model; public record AIChatRequest([property: JsonPropertyName("messages")] IList Messages) { [JsonInclude, JsonPropertyName("sessionState")] public Guid? SessionState; [JsonInclude, JsonPropertyName("context")] public BinaryData? Context; } ================================================ FILE: samples/backend/csharp/Model/AIChatRole.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json.Serialization; using Backend.Converters; namespace Backend.Model; [JsonConverter(typeof(JsonCamelCaseEnumConverter))] public enum AIChatRole { System, Assistant, User } ================================================ FILE: samples/backend/csharp/Program.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text.Json; using System.Text.Json.Serialization; using Backend.Interfaces; using Backend.Model; using Backend.Services; var builder = WebApplication.CreateBuilder(args); builder.Services.AddSingleton>(new InMemoryStore()); builder.Services.AddSingleton(new EnvVarSecretStore()); builder.Services.AddSingleton(); builder.Services .AddControllers() .AddJsonOptions(o => o.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase))); var app = builder.Build(); // Configure the HTTP request pipeline. if (!app.Environment.IsDevelopment()) { app.UseExceptionHandler("/Error"); // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. app.UseHsts(); app.UseHttpsRedirection(); } app.UseStaticFiles(); app.UseRouting(); app.UseAuthorization(); app.MapControllers(); app.Run(); ================================================ FILE: samples/backend/csharp/Properties/launchSettings.json ================================================ { "$schema": "http://json.schemastore.org/launchsettings.json", "iisSettings": { "windowsAuthentication": false, "anonymousAuthentication": true, "iisExpress": { "applicationUrl": "http://localhost:21376", "sslPort": 44324 } }, "profiles": { "http": { "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": true, "applicationUrl": "http://localhost:3000", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "https": { "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": true, "applicationUrl": "https://localhost:7281;http://localhost:5094", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "IIS Express": { "commandName": "IISExpress", "launchBrowser": true, "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } } } } ================================================ FILE: samples/backend/csharp/Services/EnvVarSecretStore.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Backend.Interfaces; namespace Backend.Services; public class EnvVarSecretStore : ISecretStore { public Task GetSecretAsync(string secretName, CancellationToken cancellationToken) { #if !DEBUG throw new ApplicationException("EnvVarSecretStore should not be used in production."); #else return Task.FromResult(Environment.GetEnvironmentVariable(secretName) ?? ""); #endif } } ================================================ FILE: samples/backend/csharp/Services/InMemoryStore.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Backend.Interfaces; public class InMemoryStore : IStateStore { private readonly Dictionary _store = new Dictionary(); public Task GetStateAsync(Guid sessionId) { _store.TryGetValue(sessionId, out var state); return Task.FromResult(state); } public Task SetStateAsync(Guid sessionId, T state) { _store[sessionId] = state; return Task.CompletedTask; } public Task RemoveStateAsync(Guid sessionId) { _store.Remove(sessionId); return Task.CompletedTask; } } ================================================ FILE: samples/backend/csharp/Services/KeyVaultSecretStore.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using Azure.Security.KeyVault.Secrets; using Backend.Interfaces; namespace Backend.Services; public class KeyVaultSecretStore : ISecretStore { private readonly SecretClient _secretClient; public KeyVaultSecretStore(SecretClient secretClient) { _secretClient = secretClient; } public async Task GetSecretAsync(string secretName, CancellationToken cancellationToken) { KeyVaultSecret secret = await _secretClient.GetSecretAsync(secretName, cancellationToken: cancellationToken); return secret.Value; } } ================================================ FILE: samples/backend/csharp/Services/SemanticKernelApp.cs ================================================ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. using System.Text; using Azure.Identity; using Microsoft.SemanticKernel; using Backend.Interfaces; using Backend.Model; namespace Backend.Services; internal record LLMConfig; internal record OpenAIConfig(string Model, string Key): LLMConfig; internal record AzureOpenAIConfig(string Deployment, string Endpoint): LLMConfig; internal struct SemanticKernelConfig { internal LLMConfig LLMConfig { get; private init; } internal static async Task CreateAsync(ISecretStore secretStore, CancellationToken cancellationToken) { var useAzureOpenAI = await secretStore.GetSecretAsync("UseAzureOpenAI", cancellationToken).ContinueWith(task => bool.Parse(task.Result)); if (useAzureOpenAI) { var azureDeployment = await secretStore.GetSecretAsync("AzureDeployment", cancellationToken); var azureEndpoint = await secretStore.GetSecretAsync("AzureEndpoint", cancellationToken); return new SemanticKernelConfig { LLMConfig = new AzureOpenAIConfig(azureDeployment, azureEndpoint), }; } else { var apiKey = await secretStore.GetSecretAsync("APIKey", cancellationToken); var model = await secretStore.GetSecretAsync("Model", cancellationToken); return new SemanticKernelConfig { LLMConfig = new OpenAIConfig(model, apiKey), }; } } } internal class SemanticKernelSession : ISemanticKernelSession { private readonly Kernel _kernel; private readonly IStateStore _stateStore; private readonly KernelFunction _chatFunction; public Guid Id { get; private set; } internal SemanticKernelSession(Kernel kernel, IStateStore stateStore, Guid sessionId) { _kernel = kernel; _stateStore = stateStore; _chatFunction = _kernel.CreateFunctionFromPrompt(prompt); Id = sessionId; } const string prompt = @" ChatBot can have a conversation with you about any topic. It can give explicit instructions or say 'I don't know' if it does not know the answer. {{$history}} User: {{$userInput}} ChatBot:"; public async Task ProcessRequest(AIChatRequest message) { var userInput = message.Messages.Last(); string history = await _stateStore.GetStateAsync(Id) ?? ""; /* TODO: Add support for text+image content */ var arguments = new KernelArguments() { ["history"] = history, ["userInput"] = userInput.Content, }; var botResponse = await _chatFunction.InvokeAsync(_kernel, arguments); var updatedHistory = $"{history}\nUser: {userInput.Content}\nChatBot: {botResponse}"; await _stateStore.SetStateAsync(Id, updatedHistory); return new AIChatCompletion(Message: new AIChatMessage { Role = AIChatRole.Assistant, Content = $"{botResponse}", }) { SessionState = Id, }; } public async IAsyncEnumerable ProcessStreamingRequest(AIChatRequest message) { var userInput = message.Messages.Last(); string history = await _stateStore.GetStateAsync(Id) ?? ""; var arguments = new KernelArguments() { ["history"] = history, ["userInput"] = userInput.Content, }; var streamedBotResponse = _chatFunction.InvokeStreamingAsync(_kernel, arguments); StringBuilder response = new(); await foreach (var botResponse in streamedBotResponse) { response.Append(botResponse); yield return new AIChatCompletionDelta(Delta: new AIChatMessageDelta { Role = AIChatRole.Assistant, Content = $"{botResponse}", }) { SessionState = Id, }; } var updatedHistory = $"{history}\nUser: {userInput.Content}\nChatBot: {response}"; await _stateStore.SetStateAsync(Id, updatedHistory); } } public class SemanticKernelApp : ISemanticKernelApp { private readonly ISecretStore _secretStore; private readonly IStateStore _stateStore; private readonly Lazy> _kernel; private async Task InitKernel() { var config = await SemanticKernelConfig.CreateAsync(_secretStore, CancellationToken.None); var builder = Kernel.CreateBuilder(); if (config.LLMConfig is AzureOpenAIConfig azureOpenAIConfig) { if (azureOpenAIConfig.Deployment is null || azureOpenAIConfig.Endpoint is null) { throw new InvalidOperationException("AzureOpenAI is enabled but AzureDeployment and AzureEndpoint are not set."); } builder.AddAzureOpenAIChatCompletion(azureOpenAIConfig.Deployment, azureOpenAIConfig.Endpoint, new DefaultAzureCredential()); } else if (config.LLMConfig is OpenAIConfig openAIConfig) { if (openAIConfig.Model is null || openAIConfig.Key is null) { throw new InvalidOperationException("AzureOpenAI is disabled but Model and APIKey are not set."); } builder.AddOpenAIChatCompletion(openAIConfig.Model, openAIConfig.Key); } else { throw new InvalidOperationException("Unsupported LLMConfig type."); } return builder.Build(); } public SemanticKernelApp(ISecretStore secretStore, IStateStore stateStore) { _secretStore = secretStore; _stateStore = stateStore; _kernel = new(() => Task.Run(InitKernel)); } public async Task CreateSession(Guid sessionId) { var kernel = await _kernel.Value; return new SemanticKernelSession(kernel, _stateStore, sessionId); } public async Task GetSession(Guid sessionId) { var kernel = await _kernel.Value; var state = await _stateStore.GetStateAsync(sessionId); if (state is null) { throw new KeyNotFoundException($"Session {sessionId} not found."); } return new SemanticKernelSession(kernel, _stateStore, sessionId); } } ================================================ FILE: samples/backend/csharp/appsettings.Development.json ================================================ { "DetailedErrors": true, "Logging": { "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Warning" } } } ================================================ FILE: samples/backend/csharp/appsettings.json ================================================ { "Logging": { "LogLevel": { "Default": "Information", "Microsoft.AspNetCore": "Warning" } }, "AllowedHosts": "*" } ================================================ FILE: samples/backend/js/expressjs/.gitignore ================================================ .env dist ================================================ FILE: samples/backend/js/expressjs/package.json ================================================ { "name": "chat-endpoint", "version": "1.0.0", "description": "A ExpressJS-based reference implementation of a chat endpoint.", "main": "dist/index.js", "scripts": { "build": "tsc", "start": "node dist/index.js", "dev": "nodemon src/index.ts", "format": "prettier --write src" }, "author": "Gerardo Lecaros ", "license": "MIT", "dependencies": { "@azure/identity": "^4.2.1", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.19.1", "formidable": "^3.5.1", "lodash": "^4.17.21", "openai": "^4.52.0", "uuid": "^9.0.1" }, "devDependencies": { "@microsoft/ai-chat-protocol": "^1.0.0-beta.20240610.1", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/formidable": "^3.4.5", "@types/lodash": "^4.17.5", "@types/node": "^20.11.30", "@types/uuid": "^9.0.8", "nodemon": "^3.1.0", "prettier": "^3.2.5", "ts-node": "^10.9.2", "typescript": "^5.4.3" } } ================================================ FILE: samples/backend/js/expressjs/src/config.ts ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import dotenv from "dotenv"; dotenv.config(); export enum ConfigParameter { port, azureOpenAIEndpoint, azureOpenAIDeployment, redisUrl, systemPrompt, stateTTL, } export function getConfig(parameter: ConfigParameter): string { const getValue = () => { switch (parameter) { case ConfigParameter.azureOpenAIEndpoint: { return process.env.AZURE_OPENAI_ENDPOINT; } case ConfigParameter.azureOpenAIDeployment: { return process.env.AZURE_OPENAI_DEPLOYMENT; } case ConfigParameter.port: { return process.env.PORT; } case ConfigParameter.systemPrompt: { return process.env.SYSTEM_PROMPT; } case ConfigParameter.stateTTL: { return process.env.STATE_TTL; } default: { throw new Error("Unsupported config parameter."); } } }; const value = getValue(); if (!value) { throw new Error("Not found."); } return value; } ================================================ FILE: samples/backend/js/expressjs/src/index.ts ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import express, { Express } from "express"; import * as cors from "cors"; import chat from "./routes/chat"; import { ConfigParameter, getConfig } from "./config"; const app: Express = express(); const port = getConfig(ConfigParameter.port); app.use(cors.default()); app.use("/api/chat", chat); app.listen(port, () => { console.log(`[server]: Server is running at http://localhost:${port}`); }); ================================================ FILE: samples/backend/js/expressjs/src/routes/chat.ts ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import { DefaultAzureCredential, getBearerTokenProvider, } from "@azure/identity"; import express, { Router, Request } from "express"; import { v4 as uuid } from "uuid"; import formidable from "formidable"; import _ from "lodash"; import fs from "fs"; import { ConfigParameter, getConfig } from "../config"; import { AIChatCompletion, AIChatCompletionDelta, AIChatCompletionRequest, AIChatMessage, AIChatRole, } from "@microsoft/ai-chat-protocol"; import { StateStore } from "../state-store"; import { AzureOpenAI } from "openai"; import { ChatCompletionContentPartImage, ChatCompletionMessageParam, } from "openai/resources"; declare global { namespace Express { interface Request { sessionState: string; } } } const chat = Router(); const client = new AzureOpenAI({ apiVersion: "2024-05-01-preview", endpoint: getConfig(ConfigParameter.azureOpenAIEndpoint), azureADTokenProvider: getBearerTokenProvider( new DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default", ), }); const stateStore = new StateStore(); type UnknownRequest = Request<{}, {}, unknown>; type ChatRequest = Request<{}, {}, AIChatCompletionRequest>; async function readFile(filepath: string): Promise { const data = await fs.promises.readFile(filepath); await fs.promises.unlink(filepath); return data; } async function readJson(filepath: string): Promise { const buffer = await readFile(filepath); const data = buffer.toString("utf-8"); return JSON.parse(data); } const jsonMiddleware = express.json(); chat.use(async (req: UnknownRequest, res, next) => { if (req.is("multipart/form-data")) { try { const form = formidable(); const [, files] = await form.parse(req); const [jsonFile] = files.json as formidable.File[]; const json = await readJson(jsonFile.filepath); for (let key of Object.keys(files)) { if (key === "json") { continue; } const [file] = files[key] as formidable.File[]; const data = await readFile(file.filepath); _.set(json, `${key}.data`, data); _.set(json, `${key}.contentType`, file.mimetype); } req.body = json; req.headers["content-type"] = "application/json"; return next(); } catch (error) { return next(error); } } else if (req.is("application/json")) { return jsonMiddleware(req, res, next); } return next(); }); chat.use((req: ChatRequest, res, next) => { const request = req.body; if (request.sessionState && typeof request.sessionState == "string") { req.sessionState = request.sessionState as string; try { const history = stateStore.read(request.sessionState); console.info(`Loaded history for session ${request.sessionState}`); req.body.messages = [...history, ...request.messages]; return next(); } catch (error) { console.error( `Failed to load history for session ${request.sessionState}: ${error}`, ); } } else { req.sessionState = uuid(); } request.messages = [ { role: "system", content: getConfig(ConfigParameter.systemPrompt), }, ...request.messages, ]; return next(); }); function toOpenAIMessage(message: AIChatMessage): ChatCompletionMessageParam { if (message.files && message.files.length > 0 && message.role === "user") { const fileContent: ChatCompletionContentPartImage[] = message.files.map( (file) => ({ type: "image_url", image_url: { url: `data:${file.contentType};base64,${file.data.toString("base64")}`, }, }), ); return { role: message.role, content: [ ...fileContent, { type: "text", text: message.content, }, ], }; } return { role: message.role, content: message.content, }; } chat.post("/", async (req: ChatRequest, res, next) => { try { const response = await client.chat.completions.create({ model: getConfig(ConfigParameter.azureOpenAIDeployment), messages: req.body.messages.map(toOpenAIMessage), }); const choice = response.choices[0]; const responseMessage: AIChatMessage = { role: (choice?.message?.role ?? undefined) as AIChatRole, content: choice?.message?.content ?? "", }; stateStore.save(req.sessionState, [...req.body.messages, responseMessage]); const completion: AIChatCompletion = { message: responseMessage, sessionState: req.sessionState, }; res.json(completion); } catch (error) { return next(error); } }); chat.post( "/stream", async (req: Request<{}, {}, AIChatCompletionRequest>, res, next) => { try { const response = await client.chat.completions.create({ stream: true, model: getConfig(ConfigParameter.azureOpenAIDeployment), messages: req.body.messages.map(toOpenAIMessage), }); res.contentType("application/jsonl"); const responseMessage: AIChatMessage = { role: "assistant", content: "", }; for await (const event of response) { const choice = event.choices[0]; if (choice && choice.delta) { const delta = choice.delta; const completion: AIChatCompletionDelta = { delta: { content: delta.content ?? undefined, role: (delta.role ?? undefined) as AIChatRole, }, sessionState: req.sessionState, }; responseMessage.role = completion.delta.role ?? responseMessage.role; responseMessage.content += completion.delta.content ?? ""; res.write(JSON.stringify(completion) + "\r\n"); } } stateStore.save(req.sessionState, [ ...req.body.messages, responseMessage, ]); res.end(); } catch (error) { return next(error); } }, ); export default chat; ================================================ FILE: samples/backend/js/expressjs/src/state-store.ts ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. export class StateStore { private store: Record; constructor() { this.store = {}; } public read(key: string): T { const state = this.store[key]; if (!state) { throw new Error("Not found."); } return this.store[key]; } public async save(key: string, state: T) { this.store[key] = state; } } ================================================ FILE: samples/backend/js/expressjs/tsconfig.json ================================================ { "compilerOptions": { /* Visit https://aka.ms/tsconfig to read more about this file */ /* Projects */ // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ // "jsx": "preserve", /* Specify what JSX code is generated. */ // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ "module": "commonjs", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */ // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ // "resolveJsonModule": true, /* Enable importing .json files. */ // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ /* JavaScript Support */ // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ "outDir": "./dist", /* Specify an output folder for all emitted files. */ // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ // "newLine": "crlf", /* Set the newline character for emitting files. */ // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ /* Type Checking */ "strict": true, /* Enable all strict type-checking options. */ // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ /* Completeness */ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ } } ================================================ FILE: samples/backend/python/quart/.gitignore ================================================ .env ================================================ FILE: samples/backend/python/quart/__init__.py ================================================ import base64 import os import re from typing import Optional from azure.identity import DefaultAzureCredential, get_bearer_token_provider from dotenv import load_dotenv from openai import AsyncAzureOpenAI from pydantic import BaseModel from model import ( AIChatCompletion, AIChatCompletionDelta, AIChatError, AIChatFile, AIChatMessage, AIChatMessageDelta, AIChatRequest, AIChatRole, ) from quart import Quart, jsonify, request, stream_with_context load_dotenv() PORT = os.getenv("PORT", 3000) AZURE_OPENAI_ENDPOINT = os.getenv("AZURE_OPENAI_ENDPOINT") AZURE_OPENAI_DEPLOYMENT = os.getenv("AZURE_OPENAI_DEPLOYMENT") AZURE_OPENAI_API_VERSION = os.getenv("AZURE_OPENAI_API_VERSION") token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default") client = AsyncAzureOpenAI( api_version=AZURE_OPENAI_API_VERSION, azure_endpoint=AZURE_OPENAI_ENDPOINT, azure_ad_token_provider=token_provider, ) app = Quart(__name__) def run() -> None: app.run(port=PORT) def get_file_position(file_key: str) -> tuple[int, int, str]: """ Extracts the message and file indices from a given file key. The function expects file keys in the format "messages[].files[]", where and are integers. It parses these indices from the file key and returns them along with the original file key as a tuple. Args: file_key (str): The key representing a file's position in the message structure, expected to follow the specific format mentioned above. Returns: tuple[int, int, str]: A tuple containing the message index, file index, and the original file key if the key matches the expected format. Raises: ValueError: If the file key does not match the expected format. """ match = re.match(r"messages\[(\d+)\]\.files\[(\d+)\]", file_key) if match: message_index, file_index = map(int, match.groups()) return message_index, file_index, file_key raise ValueError(f"Invalid file name: {file_key}") def reconstruct_multipart_request(form: dict, files: dict): """ Reconstructs an AIChatRequest object from multipart form data. This function takes form data and a dictionary of files, then reconstructs the AIChatRequest object by parsing the JSON content from the form and attaching the files to their corresponding messages based on their keys. Args: form (dict): A dictionary containing the form data, expected to have a "json" key with the JSON representation of the AIChatRequest. files (dict): A dictionary where keys are file keys in the format "messages[].files[]" and values are the file objects. Returns: AIChatRequest: The reconstructed AIChatRequest object with files attached to the appropriate messages. Raises: ValueError: If any file key does not match the expected format, or if the indices in the file keys do not correspond to valid positions in the reconstructed AIChatRequest object. """ json_content = form["json"] chat_request = AIChatRequest.model_validate_json(json_content) file_positions = sorted([get_file_position(file_key) for file_key in files]) for message_index, file_index, file_key in file_positions: file = files[file_key] if len(chat_request.messages) <= message_index: raise ValueError(f"Invalid message index: {file_key}") if chat_request.messages[message_index].files is None: chat_request.messages[message_index].files = [] if len(chat_request.messages[message_index].files) != file_index: raise ValueError(f"Invalid file index: {file_key}") chat_request.messages[message_index].files.append(AIChatFile(content_type=file.content_type, data=file.read())) return chat_request def to_openai_message(chat_message: AIChatMessage): if chat_message.files is None: return { "role": chat_message.role.value, "content": chat_message.content, } def encode_file_to_data_url(file: AIChatFile): base64_data = base64.b64encode(file.data).decode("utf-8") return f"data:{file.content_type};base64,{base64_data}" images = [ {"type": "image_url", "image_url": {"url": encode_file_to_data_url(file)}} for file in chat_message.files if file.content_type.startswith("image/") ] return { "role": chat_message.role.value, "content": [{"type": "text", "text": chat_message.content}] + images, } @app.route("/api/chat/", methods=["POST"]) async def process_message(): try: if request.content_type.startswith("multipart/form-data"): form = await request.form files = await request.files chat_request = reconstruct_multipart_request(form, files) elif request.content_type.startswith("application/json"): chat_request_data = await request.data chat_request = AIChatRequest.model_validate_json(chat_request_data) else: return jsonify({"error": "Unsupported Media Type"}), 415 completion = await client.chat.completions.create( model=AZURE_OPENAI_DEPLOYMENT, messages=[to_openai_message(message) for message in chat_request.messages], ) message = completion.choices[0].message response = AIChatCompletion( message=AIChatMessage( role=AIChatRole(message.role), content=message.content, ), ) return jsonify(response.model_dump()) except Exception as e: return ( jsonify(AIChatError(code="internal_error", message=str(e)).model_dump()), 500, ) def object_to_json_line(obj: BaseModel): return f"{obj.model_dump_json()}\r\n" @app.route("/api/chat/stream", methods=["POST"]) async def process_message_stream(): @stream_with_context async def async_generator(): try: if request.content_type.startswith("multipart/form-data"): form = await request.form files = await request.files chat_request = reconstruct_multipart_request(form, files) elif request.content_type.startswith("application/json"): chat_request_data = await request.data chat_request = AIChatRequest.model_validate_json(chat_request_data) else: yield object_to_json_line(AIChatError(code="unsupported_media_type", message="Unsupported Media Type")) return stream = await client.chat.completions.create( model=AZURE_OPENAI_DEPLOYMENT, stream=True, messages=[to_openai_message(message) for message in chat_request.messages], ) async for chunk in stream: if len(chunk.choices) == 0: continue delta = chunk.choices[0].delta response_chunk = AIChatCompletionDelta( delta=AIChatMessageDelta( content=delta.content, role=delta.role, ), ) yield object_to_json_line(response_chunk) except Exception as e: error = AIChatError(code="internal_error", message=str(e)) yield object_to_json_line(error) return async_generator(), 200, {"Content-Type": "application/jsonl"} if __name__ == "__main__": run() ================================================ FILE: samples/backend/python/quart/model/__init__.py ================================================ from .model import ( AIChatCompletion, AIChatCompletionDelta, AIChatCompletionOptions, AIChatError, AIChatErrorResponse, AIChatFile, AIChatMessage, AIChatMessageDelta, AIChatRequest, AIChatRole, ) __all__ = [ "AIChatRequest", "AIChatErrorResponse", "AIChatCompletion", "AIChatCompletionDelta", "AIChatCompletionOptions", "AIChatError", "AIChatFile", "AIChatMessage", "AIChatMessageDelta", "AIChatRole", ] ================================================ FILE: samples/backend/python/quart/model/model.py ================================================ from enum import Enum from typing import Any, Optional from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel class ChatModel(BaseModel): model_config = ConfigDict( alias_generator=to_camel, populate_by_name=True, from_attributes=True, ) class AIChatRole(str, Enum): USER = "user" ASSISTANT = "assistant" SYSTEM = "system" class AIChatFile(ChatModel): content_type: str = Field(serialization_alias="contentType") data: bytes class AIChatMessage(ChatModel): role: AIChatRole content: str context: Optional[dict[str, Any]] = None files: Optional[list[AIChatFile]] = None class AIChatMessageDelta(ChatModel): role: Optional[AIChatRole] = None content: Optional[str] = None context: Optional[dict[str, Any]] = None class AIChatCompletion(ChatModel): message: AIChatMessage session_state: Optional[Any] = Field(serialization_alias="sessionState", default=None) context: Optional[dict[str, Any]] = None class AIChatCompletionDelta(ChatModel): delta: AIChatMessageDelta session_state: Optional[Any] = Field(serialization_alias="sessionState", default=None) context: Optional[dict[str, Any]] = None class AIChatCompletionOptions(ChatModel): context: Optional[dict[str, Any]] = None session_state: Optional[Any] = Field(serialization_alias="sessionState", default=None) class AIChatError(ChatModel): code: str message: str class AIChatErrorResponse(ChatModel): error: AIChatError class AIChatRequest(ChatModel): messages: list[AIChatMessage] session_state: Optional[Any] = Field(serialization_alias="sessionState", default=None) context: Optional[bytes] = None ================================================ FILE: samples/backend/python/quart/pyproject.toml ================================================ [tool.ruff] line-length = 120 target-version = "py312" [tool.ruff.lint] select = ["E", "F", "I", "UP"] extend-ignore = ["UP007"] [tool.black] line-length = 120 target-version = ["py312"] ================================================ FILE: samples/backend/python/quart/requirements-dev.txt ================================================ -r requirements.txt ruff black ================================================ FILE: samples/backend/python/quart/requirements.txt ================================================ openai quart[dotenv] azure-identity pydantic ================================================ FILE: samples/frontend/js/react/.eslintrc.cjs ================================================ module.exports = { root: true, env: { browser: true, es2020: true }, extends: [ 'eslint:recommended', 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', ], ignorePatterns: ['dist', '.eslintrc.cjs'], parser: '@typescript-eslint/parser', plugins: ['react-refresh'], rules: { 'react-refresh/only-export-components': [ 'warn', { allowConstantExport: true }, ], }, } ================================================ FILE: samples/frontend/js/react/.gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* pnpm-debug.log* lerna-debug.log* node_modules dist dist-ssr *.local # Editor directories and files .vscode/* !.vscode/extensions.json .idea .DS_Store *.suo *.ntvs* *.njsproj *.sln *.sw? ================================================ FILE: samples/frontend/js/react/README.md ================================================ # Chat Protocol Now that you are here, you can start chatting with your endpoint, just type and press the **`Send`** button or just press **`Shift+Enter`** to send your message. ## Installation To integrate the chat protocol in your project, install the `@microsoft/ai-chat-protocol` package using npm: ```bash npm install -s @microsoft/ai-chat-protocol ``` ## Usage ### Streaming Requests For streaming requests, use the `getStreamedCompletion` method. Here's an example: ```typescript try { const result = await client.getStreamedCompletion([message], { sessionState: sessionState }); for await (const response of result) { // Note: It is expected that you update your sessionState with the value you receive from your endpoint. // Handle your streaming responses here. } } catch (e) { if (isChatError(e)) { // Handle your chat error here. } } ``` ### Non-Streaming Requests For non-streaming requests, you can use the `getCompletion` method. Here's an example: ```typescript try { const result = await client.getCompletion([message], { sessionState: sessionState }); // Handle your result here. } catch (e) { if (isChatError(e)) { // Handle your chat error here. } } ``` ================================================ FILE: samples/frontend/js/react/index.html ================================================ Chat Protocol Sample
================================================ FILE: samples/frontend/js/react/package.json ================================================ { "name": "chat-sample-react", "private": true, "version": "0.0.0", "type": "module", "scripts": { "dev": "vite --host", "build": "tsc && vite build", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "preview": "vite preview", "format": "prettier --write src" }, "dependencies": { "@fluentui/react-components": "9.46.8", "@microsoft/ai-chat-protocol": "^1.0.0-beta.20240610.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-markdown": "^9.0.1", "react-syntax-highlighter": "^15.5.0", "react-textarea-autosize": "^8.5.3", "remark-gfm": "^4.0.0" }, "devDependencies": { "@types/react": "^18.2.64", "@types/react-dom": "^18.2.21", "@typescript-eslint/eslint-plugin": "^7.1.1", "@typescript-eslint/parser": "^7.1.1", "@vitejs/plugin-react": "^4.2.1", "eslint": "^8.57.0", "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.7", "prettier": "^3.2.5", "typescript": "^5.2.2", "vite": "^5.1.6" } } ================================================ FILE: samples/frontend/js/react/src/App.module.css ================================================ /** * Copyright (c) Microsoft Corporation. * Licensed under the MIT License. */ html, body { margin: 0; padding: 0; height: 100%; } .appContainer { display: flex; margin: 0; padding: 0; height: 100vh; overflow: auto; } ================================================ FILE: samples/frontend/js/react/src/App.tsx ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import { FluentProvider, webLightTheme } from "@fluentui/react-components"; import Chat from "./Chat.tsx"; import Readme from "./Readme.tsx"; import styles from "./App.module.css"; function App() { return (
); } export default App; ================================================ FILE: samples/frontend/js/react/src/Chat.module.css ================================================ /** * Copyright (c) Microsoft Corporation. * Licensed under the MIT License. */ .chatWindow { display: flex; flex-direction: column; height: 100%; } .messages { flex-grow: 1; overflow-y: auto; padding: 10px; } .userMessage { display: flex; justify-content: flex-end; } .assistantMessage { display: flex; justify-content: flex-start; } .messageBubble { max-width: 60%; margin: 5px; padding: 10px; border-radius: 10px; } .userMessage .messageBubble { background-color: #dcf8c6; /* light green */ } .assistantMessage .messageBubble { background-color: #ece5dd; /* light gray */ } .inputArea { display: flex; border-top: 1px solid #ece5dd; padding: 10px; flex-shrink: 0; align-items: flex-end; } .inputArea textarea { flex-grow: 1; border: none; border-radius: 20px; padding: 10px; margin-right: 10px; resize: none; } .inputArea > div { display: flex; gap: 10px; } .inputArea button { margin-right: 10px; } .caution { padding: 10px 10px 10px 40px; border: 1px solid #d1d5da; background-color: #fdd; background-image: url("./assets/caution.svg"); background-repeat: no-repeat; background-position: 10px center; background-size: 20px; color: #24292e; font-weight: 600; display: flex; align-items: center; border-radius: 5px; width: 80%; margin-left: auto; margin-right: auto; } .buttons { display: flex; align-items: center; } ================================================ FILE: samples/frontend/js/react/src/Chat.tsx ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import { Button, ToggleButton } from "@fluentui/react-components"; import { AIChatMessage, AIChatProtocolClient, AIChatError, } from "@microsoft/ai-chat-protocol"; import { useEffect, useId, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import TextareaAutosize from "react-textarea-autosize"; import styles from "./Chat.module.css"; import gfm from "remark-gfm"; type ChatEntry = (AIChatMessage & { dataUrl?: string }) | AIChatError; function isChatError(entry: unknown): entry is AIChatError { return (entry as AIChatError).code !== undefined; } interface FileInput { data: Uint8Array; name: string; type: string; } function toBase64DataUrl( arr?: Uint8Array, contentType?: string, ): Promise { return new Promise((resolve, reject) => { if (!arr) { resolve(undefined); return; } const blob = new Blob([arr], { type: contentType }); const reader = new FileReader(); reader.onerror = reject; reader.onload = (event) => { resolve(event.target?.result as string); }; reader.readAsDataURL(blob); }); } export default function Chat({ style }: { style: React.CSSProperties }) { const client = new AIChatProtocolClient("/api/chat/"); const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); const [streaming, setStreaming] = useState(false); const inputId = useId(); const [sessionState, setSessionState] = useState(undefined); const messagesEndRef = useRef(null); const [selectedFile, setSelectedFile] = useState( undefined, ); const fileInputRef = useRef(null); function isArrayBuffer(value: unknown): value is ArrayBuffer { return value instanceof ArrayBuffer; } const handleFileChange = (e: React.ChangeEvent) => { if (e.target?.files && e.target.files.length > 0) { const file = e.target.files[0]; const reader = new FileReader(); reader.onload = (loadEvent) => { const arrayBuffer = loadEvent!.target!.result; if (!isArrayBuffer(arrayBuffer)) { setSelectedFile(undefined); return; } const fileUint8Array = new Uint8Array(arrayBuffer); setSelectedFile({ data: fileUint8Array, name: file.name, type: file.type, }); }; reader.readAsArrayBuffer(file); } }; const clearSelectedFile = () => { setSelectedFile(undefined); if (fileInputRef.current) { fileInputRef.current.value = ""; // Reset file input } }; const scrollToBottom = () => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); }; useEffect(scrollToBottom, [messages]); const sendMessage = async () => { const message: AIChatMessage = { role: "user", content: input, files: selectedFile ? [ { data: selectedFile.data, contentType: selectedFile.type, }, ] : undefined, }; const dataUrl = await toBase64DataUrl( selectedFile?.data, selectedFile?.type, ); const updatedMessages: ChatEntry[] = [ ...messages, { ...message, files: undefined, dataUrl: dataUrl, }, ]; setMessages(updatedMessages); setInput(""); setSelectedFile(undefined); try { if (streaming) { const result = await client.getStreamedCompletion([message], { sessionState: sessionState, }); const latestMessage: AIChatMessage = { content: "", role: "assistant" }; for await (const response of result) { if (response.sessionState) { setSessionState(response.sessionState); } if (!response.delta) { continue; } if (response.delta.role) { latestMessage.role = response.delta.role; } if (response.delta.content) { latestMessage.content += response.delta.content; setMessages([...updatedMessages, latestMessage]); } } } else { const result = await client.getCompletion([message], { sessionState: sessionState, }); setSessionState(result.sessionState); setMessages([...updatedMessages, result.message]); } } catch (e) { if (isChatError(e)) { setMessages([...updatedMessages, e]); } } }; const getClassName = (message: ChatEntry) => { if (isChatError(message)) { return styles.caution; } return message.role === "user" ? styles.userMessage : styles.assistantMessage; }; const getErrorMessage = (message: AIChatError) => { return `${message.code}: ${message.message}`; }; return (
{messages.map((message) => (
{isChatError(message) ? ( <>{getErrorMessage(message)} ) : ( <>
{message.content} {message.dataUrl && ( Message Attachment )}
)}
))}
setInput(e.target.value)} onKeyDown={(e) => { if (e.key === "Enter" && e.shiftKey) { e.preventDefault(); sendMessage(); } }} minRows={1} maxRows={4} /> {selectedFile && (
File: {selectedFile.name}
)} setStreaming(!streaming)} > Streaming
); } ================================================ FILE: samples/frontend/js/react/src/Readme.tsx ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import React, { useEffect, useState } from "react"; import ReactMarkdown from "react-markdown"; import gfm from "remark-gfm"; import { Prism } from "react-syntax-highlighter"; import readmeContent from "../README.md"; const components = { code(props: any) { const { children, className, node, ...rest } = props; const match = /language-(\w+)/.exec(className || ""); return match ? ( ) : ( {children} ); }, }; function Readme({ style }: { style: React.CSSProperties }) { const [markdown, setMarkdown] = useState(""); useEffect(() => { fetch(readmeContent) .then((response) => response.text()) .then((text) => setMarkdown(text)); }, []); return (
{markdown}
); } export default Readme; ================================================ FILE: samples/frontend/js/react/src/globals.d.ts ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. declare module "*.md"; ================================================ FILE: samples/frontend/js/react/src/main.tsx ================================================ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. import React from "react"; import ReactDOM from "react-dom/client"; import App from "./App.tsx"; ReactDOM.createRoot(document.getElementById("root")!).render( , ); ================================================ FILE: samples/frontend/js/react/src/vite-env.d.ts ================================================ /// ================================================ FILE: samples/frontend/js/react/tsconfig.json ================================================ { "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true, "lib": ["ES2020", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, /* Bundler mode */ "moduleResolution": "bundler", "allowImportingTsExtensions": true, "resolveJsonModule": true, "isolatedModules": true, "noEmit": true, "jsx": "react-jsx", /* Linting */ "strict": true, "noUnusedLocals": true, "noUnusedParameters": true, "noFallthroughCasesInSwitch": true }, "include": ["src"], "references": [{ "path": "./tsconfig.node.json" }] } ================================================ FILE: samples/frontend/js/react/tsconfig.node.json ================================================ { "compilerOptions": { "composite": true, "skipLibCheck": true, "module": "ESNext", "moduleResolution": "bundler", "allowSyntheticDefaultImports": true, "strict": true }, "include": ["vite.config.ts"] } ================================================ FILE: samples/frontend/js/react/vite.config.ts ================================================ import { defineConfig } from 'vite' import react from '@vitejs/plugin-react' // https://vitejs.dev/config/ export default defineConfig({ plugins: [react()], assetsInclude: ['**/*.md'], server: { proxy: { '/api': { target: 'http://localhost:3000', }, }, }, }); ================================================ FILE: sdk/js/.gitignore ================================================ !packages/* dist .tshy .tshy-build *.tgz ================================================ FILE: spec/.gitignore ================================================ node_modules tsp-output ================================================ FILE: spec/README.md ================================================ # Microsoft AI Chat Protocol API Specification (Version 2024-05-29) ## Rationale The AI Chat Protocol API Specification is an effort to standarize API contracts across AI solutions and languages. By having a unified approach, AI application components become easily compatible and interoperable with one another. Additionally, this allows for a consistent API surface to perform AI evaluations on, reducing the complexity in consuming different AI service endpoints. In this directory, the specification is defined via [TypeSpec](https://typespec.io) and available as a human-readable document via this README. This protocol is inspired by the [OpenAI ChatCompletion API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) but contains additional fields required for a chat application. The original specification was created by [Pamela Fox](https://github.com/pamelafox) and [Natalia Venditto](https://github.com/anfibiacreativa). Table of contents: * [HTTP requests to AI chat app endpoints](#http-requests-to-ai-chat-app-endpoints) * [Example request context](#example-request-context) * [HTTP responses from AI Chat App endpoints](#http-responses-from-ai-chat-app-endpoints) * [Non-streaming response](#non-streaming-response) * [Successful response](#successful-response) * [Error response](#error-response) * [Streaming response](#streaming-response) * [Successful streamed response](#successful-streamed-response) * [Error in streamed response](#error-in-streamed-response) * [Answer formatting](#answer-formatting) * [Example response context](#example-response-context) ## HTTP requests to AI Chat App endpoints An HTTP request should always be a POST request, with the following headers: * `Content-Type: application/json` * `Authorization: Bearer `: _Optional._ For applications that require authentication. The recommended path is `chat` for a non-streaming request and `chat/stream` for a streaming request. The body of the request can contain these properties, in JSON format: * `"messages"`: A list of messages, each containing "content" and "role", where "role" may be "assistant" or "user". A single-turn chat app may only contain 1 message, while a multi-turn chat app may contain multiple messages. * `"context"`: _Optional_. An object containing any additional context about the request, such as the temperature to use for the LLM. Each application may define its own context properties. See [example request context properties](#example-request-context). * `"sessionState"`: _Optional._ An object containing the "memory" for the chat app, such as a user ID. ### Usage example The example belows represents a valid and compliant request body to the chat app endpoints: ```json { "messages": [ { "content": "What is included in my Northwind Health Plus plan that is not in standard?", "role": "user" } ], "context": {}, "sessionState": null } ``` ### Example request context The request context object can contain any properties. Here are some common properties that may be of use depending on your AI application: * `"overrides"`: An object containing settings for the chat application. * `"temperature"`: The temperature to use for the LLM. * `"top"`: The number of results to return from the search engine. * `"retrieval_mode"`: The mode to use for the search engine. Can be "hybrid", "vectors", or "text". * `"semantic_ranker"`: _Specific to Azure AI Search_. Whether to use the semantic ranker for the search engine. * `"semantic_captions"`: _Specific to Azure AI Search_. Whether to use semantic captions for the search engine. * `"suggest_followup_questions"`: Whether to suggest follow-up questions for the chat app. * `"use_oid_security_filter"`: Whether to use the OID security filter for the search engine. * `"use_groups_security_filter"`: Whether to use the groups security filter for the search engine. * `"vector_fields"`: A list of fields to search for the vector search engine. * `"use_gpt4v"`: Whether to use a GPT-4V approach. * `"gpt4v_input"`: The input type to use for a GPT-4V approach. Can be "text", "textAndImages", or "images". Example of the overrides object: ```json "overrides": { "top": 3, "retrieval_mode": "text", "semantic_ranker": false, "semantic_captions": false, "suggest_followup_questions": false, "use_oid_security_filter": false, "use_groups_security_filter": false, "vector_fields": ["embedding"], "use_gpt4v": false, "gpt4v_input": "textAndImages" } ``` ## HTTP responses from AI Chat App endpoints The HTTP response should either be JSON for a non-streaming response, or [JSON Lines](https://github.com/wardi/jsonlines) ("jsonl") for a streaming response. ### Non-streaming response The response should contain this header: * `Content-Type: application/json` #### Successful response A successful response should have a status code of 200, and the body should contain a JSON object with the following properties: * `"message"`: An object containing the actual content of the response. See [Answer formatting](#answer-formatting). _Comes from the [OpenAI chat completion object](https://platform.openai.com/docs/api-reference/chat/object)._ * `"context"`: _Optional_. An object containing additional details needed for the chat app. Each application can define its own properties. See [example context properties for responses](#example-response-context). * `"sessionState"`: _Optional_. An object containing the "memory" for the chat app, such as a user ID. Here's an example JSON response: ```json { "message": { "content": "There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91].", "function_call": null, "role": "assistant", "tool_calls": null }, "context": { "data_points": { "text": [ "Northwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentiona lly false or misleading statements that the plan provider may make...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)" ] }, "thoughts": [ { "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null, "title": "Original user query" }, { "description": "Northwind Health Plus plan coverage details compared to standard plan", "props": { "has_vector": false, "use_semantic_captions": false }, "title": "Generated search query" }, { "description": [ { "captions": [], "category": null, "content": " \nTips for Avoiding Intentionally False Or Misleading Statements: \nWhen it comes to understanding a health plan, it is important to be aware of any \nintentiona lly false or misleading statements that the plan provider may make...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-233", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=91" }, { "captions": [], "category": null, "content": " It is important to \nresearch the providers and services offered in the Northwind Standard plan i n order to \ndetermine if the providers and services offered are sufficient for the employee's needs...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-232", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=91" }, { "captions": [], "category": null, "content": " Employees should keep track of their claims and follow up with \nNorthwind Health if a claim is not processed in a timely manner...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-41", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=17" } ], "props": null, "title": "Results" }, { "description": [ "{'role': 'system', 'content': \"Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\n Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n \n \n \"}", "{'role': 'user', 'content': \"What is included in my Northwind Health Plus plan that is not in standard?\n\nSources:\nNorthwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentiona lly false or misleading statements that the plan provider may make. To avoid being misled, employees should follow the following tips:(truncated) \nNorthwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs. In addition, Northwind Health may make claims that their plan offers low or no cost prescription drugs..(truncated)\"}" ], "props": null, "title": "Prompt" } ] }, "sessionState": null } ``` #### Error response An error response should have a status code of 400 or 500, and the body should contain a JSON object with the following properties: * `"error"`: A string describing the error. Here's an example JSON response for a 400-level error: ```json { "error": "Your message contains content that was flagged by the OpenAI content filter." } ``` Here's an example JSON response for a 500-level error: ```json { "error": "The app encountered an error processing your request.\nIf you are an administrator of the app, view the full error in the logs." } ``` ### Streaming response The response should contain this header: * `Content-Type: application/jsonl` #### Successful streamed response A successful response should have a status code of 200. The body of the response should contain a sequence of JSON objects, each representing a chunk of the response. The first chunk contains the `context` property, since that is available before the answer, and subsequent chunks contain parts of the answer to the question. Each JSON object should contain the following properties: * `"delta"`: An object containing the actual content of the response, a token at a time. See [Answer formatting](#answer-formatting). _Comes from the [OpenAI chat completion chunk object](https://platform.openai.com/docs/api-reference/chat/streaming)._ * `"context"`: _Optional_. An object containing additional details needed for the chat app. Each application can define its own properties. See [example response context properties](#example-response-context). * `"sessionState"`: _Optional_. An object containing the "memory" for the chat app, such as a user ID. Here's an example of the first three JSON objects in a streaming response: ```json { "delta": { "role": "assistant" }, "context": { "data_points": { "text": [ "Benefit_Options.pdf#page=3: The plans also cover preventive care services such as mammograms, colonoscopies, and other cancer screenings...(truncated)", "Benefit_Options.pdf#page=3: Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, doctor visits,...(truncated)", "Benefit_Options.pdf#page=3: With Northwind Health Plus, you can choose from a variety of in -network providers, including primary care physicians,...(truncated)" ] }, "thoughts": [ { "title": "Original user query", "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null }, { "title": "Generated search query", "description": "Northwind Health Plus plan standard", "props": { "use_semantic_captions": false, "has_vector": false } }, { "title": "Results", "description": [ { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": " The plans also cover preventive care services such as mammograms, colonoscopies, and \nother cancer screenings...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] }, { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-3", "content": " \nBoth plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, \ndoctor visits,...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] }, { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-1", "content": " With Northwind Health Plus, you can choose \nfrom a variety of in -network providers, including primary care physicians,...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] } ], "props": null }, { "title": "Prompt", "description": [ "{'role': 'system', 'content': \"Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\\n Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\\n \\n \\n \"}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?'}", "{'role': 'assistant', 'content': 'There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91].'}", "{'role': 'user', 'content': \"What is included in my Northwind Health Plus plan that is not in standard?\\n\\nSources:\\nBenefit_Options.pdf#page=3: The plans also cover preventive care services such as mammograms, colonoscopies, and other cancer screenings...(truncated)\\nBenefit_Options.pdf#page=3: Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, doctor visits,...(truncated)\\nBenefit_Options.pdf#page=3: With Northwind Health Plus, you can choose from a variety of in -network providers, including primary care physicians,...(truncated)\"}" ], "props": null } ] }, "sessionState": null, }{ "delta": { "content": null, "function_call": null, "role": "assistant", "tool_calls": null } }{ "delta": { "content": "The", "function_call": null, "role": null, "tool_calls": null } } ``` #### Error in streamed response If an error is encountered before the stream begins, then the response may look like a non-streaming error response. However, if an error is encountered during the stream, then the server will have already sent a 200 response, and will send a chunk with an error object. Typically that would be the last chunk, but it may not be. Here's an example of an error chunk: ```json { "error": "The app encountered an error processing your request.\nIf you are an administrator of the app, view the full error in the logs." } ``` ### Answer formatting To support the display of citations, the answer from the LLM should contain source information in square brackets, such as `[info1.txt]`. Here's a full example of an answer with citation: ```text There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91]. ``` ### Example response context The response context object can contain any properties. Here are some common properties that may be of use depending on your AI application along with some best practices: * `"followup_questions"`: A list of follow-up questions to ask the user. Example: ```json "followup_questions": [ "What types of prescription drugs are covered?", "Which services have lower out-of-pocket costs?" ] ``` If a client receives this property and the user has requested follow-up questions, the client should prompt the user with clickable versions of the questions. [See image](images/followup.png) * `"data_points"`: An object containing text and/or image data chunks, a list in the `"text"` or `"images"` properties. Example: ```json "data_points": { "text": [ "Northwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentionally false or misleading statements that the plan provider may make...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)" ] }, ``` Example with images: ```json "data_points": { "images": [ { "detail": "auto", "url": "data:image/png;base64,iVBOR1BORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAACklEQVR4nGMAAQAABQABDQ0tuhsAAAAASUVORK5CYII=" } ], "text": [ "Financial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing other...(truncated)" ] }, ``` If a client receives this property, the client should display the data points in a perusable format. [See image](images/data_points.png) * `"thoughts"`: A list describing each step of the backend. Each step should contain: * `"title"`: A string describing the step. * `"description"`: A string or list of strings describing the step. * `"props"`: _Optional_. An object containing additional properties for the step. Example: ```json "thoughts": [ { "title": "Original user query", "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null }, { "title": "Generated search query", "description": "Northwind Health Plus plan coverage details", "props": { "has_vector": false, "use_semantic_captions": false } }, { "title": "Results", "description": [ { "captions": [], "category": null, "content": " \n\u2022 Understand your coverage limits, and know what services are covered and what services \nare not covered...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-249", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Health_Plus_Benefits_Details.pdf", "sourcepage": "Northwind_Health_Plus_Benefits_Details.pdf#page=100" }, { "captions": [], "category": null, "content": " Employees should keep track of their claims and follow up with \nNorthwind Health if a claim is not processed in a timely manner...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-41", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=17" }, { "captions": [], "category": null, "content": " It is important to talk to your doctor or \nhealth care provider to make su re that you understand the details of the clinical trial before \nyou decide to participate...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-57", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Health_Plus_Benefits_Details.pdf", "sourcepage": "Northwind_Health_Plus_Benefits_Details.pdf#page=24" } ], "props": null }, { "title": "Prompt", "description": [ "{'role': 'system', 'content': 'Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\\n Answer ONLY with the facts listed in the list of sources below. If there isn\\'t enough information below, say you don\\'t know. Do not generate answers that don\\'t use the sources below. If asking a clarifying question to the user would help, ask the question.\\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don\\'t combine sources, list each source separately, for example [info1.txt][info2.pdf].\\n Generate 3 very brief follow-up questions that the user would likely ask next.\\n Enclose the follow-up questions in double angle brackets. Example:\\n <>\\n <>\\n <>\\n Do no repeat questions that have already been asked.\\n Make sure the last question ends with \">>\".\\n \\n \\n '}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?'}", "{'role': 'assistant', 'content': 'The Northwind Health Plus plan includes coverage for prescription drugs, but it is important to read the plan details to determine which prescription drugs are covered and what the associated costs are [Northwind_Standard_Benefits_Details.pdf#page=91]. Additionally, employees should select in-network providers to maximize coverage and avoid unexpected costs, submit claims as soon as possible after a service is rendered, and track claims and follow up with Northwind Health if a claim is not processed in a timely manner [Northwind_Standard_Benefits_Details.pdf#page=17].\\n\\n'}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?\\n\\nSources:\\nNorthwind_Health_Plus_Benefits_Details.pdf#page=100: \u2022 Understand your coverage limits, and know what services are covered and what services are not covered...(truncated)\\nNorthwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)\\nNorthwind_Health_Plus_Benefits_Details.pdf#page=24: It is important to talk to your doctor or health care provider..(truncated)'}" ], "props": null } ] ``` If a client receives this property, the client should display the thoughts in a debug display or to the end-user as specified by the design system. [See image](images/thoughts.png) ## Summary The Microsoft AI Chat Protocol API Specification details a consistent pattern for requests and responses to an AI service endpoint, allowing for consistent service consumption and evaluations. Any comments or feedback can be left as an [issue](https://github.com/microsoft/ai-chat-protocol/issues/new). ================================================ FILE: spec/legacy/2024-01-28.md ================================================ # HTTP protocol for AI chat apps (Version 2024-01-28) > Note: This API spec is an older version. If you're adding the AI Chat Protocol to your application, follow the most recent spec located under `/spec`. We are standardizing on a common HTTP protocol across AI chat app solutions and tools, to make them more compatible with each other. By agreeing on a common protocol, developers can swap different components, like using the Python backend [azure-search-openai-demo](https://github.com/Azure-Samples/azure-search-openai-demo) with the Web Components frontend from [azure-search-openai-javascript](https://github.com/Azure-Samples/azure-search-openai-javascript). This protocol is inspired by the [OpenAI ChatCompletion API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api), but contains additional fields required for a chat application. Table of contents: * [HTTP requests to AI chat app endpoints](#http-requests-to-ai-chat-app-endpoints) * [Recommended request context](#recommended-request-context) * [HTTP responses from RAG chat app endpoints](#http-responses-from-rag-chat-app-endpoints) * [Non-streaming response](#non-streaming-response) * [Successful response](#successful-response) * [Error response](#error-response) * [Streaming response](#streaming-response) * [Successful response](#successful-streamed-response) * [Error response](#error-in-streamed-response) * [Answer formatting](#answer-formatting) * [Recommended response context](#recommended-response-context) ## HTTP requests to AI chat app endpoints An HTTP request should always be a POST request, with the following headers: * `Content-Type: application/json` * `Authorization: Bearer `: _Optional._ For applications that require authentication. The body of the request can contain these properties, in JSON: * `"messages"`: A list of messages, each containing "content" and "role", where "role" may be "assistant" or "user". A single-turn chat app may only contain 1 message, while a multi-turn chat app may contain multiple messages. * `"stream"`: A boolean indicating whether the response should be streamed or not. * `"context"`: _Optional_. An object containing any additional context about the request, such as the temperature to use for the LLM. Each application may define its own context properties. See [recommended request context properties](#recommended-request-context). * `"session_state"`: _Optional._ An object containing the "memory" for the chat app, such as a user ID. Here's an example JSON request: ```json { "messages": [ { "content": "What is included in my Northwind Health Plus plan that is not in standard?", "role": "user" } ], "stream": false, "context": {}, "session_state": null } ``` ### Recommended request context The request context object can contain any properties. However, here are some recommended common properties: * `"overrides"`: An object containing settings for the chat application. * `"temperature"`: The temperature to use for the LLM. * `"top"`: The number of results to return from the search engine. * `"retrieval_mode"`: The mode to use for the search engine. Can be "hybrid", "vectors", or "text". * `"semantic_ranker"`: _Specific to Azure AI Search_. Whether to use the semantic ranker for the search engine. * `"semantic_captions"`: _Specific to Azure AI Search_. Whether to use semantic captions for the search engine. * `"suggest_followup_questions"`: Whether to suggest follow-up questions for the chat app. * `"use_oid_security_filter"`: Whether to use the OID security filter for the search engine. * `"use_groups_security_filter"`: Whether to use the groups security filter for the search engine. * `"vector_fields"`: A list of fields to search for the vector search engine. * `"use_gpt4v"`: Whether to use a GPT-4V approach. * `"gpt4v_input"`: The input type to use for a GPT-4V approach. Can be "text", "textAndImages", or "images". Example: ```json "overrides": { "top": 3, "retrieval_mode": "text", "semantic_ranker": false, "semantic_captions": false, "suggest_followup_questions": false, "use_oid_security_filter": false, "use_groups_security_filter": false, "vector_fields": ["embedding"], "use_gpt4v": false, "gpt4v_input": "textAndImages" } ``` ## HTTP responses from RAG chat app endpoints The HTTP response should either be JSON for a non-streaming response, or newline-delimited JSON ("NDJSON"/"jsonlines") for a streaming response. ### Non-streaming response This response is based off the [OpenAI chat completion object](https://platform.openai.com/docs/api-reference/chat/object), with additional properties needed to display sources and citations properly. The response should contain this header: * `Content-Type: application/json` #### Successful response A successful response should have a status code of 200, and the body should contain a JSON object with the following properties: * `"choices"`: A list of responses from the LLM, typically containing only 1 response as our app sets `n=1` when requesting a completion. Each response contains: * `"message"`: An object containing the actual content of the response. See [Answer formatting](#answer-formatting). _Comes from the OpenAI response._ * `"finish_reason"`: A string representing the finish state of the response. _Comes from the OpenAI response._ * `"index"`: A number indicating which response this is (0 in the case of 1 response given). _Comes from the OpenAI response._ * `"content_filter_results"`: An object from the Azure Content Safety filter. _Same as the OpenAI response, but will only be returned when using Azure OpenAI, not openai.com OpenAI._ * `"context"`: _Optional_. An object containing additional details needed for the chat app. Each application can define its own properties. See [recommended response context properties](#recommended-response-context). * `"session_state"`: _Optional_. An object containing the "memory" for the chat app, such as a user ID. * `"created"`: The Unix timestamp (in seconds) of when the chat completion was created. _Comes from the OpenAI response._ * `"id"`: A unique identifier for the chat completion. _Comes from the OpenAI response._ * `"model"`: The model used for the completion, such as "gpt-35-turbo". _Comes from the OpenAI response._ * `"object"`: The object type, always "chat.completion". _Comes from the OpenAI response._ * `"prompt_filter_results"`: An object from the Azure Content Safety filter. _Same as the OpenAI response, but will only be returned when using Azure OpenAI, not openai.com OpenAI._ * `"system_fingerprint"`: Represents the backend configuration that the model runs with. _Same as the OpenAI response, but will only be returned when using Azure OpenAI, not openai.com OpenAI._ * `"usage"`: Usage statistics for the completion request. _Comes from the OpenAI response._ Here's an example JSON response: ```json { "choices": [ { "content_filter_results": { "hate": { "filtered": false, "severity": "safe" }, "self_harm": { "filtered": false, "severity": "safe" }, "sexual": { "filtered": false, "severity": "safe" }, "violence": { "filtered": false, "severity": "safe" } }, "context": { "data_points": { "text": [ "Northwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentiona lly false or misleading statements that the plan provider may make...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)" ] }, "thoughts": [ { "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null, "title": "Original user query" }, { "description": "Northwind Health Plus plan coverage details compared to standard plan", "props": { "has_vector": false, "use_semantic_captions": false }, "title": "Generated search query" }, { "description": [ { "captions": [], "category": null, "content": " \nTips for Avoiding Intentionally False Or Misleading Statements: \nWhen it comes to understanding a health plan, it is important to be aware of any \nintentiona lly false or misleading statements that the plan provider may make...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-233", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=91" }, { "captions": [], "category": null, "content": " It is important to \nresearch the providers and services offered in the Northwind Standard plan i n order to \ndetermine if the providers and services offered are sufficient for the employee's needs...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-232", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=91" }, { "captions": [], "category": null, "content": " Employees should keep track of their claims and follow up with \nNorthwind Health if a claim is not processed in a timely manner...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-41", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=17" } ], "props": null, "title": "Results" }, { "description": [ "{'role': 'system', 'content': \"Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\n Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\n \n \n \"}", "{'role': 'user', 'content': \"What is included in my Northwind Health Plus plan that is not in standard?\n\nSources:\nNorthwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentiona lly false or misleading statements that the plan provider may make. To avoid being misled, employees should follow the following tips:(truncated) \nNorthwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs. In addition, Northwind Health may make claims that their plan offers low or no cost prescription drugs..(truncated)\"}" ], "props": null, "title": "Prompt" } ] }, "finish_reason": "stop", "index": 0, "message": { "content": "There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91].", "function_call": null, "role": "assistant", "tool_calls": null }, "session_state": null } ], "created": 1706301586, "id": "chatcmpl-8lNHGormHX5fhozITxASIufDZno9D", "model": "gpt-35-turbo", "object": "chat.completion", "prompt_filter_results": [ { "content_filter_results": { "hate": { "filtered": false, "severity": "safe" }, "self_harm": { "filtered": false, "severity": "safe" }, "sexual": { "filtered": false, "severity": "safe" }, "violence": { "filtered": false, "severity": "safe" } }, "prompt_index": 0 } ], "system_fingerprint": null, "usage": { "completion_tokens": 64, "prompt_tokens": 967, "total_tokens": 1031 } } ``` #### Error response An error response should have a status code of 400 or 500, and the body should contain a JSON object with the following properties: * `"error"`: A string describing the error. Here's an example JSON response for a 500-level error: ```json { "error": "The app encountered an error processing your request.\nIf you are an administrator of the app, view the full error in the logs." } ``` Here's an example JSON response for a 400-level error: ```json { "error": "Your message contains content that was flagged by the OpenAI content filter." } ``` ### Streaming response This response is based off the [OpenAI chat completion chunk object](https://platform.openai.com/docs/api-reference/chat/streaming), with additional properties needed to display sources and citations properly. The response should contain these headers: * `Content-Type: application/json-lines` * `Transfer-Encoding: chunked` #### Successful streamed response A successful response should have a status code of 200. The body of the response should contain a sequence of JSON objects, each representing a chunk of the response. The first chunk contains a choice with the `context` property, since that is available before the answer, and subsequent chunks contain parts of the answer to the question. Each JSON object should contain the following properties: * `"choices"`: A list of responses from the LLM, typically containing only 1 response as our app sets `n=1` when requesting a completion. Each response contains: * `"delta"`: An object containing the actual content of the response, a token at a time. See [Answer formatting](#answer-formatting). _Comes from the OpenAI response._ * `"finish_reason"`: A string representing the finish state of the response. _Comes from the OpenAI response._ * `"index"`: A number indicating which response this is (0 in the case of 1 response given). _Comes from the OpenAI response._ * `"content_filter_results"`: An object from the Azure Content Safety filter. _Comes from the OpenAI response, but will only be returned when using Azure OpenAI, not openai.com OpenAI._ * `"context"`: _Optional_. An object containing additional details needed for the chat app. Each application can define its own properties. See [recommended response context properties](#recommended-response-context). * `"session_state"`: _Optional_. An object containing the "memory" for the chat app, such as a user ID. * `"created"`: The Unix timestamp (in seconds) of when the chat completion was created. _Comes from the OpenAI response._ * `"id"`: A unique identifier for the chat completion. _Comes from the OpenAI response._ * `"model"`: The model used for the completion, such as "gpt-35-turbo". _Comes from the OpenAI response._ * `"object"`: The object type, always "chat.completion". _Comes from the OpenAI response._ * `"prompt_filter_results"`: An object from the Azure Content Safety filter. _Comes from the OpenAI response, but will only be returned when using Azure OpenAI, not openai.com OpenAI._ * `"system_fingerprint"`: Represents the backend configuration that the model runs with. _Comes from the OpenAI response._ * `"usage"`: Usage statistics for the completion request. _Comes from the OpenAI response._ Here's an example of the first three JSON objects in a streaming response: ```json { "choices": [ { "delta": { "role": "assistant" }, "context": { "data_points": { "text": [ "Benefit_Options.pdf#page=3: The plans also cover preventive care services such as mammograms, colonoscopies, and other cancer screenings...(truncated)", "Benefit_Options.pdf#page=3: Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, doctor visits,...(truncated)", "Benefit_Options.pdf#page=3: With Northwind Health Plus, you can choose from a variety of in -network providers, including primary care physicians,...(truncated)" ] }, "thoughts": [ { "title": "Original user query", "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null }, { "title": "Generated search query", "description": "Northwind Health Plus plan standard", "props": { "use_semantic_captions": false, "has_vector": false } }, { "title": "Results", "description": [ { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-2", "content": " The plans also cover preventive care services such as mammograms, colonoscopies, and \nother cancer screenings...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] }, { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-3", "content": " \nBoth plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, \ndoctor visits,...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] }, { "id": "file-Benefit_Options_pdf-42656E656669745F4F7074696F6E732E706466-page-1", "content": " With Northwind Health Plus, you can choose \nfrom a variety of in -network providers, including primary care physicians,...(truncated)", "embedding": null, "imageEmbedding": null, "category": null, "sourcepage": "Benefit_Options.pdf#page=3", "sourcefile": "Benefit_Options.pdf", "oids": [], "groups": [], "captions": [] } ], "props": null }, { "title": "Prompt", "description": [ "{'role': 'system', 'content': \"Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\\n Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question.\\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf].\\n \\n \\n \"}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?'}", "{'role': 'assistant', 'content': 'There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91].'}", "{'role': 'user', 'content': \"What is included in my Northwind Health Plus plan that is not in standard?\\n\\nSources:\\nBenefit_Options.pdf#page=3: The plans also cover preventive care services such as mammograms, colonoscopies, and other cancer screenings...(truncated)\\nBenefit_Options.pdf#page=3: Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, doctor visits,...(truncated)\\nBenefit_Options.pdf#page=3: With Northwind Health Plus, you can choose from a variety of in -network providers, including primary care physicians,...(truncated)\"}" ], "props": null } ] }, "session_state": null, "finish_reason": null, "index": 0 } ], "object": "chat.completion.chunk" }{ "id": "chatcmpl-8lNX48CGv9kW7vXTCa9Jb0J4RfnlQ", "choices": [ { "delta": { "content": null, "function_call": null, "role": "assistant", "tool_calls": null }, "finish_reason": null, "index": 0, "content_filter_results": {} } ], "created": 1706302566, "model": "gpt-35-turbo", "object": "chat.completion.chunk", "system_fingerprint": null }{ "id": "chatcmpl-8lNX48CGv9kW7vXTCa9Jb0J4RfnlQ", "choices": [ { "delta": { "content": "The", "function_call": null, "role": null, "tool_calls": null }, "finish_reason": null, "index": 0, "content_filter_results": { "hate": { "filtered": false, "severity": "safe" }, "self_harm": { "filtered": false, "severity": "safe" }, "sexual": { "filtered": false, "severity": "safe" }, "violence": { "filtered": false, "severity": "safe" } } } ], "created": 1706302566, "model": "gpt-35-turbo", "object": "chat.completion.chunk", "system_fingerprint": null } ``` #### Error in streamed response If an error is encountered before the stream begins, then the response may look like a non-streaming error response. However, if an error is encountered during the stream, then the server will have already sent a 200 response, and will send a chunk with an error object. Typically that would be the last chunk, but it may not be. Here's an example of an error chunk: ```json { "error": "The app encountered an error processing your request.\nIf you are an administrator of the app, view the full error in the logs." } ``` ### Answer formatting To support the display of citations, the answer from the LLM should contain source information in square brackets, such as `[info1.txt]`. Here's a full example of an answer with citation: ```text There is no specific information provided about what is included in the Northwind Health Plus plan that is not in the standard plan. It is recommended to read the plan details carefully and ask questions to understand the specific benefits of the Northwind Health Plus plan [Northwind_Standard_Benefits_Details.pdf#page=91]. ``` ### Recommended response context The response context object can contain any properties. However, here are some recommended properties: * `"followup_questions"`: A list of follow-up questions to ask the user. Example: ```json "followup_questions": [ "What types of prescription drugs are covered?", "Which services have lower out-of-pocket costs?" ] ``` If a client receives this property and the user has requested follow-up questions, the client should prompt the user with clickable versions of the questions. [See image](images/followup.png) * `"data_points"`: An object containing text and/or image data chunks, a list in the `"text"` or `"images"` properties. Example: ```json "data_points": { "text": [ "Northwind_Standard_Benefits_Details.pdf#page=91: Tips for Avoiding Intentionally False Or Misleading Statements: When it comes to understanding a health plan, it is important to be aware of any intentionally false or misleading statements that the plan provider may make...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=91: It is important to research the providers and services offered in the Northwind Standard plan in order to determine if the providers and services offered are sufficient for the employee's needs...(truncated)", "Northwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)" ] }, ``` Example with images: ```json "data_points": { "images": [ { "detail": "auto", "url": "data:image/png;base64,iVBOR1BORw0KGgoAAAANSUhEUgAAAAEAAAABAQAAAAA3bvkkAAAACklEQVR4nGMAAQAABQABDQ0tuhsAAAAASUVORK5CYII=" } ], "text": [ "Financial Market Analysis Report 2023-6.png: 31 Financial markets are interconnected, with movements in one segment often influencing other...(truncated)" ] }, ``` If a client receives this property, the client should display the data points in a perusable format. [See image](images/datapoints.png) * `"thoughts"`: A list describing each step of the backend. Each step should contain: * `"title"`: A string describing the step. * `"description"`: A string or list of strings describing the step. * `"props"`: _Optional_. An object containing additional properties for the step. Example: ```json "thoughts": [ { "title": "Original user query", "description": "What is included in my Northwind Health Plus plan that is not in standard?", "props": null }, { "title": "Generated search query", "description": "Northwind Health Plus plan coverage details", "props": { "has_vector": false, "use_semantic_captions": false } }, { "title": "Results", "description": [ { "captions": [], "category": null, "content": " \n\u2022 Understand your coverage limits, and know what services are covered and what services \nare not covered...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-249", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Health_Plus_Benefits_Details.pdf", "sourcepage": "Northwind_Health_Plus_Benefits_Details.pdf#page=100" }, { "captions": [], "category": null, "content": " Employees should keep track of their claims and follow up with \nNorthwind Health if a claim is not processed in a timely manner...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Standard_Benefits_Details_pdf-4E6F72746877696E645F5374616E646172645F42656E65666974735F44657461696C732E706466-page-41", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Standard_Benefits_Details.pdf", "sourcepage": "Northwind_Standard_Benefits_Details.pdf#page=17" }, { "captions": [], "category": null, "content": " It is important to talk to your doctor or \nhealth care provider to make su re that you understand the details of the clinical trial before \nyou decide to participate...(truncated)", "embedding": null, "groups": [], "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-57", "imageEmbedding": null, "oids": [], "sourcefile": "Northwind_Health_Plus_Benefits_Details.pdf", "sourcepage": "Northwind_Health_Plus_Benefits_Details.pdf#page=24" } ], "props": null }, { "title": "Prompt", "description": [ "{'role': 'system', 'content': 'Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.\\n Answer ONLY with the facts listed in the list of sources below. If there isn\\'t enough information below, say you don\\'t know. Do not generate answers that don\\'t use the sources below. If asking a clarifying question to the user would help, ask the question.\\n For tabular information return it as an html table. Do not return markdown format. If the question is not in English, answer in the language used in the question.\\n Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don\\'t combine sources, list each source separately, for example [info1.txt][info2.pdf].\\n Generate 3 very brief follow-up questions that the user would likely ask next.\\n Enclose the follow-up questions in double angle brackets. Example:\\n <>\\n <>\\n <>\\n Do no repeat questions that have already been asked.\\n Make sure the last question ends with \">>\".\\n \\n \\n '}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?'}", "{'role': 'assistant', 'content': 'The Northwind Health Plus plan includes coverage for prescription drugs, but it is important to read the plan details to determine which prescription drugs are covered and what the associated costs are [Northwind_Standard_Benefits_Details.pdf#page=91]. Additionally, employees should select in-network providers to maximize coverage and avoid unexpected costs, submit claims as soon as possible after a service is rendered, and track claims and follow up with Northwind Health if a claim is not processed in a timely manner [Northwind_Standard_Benefits_Details.pdf#page=17].\\n\\n'}", "{'role': 'user', 'content': 'What is included in my Northwind Health Plus plan that is not in standard?\\n\\nSources:\\nNorthwind_Health_Plus_Benefits_Details.pdf#page=100: \u2022 Understand your coverage limits, and know what services are covered and what services are not covered...(truncated)\\nNorthwind_Standard_Benefits_Details.pdf#page=17: Employees should keep track of their claims and follow up with Northwind Health if a claim is not processed in a timely manner...(truncated)\\nNorthwind_Health_Plus_Benefits_Details.pdf#page=24: It is important to talk to your doctor or health care provider..(truncated)'}" ], "props": null } ] ``` If a client receives this property, the client should display the thoughts in a debug display. [See image](images/thoughts.png) ## Sample applications The following applications support at least a subset of the schema described above: Markdown table: | Application | Supports schema | Description | | ----------- | --------------- | ----------- | | [azure-search-openai-demo](https://www.github.com/Azure-samples/azure-search-openai-demo) | Supports the full schema, including all recommended context properties. | A RAG chat app that uses Azure AI Search and OpenAI. | | [azure-search-openai-javascript](https://www.github.com/Azure-samples/azure-search-openai-javascript) | Supports most of the schema, but not all recommended context properties (like for GPT-4V feature). | A RAG chat app that uses Azure AI Search and OpenAI. | | [chatgpt-backend-fastapi](https://github.com/pamelafox/chatgpt-backend-fastapi/) | Supports messages and stream, but no context properties. | A simple chat app using OpenAI. | | [chatgpt-quickstart](https://github.com/Azure-Samples/chatgpt-quickstart) | Supports messages, but no context properties, and only supports stream = True. | A simple chat app using OpenAI. | ================================================ FILE: spec/main.tsp ================================================ import "@typespec/http"; import "@typespec/rest"; import "@typespec/openapi3"; import "./operations.tsp"; using TypeSpec.Http; @service({ title: "Chat Protocol", }) @server( "{endpoint}", "Chat Protocol enabled endpoint", { endpoint: string, } ) namespace AI.Chat; ================================================ FILE: spec/models.tsp ================================================ import "@typespec/http"; using TypeSpec.Http; model ContextProp { context?: Record; } model SessionStateProp { sessionState?: unknown; } enum AIChatRole { assistant, user, system, } model AIChatMessage { role: AIChatRole; content: string; ...ContextProp; } model AIChatMessageDelta { role?: AIChatRole; content?: string; ...ContextProp; } model AIChatCompletion { message: AIChatMessage; ...ContextProp; ...SessionStateProp; } model AIChatCompletionDelta { @header contentType: "application/jsonl"; delta: AIChatMessageDelta; ...ContextProp; ...SessionStateProp; } model AIChatCompletionRequest { messages: AIChatMessage[]; ...ContextProp; ...SessionStateProp; } model AIChatCompletionRequestMultipart { json: HttpPart, files: HttpPart[], } model AIChatErrorResponse { error: AIChatError } model AIChatError { code: string; message: string; } ================================================ FILE: spec/operations.tsp ================================================ import "@typespec/http"; import "./models.tsp"; using TypeSpec.Http; namespace AI.Chat; alias Response = T | AIChatErrorResponse; op getCompletion(@header contentType: "application/json" | "multipart/form-data", body: AIChatCompletionRequest | AIChatCompletionRequestMultipart): Response; @overload(getCompletion) @post op getCompletionJSON(@header contentType: "application/json", @body body: AIChatCompletionRequest): Response; @overload(getCompletion) @post op getCompletionMultipart(@header contentType: "multipart/form-data", @multipartBody body: AIChatCompletionRequestMultipart): Response; @post @route("/stream") op getStreamedCompletion(@header contentType: "application/json" | "multipart/form-data", body: AIChatCompletionRequest | AIChatCompletionRequestMultipart): Response; @overload(getStreamedCompletion) @post @route("/stream") op getStreamedCompletionJSON(@header contentType: "application/json", @body body: AIChatCompletionRequest): Response; @overload(getStreamedCompletion) @post @route("/stream") op getStreamedCompletionMultipart(@header contentType: "multipart/form-data", @multipartBody body: AIChatCompletionRequestMultipart): Response; ================================================ FILE: spec/package.json ================================================ { "name": "chat-protocol", "version": "0.1.0", "type": "module", "scripts": { "build": "tsp compile .", "format": "tsp format **/*.tsp" }, "dependencies": { "@typespec/compiler": "^0.57.0", "@typespec/http": "^0.57.0", "@typespec/rest": "^0.57.0", "@typespec/openapi3": "^0.57.0" }, "private": true } ================================================ FILE: spec/tspconfig.yaml ================================================ emit: - "@typespec/openapi3"