Repository: InftyAI/Awesome-LLMOps Branch: main Commit: bdc0be33764d Files: 51 Total size: 418.2 KB Directory structure: gitextract_c9q67u90/ ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ └── REQUEST.md │ └── workflows/ │ ├── kube-workflow-init.yaml │ ├── kube-workflow.yaml │ ├── landscape.yml │ └── project-request.yaml ├── .gitignore ├── CNAME ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── project_request.py ├── requirements.txt └── website/ ├── README.md ├── data.yml ├── guide.yml ├── logos/ │ ├── aide-ml │ ├── aider │ ├── axlearn │ ├── baml │ ├── beta9 │ ├── bifrost │ ├── checkpoint-engine │ ├── cordumio │ ├── crush │ ├── deepeval │ ├── evidently │ ├── evoagentx │ ├── gemini-cli │ ├── goose │ ├── higress.avif │ ├── kvcached │ ├── loongflow │ ├── lunaary │ ├── magentic-ui │ ├── mle-bench │ ├── modular │ ├── open-swe │ ├── opencode │ ├── petals │ ├── posthog │ ├── rllm │ ├── slime │ ├── terminal-bench │ ├── tongyi-deep-research │ ├── verl │ └── xinference └── settings.yml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: InftyAI ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry polar: # Replace with a single Polar username buy_me_a_coffee: # Replace with a single Buy Me a Coffee username thanks_dev: # Replace with a single thanks.dev username custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] ================================================ FILE: .github/ISSUE_TEMPLATE/REQUEST.md ================================================ --- name: Project Request about: Suggest a project to join the list title: 'Project Request: PROJECT_NAME' labels: documentation assignees: '' --- ## Project Information - Project name: - Github URL: - Homepage URL: - Logo URL: ## Category - Inference - [ ] Inference Engine - [ ] Inference Platform - [ ] Middleware - [ ] LLM Router - [ ] AI Gateway - [ ] Output - [ ] Simulator - [ ] Benchmark - Orchestration - [ ] Application Framework - [ ] Agent Framework - [ ] Evolutionary Framework - [ ] RAG - [ ] Workflow - Runtime - [ ] AI Terminal - [ ] AI Agent - [ ] Chatbot - [ ] Code Agent - [ ] Database - [ ] Evolve Agent - [ ] Observation - [ ] Sandbox - [ ] Tool - Training - [ ] Framework - [ ] FineTune - [ ] RLHF - [ ] Agentic RL - [ ] Benchmark - [ ] Workflow ================================================ FILE: .github/workflows/kube-workflow-init.yaml ================================================ name: Workflow As Kubernetes Initialization on: workflow_dispatch: jobs: call-workflow: uses: kerthcet/github-workflow-as-kube/.github/workflows/workflow-as-kubernetes-init.yaml@v0.1.12 secrets: AGENT_TOKEN: ${{ secrets.AGENT_TOKEN }} ================================================ FILE: .github/workflows/kube-workflow.yaml ================================================ name: Workflow As Kubernetes on: issues: types: - opened issue_comment: types: - created pull_request_target: types: - opened - labeled - unlabeled - synchronize jobs: call-workflow: uses: kerthcet/github-workflow-as-kube/.github/workflows/workflow-as-kubernetes.yaml@v0.1.12 secrets: AGENT_TOKEN: ${{ secrets.AGENT_TOKEN }} ================================================ FILE: .github/workflows/landscape.yml ================================================ name: Build and Deploy Landscape on: push: branches: [ main ] pull_request: branches: [ main ] # Allow manual triggering workflow_dispatch: permissions: contents: read pages: write id-token: write # Allow only one concurrent deployment concurrency: group: "pages" cancel-in-progress: true jobs: build: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - name: Setup Pages uses: actions/configure-pages@v4 if: github.ref == 'refs/heads/main' - name: Install landscape2 run: | curl --proto '=https' --tlsv1.2 -LsSf \ https://github.com/cncf/landscape2/releases/download/v0.13.0/landscape2-installer.sh | sh echo "$HOME/.cargo/bin" >> $GITHUB_PATH - name: Cache landscape data uses: actions/cache@v3 with: path: .cache key: ${{ runner.os }}-landscape-cache-${{ hashFiles('website/data.yml', 'website/settings.yml') }} restore-keys: | ${{ runner.os }}-landscape-cache- - name: Validate landscape files run: make validate - name: Build landscape run: make build - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 if: github.ref == 'refs/heads/main' with: path: build deploy: if: github.ref == 'refs/heads/main' environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest needs: build steps: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 ================================================ FILE: .github/workflows/project-request.yaml ================================================ name: Project Request on: issues: types: [opened, edited] env: GH_TOKEN: ${{ secrets.AGENT_TOKEN }} jobs: process-project-request: if: contains(github.event.issue.title, 'Project Request:') runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.10' - name: Install dependencies run: pip install -r requirements.txt - name: Parse issue body id: parse-issue uses: actions/github-script@v6 with: script: | const issueBody = context.payload.issue.body; const issueAuthorId = context.payload.issue.user.login; // First remove HTML comments from the issue body const bodyWithoutComments = issueBody.replace(//g, ''); // Extract Project Information // Project name const projectNameMatch = bodyWithoutComments.match(/- Project name:([^\n]+)/i); const projectName = projectNameMatch ? projectNameMatch[1].trim() : null; // Github URL const githubUrlMatch = bodyWithoutComments.match(/- Github URL:([^\n]+)/i); const githubUrl = githubUrlMatch ? githubUrlMatch[1].trim() : null; // Homepage URL const homepageUrlMatch = bodyWithoutComments.match(/- Homepage URL:([^\n]+)/i); const homepageUrl = homepageUrlMatch ? homepageUrlMatch[1].trim() : null; // Logo URL const logoUrlMatch = bodyWithoutComments.match(/- Logo URL:([^\n]+)/i); const logoUrl = logoUrlMatch ? logoUrlMatch[1].trim() : null; // Logo Name (optional) const logoNameMatch = bodyWithoutComments.match(/- Logo Name:([^\n]+)/i); const logoName = logoNameMatch ? logoNameMatch[1].trim() : null; // Extract Category - Find the first checked box with [x] and its parent category let category = null; const lines = bodyWithoutComments.split('\n'); let parentCategory = null; for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); // Check if this is a main category line if (line.startsWith('- ') && !line.includes('[')) { parentCategory = line.substring(2).trim(); } // Check if this is a checked subcategory if (line.match(/- \[x\] (.+)/)) { const subcategory = line.match(/- \[x\] (.+)/)[1].trim(); if (parentCategory) { category = `${parentCategory}/${subcategory}`; break; } else { category = subcategory; break; } } } // Set outputs core.setOutput('repo-url', githubUrl); core.setOutput('project-name', projectName); core.setOutput('category', category); core.setOutput('logo-url', logoUrl); core.setOutput('logo-name', logoName); core.setOutput('homepage-url', homepageUrl); core.setOutput("author-id", issueAuthorId); // Log the extracted information console.log(`Github URL: ${githubUrl}`); console.log(`Project Name: ${projectName}`); console.log(`Homepage URL: ${homepageUrl}`); console.log(`Logo URL: ${logoUrl}`); console.log(`Logo Name: ${logoName}`); console.log(`Category: ${category}`); console.log(`Author ID: ${issueAuthorId}`); // Define README_ONLY_CATEGORIES const README_ONLY_CATEGORIES = ["MCP/MCP Server", "MCP/MCP Client"]; // Check if the category is in README_ONLY_CATEGORIES const isReadmeOnly = README_ONLY_CATEGORIES.some(c => c.toLowerCase() === category?.toLowerCase()); // Check if we have all required information if (!githubUrl || !projectName || !category) { core.setFailed('Missing required information in the issue'); await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: '❌ Could not process this request. Please make sure you have provided the GitHub repo URL, project name, and selected a category.' }); return false; } // For non-README_ONLY_CATEGORIES, check for homepage_url if (!isReadmeOnly && !homepageUrl) { core.setFailed('Missing required information in the issue'); await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: `❌ Could not process this request. Please make sure you have provided the homepage URL.` }); return false; } return true; - name: Check for existing PR if: steps.parse-issue.outputs.result == 'true' id: check-pr uses: actions/github-script@v6 with: github-token: ${{ secrets.AGENT_TOKEN }} script: | const { owner, repo } = context.repo; const issueNumber = context.issue.number; const branchName = `add-project-issue-${issueNumber}`; // Search for existing PRs that reference this issue const prs = await github.rest.pulls.list({ owner, repo, state: 'open' }); // Look for PRs with the expected branch name or that reference this issue let existingPR = null; for (const pr of prs.data) { if (pr.head.ref === branchName || pr.body.includes(`fixes #${issueNumber}`)) { existingPR = pr; break; } } if (existingPR) { console.log(`Found existing PR #${existingPR.number} for issue #${issueNumber}`); core.setOutput('pr-exists', 'true'); core.setOutput('pr-number', existingPR.number.toString()); } else { console.log(`No existing PR found for issue #${issueNumber}`); core.setOutput('pr-exists', 'false'); } // Always set the branch name for later steps core.setOutput('branch-name', branchName); - name: Create or update branch if: steps.parse-issue.outputs.result == 'true' run: | git config --global user.name "InftyAI-Agent" git config --global user.email "143625840+InftyAI-Agent@users.noreply.github.com" BRANCH_NAME="${{ steps.check-pr.outputs.branch-name }}" # Check if the branch already exists on remote if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then echo "Branch $BRANCH_NAME already exists on remote, updating it" git fetch origin git checkout $BRANCH_NAME || git checkout -b $BRANCH_NAME --track origin/$BRANCH_NAME # Reset to main to get latest changes and avoid conflicts git reset --hard origin/main else echo "Creating new branch $BRANCH_NAME" git checkout -b $BRANCH_NAME fi echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV - name: Run project_request.py script run: | # Define README_ONLY_CATEGORIES README_ONLY_CATEGORIES=("MCP/MCP Server" "MCP/MCP Client") CATEGORY="${{ steps.parse-issue.outputs.category }}" # Check if category is in README_ONLY_CATEGORIES IS_README_ONLY=false for c in "${README_ONLY_CATEGORIES[@]}"; do if [ "${c,,}" = "${CATEGORY,,}" ]; then IS_README_ONLY=true break fi done # Build command based on category type CMD="python project_request.py \ --category \"$CATEGORY\" \ --repo_url \"${{ steps.parse-issue.outputs.repo-url }}\" \ --name \"${{ steps.parse-issue.outputs.project-name }}\"" # Add logo_url and homepage_url if provided or required if [ "$IS_README_ONLY" = false ] || [ -n "${{ steps.parse-issue.outputs.logo-url }}" ]; then CMD="$CMD --logo_url \"${{ steps.parse-issue.outputs.logo-url }}\"" fi if [ "$IS_README_ONLY" = false ] || [ -n "${{ steps.parse-issue.outputs.homepage-url }}" ]; then CMD="$CMD --homepage_url \"${{ steps.parse-issue.outputs.homepage-url }}\"" fi # Add logo_name if provided if [ -n "${{ steps.parse-issue.outputs.logo-name }}" ]; then CMD="$CMD --logo_name \"${{ steps.parse-issue.outputs.logo-name }}\"" fi echo "Running command: $CMD" eval $CMD - name: Commit changes env: ISSUE_AUTHOR: ${{ steps.parse-issue.outputs.author-id }} run: | git add . git commit -m "Add ${{ steps.parse-issue.outputs.repo-url }} to ${{ steps.parse-issue.outputs.category }} category Co-authored-by: $ISSUE_AUTHOR <$ISSUE_AUTHOR@users.noreply.github.com>" git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/InftyAI/Awesome-LLMOps.git # Check if we need to force push (if branch already exists) if git ls-remote --heads origin $BRANCH_NAME | grep -q $BRANCH_NAME; then git push --force origin $BRANCH_NAME else git push --set-upstream origin $BRANCH_NAME fi - name: Create or Update Pull Request uses: actions/github-script@v6 with: github-token: ${{ secrets.AGENT_TOKEN }} script: | const { owner, repo } = context.repo; const issueNumber = context.issue.number; const branchName = process.env.BRANCH_NAME; const prExists = '${{ steps.check-pr.outputs.pr-exists }}' === 'true'; const existingPrNumber = '${{ steps.check-pr.outputs.pr-number }}'; const repoUrl = '${{ steps.parse-issue.outputs.repo-url }}'; const category = '${{ steps.parse-issue.outputs.category }}'; // Get the repository name from the URL const repoName = repoUrl.split('/').pop(); // Get the user who created the issue const issue = await github.rest.issues.get({ owner, repo, issue_number: issueNumber }); const issueAuthor = issue.data.user.login; let pr; if (prExists) { // PR already exists, no need to create a new one console.log(`Using existing PR #${existingPrNumber}`); // Add comment to the PR about the update await github.rest.issues.createComment({ issue_number: parseInt(existingPrNumber), owner, repo, body: `♻️ PR updated with latest changes from issue #${issueNumber}` }); // Add comment to the issue await github.rest.issues.createComment({ issue_number: issueNumber, owner, repo, body: `✅ Pull Request #${existingPrNumber} has been updated with your changes` }); } else { // Create a new PR pr = await github.rest.pulls.create({ owner, repo, title: `Add ${repoName} to ${category}`, body: `fixes #${issueNumber}, Co-authored-by: @${issueAuthor}`, head: branchName, base: 'main' }); // Add comment to the issue await github.rest.issues.createComment({ issue_number: issueNumber, owner, repo, body: `✅ Pull Request created: #${pr.data.number}` }); } ================================================ FILE: .gitignore ================================================ .DS_Store .cache/ build/ ================================================ FILE: CNAME ================================================ awesome-llmops.inftyai.com ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Code of Conduct 👋 Welcome to InftyAI community ! - [Scope](#scope) - [Our Standards](#our-standards) ## Scope This code of conduct applies within project and community spaces. ## Our Standards Examples of behavior that contributes to a positive environment include but are not limited to: - Demonstrating empathy and kindness toward other people - Being respectful of differing opinions, viewpoints, and experiences - Giving and gracefully accepting constructive feedback - Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience - Focusing on what is best not just for us as individuals, but for the overall community - Using welcoming and inclusive language Examples of unacceptable behavior include but are not limited to: - The use of sexualized language or imagery - Trolling, insulting or derogatory comments, and personal or political attacks - Public or private harassment in any form - Publishing others' private information, such as a physical or email address, without their explicit permission - Violence, threatening violence, or encouraging others to engage in violent behavior - Stalking or following someone without their consent - Unwelcome physical contact - Unwelcome sexual or romantic attention or advances - Other conduct which could reasonably be considered inappropriate in a professional setting The following behaviors are also prohibited: - Providing knowingly false or misleading information in connection with a Code of Conduct investigation or otherwise intentionally tampering with an investigation. - Retaliating against a person because they reported an incident or provided information about an incident as a witness. ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing 👋 Welcome to InftyAI community ! - [Before you get started](#before-you-get-started) - [Code of Conduct](#code-of-conduct) - [Getting started](#getting-started) - [PullRequests](#pull-requests) - [Code Review](#code-review) ## Before you get started ### Code of Conduct Please make sure to read and observe our [Code of Conduct](/CODE_OF_CONDUCT.md) first. ## Getting started 🚀 **All kinds of contributions are welcomed !** - Fix documents & Typos - Report & fix bugs - New features - Issues & discussions - ... ### Pull Requests Pull requests are often called simply "PR". Please follows the standard [github pull request](https://help.github.com/articles/about-pull-requests/) process. To submit a proposed change, please develop the code and add new test cases. ### Code Review To make it easier for your PR to receive reviews, consider the reviewers will need you to: - Follow [good coding guidelines](https://github.com/golang/go/wiki/CodeReviewComments). - Write [good commit messages](https://chris.beams.io/posts/git-commit/). - Break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue. ### How to Add a New Project #### Option 1: Using GitHub Issues (Recommended) The easiest way to add a new project is by creating a Project Request issue: 1. Go to the [Issues tab](https://github.com/InftyAI/Awesome-LLMOps/issues) 2. Click "New Issue" and select "Project Request" 3. Fill out the template with your project information 4. Submit the issue GitHub Actions workflow will automatically process your request, create a PR, and add the project to the appropriate category. #### Option 2: Manual Addition To add a new project to the landscape, follow these steps: 1. **Prepare the project logo**: - Create or obtain a logo for the project (PNG or SVG format recommended) - Image should be square or have transparent background - Place the logo file in the `logos/` directory with a descriptive name 2. **Update `data.yml`**: - Find the appropriate category and subcategory for your project - Add a new entry under the `items` section with the following format: ```yaml - name: Project Name description: A brief description of the project (1-2 sentences) homepage_url: https://github.com/org/repo logo: project-logo.png repo_url: https://github.com/org/repo ``` 3. **Update the main README.md**: - Add the project to the appropriate category in the main README.md ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ # Makefile for Awesome-LLMOps Landscape # See: https://github.com/cncf/landscape2 for reference # Configuration DATA_FILE := website/data.yml SETTINGS_FILE := website/settings.yml GUIDE_FILE := website/guide.yml LOGOS_PATH := website/logos OUTPUT_DIR := build CACHE_DIR := .cache LANDSCAPE2_VERSION := latest CONTAINER_NAME := awesome-llmops-landscape # Detect OS for installation UNAME_S := $(shell uname -s) LANDSCAPE2_BIN := $(shell command -v landscape2 2> /dev/null) # The image only supports this amd64 platform DOCKER_PLATFORM := --platform linux/amd64 # Default target .PHONY: help help: @echo "Usage: make [target]" @echo "Targets:" @echo " install Install landscape2" @echo " validate Validate data and settings files" @echo " build Build the landscape" @echo " serve Serve the landscape website locally" @echo " docker-build Build the landscape using Docker" @echo " docker-serve Serve the landscape using Docker" @echo " docker-stop Stop the Docker container serving the landscape" @echo " run Run complete workflow (install, validate, build, serve)" @echo " clean Clean build artifacts and Docker containers" all: install validate build # Install landscape2 based on detected OS .PHONY: install install: @echo "Installing landscape2..." ifeq ($(LANDSCAPE2_BIN),) ifeq ($(UNAME_S),Darwin) @echo "Installing via Homebrew on macOS..." brew install cncf/landscape2/landscape2 else ifeq ($(UNAME_S),Linux) @echo "Installing via curl script on Linux..." curl --proto '=https' --tlsv1.2 -LsSf https://github.com/cncf/landscape2/releases/download/$(LANDSCAPE2_VERSION)/landscape2-installer.sh | sh else ifeq ($(shell echo "$(UNAME_S)" | grep -c "MINGW\|MSYS\|CYGWIN"),1) @echo "Installing via PowerShell on Windows..." powershell -Command "irm https://github.com/cncf/landscape2/releases/download/$(LANDSCAPE2_VERSION)/landscape2-installer.ps1 | iex" else @echo "Unsupported OS for direct installation. Please install manually:" @echo "See: https://github.com/cncf/landscape2#installation" @exit 1 endif else @echo "landscape2 is already installed." endif # Validate data and settings files .PHONY: validate validate: @echo "Validating data and settings files..." landscape2 validate data --data-file $(DATA_FILE) landscape2 validate settings --settings-file $(SETTINGS_FILE) landscape2 validate guide --guide-file $(GUIDE_FILE) @echo "Validation completed successfully ✓" # Build the landscape .PHONY: build build: @echo "Building landscape website..." @mkdir -p $(OUTPUT_DIR) $(CACHE_DIR) landscape2 build \ --data-file $(DATA_FILE) \ --settings-file $(SETTINGS_FILE) \ --guide-file $(GUIDE_FILE) \ --logos-path $(LOGOS_PATH) \ --output-dir $(OUTPUT_DIR) \ --cache-dir $(CACHE_DIR) @echo "Build completed ✓" # Serve the landscape locally .PHONY: serve serve: @echo "Serving landscape website on http://127.0.0.1:8000 ..." landscape2 serve --landscape-dir $(OUTPUT_DIR) # Clean build artifacts .PHONY: clean clean: @echo "Cleaning build artifacts and Docker containers..." rm -rf $(OUTPUT_DIR) @echo "Stopping any running landscape Docker containers..." -docker stop $(CONTAINER_NAME) 2>/dev/null || true -docker rm $(CONTAINER_NAME) 2>/dev/null || true # Full workflow: install, validate, build, and serve .PHONY: run run: install validate build serve # Stop Docker container if running .PHONY: docker-stop docker-stop: @echo "Stopping any running landscape Docker containers..." -docker stop $(CONTAINER_NAME) 2>/dev/null || true -docker rm $(CONTAINER_NAME) 2>/dev/null || true # Docker-based alternatives (useful for CI/CD) .PHONY: docker-build docker-build: @echo "Building landscape using Docker ..." @mkdir -p $(OUTPUT_DIR) $(CACHE_DIR) docker run --rm $(DOCKER_PLATFORM) -v $(PWD):/landscape public.ecr.aws/g6m3a0y9/landscape2:latest \ landscape2 build \ --data-file /landscape/$(DATA_FILE) \ --settings-file /landscape/$(SETTINGS_FILE) \ --guide-file /landscape/$(GUIDE_FILE) \ --logos-path /landscape/$(LOGOS_PATH) \ --output-dir /landscape/$(OUTPUT_DIR) \ --cache-dir /landscape/$(CACHE_DIR) @echo "Docker build completed ✓" .PHONY: docker-serve docker-serve: docker-stop @echo "Serving landscape using Docker on http://localhost:8000 ..." docker run --rm $(DOCKER_PLATFORM) -p 8000:8000 --name $(CONTAINER_NAME) -v $(PWD):/landscape public.ecr.aws/g6m3a0y9/landscape2:latest \ landscape2 serve \ --addr 0.0.0.0:8000 \ --landscape-dir /landscape/$(OUTPUT_DIR) @echo "Docker container stopped" ================================================ FILE: OWNERS ================================================ approvers: - cr7258 - kerthcet - samzong reviewers: - cr7258 - kerthcet - samzong ================================================ FILE: README.md ================================================ # Awesome-LLMOps [![Awesome](https://awesome.re/badge.svg)](https://awesome.re) 🎉 An awesome & curated list of best LLMOps tools. > More than welcome to add a new project by simply opening an issue. ## Table of Contents - [Inference](#inference) - [Inference Engine](#inference-engine) - [Inference Platform](#inference-platform) - [Middleware](#middleware) - [LLM Router](#llm-router) - [AI Gateway](#ai-gateway) - [Output](#output) - [Simulator](#simulator) - [Benchmark](#benchmark) - [Orchestration](#orchestration) - [Agent Framework](#agent-framework) - [Application Framework](#application-framework) - [RAG](#rag) - [Workflow](#workflow) - [Evolutionary Framework](#evolutionary-framework) - [Runtime](#runtime) - [AI Terminal](#ai-terminal) - [AI Agent](#ai-agent) - [Chatbot](#chatbot) - [Code Agent](#code-agent) - [Evolve Agent](#evolve-agent) - [Database](#database) - [Observation](#observation) - [Sandbox](#sandbox) - [Tool](#tool) - [Training](#training) - [Framework](#framework) - [FineTune](#finetune) - [RLHF](#rlhf) - [Agentic RL](#agentic-rl) - [Benchmark](#benchmark) - [Workflow](#workflow) ## Inference ### Inference Engine * **[Cortex.cpp](https://github.com/janhq/cortex.cpp)**: Local AI API Platform. ![Stars](https://img.shields.io/github/stars/janhq/cortex.cpp.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/janhq/cortex.cpp?color=green) ![LastCommit](https://img.shields.io/github/last-commit/janhq/cortex.cpp?color=green) * **[DeepSpeed-MII](https://github.com/microsoft/DeepSpeed-MII)**: MII makes low-latency and high-throughput inference possible, powered by DeepSpeed. ![Stars](https://img.shields.io/github/stars/microsoft/deepspeed-mii.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/deepspeed-mii?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/deepspeed-mii?color=green) * **[llama-box](https://github.com/gpustack/llama-box)**: LM inference server implementation based on *.cpp. ![Stars](https://img.shields.io/github/stars/gpustack/llama-box.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/gpustack/llama-box?color=green) ![LastCommit](https://img.shields.io/github/last-commit/gpustack/llama-box?color=green) * **[Nvidia Dynamo](https://github.com/ai-dynamo/dynamo)**: A Datacenter Scale Distributed Inference Serving Framework. ![Stars](https://img.shields.io/github/stars/ai-dynamo/dynamo.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ai-dynamo/dynamo?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ai-dynamo/dynamo?color=green) * **[ipex-llm](https://github.com/intel-analytics/ipex-llm)**: Accelerate local LLM inference and finetuning (LLaMA, Mistral, ChatGLM, Qwen, DeepSeek, Mixtral, Gemma, Phi, MiniCPM, Qwen-VL, MiniCPM-V, etc.) on Intel XPU (e.g., local PC with iGPU and NPU, discrete GPU such as Arc, Flex and Max); seamlessly integrate with llama.cpp, Ollama, HuggingFace, LangChain, LlamaIndex, vLLM, DeepSpeed, Axolotl, etc. ![Stars](https://img.shields.io/github/stars/intel-analytics/ipex-llm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/intel-analytics/ipex-llm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/intel-analytics/ipex-llm?color=green) * **[LMDeploy](https://github.com/InternLM/lmdeploy)**: LMDeploy is a toolkit for compressing, deploying, and serving LLMs. ![Stars](https://img.shields.io/github/stars/internlm/lmdeploy.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/internlm/lmdeploy?color=green) ![LastCommit](https://img.shields.io/github/last-commit/internlm/lmdeploy?color=green) * **[LoRAX](https://github.com/predibase/lorax)**: Multi-LoRA inference server that scales to 1000s of fine-tuned LLMs. ![Stars](https://img.shields.io/github/stars/predibase/lorax.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/predibase/lorax?color=green) ![LastCommit](https://img.shields.io/github/last-commit/predibase/lorax?color=green) ![Tag](https://img.shields.io/badge/lora-orange) * **[llama.cpp](https://github.com/ggerganov/llama.cpp)**: LLM inference in C/C++. ![Stars](https://img.shields.io/github/stars/ggerganov/llama.cpp.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ggerganov/llama.cpp?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ggerganov/llama.cpp?color=green) * **[Llumnix](https://github.com/AlibabaPAI/llumnix)**: Efficient and easy multi-instance LLM serving. ![Stars](https://img.shields.io/github/stars/alibabapai/llumnix.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/alibabapai/llumnix?color=green) ![LastCommit](https://img.shields.io/github/last-commit/alibabapai/llumnix?color=green) * **[MInference](https://github.com/microsoft/minference)**: [NeurIPS'24 Spotlight, ICLR'25] To speed up Long-context LLMs' inference, approximate and dynamic sparse calculate the attention, which reduces inference latency by up to 10x for pre-filling on an A100 while maintaining accuracy. ![Stars](https://img.shields.io/github/stars/microsoft/minference.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/minference?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/minference?color=green) ![Tag](https://img.shields.io/badge/long_context-orange) * **[MLC LLM](https://github.com/mlc-ai/mlc-llm)**: Universal LLM Deployment Engine with ML Compilation. ![Stars](https://img.shields.io/github/stars/mlc-ai/mlc-llm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mlc-ai/mlc-llm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mlc-ai/mlc-llm?color=green) * **[MLServer](https://github.com/SeldonIO/MLServer)**: An inference server for your machine learning models, including support for multiple frameworks, multi-model serving and more. ![Stars](https://img.shields.io/github/stars/seldonio/mlserver.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/seldonio/mlserver?color=green) ![LastCommit](https://img.shields.io/github/last-commit/seldonio/mlserver?color=green) * **[Ollama](https://github.com/ollama/ollama)**: Get up and running with Llama 3.3, DeepSeek-R1, Phi-4, Gemma 3, and other large language models. ![Stars](https://img.shields.io/github/stars/ollama/ollama.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ollama/ollama?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ollama/ollama?color=green) * **[OpenLLM](https://github.com/bentoml/OpenLLM)**: Run any open-source LLMs, such as DeepSeek and Llama, as OpenAI compatible API endpoint in the cloud. ![Stars](https://img.shields.io/github/stars/bentoml/openllm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/bentoml/openllm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/bentoml/openllm?color=green) * **[OpenVINO](https://github.com/openvinotoolkit/openvino)**: OpenVINO™ is an open source toolkit for optimizing and deploying AI inference. ![Stars](https://img.shields.io/github/stars/openvinotoolkit/openvino.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openvinotoolkit/openvino?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openvinotoolkit/openvino?color=green) * **[Petals](https://github.com/bigscience-workshop/petals)**: 🌸 Run LLMs at home, BitTorrent-style. Fine-tuning and inference up to 10x faster than offloading ![Stars](https://img.shields.io/github/stars/bigscience-workshop/petals.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/bigscience-workshop/petals?color=green) ![LastCommit](https://img.shields.io/github/last-commit/bigscience-workshop/petals?color=green) * **[Ratchet](https://github.com/huggingface/ratchet)**: A cross-platform browser ML framework. ![Stars](https://img.shields.io/github/stars/huggingface/ratchet.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/huggingface/ratchet?color=green) ![LastCommit](https://img.shields.io/github/last-commit/huggingface/ratchet?color=green) ![Tag](https://img.shields.io/badge/browser-orange) * **[SGLang](https://github.com/sgl-project/sglang)**: SGLang is a fast serving framework for large language models and vision language models. ![Stars](https://img.shields.io/github/stars/sgl-project/sglang.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/sgl-project/sglang?color=green) ![LastCommit](https://img.shields.io/github/last-commit/sgl-project/sglang?color=green) * **[TinyGrad](https://github.com/tinygrad/tinygrad)**: You like pytorch? You like micrograd? You love tinygrad! ❤️ ![Stars](https://img.shields.io/github/stars/tinygrad/tinygrad.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/tinygrad/tinygrad?color=green) ![LastCommit](https://img.shields.io/github/last-commit/tinygrad/tinygrad?color=green) * **[transformers.js](https://github.com/huggingface/transformers.js)**: State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server! ![Stars](https://img.shields.io/github/stars/huggingface/transformers.js.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/huggingface/transformers.js?color=green) ![LastCommit](https://img.shields.io/github/last-commit/huggingface/transformers.js?color=green) ![Tag](https://img.shields.io/badge/browser-orange) * **[Triton Inference Server](https://github.com/triton-inference-server/server)**: The Triton Inference Server provides an optimized cloud and edge inferencing solution. ![Stars](https://img.shields.io/github/stars/triton-inference-server/server.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/triton-inference-server/server?color=green) ![LastCommit](https://img.shields.io/github/last-commit/triton-inference-server/server?color=green) * **[Text Generation Inference](https://github.com/huggingface/text-generation-inference)**: Large Language Model Text Generation Inference. ![Stars](https://img.shields.io/github/stars/huggingface/text-generation-inference.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/huggingface/text-generation-inference?color=green) ![LastCommit](https://img.shields.io/github/last-commit/huggingface/text-generation-inference?color=green) * **[vLLM](https://github.com/vllm-project/vllm)**: A high-throughput and memory-efficient inference and serving engine for LLMs. ![Stars](https://img.shields.io/github/stars/vllm-project/vllm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/vllm-project/vllm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/vllm-project/vllm?color=green) * **[web-llm](https://github.com/mlc-ai/web-llm)**: High-performance In-browser LLM Inference Engine. ![Stars](https://img.shields.io/github/stars/mlc-ai/web-llm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mlc-ai/web-llm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mlc-ai/web-llm?color=green) ![Tag](https://img.shields.io/badge/browser-orange) * **[Xinference](https://github.com/xorbitsai/inference)**: Replace OpenAI GPT with another LLM in your app by changing a single line of code. Xinference gives you the freedom to use any LLM you need. With Xinference, you're empowered to run inference with any open-source language models, speech recognition models, and multimodal models, whether in the cloud, on-premises, or even on your laptop. ![Stars](https://img.shields.io/github/stars/xorbitsai/inference.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/xorbitsai/inference?color=green) ![LastCommit](https://img.shields.io/github/last-commit/xorbitsai/inference?color=green) * **[zml](https://github.com/zml/zml)**: Any model. Any hardware. Zero compromise. Built with @ziglang / @openxla / MLIR / @bazelbuild. ![Stars](https://img.shields.io/github/stars/zml/zml.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/zml/zml?color=green) ![LastCommit](https://img.shields.io/github/last-commit/zml/zml?color=green) ### Inference Platform * **[AIBrix](https://github.com/vllm-project/aibrix)**: Cost-efficient and pluggable Infrastructure components for GenAI inference. ![Stars](https://img.shields.io/github/stars/vllm-project/aibrix.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/vllm-project/aibrix?color=green) ![LastCommit](https://img.shields.io/github/last-commit/vllm-project/aibrix?color=green) * **[BentoML](https://github.com/bentoml/BentoML)**: The easiest way to serve AI apps and models - Build Model Inference APIs, Job queues, LLM apps, Multi-model pipelines, and more! ![Stars](https://img.shields.io/github/stars/bentoml/bentoml.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/bentoml/bentoml?color=green) ![LastCommit](https://img.shields.io/github/last-commit/bentoml/bentoml?color=green) * **[beta9](https://github.com/beam-cloud/beta9)**: Ultrafast serverless GPU inference, sandboxes, and background jobs ![Stars](https://img.shields.io/github/stars/beam-cloud/beta9.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/beam-cloud/beta9?color=green) ![LastCommit](https://img.shields.io/github/last-commit/beam-cloud/beta9?color=green) * **[Kaito](https://github.com/kaito-project/Kaito)**: Kubernetes operator for large-model inference and fine-tuning, with GPU auto-provisioning, container-based hosting, and CRD-based orchestration. ![Stars](https://img.shields.io/github/stars/kaito-project/Kaito.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kaito-project/Kaito?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kaito-project/Kaito?color=green) * **[Kserve](https://github.com/kserve/kserve)**: Standardized Serverless ML Inference Platform on Kubernetes. ![Stars](https://img.shields.io/github/stars/kserve/kserve.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kserve/kserve?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kserve/kserve?color=green) * **[KubeAI](https://github.com/substratusai/kubeai)**: AI Inference Operator for Kubernetes. The easiest way to serve ML models in production. Supports VLMs, LLMs, embeddings, and speech-to-text. ![Stars](https://img.shields.io/github/stars/substratusai/kubeai.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/substratusai/kubeai?color=green) ![LastCommit](https://img.shields.io/github/last-commit/substratusai/kubeai?color=green) * **[llm-d](https://github.com/llm-d/llm-d)**: llm-d is a Kubernetes-native high-performance distributed LLM inference framework ![Stars](https://img.shields.io/github/stars/llm-d/llm-d.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/llm-d/llm-d?color=green) ![LastCommit](https://img.shields.io/github/last-commit/llm-d/llm-d?color=green) * **[llmaz](https://github.com/InftyAI/llmaz)**: ☸️ Easy, advanced inference platform for large language models on Kubernetes. 🌟 Star to support our work! ![Stars](https://img.shields.io/github/stars/inftyai/llmaz.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/inftyai/llmaz?color=green) ![LastCommit](https://img.shields.io/github/last-commit/inftyai/llmaz?color=green) * **[Modular](https://github.com/modular/modular)**: The Modular Platform (includes MAX & Mojo) ![Stars](https://img.shields.io/github/stars/modular/modular.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/modular/modular?color=green) ![LastCommit](https://img.shields.io/github/last-commit/modular/modular?color=green) * **[Mooncake](https://github.com/kvcache-ai/Mooncake)**: Mooncake is the serving platform for Kimi, a leading LLM service provided by Moonshot AI. ![Stars](https://img.shields.io/github/stars/kvcache-ai/mooncake.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kvcache-ai/mooncake?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kvcache-ai/mooncake?color=green) * **[OME](https://github.com/sgl-project/ome)**: OME is a Kubernetes operator for enterprise-grade management and serving of Large Language Models (LLMs) ![Stars](https://img.shields.io/github/stars/sgl-project/ome.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/sgl-project/ome?color=green) ![LastCommit](https://img.shields.io/github/last-commit/sgl-project/ome?color=green) ### Middleware * **[Checkpoint Engine](https://github.com/MoonshotAI/checkpoint-engine)**: Checkpoint-engine is a simple middleware to update model weights in LLM inference engines ![Stars](https://img.shields.io/github/stars/MoonshotAI/checkpoint-engine.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/MoonshotAI/checkpoint-engine?color=green) ![LastCommit](https://img.shields.io/github/last-commit/MoonshotAI/checkpoint-engine?color=green) * **[kvcached](https://github.com/ovg-project/kvcached)**: Virtualized Elastic KV Cache for Dynamic GPU Sharing and Beyond ![Stars](https://img.shields.io/github/stars/ovg-project/kvcached.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ovg-project/kvcached?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ovg-project/kvcached?color=green) * **[LMCache](https://github.com/LMCache/LMCache)**: 10x Faster Long-Context LLM By Smart KV Cache Optimizations. ![Stars](https://img.shields.io/github/stars/lmcache/lmcache.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/lmcache/lmcache?color=green) ![LastCommit](https://img.shields.io/github/last-commit/lmcache/lmcache?color=green) ![Tag](https://img.shields.io/badge/kvcache-orange) ### LLM Router * **[AI Gateway](https://github.com/Portkey-AI/gateway)**: A blazing fast AI Gateway with integrated guardrails. Route to 200+ LLMs, 50+ AI Guardrails with 1 fast & friendly API. ![Stars](https://img.shields.io/github/stars/portkey-ai/gateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/portkey-ai/gateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/portkey-ai/gateway?color=green) * **[bifrost](https://github.com/maximhq/bifrost)**: Fastest LLM gateway (50x faster than LiteLLM) with adaptive load balancer, cluster mode, guardrails, 1000+ models support & <100 µs overhead at 5k RPS. ![Stars](https://img.shields.io/github/stars/maximhq/bifrost.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/maximhq/bifrost?color=green) ![LastCommit](https://img.shields.io/github/last-commit/maximhq/bifrost?color=green) * **[Ferro Labs AI Gateway](https://github.com/ferro-labs/ai-gateway)**: One API for 25+ LLMs, OpenAI, Anthropic, Bedrock, Azure. Caching, guardrails & cost controls. Go-native LiteLLM & Kong AI Gateway alternative. ![Stars](https://img.shields.io/github/stars/ferro-labs/ai-gateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ferro-labs/ai-gateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ferro-labs/ai-gateway?color=green) * **[Hebo AI Gateway](https://github.com/8monkey-ai/hebo-gateway/)**: OpenAI-compatible /chat/completions, /embeddings & /models endpoints ![Stars](https://img.shields.io/github/stars/8monkey-ai/hebo-gateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/8monkey-ai/hebo-gateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/8monkey-ai/hebo-gateway?color=green) * **[LiteLLM](https://github.com/BerriAI/litellm)**: Python SDK, Proxy Server (LLM Gateway) to call 100+ LLM APIs in OpenAI format - [Bedrock, Azure, OpenAI, VertexAI, Cohere, Anthropic, Sagemaker, HuggingFace, Replicate, Groq]. ![Stars](https://img.shields.io/github/stars/berriai/litellm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/berriai/litellm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/berriai/litellm?color=green) * **[LLMRouter](https://github.com/ulab-uiuc/LLMRouter)**: LLMRouter: An Open-Source Library for LLM Routing ![Stars](https://img.shields.io/github/stars/ulab-uiuc/LLMRouter.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ulab-uiuc/LLMRouter?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ulab-uiuc/LLMRouter?color=green) * **[RouteLLM](https://github.com/lm-sys/routellm)**: A framework for serving and evaluating LLM routers - save LLM costs without compromising quality. ![Stars](https://img.shields.io/github/stars/lm-sys/routellm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/lm-sys/routellm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/lm-sys/routellm?color=green) * **[vLLM Semantic Router](https://github.com/vllm-project/semantic-router)**: Intelligent Mixture-of-Models Router for Efficient LLM Inference ![Stars](https://img.shields.io/github/stars/vllm-project/semantic-router.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/vllm-project/semantic-router?color=green) ![LastCommit](https://img.shields.io/github/last-commit/vllm-project/semantic-router?color=green) ### AI Gateway * **[agentgateway](https://github.com/agentgateway/agentgateway)**: Next Generation Agentic Proxy for AI Agents and MCP servers ![Stars](https://img.shields.io/github/stars/agentgateway/agentgateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/agentgateway/agentgateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/agentgateway/agentgateway?color=green) * **[APISIX](https://github.com/apache/apisix)**: The Cloud-Native API Gateway and AI Gateway with extensive plugin system and AI capabilities. ![Stars](https://img.shields.io/github/stars/apache/apisix.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/apache/apisix?color=green) ![LastCommit](https://img.shields.io/github/last-commit/apache/apisix?color=green) * **[Envoy AI Gateway](https://github.com/envoyproxy/ai-gateway)**: Envoy AI Gateway is an open source project for using Envoy Gateway to handle request traffic from application clients to Generative AI services. ![Stars](https://img.shields.io/github/stars/envoyproxy/ai-gateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/envoyproxy/ai-gateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/envoyproxy/ai-gateway?color=green) * **[Higress](https://github.com/alibaba/higress)**: 🤖 AI Gateway | AI Native API Gateway. ![Stars](https://img.shields.io/github/stars/alibaba/higress.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/alibaba/higress?color=green) ![LastCommit](https://img.shields.io/github/last-commit/alibaba/higress?color=green) * **[kgateway](https://github.com/kgateway-dev/kgateway)**: The Cloud-Native API Gateway and AI Gateway. ![Stars](https://img.shields.io/github/stars/kgateway-dev/kgateway.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kgateway-dev/kgateway?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kgateway-dev/kgateway?color=green) * **[Kong](https://github.com/Kong/kong)**: 🦍 The Cloud-Native API Gateway and AI Gateway. ![Stars](https://img.shields.io/github/stars/Kong/kong.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Kong/kong?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Kong/kong?color=green) * **[gateway-api-inference-extension](https://github.com/kubernetes-sigs/gateway-api-inference-extension)**: Gateway API Inference Extension. ![Stars](https://img.shields.io/github/stars/kubernetes-sigs/gateway-api-inference-extension.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kubernetes-sigs/gateway-api-inference-extension?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kubernetes-sigs/gateway-api-inference-extension?color=green) ### Output * **[BAML](https://github.com/boundaryml/baml)**: The AI framework that adds the engineering to prompt engineering (Python/TS/Ruby/Java/C#/Rust/Go compatible) ![Stars](https://img.shields.io/github/stars/boundaryml/baml.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/boundaryml/baml?color=green) ![LastCommit](https://img.shields.io/github/last-commit/boundaryml/baml?color=green) * **[Instructor](https://github.com/instructor-ai/instructor)**: structured outputs for llms. ![Stars](https://img.shields.io/github/stars/instructor-ai/instructor.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/instructor-ai/instructor?color=green) ![LastCommit](https://img.shields.io/github/last-commit/instructor-ai/instructor?color=green) * **[Outlines](https://github.com/dottxt-ai/outlines)**: Structured Text Generation. ![Stars](https://img.shields.io/github/stars/dottxt-ai/outlines.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/dottxt-ai/outlines?color=green) ![LastCommit](https://img.shields.io/github/last-commit/dottxt-ai/outlines?color=green) * **[XGrammar](https://github.com/mlc-ai/xgrammar)**: Fast, Flexible and Portable Structured Generation ![Stars](https://img.shields.io/github/stars/mlc-ai/xgrammar.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mlc-ai/xgrammar?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mlc-ai/xgrammar?color=green) ### Simulator * **[Vidur](https://github.com/microsoft/vidur)**: A large-scale simulation framework for LLM inference ![Stars](https://img.shields.io/github/stars/microsoft/vidur.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/vidur?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/vidur?color=green) ### Benchmark * **[genai-bench](https://github.com/sgl-project/genai-bench)**: Genai-bench is a powerful benchmark tool designed for comprehensive token-level performance evaluation of large language model (LLM) serving systems. ![Stars](https://img.shields.io/github/stars/sgl-project/genai-bench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/sgl-project/genai-bench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/sgl-project/genai-bench?color=green) * **[Inference Benchmark](https://github.com/AI-Hypercomputer/inference-benchmark)**: A model server agnostic inference benchmarking tool that can be used to benchmark LLMs running on differet infrastructure like GPU and TPU. It can also be run on a GKE cluster as a container. ![Stars](https://img.shields.io/github/stars/AI-Hypercomputer/inference-benchmark.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/AI-Hypercomputer/inference-benchmark?color=green) ![LastCommit](https://img.shields.io/github/last-commit/AI-Hypercomputer/inference-benchmark?color=green) * **[Inference Perf](https://github.com/kubernetes-sigs/inference-perf)**: GenAI inference performance benchmarking tool ![Stars](https://img.shields.io/github/stars/kubernetes-sigs/inference-perf.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kubernetes-sigs/inference-perf?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kubernetes-sigs/inference-perf?color=green) ## Orchestration ### Application Framework * **[Cordum.io](https://github.com/cordum-io/cordum)**: Cordum (cordum.io) is a platform-only control plane for autonomous AI Agents and external workers. It uses NATS for the bus, Redis for state and payload pointers, and CAP v2 wire contracts for jobs, results, and heartbeats. Workers and product packs live outside this repo.Core cordum ![Stars](https://img.shields.io/github/stars/cordum-io/cordum.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/cordum-io/cordum?color=green) ![LastCommit](https://img.shields.io/github/last-commit/cordum-io/cordum?color=green) * **[DeepEval](https://github.com/confident-ai/deepeval)**: The LLM Evaluation Framework ![Stars](https://img.shields.io/github/stars/confident-ai/deepeval.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/confident-ai/deepeval?color=green) ![LastCommit](https://img.shields.io/github/last-commit/confident-ai/deepeval?color=green) * **[Evidently](https://github.com/evidentlyai/evidently)**: Evidently is ​​an open-source ML and LLM observability framework. Evaluate, test, and monitor any AI-powered system or data pipeline. From tabular data to Gen AI. 100+ metrics. ![Stars](https://img.shields.io/github/stars/evidentlyai/evidently.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/evidentlyai/evidently?color=green) ![LastCommit](https://img.shields.io/github/last-commit/evidentlyai/evidently?color=green) * **[Langfuse](https://github.com/langfuse/langfuse)**: 🪢 Open source LLM engineering platform: LLM Observability, metrics, evals, prompt management, playground, datasets. Integrates with OpenTelemetry, Langchain, OpenAI SDK, LiteLLM, and more. 🍊YC W23 ![Stars](https://img.shields.io/github/stars/langfuse/langfuse.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/langfuse/langfuse?color=green) ![LastCommit](https://img.shields.io/github/last-commit/langfuse/langfuse?color=green) * **[Helicone](https://github.com/helicone/helicone)**: 🧊 Open source LLM observability platform. One line of code to monitor, evaluate, and experiment. YC W23 🍓 ![Stars](https://img.shields.io/github/stars/helicone/helicone.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/helicone/helicone?color=green) ![LastCommit](https://img.shields.io/github/last-commit/helicone/helicone?color=green) * **[lunaary](https://github.com/lunary-ai/lunary)**: The production toolkit for LLMs. Observability, prompt management and evaluations. ![Stars](https://img.shields.io/github/stars/lunary-ai/lunary.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/lunary-ai/lunary?color=green) ![LastCommit](https://img.shields.io/github/last-commit/lunary-ai/lunary?color=green) * **[Neurolink](https://github.com/juspay/neurolink)**: Universal AI Development Platform with MCP server integration, multi-provider support, and professional CLI. Build, test, and deploy AI applications with multiple ai providers. ![Stars](https://img.shields.io/github/stars/juspay/neurolink.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/juspay/neurolink?color=green) ![LastCommit](https://img.shields.io/github/last-commit/juspay/neurolink?color=green) * **[OpenLIT](https://github.com/openlit/openlit)**: Open source platform for AI Engineering: OpenTelemetry-native LLM Observability, GPU Monitoring, Guardrails, Evaluations, Prompt Management, Vault, Playground. 🚀💻 Integrates with 50+ LLM Providers, VectorDBs, Agent Frameworks and GPUs. ![Stars](https://img.shields.io/github/stars/openlit/openlit.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openlit/openlit?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openlit/openlit?color=green) * **[phoenix](https://github.com/arize-ai/phoenix)**: AI Observability & Evaluation. ![Stars](https://img.shields.io/github/stars/arize-ai/phoenix.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/arize-ai/phoenix?color=green) ![LastCommit](https://img.shields.io/github/last-commit/arize-ai/phoenix?color=green) * **[PostHog](https://github.com/PostHog/posthog)**: 🦔 PostHog provides open-source web & product analytics, session recording, feature flagging and A/B testing that you can self-host. Get started - free. ![Stars](https://img.shields.io/github/stars/PostHog/posthog.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/PostHog/posthog?color=green) ![LastCommit](https://img.shields.io/github/last-commit/PostHog/posthog?color=green) * **[ragas](https://github.com/explodinggradients/ragas)**: Supercharge Your LLM Application Evaluations 🚀 ![Stars](https://img.shields.io/github/stars/explodinggradients/ragas.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/explodinggradients/ragas?color=green) ![LastCommit](https://img.shields.io/github/last-commit/explodinggradients/ragas?color=green) * **[Weave](https://github.com/wandb/weave)**: Weave is a toolkit for developing AI-powered applications, built by Weights & Biases. ![Stars](https://img.shields.io/github/stars/wandb/weave.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/wandb/weave?color=green) ![LastCommit](https://img.shields.io/github/last-commit/wandb/weave?color=green) ### Agent Framework * **[Agent Development Kit (ADK)](https://github.com/google/adk-python)**: An open-source, code-first Python toolkit for building, evaluating, and deploying sophisticated AI agents with flexibility and control. ![Stars](https://img.shields.io/github/stars/google/adk-python.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/google/adk-python?color=green) ![LastCommit](https://img.shields.io/github/last-commit/google/adk-python?color=green) * **[Agent Shadow Brain](https://github.com/theihtisham/agent-shadow-brain)**: Self-evolving AI coding intelligence with infinite memory (TurboQuant), genetic algorithm evolution, predictive bug detection, PageRank knowledge graphs, and swarm intelligence. The world's first autonomous coding brain. ![Stars](https://img.shields.io/github/stars/theihtisham/agent-shadow-brain.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/theihtisham/agent-shadow-brain?color=green) ![LastCommit](https://img.shields.io/github/last-commit/theihtisham/agent-shadow-brain?color=green) * **[AgentField](https://github.com/Agent-Field/agentfield)**: Framework for AI Backend. Build and run AI agents like microservices - scalable, observable, and identity-aware from day one. ![Stars](https://img.shields.io/github/stars/Agent-Field/agentfield.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Agent-Field/agentfield?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Agent-Field/agentfield?color=green) * **[Agno](https://github.com/agno-agi/agno)**: Open-source framework for building multi-agent systems with memory, knowledge and reasoning. ![Stars](https://img.shields.io/github/stars/agno-agi/agno.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/agno-agi/agno?color=green) ![LastCommit](https://img.shields.io/github/last-commit/agno-agi/agno?color=green) * **[autogen](https://github.com/microsoft/autogen)**: A programming framework for agentic AI ![Stars](https://img.shields.io/github/stars/microsoft/autogen.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/autogen?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/autogen?color=green) * **[AutoGPT](https://github.com/Significant-Gravitas/AutoGPT)**: AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters. ![Stars](https://img.shields.io/github/stars/significant-gravitas/autogpt.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/significant-gravitas/autogpt?color=green) ![LastCommit](https://img.shields.io/github/last-commit/significant-gravitas/autogpt?color=green) * **[CAMEL](https://github.com/camel-ai/camel)**: 🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. ![Stars](https://img.shields.io/github/stars/camel-ai/camel.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/camel-ai/camel?color=green) ![LastCommit](https://img.shields.io/github/last-commit/camel-ai/camel?color=green) * **[crewAI](https://github.com/crewAIInc/crewAI)**: Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks. ![Stars](https://img.shields.io/github/stars/crewAIInc/crewAI.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/crewAIInc/crewAI?color=green) ![LastCommit](https://img.shields.io/github/last-commit/crewAIInc/crewAI?color=green) * **[fast-agent](https://github.com/evalstate/fast-agent)**: Define, Prompt and Test MCP enabled Agents and Workflows ![Stars](https://img.shields.io/github/stars/evalstate/fast-agent.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/evalstate/fast-agent?color=green) ![LastCommit](https://img.shields.io/github/last-commit/evalstate/fast-agent?color=green) * **[Flowise](https://github.com/FlowiseAI/Flowise)**: Build AI Agents, Visually ![Stars](https://img.shields.io/github/stars/flowiseai/flowise.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/flowiseai/flowise?color=green) ![LastCommit](https://img.shields.io/github/last-commit/flowiseai/flowise?color=green) * **[kagent](https://github.com/kagent-dev/kagent)**: kagent is a kubernetes native framework for building AI agents. ![Stars](https://img.shields.io/github/stars/kagent-dev/kagent.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kagent-dev/kagent?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kagent-dev/kagent?color=green) ![Tag](https://img.shields.io/badge/kubernetes-orange) * **[LangGraph](https://github.com/langchain-ai/langgraph)**: Build resilient language agents as graphs. ![Stars](https://img.shields.io/github/stars/langchain-ai/langgraph.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/langchain-ai/langgraph?color=green) ![LastCommit](https://img.shields.io/github/last-commit/langchain-ai/langgraph?color=green) * **[MetaGPT](https://github.com/geekan/MetaGPT)**: 🌟 The Multi-Agent Framework: First AI Software Company, Towards Natural Language Programming. ![Stars](https://img.shields.io/github/stars/geekan/metagpt.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/geekan/metagpt?color=green) ![LastCommit](https://img.shields.io/github/last-commit/geekan/metagpt?color=green) * **[OpenAI Agents SDK](https://github.com/openai/openai-agents-python)**: A lightweight, powerful framework for multi-agent workflows. ![Stars](https://img.shields.io/github/stars/openai/openai-agents-python.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openai/openai-agents-python?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openai/openai-agents-python?color=green) * **[PydanticAI](https://github.com/pydantic/pydantic-ai)**: GenAI Agent Framework, the Pydantic way ![Stars](https://img.shields.io/github/stars/pydantic/pydantic-ai.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/pydantic/pydantic-ai?color=green) ![LastCommit](https://img.shields.io/github/last-commit/pydantic/pydantic-ai?color=green) * **[Qwen-Agent](https://github.com/QwenLM/Qwen-Agent)**: Agent framework and applications built upon Qwen>=3.0, featuring Function Calling, MCP, Code Interpreter, RAG, Chrome extension, etc. ![Stars](https://img.shields.io/github/stars/QwenLM/Qwen-Agent.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/QwenLM/Qwen-Agent?color=green) ![LastCommit](https://img.shields.io/github/last-commit/QwenLM/Qwen-Agent?color=green) * **[Semantic Kernel](https://github.com/microsoft/semantic-kernel)**: Integrate cutting-edge LLM technology quickly and easily into your apps. ![Stars](https://img.shields.io/github/stars/microsoft/semantic-kernel.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/semantic-kernel?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/semantic-kernel?color=green) * **[Suna](https://github.com/kortix-ai/suna)**: Suna - Open Source Generalist AI Agent ![Stars](https://img.shields.io/github/stars/kortix-ai/suna.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kortix-ai/suna?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kortix-ai/suna?color=green) * **[Swarm](https://github.com/openai/swarm)**: Educational framework exploring ergonomic, lightweight multi-agent orchestration. Managed by OpenAI Solution team. ![Stars](https://img.shields.io/github/stars/openai/swarm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openai/swarm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openai/swarm?color=green) ![Tag](https://img.shields.io/badge/experimental-slategray) ### Evolutionary Framework * **[AI-Researcher](https://github.com/HKUDS/AI-Researcher)**: [NeurIPS2025] "AI-Researcher: Autonomous Scientific Innovation" -- A production-ready version: https://novix.science/chat ![Stars](https://img.shields.io/github/stars/HKUDS/AI-Researcher.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/HKUDS/AI-Researcher?color=green) ![LastCommit](https://img.shields.io/github/last-commit/HKUDS/AI-Researcher?color=green) * **[AIDE ML](https://github.com/WecoAI/aideml)**: AIDE: AI-Driven Exploration in the Space of Code. The machine Learning engineering agent that automates AI R&D. ![Stars](https://img.shields.io/github/stars/WecoAI/aideml.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/WecoAI/aideml?color=green) ![LastCommit](https://img.shields.io/github/last-commit/WecoAI/aideml?color=green) * **[LoongFlow](https://github.com/baidu-baige/LoongFlow)**: LoongFlow: A Thinking & Learning Framework for Expert-Grade AI Agents. ![Stars](https://img.shields.io/github/stars/baidu-baige/LoongFlow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/baidu-baige/LoongFlow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/baidu-baige/LoongFlow?color=green) * **[OpenEvolve](https://github.com/codelion/openevolve)**: Open-source implementation of AlphaEvolve ![Stars](https://img.shields.io/github/stars/codelion/openevolve.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/codelion/openevolve?color=green) ![LastCommit](https://img.shields.io/github/last-commit/codelion/openevolve?color=green) * **[ShinkaEvolve](https://github.com/SakanaAI/ShinkaEvolve)**: ShinkaEvolve: Towards Open-Ended and Sample-Efficient Program Evolution ![Stars](https://img.shields.io/github/stars/SakanaAI/ShinkaEvolve.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/SakanaAI/ShinkaEvolve?color=green) ![LastCommit](https://img.shields.io/github/last-commit/SakanaAI/ShinkaEvolve?color=green) * **[SkyDiscover](https://github.com/skydiscover-ai/skydiscover#-benchmark-performance)**: AI-Driven Scientific and Algorithmic Discovery ![Stars](https://img.shields.io/github/stars/skydiscover-ai/skydiscover.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/skydiscover-ai/skydiscover?color=green) ![LastCommit](https://img.shields.io/github/last-commit/skydiscover-ai/skydiscover?color=green) ### RAG * **[GraphRAG](https://github.com/microsoft/graphrag)**: A modular graph-based Retrieval-Augmented Generation (RAG) system. ![Stars](https://img.shields.io/github/stars/microsoft/graphrag.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/graphrag?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/graphrag?color=green) * **[LightRAG](https://github.com/HKUDS/LightRAG)**: "LightRAG: Simple and Fast Retrieval-Augmented Generation" ![Stars](https://img.shields.io/github/stars/HKUDS/LightRAG.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/HKUDS/LightRAG?color=green) ![LastCommit](https://img.shields.io/github/last-commit/HKUDS/LightRAG?color=green) * **[quivr](https://github.com/QuivrHQ/quivr)**: Opiniated RAG for integrating GenAI in your apps 🧠 Focus on your product rather than the RAG. Easy integration in existing products with customisation! Any LLM: GPT4, Groq, Llama. Any Vectorstore: PGVector, Faiss. Any Files. Anyway you want. ![Stars](https://img.shields.io/github/stars/QuivrHQ/quivr.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/QuivrHQ/quivr?color=green) ![LastCommit](https://img.shields.io/github/last-commit/QuivrHQ/quivr?color=green) * **[RAG-Anything](https://github.com/HKUDS/RAG-Anything)**: "RAG-Anything: All-in-One RAG Framework" ![Stars](https://img.shields.io/github/stars/HKUDS/RAG-Anything.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/HKUDS/RAG-Anything?color=green) ![LastCommit](https://img.shields.io/github/last-commit/HKUDS/RAG-Anything?color=green) * **[RAGFlow](https://github.com/infiniflow/ragflow)**: RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. ![Stars](https://img.shields.io/github/stars/infiniflow/ragflow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/infiniflow/ragflow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/infiniflow/ragflow?color=green) ### Workflow * **[Dify](https://github.com/langgenius/dify)**: Production-ready platform for agentic workflow development. ![Stars](https://img.shields.io/github/stars/langgenius/dify.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/langgenius/dify?color=green) ![LastCommit](https://img.shields.io/github/last-commit/langgenius/dify?color=green) * **[FastGPT](https://github.com/labring/FastGPT)**: FastGPT is a knowledge-based platform built on the LLMs, offers a comprehensive suite of out-of-the-box capabilities such as data processing, RAG retrieval, and visual AI workflow orchestration, letting you easily develop and deploy complex question-answering systems without the need for extensive setup or configuration. ![Stars](https://img.shields.io/github/stars/labring/FastGPT.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/labring/FastGPT?color=green) ![LastCommit](https://img.shields.io/github/last-commit/labring/FastGPT?color=green) * **[Haystack](https://github.com/deepset-ai/haystack)**: AI orchestration framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data. With advanced retrieval methods, it's best suited for building RAG, question answering, semantic search or conversational agent chatbots. ![Stars](https://img.shields.io/github/stars/deepset-ai/haystack.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/deepset-ai/haystack?color=green) ![LastCommit](https://img.shields.io/github/last-commit/deepset-ai/haystack?color=green) * **[Inference](https://github.com/roboflow/inference)**: Turn any computer or edge device into a command center for your computer vision projects. ![Stars](https://img.shields.io/github/stars/roboflow/inference.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/roboflow/inference?color=green) ![LastCommit](https://img.shields.io/github/last-commit/roboflow/inference?color=green) ![Tag](https://img.shields.io/badge/computer_vision-orange) * **[LangChain](https://github.com/langchain-ai/langchain)**: 🦜🔗 Build context-aware reasoning applications. ![Stars](https://img.shields.io/github/stars/langchain-ai/langchain.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/langchain-ai/langchain?color=green) ![LastCommit](https://img.shields.io/github/last-commit/langchain-ai/langchain?color=green) * **[LlamaIndex](https://github.com/run-llama/llama_index)**: LlamaIndex is the leading framework for building LLM-powered agents over your data. ![Stars](https://img.shields.io/github/stars/run-llama/llama_index.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/run-llama/llama_index?color=green) ![LastCommit](https://img.shields.io/github/last-commit/run-llama/llama_index?color=green) ## Runtime ### AI Terminal * **[aider](https://github.com/Aider-AI/aider)**: aider is AI pair programming in your terminal ![Stars](https://img.shields.io/github/stars/Aider-AI/aider.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Aider-AI/aider?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Aider-AI/aider?color=green) * **[Codex](https://github.com/openai/codex)**: Lightweight coding agent that runs in your terminal ![Stars](https://img.shields.io/github/stars/openai/codex.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openai/codex?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openai/codex?color=green) ![Tag](https://img.shields.io/badge/coding-orange) * **[Crush](https://github.com/charmbracelet/crush)**: The glamourous AI coding agent for your favourite terminal 💘 ![Stars](https://img.shields.io/github/stars/charmbracelet/crush.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/charmbracelet/crush?color=green) ![LastCommit](https://img.shields.io/github/last-commit/charmbracelet/crush?color=green) * **[Gemini CLI](https://github.com/google-gemini/gemini-cli)**: An open-source AI agent that brings the power of Gemini directly into your terminal. ![Stars](https://img.shields.io/github/stars/google-gemini/gemini-cli.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/google-gemini/gemini-cli?color=green) ![LastCommit](https://img.shields.io/github/last-commit/google-gemini/gemini-cli?color=green) * **[OpenCode](https://github.com/sst/opencode)**: The AI coding agent built for the terminal. ![Stars](https://img.shields.io/github/stars/sst/opencode.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/sst/opencode?color=green) ![LastCommit](https://img.shields.io/github/last-commit/sst/opencode?color=green) * **[Stakpak](https://github.com/stakpak/agent)**: DevOps agent that won't accidentally tweet your AWS credentials 🦀 ![Stars](https://img.shields.io/github/stars/stakpak/agent.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/stakpak/agent?color=green) ![LastCommit](https://img.shields.io/github/last-commit/stakpak/agent?color=green) ### AI Agent * **[goose](https://github.com/block/goose)**: an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM ![Stars](https://img.shields.io/github/stars/block/goose.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/block/goose?color=green) ![LastCommit](https://img.shields.io/github/last-commit/block/goose?color=green) * **[Magentic-UI](https://github.com/microsoft/magentic-ui)**: A research prototype of a human-centered web agent ![Stars](https://img.shields.io/github/stars/microsoft/magentic-ui.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/microsoft/magentic-ui?color=green) ![LastCommit](https://img.shields.io/github/last-commit/microsoft/magentic-ui?color=green) * **[OpenManus](https://github.com/FoundationAgents/OpenManus)**: No fortress, purely open ground. OpenManus is Coming. ![Stars](https://img.shields.io/github/stars/FoundationAgents/openmanus.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/FoundationAgents/openmanus?color=green) ![LastCommit](https://img.shields.io/github/last-commit/FoundationAgents/openmanus?color=green) * **[Tongyi Deep Research](https://github.com/Alibaba-NLP/DeepResearch)**: Tongyi DeepResearch, the Leading Open-source DeepResearch Agent ![Stars](https://img.shields.io/github/stars/Alibaba-NLP/DeepResearch.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Alibaba-NLP/DeepResearch?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Alibaba-NLP/DeepResearch?color=green) ### Code Agent * **[Continue](https://github.com/continuedev/continue)**: ⏩ Create, share, and use custom AI code assistants with our open-source IDE extensions and hub of models, rules, prompts, docs, and other building blocks. ![Stars](https://img.shields.io/github/stars/continuedev/continue.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/continuedev/continue?color=green) ![LastCommit](https://img.shields.io/github/last-commit/continuedev/continue?color=green) * **[Open SWE](https://github.com/langchain-ai/open-swe)**: An Open-Source Asynchronous Coding Agent ![Stars](https://img.shields.io/github/stars/langchain-ai/open-swe.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/langchain-ai/open-swe?color=green) ![LastCommit](https://img.shields.io/github/last-commit/langchain-ai/open-swe?color=green) * **[SWE-agent](https://github.com/SWE-agent/SWE-agent)**: SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024] ![Stars](https://img.shields.io/github/stars/SWE-agent/SWE-agent.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/SWE-agent/SWE-agent?color=green) ![LastCommit](https://img.shields.io/github/last-commit/SWE-agent/SWE-agent?color=green) * **[Tabby](https://github.com/TabbyML/tabby)**: Self-hosted AI coding assistant. ![Stars](https://img.shields.io/github/stars/tabbyml/tabby.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/tabbyml/tabby?color=green) ![LastCommit](https://img.shields.io/github/last-commit/tabbyml/tabby?color=green) ### Tool * **[Beads](https://github.com/steveyegge/beads)**: Beads - A memory upgrade for your coding agent ![Stars](https://img.shields.io/github/stars/steveyegge/beads.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/steveyegge/beads?color=green) ![LastCommit](https://img.shields.io/github/last-commit/steveyegge/beads?color=green) * **[Browser Use](https://github.com/browser-use/browser-use)**: Make websites accessible for AI agents. ![Stars](https://img.shields.io/github/stars/browser-use/browser-use.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/browser-use/browser-use?color=green) ![LastCommit](https://img.shields.io/github/last-commit/browser-use/browser-use?color=green) * **[Graphiti](https://github.com/getzep/graphiti)**: Build Real-Time Knowledge Graphs for AI Agents. ![Stars](https://img.shields.io/github/stars/getzep/graphiti.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/getzep/graphiti?color=green) ![LastCommit](https://img.shields.io/github/last-commit/getzep/graphiti?color=green) * **[Mem0](https://github.com/mem0ai/mem0)**: The Memory layer for AI Agents. ![Stars](https://img.shields.io/github/stars/mem0ai/mem0.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mem0ai/mem0?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mem0ai/mem0?color=green) * **[OpenAI CUA](https://github.com/openai/openai-cua-sample-app)**: Computer Using Agent Sample App. ![Stars](https://img.shields.io/github/stars/openai/openai-cua-sample-app.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openai/openai-cua-sample-app?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openai/openai-cua-sample-app?color=green) * **[SidClaw](https://github.com/sidclawhq/platform)**: The approval and accountability layer for AI agents. Identity → Policy → Approval → Trace. 13 framework integrations. Free during early access. ![Stars](https://img.shields.io/github/stars/sidclawhq/platform.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/sidclawhq/platform?color=green) ![LastCommit](https://img.shields.io/github/last-commit/sidclawhq/platform?color=green) * **[WritBase](https://github.com/Writbase/writbase)**: MCP-native task management for AI agent fleets ![Stars](https://img.shields.io/github/stars/Writbase/writbase.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Writbase/writbase?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Writbase/writbase?color=green) ### Chatbot * **[5ire](https://github.com/nanbingxyz/5ire)**: 5ire is a cross-platform desktop AI assistant, MCP client. It compatible with major service providers, supports local knowledge base and tools via model context protocol servers. ![Stars](https://img.shields.io/github/stars/nanbingxyz/5ire.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/nanbingxyz/5ire?color=green) ![LastCommit](https://img.shields.io/github/last-commit/nanbingxyz/5ire?color=green) * **[AnythingLLM](https://github.com/Mintplex-Labs/anything-llm)**: The all-in-one Desktop & Docker AI application with built-in RAG, AI agents, No-code agent builder, MCP compatibility, and more. ![Stars](https://img.shields.io/github/stars/Mintplex-Labs/anything-llm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/Mintplex-Labs/anything-llm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/Mintplex-Labs/anything-llm?color=green) * **[Chat SDK](https://github.com/vercel/ai-chatbot)**: A full-featured, hackable Next.js AI chatbot built by Vercel ![Stars](https://img.shields.io/github/stars/vercel/ai-chatbot.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/vercel/ai-chatbot?color=green) ![LastCommit](https://img.shields.io/github/last-commit/vercel/ai-chatbot?color=green) * **[Chatbot UI](https://github.com/mckaywrigley/chatbot-ui)**: AI chat for any model. ![Stars](https://img.shields.io/github/stars/mckaywrigley/chatbot-ui.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mckaywrigley/chatbot-ui?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mckaywrigley/chatbot-ui?color=green) * **[Cherry Studio](https://github.com/CherryHQ/cherry-studio)**: 🍒 Cherry Studio is a desktop client that supports for multiple LLM providers. Support deepseek-r1. ![Stars](https://img.shields.io/github/stars/CherryHQ/cherry-studio.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/CherryHQ/cherry-studio?color=green) ![LastCommit](https://img.shields.io/github/last-commit/CherryHQ/cherry-studio?color=green) * **[FastChat](https://github.com/lm-sys/FastChat)**: An open platform for training, serving, and evaluating large language models. Release repo for Vicuna and Chatbot Arena. ![Stars](https://img.shields.io/github/stars/lm-sys/fastchat.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/lm-sys/fastchat?color=green) ![LastCommit](https://img.shields.io/github/last-commit/lm-sys/fastchat?color=green) * **[Gradio](https://github.com/gradio-app/gradio)**: Build and share delightful machine learning apps, all in Python. 🌟 Star to support our work! ![Stars](https://img.shields.io/github/stars/gradio-app/gradio.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/gradio-app/gradio?color=green) ![LastCommit](https://img.shields.io/github/last-commit/gradio-app/gradio?color=green) * **[Jan](https://github.com/janhq/jan)**: Jan is an open source alternative to ChatGPT that runs 100% offline on your computer. ![Stars](https://img.shields.io/github/stars/janhq/jan.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/janhq/jan?color=green) ![LastCommit](https://img.shields.io/github/last-commit/janhq/jan?color=green) * **[LLM](https://github.com/simonw/llm)**: Access large language models from the command-line ![Stars](https://img.shields.io/github/stars/simonw/llm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/simonw/llm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/simonw/llm?color=green) * **[Lobe Chat](https://github.com/lobehub/lobe-chat)**: 🤯 Lobe Chat - an open-source, modern-design AI chat framework. Supports Multi AI Providers( OpenAI / Claude 3 / Gemini / Ollama / DeepSeek / Qwen), Knowledge Base (file upload / knowledge management / RAG ), Multi-Modals (Plugins/Artifacts) and Thinking. One-click FREE deployment of your private ChatGPT/ Claude / DeepSeek application. ![Stars](https://img.shields.io/github/stars/lobehub/lobe-chat.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/lobehub/lobe-chat?color=green) ![LastCommit](https://img.shields.io/github/last-commit/lobehub/lobe-chat?color=green) * **[NextChat](https://github.com/ChatGPTNextWeb/NextChat)**: ✨ Light and Fast AI Assistant. Support: Web | iOS | MacOS | Android | Linux | Windows. ![Stars](https://img.shields.io/github/stars/chatgptnextweb/nextchat.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/chatgptnextweb/nextchat?color=green) ![LastCommit](https://img.shields.io/github/last-commit/chatgptnextweb/nextchat?color=green) * **[opcode](https://github.com/getAsterisk/opcode)**: A powerful GUI app and Toolkit for Claude Code - Create custom agents, manage interactive Claude Code sessions, run secure background agents, and more. ![Stars](https://img.shields.io/github/stars/getAsterisk/opcode.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/getAsterisk/opcode?color=green) ![LastCommit](https://img.shields.io/github/last-commit/getAsterisk/opcode?color=green) * **[Open WebUI](https://github.com/open-webui/open-webui)**: User-friendly AI Interface (Supports Ollama, OpenAI API, ...). ![Stars](https://img.shields.io/github/stars/open-webui/open-webui.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/open-webui/open-webui?color=green) ![LastCommit](https://img.shields.io/github/last-commit/open-webui/open-webui?color=green) * **[PrivateGPT](https://github.com/zylon-ai/private-gpt)**: Interact with your documents using the power of GPT, 100% privately, no data leaks. ![Stars](https://img.shields.io/github/stars/zylon-ai/private-gpt.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/zylon-ai/private-gpt?color=green) ![LastCommit](https://img.shields.io/github/last-commit/zylon-ai/private-gpt?color=green) ### Evolve Agent * **[AgentEvolver](https://github.com/modelscope/AgentEvolver)**: AgentEvolver: Towards Efficient Self-Evolving Agent System ![Stars](https://img.shields.io/github/stars/modelscope/AgentEvolver.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/modelscope/AgentEvolver?color=green) ![LastCommit](https://img.shields.io/github/last-commit/modelscope/AgentEvolver?color=green) * **[EvoAgentX](https://github.com/EvoAgentX/EvoAgentX)**: 🚀 EvoAgentX: Building a Self-Evolving Ecosystem of AI Agents ![Stars](https://img.shields.io/github/stars/EvoAgentX/EvoAgentX.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/EvoAgentX/EvoAgentX?color=green) ![LastCommit](https://img.shields.io/github/last-commit/EvoAgentX/EvoAgentX?color=green) ### Database * **[chroma](https://github.com/chroma-core/chroma)**: the AI-native open-source embedding database. ![Stars](https://img.shields.io/github/stars/chroma-core/chroma.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/chroma-core/chroma?color=green) ![LastCommit](https://img.shields.io/github/last-commit/chroma-core/chroma?color=green) * **[deeplake](https://github.com/activeloopai/deeplake)**: Database for AI. Store Vectors, Images, Texts, Videos, etc. Use with LLMs/LangChain. Store, query, version, & visualize any AI data. Stream data in real-time to PyTorch/TensorFlow. ![Stars](https://img.shields.io/github/stars/activeloopai/deeplake.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/activeloopai/deeplake?color=green) ![LastCommit](https://img.shields.io/github/last-commit/activeloopai/deeplake?color=green) * **[Faiss](https://github.com/facebookresearch/faiss)**: A library for efficient similarity search and clustering of dense vectors. ![Stars](https://img.shields.io/github/stars/facebookresearch/faiss.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/facebookresearch/faiss?color=green) ![LastCommit](https://img.shields.io/github/last-commit/facebookresearch/faiss?color=green) * **[Hindsight](https://github.com/vectorize-io/hindsight)**: Hindsight: Agent Memory That Learns ![Stars](https://img.shields.io/github/stars/vectorize-io/hindsight.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/vectorize-io/hindsight?color=green) ![LastCommit](https://img.shields.io/github/last-commit/vectorize-io/hindsight?color=green) * **[milvus](https://github.com/milvus-io/milvus)**: Milvus is a high-performance, cloud-native vector database built for scalable vector ANN search. ![Stars](https://img.shields.io/github/stars/milvus-io/milvus.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/milvus-io/milvus?color=green) ![LastCommit](https://img.shields.io/github/last-commit/milvus-io/milvus?color=green) * **[weaviate](https://github.com/weaviate/weaviate)**: Weaviate is an open-source vector database that stores both objects and vectors, allowing for the combination of vector search with structured filtering with the fault tolerance and scalability of a cloud-native database​. ![Stars](https://img.shields.io/github/stars/weaviate/weaviate.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/weaviate/weaviate?color=green) ![LastCommit](https://img.shields.io/github/last-commit/weaviate/weaviate?color=green) ### Sandbox * **[Daytona](https://github.com/daytonaio/daytona)**: Daytona is a Secure and Elastic Infrastructure for Running AI-Generated Code. ![Stars](https://img.shields.io/github/stars/daytonaio/daytona.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/daytonaio/daytona?color=green) ![LastCommit](https://img.shields.io/github/last-commit/daytonaio/daytona?color=green) * **[E2B](https://github.com/e2b-dev/E2B)**: Secure open source cloud runtime for AI apps & AI agents. ![Stars](https://img.shields.io/github/stars/e2b-dev/E2B.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/e2b-dev/E2B?color=green) ![LastCommit](https://img.shields.io/github/last-commit/e2b-dev/E2B?color=green) ### Observation * **[OpenLLMetry](https://github.com/traceloop/openllmetry)**: Open-source observability for your LLM application, based on OpenTelemetry. ![Stars](https://img.shields.io/github/stars/traceloop/openllmetry.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/traceloop/openllmetry?color=green) ![LastCommit](https://img.shields.io/github/last-commit/traceloop/openllmetry?color=green) * **[wandb](https://github.com/wandb/wandb)**: The AI developer platform. Use Weights & Biases to train and fine-tune models, and manage models from experimentation to production. ![Stars](https://img.shields.io/github/stars/wandb/wandb.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/wandb/wandb?color=green) ![LastCommit](https://img.shields.io/github/last-commit/wandb/wandb?color=green) ## Training ### Framework * **[AXLearn](https://github.com/apple/axlearn)**: An Extensible Deep Learning Library ![Stars](https://img.shields.io/github/stars/apple/axlearn.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/apple/axlearn?color=green) ![LastCommit](https://img.shields.io/github/last-commit/apple/axlearn?color=green) * **[Candle](https://github.com/huggingface/candle)**: Minimalist ML framework for Rust. ![Stars](https://img.shields.io/github/stars/huggingface/candle.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/huggingface/candle?color=green) ![LastCommit](https://img.shields.io/github/last-commit/huggingface/candle?color=green) * **[ColossalAI](https://github.com/hpcaitech/ColossalAI)**: Making large AI models cheaper, faster and more accessible. ![Stars](https://img.shields.io/github/stars/hpcaitech/ColossalAI.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/hpcaitech/ColossalAI?color=green) ![LastCommit](https://img.shields.io/github/last-commit/hpcaitech/ColossalAI?color=green) * **[DLRover](https://github.com/intelligent-machine-learning/dlrover)**: DLRover: An Automatic Distributed Deep Learning System ![Stars](https://img.shields.io/github/stars/intelligent-machine-learning/dlrover.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/intelligent-machine-learning/dlrover?color=green) ![LastCommit](https://img.shields.io/github/last-commit/intelligent-machine-learning/dlrover?color=green) * **[Ludwig](https://github.com/ludwig-ai/ludwig)**: Low-code framework for building custom LLMs, neural networks, and other AI models. ![Stars](https://img.shields.io/github/stars/ludwig-ai/ludwig.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ludwig-ai/ludwig?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ludwig-ai/ludwig?color=green) * **[MaxText](https://github.com/google/maxtext)**: A simple, performant and scalable Jax LLM! ![Stars](https://img.shields.io/github/stars/google/maxtext.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/google/maxtext?color=green) ![LastCommit](https://img.shields.io/github/last-commit/google/maxtext?color=green) * **[MLX](https://github.com/ml-explore/mlx)**: MLX: An array framework for Apple silicon. ![Stars](https://img.shields.io/github/stars/ml-explore/mlx.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ml-explore/mlx?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ml-explore/mlx?color=green) ### FineTune * **[Axolotl](https://github.com/axolotl-ai-cloud/axolotl)**: Go ahead and axolotl questions. ![Stars](https://img.shields.io/github/stars/axolotl-ai-cloud/axolotl.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/axolotl-ai-cloud/axolotl?color=green) ![LastCommit](https://img.shields.io/github/last-commit/axolotl-ai-cloud/axolotl?color=green) * **[EasyLM](https://github.com/young-geng/EasyLM)**: Large language models (LLMs) made easy, EasyLM is a one stop solution for pre-training, finetuning, evaluating and serving LLMs in JAX/Flax. ![Stars](https://img.shields.io/github/stars/young-geng/EasyLM.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/young-geng/EasyLM?color=green) ![LastCommit](https://img.shields.io/github/last-commit/young-geng/EasyLM?color=green) * **[LLaMa-Factory](https://github.com/hiyouga/LLaMA-Factory)**: Unified Efficient Fine-Tuning of 100+ LLMs & VLMs (ACL 2024). ![Stars](https://img.shields.io/github/stars/hiyouga/llama-factory.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/hiyouga/llama-factory?color=green) ![LastCommit](https://img.shields.io/github/last-commit/hiyouga/llama-factory?color=green) * **[LMFlow](https://github.com/OptimalScale/LMFlow)**: An Extensible Toolkit for Finetuning and Inference of Large Foundation Models. Large Models for All. ![Stars](https://img.shields.io/github/stars/optimalscale/lmflow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/optimalscale/lmflow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/optimalscale/lmflow?color=green) * **[maestro](https://github.com/roboflow/maestro)**: streamline the fine-tuning process for multimodal models: PaliGemma 2, Florence-2, and Qwen2.5-VL. ![Stars](https://img.shields.io/github/stars/roboflow/maestro.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/roboflow/maestro?color=green) ![LastCommit](https://img.shields.io/github/last-commit/roboflow/maestro?color=green) * **[MLX-VLM](https://github.com/Blaizzy/mlx-vlm)**: MLX-VLM is a package for inference and fine-tuning of Vision Language Models (VLMs) on your Mac using MLX. ![Stars](https://img.shields.io/github/stars/blaizzy/mlx-vlm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/blaizzy/mlx-vlm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/blaizzy/mlx-vlm?color=green) * **[Swift](https://github.com/modelscope/ms-swift)**: Use PEFT or Full-parameter to finetune 450+ LLMs (Qwen2.5, InternLM3, GLM4, Llama3.3, Mistral, Yi1.5, Baichuan2, DeepSeek-R1, ...) and 150+ MLLMs (Qwen2.5-VL, Qwen2-Audio, Llama3.2-Vision, Llava, InternVL2.5, MiniCPM-V-2.6, GLM4v, Xcomposer2.5, Yi-VL, DeepSeek-VL2, Phi3.5-Vision, GOT-OCR2, ...). ![Stars](https://img.shields.io/github/stars/modelscope/ms-swift?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/modelscope/ms-swift?color=green) ![LastCommit](https://img.shields.io/github/last-commit/modelscope/ms-swift?color=green) * **[torchtune](https://github.com/pytorch/torchtune)**: PyTorch native post-training library. ![Stars](https://img.shields.io/github/stars/pytorch/torchtune.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/pytorch/torchtune?color=green) ![LastCommit](https://img.shields.io/github/last-commit/pytorch/torchtune?color=green) * **[Transformer Lab](https://github.com/transformerlab/transformerlab-app)**: Open Source Application for Advanced LLM Engineering: interact, train, fine-tune, and evaluate large language models on your own computer. ![Stars](https://img.shields.io/github/stars/transformerlab/transformerlab-app.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/transformerlab/transformerlab-app?color=green) ![LastCommit](https://img.shields.io/github/last-commit/transformerlab/transformerlab-app?color=green) * **[unsloth](https://github.com/unslothai/unsloth)**: Finetune Llama 3.3, DeepSeek-R1 & Reasoning LLMs 2x faster with 70% less memory! 🦥 ![Stars](https://img.shields.io/github/stars/unslothai/unsloth.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/unslothai/unsloth?color=green) ![LastCommit](https://img.shields.io/github/last-commit/unslothai/unsloth?color=green) ### RLHF * **[OpenRLHF](https://github.com/OpenLLMAI/OpenRLHF)**: An Easy-to-use, Scalable and High-performance RLHF Framework (70B+ PPO Full Tuning & Iterative DPO & LoRA & RingAttention & RFT). ![Stars](https://img.shields.io/github/stars/openllmai/openrlhf.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openllmai/openrlhf?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openllmai/openrlhf?color=green) * **[Self-RLHF](https://github.com/PKU-Alignment/safe-rlhf)**: Safe RLHF: Constrained Value Alignment via Safe Reinforcement Learning from Human Feedback. ![Stars](https://img.shields.io/github/stars/pku-alignment/safe-rlhf.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/pku-alignment/safe-rlhf?color=green) ![LastCommit](https://img.shields.io/github/last-commit/pku-alignment/safe-rlhf?color=green) ### Agentic RL * **[AReaL](https://github.com/inclusionAI/AReaL)**: Lightning-Fast RL for LLM Reasoning and Agents. Made Simple & Flexible. ![Stars](https://img.shields.io/github/stars/inclusionAI/AReaL.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/inclusionAI/AReaL?color=green) ![LastCommit](https://img.shields.io/github/last-commit/inclusionAI/AReaL?color=green) * **[rLLM](https://github.com/rllm-org/rllm)**: Democratizing Reinforcement Learning for LLMs ![Stars](https://img.shields.io/github/stars/rllm-org/rllm.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/rllm-org/rllm?color=green) ![LastCommit](https://img.shields.io/github/last-commit/rllm-org/rllm?color=green) * **[slime](https://github.com/THUDM/slime)**: slime is an LLM post-training framework for RL Scaling. ![Stars](https://img.shields.io/github/stars/thudm/slime.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/thudm/slime?color=green) ![LastCommit](https://img.shields.io/github/last-commit/thudm/slime?color=green) * **[verl](https://github.com/volcengine/verl)**: verl: Volcano Engine Reinforcement Learning for LLMs ![Stars](https://img.shields.io/github/stars/volcengine/verl.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/volcengine/verl?color=green) ![LastCommit](https://img.shields.io/github/last-commit/volcengine/verl?color=green) ### Benchmark * **[AgentBench](https://github.com/THUDM/AgentBench)**: A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24). ![Stars](https://img.shields.io/github/stars/thudm/agentbench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/thudm/agentbench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/thudm/agentbench?color=green) * **[ASQI Engineer](https://github.com/asqi-engineer/asqi-engineer)**: ASQI (AI Solutions Quality Index) Engineer - run containerised AI tests and map to score cards! ![Stars](https://img.shields.io/github/stars/asqi-engineer/asqi-engineer.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/asqi-engineer/asqi-engineer?color=green) ![LastCommit](https://img.shields.io/github/last-commit/asqi-engineer/asqi-engineer?color=green) * **[LiveBench](https://github.com/livebench/livebench)**: LiveBench: A Challenging, Contamination-Free LLM Benchmark ![Stars](https://img.shields.io/github/stars/livebench/livebench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/livebench/livebench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/livebench/livebench?color=green) * **[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness)**: A framework for few-shot evaluation of language models. ![Stars](https://img.shields.io/github/stars/eleutherai/lm-evaluation-harness.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/eleutherai/lm-evaluation-harness?color=green) ![LastCommit](https://img.shields.io/github/last-commit/eleutherai/lm-evaluation-harness?color=green) * **[LongBench](https://github.com/THUDM/LongBench)**: LongBench v2 and LongBench (ACL 2024). ![Stars](https://img.shields.io/github/stars/thudm/longbench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/thudm/longbench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/thudm/longbench?color=green) * **[MLE-bench](https://github.com/openai/mle-bench/)**: MLE-bench is a benchmark for measuring how well AI agents perform at machine learning engineering ![Stars](https://img.shields.io/github/stars/openai/mle-bench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/openai/mle-bench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/openai/mle-bench?color=green) * **[OpenCompass](https://github.com/open-compass/opencompass)**: OpenCompass is an LLM evaluation platform, supporting a wide range of models (Llama3, Mistral, InternLM2,GPT-4,LLaMa2, Qwen,GLM, Claude, etc) over 100+ datasets. ![Stars](https://img.shields.io/github/stars/open-compass/opencompass.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/open-compass/opencompass?color=green) ![LastCommit](https://img.shields.io/github/last-commit/open-compass/opencompass?color=green) * **[opik](https://github.com/comet-ml/opik)**: Debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards. ![Stars](https://img.shields.io/github/stars/comet-ml/opik.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/comet-ml/opik?color=green) ![LastCommit](https://img.shields.io/github/last-commit/comet-ml/opik?color=green) * **[skill-optimizer](https://github.com/fastxyz/skill-optimizer)**: Benchmark and self-optimize SDK/CLI/MCP guidance so every agent model can use your tool reliably. ![Stars](https://img.shields.io/github/stars/fastxyz/skill-optimizer.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/fastxyz/skill-optimizer?color=green) ![LastCommit](https://img.shields.io/github/last-commit/fastxyz/skill-optimizer?color=green) * **[terminal-bench](https://github.com/laude-institute/terminal-bench)**: A benchmark for LLMs on complicated tasks in the terminal ![Stars](https://img.shields.io/github/stars/laude-institute/terminal-bench.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/laude-institute/terminal-bench?color=green) ![LastCommit](https://img.shields.io/github/last-commit/laude-institute/terminal-bench?color=green) ### Workflow * **[Flyte](https://github.com/flyteorg/flyte)**: Scalable and flexible workflow orchestration platform that seamlessly unifies data, ML and analytics stacks. ![Stars](https://img.shields.io/github/stars/flyteorg/flyte.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/flyteorg/flyte?color=green) ![LastCommit](https://img.shields.io/github/last-commit/flyteorg/flyte?color=green) * **[Kubeflow](https://github.com/kubeflow/kubeflow)**: Machine Learning Toolkit for Kubernetes. ![Stars](https://img.shields.io/github/stars/kubeflow/kubeflow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/kubeflow/kubeflow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/kubeflow/kubeflow?color=green) * **[Metaflow](https://github.com/Netflix/metaflow)**: Build, Deploy and Manage AI/ML Systems. ![Stars](https://img.shields.io/github/stars/netflix/metaflow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/netflix/metaflow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/netflix/metaflow?color=green) * **[MLflow](https://github.com/mlflow/mlflow)**: Open source platform for the machine learning lifecycle. ![Stars](https://img.shields.io/github/stars/mlflow/mlflow.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/mlflow/mlflow?color=green) ![LastCommit](https://img.shields.io/github/last-commit/mlflow/mlflow?color=green) * **[Polyaxon](https://github.com/polyaxon/polyaxon)**: MLOps Tools For Managing & Orchestrating The Machine Learning LifeCycle. ![Stars](https://img.shields.io/github/stars/polyaxon/polyaxon.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/polyaxon/polyaxon?color=green) ![LastCommit](https://img.shields.io/github/last-commit/polyaxon/polyaxon?color=green) * **[Ray](https://github.com/ray-project/ray)**: Ray is an AI compute engine. Ray consists of a core distributed runtime and a set of AI Libraries for accelerating ML workloads. ![Stars](https://img.shields.io/github/stars/ray-project/ray.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/ray-project/ray?color=green) ![LastCommit](https://img.shields.io/github/last-commit/ray-project/ray?color=green) * **[Seldon-Core](https://github.com/SeldonIO/seldon-core)**: An MLOps framework to package, deploy, monitor and manage thousands of production machine learning models. ![Stars](https://img.shields.io/github/stars/seldonio/seldon-core.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/seldonio/seldon-core?color=green) ![LastCommit](https://img.shields.io/github/last-commit/seldonio/seldon-core?color=green) * **[ZenML](https://github.com/zenml-io/zenml)**: ZenML 🙏: The bridge between ML and Ops. https://zenml.io. ![Stars](https://img.shields.io/github/stars/zenml-io/zenml.svg?style=flat&color=green) ![Contributors](https://img.shields.io/github/contributors/zenml-io/zenml?color=green) ![LastCommit](https://img.shields.io/github/last-commit/zenml-io/zenml?color=green) ================================================ FILE: project_request.py ================================================ import argparse import os import re import requests import sys import yaml from typing import Tuple, Dict, List, Any, Optional from urllib.parse import urlparse # Constants README_PATH = 'README.md' DATA_YML_PATH = 'website/data.yml' LOGOS_DIR = 'website/logos' # Categories that should only be added to README.md, not to website/data.yml README_ONLY_CATEGORIES = ["MCP/MCP Server", "MCP/MCP Client"] def parse_github_url(url: str) -> Tuple[str, str]: """Extract owner and repository name from a GitHub URL. Args: url: A GitHub repository URL (e.g., https://github.com/owner/repo) Returns: A tuple containing (owner, repo) strings Raises: ValueError: If the URL is not a valid GitHub repository URL """ parsed_url = urlparse(url) if 'github.com' not in parsed_url.netloc: raise ValueError(f"Not a GitHub URL: {url}") path_parts = parsed_url.path.strip('/').split('/') if len(path_parts) < 2: raise ValueError(f"Invalid GitHub repository URL: {url}") owner, repo = path_parts[0], path_parts[1] return owner, repo def get_repo_info(owner: str, repo: str) -> Dict[str, Any]: """Fetch repository information from GitHub API. Args: owner: GitHub repository owner/organization name repo: GitHub repository name Returns: Dictionary containing repository information from GitHub API Raises: Exception: If the API request fails """ api_url = f"https://api.github.com/repos/{owner}/{repo}" response = requests.get(api_url) if response.status_code != 200: raise Exception(f"Failed to fetch repository info: {response.status_code} {response.text}") return response.json() def generate_entry(repo_url: str, project_name: str) -> Tuple[str, str]: """Generate formatted entry for README.md. Args: repo_url: GitHub repository URL project_name: The project name Returns: A tuple containing: - project_name: The name of the project - entry: A formatted markdown string for the README entry """ # Parse GitHub URL owner, repo = parse_github_url(repo_url) # Get repository description repo_info = get_repo_info(owner, repo) description = repo_info.get('description', '') # Generate shields.io URLs stars_badge = f"![Stars](https://img.shields.io/github/stars/{owner}/{repo}.svg?style=flat&color=green)" contributors_badge = f"![Contributors](https://img.shields.io/github/contributors/{owner}/{repo}?color=green)" last_commit_badge = f"![LastCommit](https://img.shields.io/github/last-commit/{owner}/{repo}?color=green)" # Format the entry entry = f"* **[{project_name}]({repo_url})**: {description} {stars_badge} {contributors_badge} {last_commit_badge}" return project_name, entry def find_category(content: str, category: str) -> Tuple[int, int, List[str]]: """Find the specified category in the README content. Args: content: The full content of the README.md file category: The name of the category to find (e.g., "framework" or "orchestration/workflow") Can include a path with '/' as separator for nested categories Returns: A tuple containing: - category_start_line: The line number where the category starts - category_end_line: The line number where the category ends - lines: List of all lines in the content Raises: ValueError: If the specified category is not found """ # Define patterns for different category levels category_patterns = [ # Main category (##) re.compile(r'##\s+([^\n]+)'), # Subcategory (###) re.compile(r'###\s+([^\n]+)') ] lines = content.split('\n') category_path = [c.strip().lower() for c in category.split('/')] # If we have a path with multiple levels, we need to find each level if len(category_path) > 1: current_path = [] current_level = 0 category_start_line = -1 category_end_line = -1 for i, line in enumerate(lines): # Check if this line starts a category for pattern in category_patterns: match = pattern.match(line) if match: # Get the heading level (## = 2, ### = 3) heading_level = line.count('#') category_name = match.group(1).strip().lower() # If we're at a level we're tracking if heading_level - 2 <= len(current_path): # If we're at a lower level than current, pop levels while heading_level - 2 < len(current_path): current_path.pop() # If we're at a new level, add it if heading_level - 2 == len(current_path): current_path.append(category_name) # If we're at the same level, replace the last item else: current_path[-1] = category_name # Check if the current path matches our target path if len(current_path) == len(category_path) and all(a == b for a, b in zip(current_path, category_path)): category_start_line = i # If we already found our category and encounter another at the same or higher level, that's the end elif category_start_line != -1 and category_end_line == -1 and heading_level - 2 <= len(category_path) - 1: category_end_line = i break else: # Original single-level category search category_start_line = -1 category_end_line = -1 current_category = "" for i, line in enumerate(lines): # Check if this line starts a category for pattern in category_patterns: match = pattern.match(line) if match: # If we already found our category, this new category marks the end if category_start_line != -1 and category_end_line == -1: category_end_line = i break # Check if this is the category we're looking for current_category = match.group(1).strip().lower() if current_category == category_path[0]: category_start_line = i break # If we found the start but not the end, the category goes to the end of the file if category_start_line != -1 and category_end_line == -1: category_end_line = len(lines) if category_start_line == -1: raise ValueError(f"Category '{category}' not found in README.md") return category_start_line, category_end_line, lines def insert_entry(lines: List[str], category_start_line: int, category_end_line: int, project_name: str, new_entry: str) -> List[str]: """Insert the new entry in alphabetical order within the category. Args: lines: List of all lines in the README.md file category_start_line: The line number where the category starts category_end_line: The line number where the category ends project_name: The name of the project new_entry: The formatted entry to insert Returns: Updated list of lines with the new entry inserted in alphabetical order """ # Find the correct position to insert the new entry insert_position = category_end_line last_entry_position = -1 # Skip the category header for i in range(category_start_line + 1, category_end_line): line = lines[i] # Check if this line is an entry entry_name_match = re.search(r'\*\s+\*\*\[([^\]]+)\]', line) if entry_name_match: last_entry_position = i entry_name = entry_name_match.group(1).lower() # If the new entry comes before this entry alphabetically (case-insensitive comparison) if project_name.lower() < entry_name: insert_position = i break # If we're inserting at the end of the category if insert_position == category_end_line: # Always insert after the last entry insert_position = last_entry_position + 1 # Insert the new entry at the determined position lines.insert(insert_position, new_entry) return lines def update_website(category: str, project_name: str, repo_url: str, homepage_url: str, logo_url: str = None, logo_name: str = None) -> bool: """Update the website data.yml file and download the logo. Args: category: The name of the category to add the entry to (e.g., "Inference Engine") project_name: The name of the project repo_url: The GitHub repository URL logo_url: URL to the project logo (optional, default.png will be used if not provided) homepage_url: Custom homepage URL logo_name: Optional custom logo filename Returns: True if the update was successful, False otherwise """ try: # Load the data.yml file with open(DATA_YML_PATH, 'r', encoding='utf-8') as file: data = yaml.safe_load(file) # Parse GitHub URL owner, repo = parse_github_url(repo_url) # Get repository information repo_info = get_repo_info(owner, repo) description = repo_info.get('description', '') # Process logo logo_filename = None # If no logo_url is provided, use default.png if not logo_url: logo_filename = "default.png" else: # If logo_name is provided, use it directly if logo_name: logo_filename = logo_name else: # Extract filename from URL parsed_url = urlparse(logo_url) original_filename = os.path.basename(parsed_url.path) file_ext = os.path.splitext(original_filename)[1].lower() # Create a sanitized filename based on project name sanitized_name = project_name.lower().replace(' ', '-') sanitized_name = re.sub(r'[^\w\-]', '', sanitized_name) logo_filename = f"{sanitized_name}{file_ext}" logo_path = os.path.join(LOGOS_DIR, logo_filename) # Download the logo response = requests.get(logo_url, stream=True) response.raise_for_status() with open(logo_path, 'wb') as logo_file: for chunk in response.iter_content(chunk_size=8192): logo_file.write(chunk) print(f"Successfully downloaded logo to {logo_path}") # Parse the category path category_path = [c.strip().lower() for c in category.split('/')] target_category = category_path[-1] # Use the last part as the actual category name # Find the appropriate category and subcategory for maincategory in data['categories']: for subcategory in maincategory['subcategories']: if subcategory['name'].lower() == target_category: # Create new item entry new_item = { 'name': project_name, 'description': description, 'homepage_url': homepage_url, 'logo': logo_filename, 'repo_url': repo_url } # Add the new item to the subcategory subcategory['items'].append(new_item) # Sort items by name subcategory['items'] = sorted(subcategory['items'], key=lambda x: x['name'].lower()) # Write the updated data back to the file with open(DATA_YML_PATH, 'w', encoding='utf-8') as file: yaml.dump(data, file, sort_keys=False, default_flow_style=False, allow_unicode=True) return True except Exception as e: print(f"Error updating website data: {str(e)}") return False def update_readme(category: str, project_name: str, new_entry: str) -> bool: """Update the README.md file with the new entry. Args: category: The name of the category to add the entry to project_name: The name of the project new_entry: The formatted entry to add Returns: True if the update was successful, False otherwise """ try: with open(README_PATH, 'r', encoding='utf-8') as file: content = file.read() category_start_line, category_end_line, lines = find_category(content, category) updated_lines = insert_entry(lines, category_start_line, category_end_line, project_name, new_entry) # Write the updated content back to the file with open(README_PATH, 'w', encoding='utf-8') as file: file.write('\n'.join(updated_lines)) print(f"Successfully added {project_name} to {category} category in README.md") return True except Exception as e: print(f"Error updating README.md: {str(e)}") return False def is_readme_only(category: str) -> bool: """Check if a project should only be added to README.md and not to website/data.yml. Args: category: The category of the project Returns: True if the project should only be added to README.md, False otherwise """ # Normalize the category for case-insensitive comparison normalized_category = category.strip().lower() # Check if the category is in the README_ONLY_CATEGORIES list for readme_only_category in README_ONLY_CATEGORIES: if normalized_category == readme_only_category.lower(): return True return False def main() -> None: """Main function to parse arguments and execute the script. Command line arguments: --category/-c: The category to add the project to (e.g., "Inference/Inference Engine", "Orchestration/Workflow") Can include a path with '/' as separator for nested categories --repo_url/-r: The GitHub repository URL --name/-n: Custom project name --logo_url/-l: URL to the project logo (optional for README_ONLY_CATEGORIES) --homepage_url/-hu: Project homepage URL (optional for README_ONLY_CATEGORIES) --logo_name/-ln: Optional custom logo filename Example: python project_request.py \ --category "Inference/Inference Engine" \ --repo_url https://github.com/google/adk-python \ --name "Agent Development Kit (ADK)" \ --logo_url https://raw.githubusercontent.com/google/adk-python/main/assets/agent-development-kit.png \ --homepage_url https://google.github.io/adk-docs """ parser = argparse.ArgumentParser(description='Add a new project to the README.md file and update the website data.') parser.add_argument('--category', '-c', required=True, help='The category to add the project to (e.g., "Inference Engine", "Agent", "Orchestration/Workflow"). Can include a path with "/" as separator for nested categories.') parser.add_argument('--repo_url', '-r', required=True, help='The GitHub repository URL') parser.add_argument('--name', '-n', required=True, help='Custom project name') parser.add_argument('--logo_url', '-l', required=False, help='URL to the project logo (optional for MCP-related projects)') parser.add_argument('--homepage_url', '-hu', required=False, help='Custom homepage URL (optional for MCP-related projects)') parser.add_argument('--logo_name', '-ln', required=False, help='Optional custom logo filename') args = parser.parse_args() try: # Generate the entry for README.md project_name, entry = generate_entry(args.repo_url, args.name) # Update the README.md file readme_success = update_readme(args.category.lower(), project_name, entry) if not readme_success: print("Failed to update README.md") sys.exit(1) # Check if the project is in README_ONLY_CATEGORIES readme_only = is_readme_only(args.category) if readme_only: print(f"Category '{args.category}' is in README_ONLY_CATEGORIES. Skipping website data update.") website_success = True else: # For non-README_ONLY_CATEGORIES projects, warn if logo_url is not provided if not args.logo_url: print("Warning: No logo URL provided, using default.png") # For non-README_ONLY_CATEGORIES projects, homepage_url is required if not args.homepage_url: print("Error: --homepage_url is required for projects not in README_ONLY_CATEGORIES") sys.exit(1) # Update website website_success = update_website(args.category, args.name, args.repo_url, args.homepage_url, args.logo_url,args.logo_name) if not website_success: print("Failed to update website data") sys.exit(1) else: print(f"Successfully updated website data for {project_name}") except Exception as e: print(f"Error: {str(e)}") sys.exit(1) print(f"Successfully added {project_name} to the {args.category} category") if __name__ == '__main__': main() ================================================ FILE: requirements.txt ================================================ requests>=2.25.0 pyyaml>=6.0 ================================================ FILE: website/README.md ================================================ # Awesome-LLMOps Landscape This directory contains the configuration files and assets for the Awesome-LLMOps landscape website. The landscape is built using [landscape2](https://github.com/cncf/landscape2), a tool developed by CNCF for creating interactive landscapes. ## Overview The landscape website provides a visual representation of the Awesome-LLMOps ecosystem, categorizing projects into different groups and subcategories. It helps users discover and navigate through the various tools and projects in the LLMOps space. ## Configuration Files - `data.yml`: Contains the data structure for the landscape, including categories, subcategories, and project items. - `guide.yml`: Provides descriptive content for categories and subcategories displayed in the landscape guide. - `settings.yml`: Customizes the appearance and behavior of the landscape website. ## Directory Structure - `logos/`: Contains logo files for projects and the landscape itself. ## Running the Landscape Locally To run the landscape website locally for testing: 1. Use the commands defined in the Makefile: ``` make install # Install landscape2 make build # Build the landscape make serve # Serve the landscape locally ``` 2. Access the local website at http://127.0.0.1:8000 ## Landscape Categories The landscape currently includes the following main categories: - **Inference**: Tools and platforms for deploying and serving LLMs - **Orchestration**: Tools for orchestrating LLM workflows and agents - **Runtime**: Runtime environments and tools for LLM applications - **Training**: Tools and frameworks for training and fine-tuning LLMs Additional categories can be added by updating the `data.yml`, `guide.yml`, and `settings.yml` files. ================================================ FILE: website/data.yml ================================================ categories: - name: Inference subcategories: - name: Inference Engine items: - name: Cortex.cpp description: Local AI API Platform. homepage_url: https://cortex.so logo: cortex-cpp.svg repo_url: https://github.com/janhq/cortex.cpp - name: DeepSpeed-MII description: MII makes low-latency and high-throughput inference possible, powered by DeepSpeed. homepage_url: https://deepspeed-mii.readthedocs.io logo: deepspeed-mii.svg repo_url: https://github.com/microsoft/DeepSpeed-MII - name: ipex-llm description: Accelerate local LLM inference and finetuning on Intel XPU. homepage_url: https://github.com/intel-analytics/ipex-llm logo: intel.png repo_url: https://github.com/intel-analytics/ipex-llm - name: llama-box description: LM inference server implementation based on *.cpp. homepage_url: https://github.com/gpustack/llama-box logo: default.png repo_url: https://github.com/gpustack/llama-box - name: llama.cpp description: LLM inference in C/C++. homepage_url: https://github.com/ggerganov/llama.cpp logo: llamacpp.svg repo_url: https://github.com/ggerganov/llama.cpp - name: Llumnix description: Efficient and easy multi-instance LLM serving. homepage_url: https://github.com/AlibabaPAI/llumnix logo: alibaba.png repo_url: https://github.com/AlibabaPAI/llumnix - name: LMDeploy description: LMDeploy is a toolkit for compressing, deploying, and serving LLMs. homepage_url: http://lmdeploy.readthedocs.io/ logo: lmdeploy.svg repo_url: https://github.com/InternLM/lmdeploy - name: LoRAX description: Multi-LoRA inference server that scales to 1000s of fine-tuned LLMs. homepage_url: https://loraexchange.ai logo: lorax.png repo_url: https://github.com/predibase/lorax - name: MInference description: To speed up Long-context LLMs' inference, approximate and dynamic sparse calculate the attention. homepage_url: https://aka.ms/MInference logo: minference.png repo_url: https://github.com/microsoft/minference - name: MLC LLM description: Universal LLM Deployment Engine with ML Compilation. homepage_url: https://llm.mlc.ai/ logo: mlc.png repo_url: https://github.com/mlc-ai/mlc-llm - name: MLServer description: An inference server for your machine learning models, including support for multiple frameworks. homepage_url: https://mlserver.readthedocs.io/ logo: mlserver.png repo_url: https://github.com/SeldonIO/MLServer - name: Nvidia Dynamo description: A Datacenter Scale Distributed Inference Serving Framework. homepage_url: https://developer.nvidia.com/dynamo logo: nvidia.png repo_url: https://github.com/ai-dynamo/dynamo - name: Ollama description: Get up and running with Llama 3.3, DeepSeek-R1, Phi-4, Gemma 3, and other large language models. homepage_url: https://ollama.com logo: ollama.png repo_url: https://github.com/ollama/ollama - name: OpenLLM description: Run any open-source LLMs, such as DeepSeek and Llama, as OpenAI compatible API endpoint in the cloud. homepage_url: https://bentoml.com logo: bentoml.png repo_url: https://github.com/bentoml/OpenLLM - name: OpenVINO description: OpenVINO™ is an open source toolkit for optimizing and deploying AI inference. homepage_url: https://docs.openvino.ai logo: openvino.svg repo_url: https://github.com/openvinotoolkit/openvino - name: Petals description: 🌸 Run LLMs at home, BitTorrent-style. Fine-tuning and inference up to 10x faster than offloading homepage_url: https://petals.dev/ logo: petals repo_url: https://github.com/bigscience-workshop/petals - name: Ratchet description: A cross-platform browser ML framework. homepage_url: https://huggingface.co/ratchet-community logo: ratchet.png repo_url: https://github.com/huggingface/ratchet - name: SGLang description: SGLang is a fast serving framework for large language models and vision language models. homepage_url: https://docs.sglang.ai/ logo: sglang.png repo_url: https://github.com/sgl-project/sglang - name: Text Generation Inference description: Large Language Model Text Generation Inference. homepage_url: https://hf.co/docs/text-generation-inference logo: hf.png repo_url: https://github.com/huggingface/text-generation-inference - name: TinyGrad description: 'You like pytorch? You like micrograd? You love tinygrad! ❤️ ' homepage_url: https://github.com/tinygrad/tinygrad logo: tinygrad.svg repo_url: https://github.com/tinygrad/tinygrad - name: transformers.js description: State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server! homepage_url: https://huggingface.co/docs/transformers.js logo: transformers-js.png repo_url: https://github.com/huggingface/transformers.js - name: Triton Inference Server description: The Triton Inference Server provides an optimized cloud and edge inferencing solution. homepage_url: https://docs.nvidia.com/deeplearning/triton-inference-server/ logo: nvidia.png repo_url: https://github.com/triton-inference-server/server - name: vLLM description: A high-throughput and memory-efficient inference and serving engine for LLMs. homepage_url: https://docs.vllm.ai logo: vllm.png repo_url: https://github.com/vllm-project/vllm - name: web-llm description: High-performance In-browser LLM Inference Engine. homepage_url: https://webllm.mlc.ai logo: mlc.png repo_url: https://github.com/mlc-ai/web-llm - name: Xinference description: Replace OpenAI GPT with another LLM in your app by changing a single line of code. Xinference gives you the freedom to use any LLM you need. With Xinference, you're empowered to run inference with any open-source language models, speech recognition models, and multimodal models, whether in the cloud, on-premises, or even on your laptop. homepage_url: https://inference.readthedocs.io/ logo: xinference repo_url: https://github.com/xorbitsai/inference - name: zml description: Any model. Any hardware. Zero compromise. Built with @ziglang / @openxla / MLIR / @bazelbuild. homepage_url: https://docs.zml.ai logo: zml.svg repo_url: https://github.com/zml/zml - name: Inference Platform items: - name: AIBrix description: Cost-efficient and pluggable Infrastructure components for GenAI inference. homepage_url: https://aibrix.readthedocs.io/latest/ logo: aibrix.jpeg repo_url: https://github.com/vllm-project/aibrix - name: BentoML description: The easiest way to serve AI apps and models - Build Model Inference APIs, Job queues, LLM apps, Multi-model pipelines, and more! homepage_url: https://bentoml.com logo: bentoml.png repo_url: https://github.com/bentoml/BentoML - name: beta9 description: Ultrafast serverless GPU inference, sandboxes, and background jobs homepage_url: https://www.beam.cloud logo: beta9 repo_url: https://github.com/beam-cloud/beta9 - name: Kaito description: Kubernetes operator for large-model inference and fine-tuning, with GPU auto-provisioning, container-based hosting, and CRD-based orchestration. homepage_url: https://github.com/kaito-project/Kaito logo: kaito.png repo_url: https://github.com/kaito-project/Kaito - name: Kserve description: Standardized Serverless ML Inference Platform on Kubernetes. homepage_url: https://kserve.github.io/website/latest/ logo: kserve.png repo_url: https://github.com/kserve/kserve - name: KubeAI description: AI Inference Operator for Kubernetes. The easiest way to serve ML models in production. Supports VLMs, LLMs, embeddings, and speech-to-text. homepage_url: https://www.kubeai.org/ logo: kubeai.png repo_url: https://github.com/substratusai/kubeai - name: llm-d description: llm-d is a Kubernetes-native high-performance distributed LLM inference framework homepage_url: https://llm-d.ai logo: llm-d.png repo_url: https://github.com/llm-d/llm-d - name: llmaz description: ☸️ Easy, advanced inference platform for large language models on Kubernetes. 🌟 Star to support our work! homepage_url: https://llmaz.inftyai.com logo: llmaz.svg repo_url: https://github.com/InftyAI/llmaz - name: Modular description: The Modular Platform (includes MAX & Mojo) homepage_url: https://docs.modular.com/ logo: modular repo_url: https://github.com/modular/modular - name: Mooncake description: Mooncake is the serving platform for Kimi, a leading LLM service provided by Moonshot AI. homepage_url: https://kvcache.ai/repo/mooncake logo: mooncake.png repo_url: https://github.com/kvcache-ai/Mooncake - name: OME description: OME is a Kubernetes operator for enterprise-grade management and serving of Large Language Models (LLMs) homepage_url: https://docs.sglang.ai/ome/ logo: ome.png repo_url: https://github.com/sgl-project/ome - name: Middleware items: - name: Checkpoint Engine description: Checkpoint-engine is a simple middleware to update model weights in LLM inference engines homepage_url: https://github.com/MoonshotAI/checkpoint-engine logo: checkpoint-engine repo_url: https://github.com/MoonshotAI/checkpoint-engine - name: kvcached description: Virtualized Elastic KV Cache for Dynamic GPU Sharing and Beyond homepage_url: https://github.com/ovg-project/kvcached logo: kvcached repo_url: https://github.com/ovg-project/kvcached - name: LMCache description: 10x Faster Long-Context LLM By Smart KV Cache Optimizations. homepage_url: https://lmcache.ai/ logo: lmcache.png repo_url: https://github.com/LMCache/LMCache - name: AI Gateway items: - name: agentgateway description: Next Generation Agentic Proxy for AI Agents and MCP servers homepage_url: https://agentgateway.dev logo: agentgateway.svg repo_url: https://github.com/agentgateway/agentgateway - name: APISIX description: The Cloud-Native API Gateway and AI Gateway with extensive plugin system and AI capabilities. homepage_url: https://apisix.apache.org/ logo: apisix.png repo_url: https://github.com/apache/apisix - name: Envoy AI Gateway description: Envoy AI Gateway is an open source project for using Envoy Gateway to handle request traffic from application clients to Generative AI services. homepage_url: https://aigateway.envoyproxy.io/ logo: envoy-ai-gateway.png repo_url: https://github.com/envoyproxy/ai-gateway - name: gateway-api-inference-extension description: Gateway API Inference Extension. homepage_url: https://gateway-api-inference-extension.sigs.k8s.io/ logo: k8s.png repo_url: https://github.com/kubernetes-sigs/gateway-api-inference-extension - name: Higress description: 🤖 AI Gateway | AI Native API Gateway. homepage_url: https://higress.ai/en/ logo: higress.avif repo_url: https://github.com/alibaba/higress - name: kgateway description: The Cloud-Native API Gateway and AI Gateway. homepage_url: https://kgateway.dev/ logo: kgateway.png repo_url: https://github.com/kgateway-dev/kgateway - name: Kong description: 🦍 The Cloud-Native API Gateway and AI Gateway. homepage_url: https://docs.konghq.com/gateway/latest/ logo: kong.png repo_url: https://github.com/Kong/kong - name: LLM Router items: - name: AI Gateway description: A blazing fast AI Gateway with integrated guardrails. Route to 200+ LLMs, 50+ AI Guardrails with 1 fast & friendly API. homepage_url: https://portkey.ai/features/ai-gateway logo: portkeyai.png repo_url: https://github.com/Portkey-AI/gateway - name: bifrost description: Fastest LLM gateway (50x faster than LiteLLM) with adaptive load balancer, cluster mode, guardrails, 1000+ models support & <100 µs overhead at 5k RPS. homepage_url: https://www.getmaxim.ai/bifrost logo: bifrost repo_url: https://github.com/maximhq/bifrost - name: Ferro Labs AI Gateway description: One API for 25+ LLMs, OpenAI, Anthropic, Bedrock, Azure. Caching, guardrails & cost controls. Go-native LiteLLM & Kong AI Gateway alternative. homepage_url: https://docs.ferrolabs.ai logo: ferro-labs-ai-gateway.png repo_url: https://github.com/ferro-labs/ai-gateway - name: Hebo AI Gateway description: OpenAI-compatible /chat/completions, /embeddings & /models endpoints homepage_url: https://hebo.ai/gateway logo: hebo-ai-gateway.png repo_url: https://github.com/8monkey-ai/hebo-gateway/ - name: LiteLLM description: Python SDK, Proxy Server (LLM Gateway) to call 100+ LLM APIs in OpenAI format - [Bedrock, Azure, OpenAI, VertexAI, Cohere, Anthropic, Sagemaker, HuggingFace, Replicate, Groq]. homepage_url: https://docs.litellm.ai/docs/ logo: berriai.png repo_url: https://github.com/BerriAI/litellm - name: LLMRouter description: 'LLMRouter: An Open-Source Library for LLM Routing' homepage_url: https://ulab-uiuc.github.io/LLMRouter/ logo: llmrouter.png repo_url: https://github.com/ulab-uiuc/LLMRouter - name: RouteLLM description: A framework for serving and evaluating LLM routers - save LLM costs without compromising quality. homepage_url: https://github.com/lm-sys/routellm logo: lmsys.png repo_url: https://github.com/lm-sys/routellm - name: vLLM Semantic Router description: Intelligent Mixture-of-Models Router for Efficient LLM Inference homepage_url: https://vllm-semantic-router.com/ logo: vllm.png repo_url: https://github.com/vllm-project/semantic-router - name: Output items: - name: BAML description: The AI framework that adds the engineering to prompt engineering (Python/TS/Ruby/Java/C#/Rust/Go compatible) homepage_url: https://docs.boundaryml.com/home logo: baml repo_url: https://github.com/boundaryml/baml - name: Instructor description: Structured outputs for LLMs. homepage_url: https://python.useinstructor.com/ logo: default.png repo_url: https://github.com/instructor-ai/instructor - name: Outlines description: Structured Text Generation. homepage_url: https://dottxt-ai.github.io/outlines/ logo: outlines.png repo_url: https://github.com/dottxt-ai/outlines - name: XGrammar description: Fast, Flexible and Portable Structured Generation homepage_url: https://xgrammar.mlc.ai/docs/ logo: xgrammar.svg repo_url: https://github.com/mlc-ai/xgrammar - name: Simulator items: - name: Vidur description: A large-scale simulation framework for LLM inference homepage_url: https://github.com/microsoft/vidur logo: microsoft.png repo_url: https://github.com/microsoft/vidur - name: Benchmark items: - name: genai-bench description: Genai-bench is a powerful benchmark tool designed for comprehensive token-level performance evaluation of large language model (LLM) serving systems. homepage_url: https://docs.sglang.ai/genai-bench logo: genai-bench.png repo_url: https://github.com/sgl-project/genai-bench - name: Inference Benchmark description: A model server agnostic inference benchmarking tool that can be used to benchmark LLMs running on differet infrastructure like GPU and TPU. It can also be run on a GKE cluster as a container. homepage_url: https://github.com/AI-Hypercomputer/inference-benchmark logo: inference-benchmark.png repo_url: https://github.com/AI-Hypercomputer/inference-benchmark - name: Inference Perf description: GenAI inference performance benchmarking tool. homepage_url: https://github.com/kubernetes-sigs/inference-perf logo: k8s.png repo_url: https://github.com/kubernetes-sigs/inference-perf - name: skill-optimizer description: Benchmark and self-optimize SDK/CLI/MCP guidance so every agent model can use your tool reliably. homepage_url: https://github.com/fastxyz/skill-optimizer logo: default.png repo_url: https://github.com/fastxyz/skill-optimizer - name: Orchestration subcategories: - name: Application Framework items: - name: Cordum.io description: 'Cordum (cordum.io) is a platform-only control plane for autonomous AI Agents and external workers. It uses NATS for the bus, Redis for state and payload pointers, and CAP v2 wire contracts for jobs, results, and heartbeats. Workers and product packs live outside this repo.Core cordum ' homepage_url: https://cordum.io logo: cordumio repo_url: https://github.com/cordum-io/cordum - name: DeepEval description: The LLM Evaluation Framework homepage_url: https://deepeval.com/ logo: deepeval repo_url: https://github.com/confident-ai/deepeval - name: Evidently description: Evidently is ​​an open-source ML and LLM observability framework. Evaluate, test, and monitor any AI-powered system or data pipeline. From tabular data to Gen AI. 100+ metrics. homepage_url: https://github.com/evidentlyai/evidently logo: evidently repo_url: https://github.com/evidentlyai/evidently - name: Helicone description: Open source LLM observability platform. One line of code to monitor, evaluate, and experiment. homepage_url: https://www.helicone.ai logo: helicone.png repo_url: https://github.com/helicone/helicone - name: Langfuse description: 'Open source LLM engineering platform: LLM Observability, metrics, evals, prompt management, and more.' homepage_url: https://langfuse.com/docs logo: langfuse.png repo_url: https://github.com/langfuse/langfuse - name: lunaary description: 'The production toolkit for LLMs. Observability, prompt management and evaluations. ' homepage_url: https://lunary.ai/ logo: lunaary repo_url: https://github.com/lunary-ai/lunary - name: Neurolink description: Universal AI Development Platform with MCP server integration, multi-provider support, and professional CLI. Build, test, and deploy AI applications with multiple ai providers. homepage_url: https://juspay.github.io/neurolink/ logo: default.png repo_url: https://github.com/juspay/neurolink - name: OpenLit description: Open source Observability platform for OpenTelemetry-native LLM Observability, GPU Monitoring. homepage_url: https://openlit.io logo: openlit.png repo_url: https://github.com/openlit/openlit - name: phoenix description: AI Observability & Evaluation. homepage_url: https://docs.arize.com/phoenix logo: phoenix.png repo_url: https://github.com/arize-ai/phoenix - name: PostHog description: 🦔 PostHog provides open-source web & product analytics, session recording, feature flagging and A/B testing that you can self-host. Get started - free. homepage_url: https://posthog.com/ logo: posthog repo_url: https://github.com/PostHog/posthog - name: ragas description: Supercharge Your LLM Application Evaluations 🚀 homepage_url: https://docs.ragas.io/en/stable/ logo: ragas.png repo_url: https://github.com/explodinggradients/ragas - name: Weave description: Weave is a toolkit for developing AI-powered applications, built by Weights & Biases. homepage_url: https://weave-docs.wandb.ai/ logo: wandb.png repo_url: https://github.com/wandb/weave - name: Agent Framework items: - name: Agent Development Kit (ADK) description: An open-source, code-first Python toolkit for building, evaluating, and deploying sophisticated AI agents with flexibility and control. homepage_url: https://github.com/google/adk-python logo: agent-development-kit.png repo_url: https://github.com/google/adk-python - name: Agent Shadow Brain description: Self-evolving AI coding intelligence with infinite memory (TurboQuant), genetic algorithm evolution, predictive bug detection, PageRank knowledge graphs, and swarm intelligence. The world's first autonomous coding brain. homepage_url: https://github.com/theihtisham/agent-shadow-brain logo: default.png repo_url: https://github.com/theihtisham/agent-shadow-brain - name: AgentField description: Framework for AI Backend. Build and run AI agents like microservices - scalable, observable, and identity-aware from day one. homepage_url: https://agentfield.ai/docs/learn?utm_source=github&utm_campaign=awesome-llmops&utm_id=inftyai-project-request logo: agentfield.svg repo_url: https://github.com/Agent-Field/agentfield - name: Agno description: Build Multimodal AI Agents with memory, knowledge and tools. Simple, fast and model-agnostic. homepage_url: https://github.com/agno-agi/agno logo: agno.svg repo_url: https://github.com/agno-agi/agno - name: autogen description: 'A programming framework for agentic AI 🤖 PyPi: autogen-agentchat Discord: https://aka.ms/autogen-discord Office Hour: https://aka.ms/autogen-officehour' homepage_url: https://microsoft.github.io/autogen/stable/# logo: microsoft.png repo_url: https://github.com/microsoft/autogen - name: AutoGPT description: AutoGPT is the vision of accessible AI for everyone, providing the tools to focus on what matters. homepage_url: https://agpt.co/ logo: autogpt.png repo_url: https://github.com/Significant-Gravitas/AutoGPT - name: CAMEL description: CAMEL is the first multi-agent framework, focused on finding the scaling law of agents. homepage_url: https://www.camel-ai.org/ logo: camel.png repo_url: https://github.com/camel-ai/camel - name: crewAI description: Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks. homepage_url: https://www.crewai.com/ logo: crewai.svg repo_url: https://github.com/crewAIInc/crewAI - name: fast-agent description: Define, Prompt and Test MCP enabled Agents and Workflows homepage_url: https://github.com/evalstate/fast-agent logo: fast-agent.png repo_url: https://github.com/evalstate/fast-agent - name: Flowise description: Drag & drop UI to build your customized LLM flow. homepage_url: https://flowiseai.com logo: flowiseai.png repo_url: https://github.com/FlowiseAI/Flowise - name: kagent description: kagent is a Kubernetes-native framework for building AI agents. homepage_url: https://kagent.dev/ logo: kagent.svg repo_url: https://github.com/kagent-dev/kagent tags: - kubernetes - name: LangGraph description: Build resilient language agents as graphs. homepage_url: https://langchain-ai.github.io/langgraph/ logo: langgraph.png repo_url: https://github.com/langchain-ai/langgraph - name: MetaGPT description: MetaGPT is the first multi-agent framework for natural language programming. homepage_url: https://mgx.dev/ logo: metagpt.png repo_url: https://github.com/geekan/MetaGPT - name: OpenAI Agents SDK description: A lightweight, powerful framework for multi-agent workflows. homepage_url: https://github.com/openai/openai-agents-python logo: openai.png repo_url: https://github.com/openai/openai-agents-python - name: PydanticAI description: Agent framework/shim to use Pydantic with LLMs. homepage_url: https://ai.pydantic.dev/ logo: pydanticai.svg repo_url: https://github.com/pydantic/pydantic-ai - name: Qwen-Agent description: Agent framework and applications built upon Qwen>=3.0, featuring Function Calling, MCP, Code Interpreter, RAG, Chrome extension, etc. homepage_url: https://pypi.org/project/qwen-agent logo: qwen-agent.png repo_url: https://github.com/QwenLM/Qwen-Agent - name: Semantic Kernel description: Integrate cutting-edge LLM technology quickly and easily into your apps. homepage_url: https://aka.ms/semantic-kernel logo: microsoft.png repo_url: https://github.com/microsoft/semantic-kernel - name: Suna description: Suna is an open-source generalist AI agent. homepage_url: https://www.suna.so/ logo: suna.svg repo_url: https://github.com/kortix-ai/suna - name: Swarm description: An educational framework exploring ergonomic, lightweight multi-agent orchestration. homepage_url: https://github.com/openai/swarm logo: openai.png repo_url: https://github.com/openai/swarm tags: - experimental - name: Evolutionary Framework items: - name: AI-Researcher description: '[NeurIPS2025] "AI-Researcher: Autonomous Scientific Innovation" -- A production-ready version: https://novix.science/chat' homepage_url: https://github.com/HKUDS/AI-Researcher logo: ai-researcher.png repo_url: https://github.com/HKUDS/AI-Researcher - name: AIDE ML description: 'AIDE: AI-Driven Exploration in the Space of Code. The machine Learning engineering agent that automates AI R&D.' homepage_url: https://www.weco.ai/ logo: aide-ml repo_url: https://github.com/WecoAI/aideml - name: LoongFlow description: 'LoongFlow: A Thinking & Learning Framework for Expert-Grade AI Agents.' homepage_url: https://github.com/baidu-baige/LoongFlow logo: loongflow repo_url: https://github.com/baidu-baige/LoongFlow - name: OpenEvolve description: Open-source implementation of AlphaEvolve homepage_url: https://github.com/codelion/openevolve logo: openevolve.png repo_url: https://github.com/codelion/openevolve - name: ShinkaEvolve description: null homepage_url: https://github.com/SakanaAI/ShinkaEvolve logo: shinkaevolve.png repo_url: https://github.com/SakanaAI/ShinkaEvolve - name: SkyDiscover description: AI-Driven Scientific and Algorithmic Discovery homepage_url: https://skydiscover-ai.github.io/blog.html logo: skydiscover.png repo_url: https://github.com/skydiscover-ai/skydiscover#-benchmark-performance - name: RAG items: - name: graphrag description: A modular graph-based Retrieval-Augmented Generation (RAG) system. homepage_url: https://microsoft.github.io/graphrag/ logo: graphrag.png repo_url: https://github.com/microsoft/graphrag - name: LightRAG description: Simple and Fast Retrieval-Augmented Generation. homepage_url: https://github.com/HKUDS/LightRAG logo: lightrag.png repo_url: https://github.com/HKUDS/LightRAG - name: quivr description: 'Opiniated RAG for integrating GenAI in your apps 🧠 Focus on your product rather than the RAG. Easy integration in existing products with customisation! Any LLM: GPT4, Groq, Llama. Any Vectorstore: PGVector, Faiss. Any Files. Anyway you want.' homepage_url: https://core.quivr.com/en/latest/ logo: quivr.png repo_url: https://github.com/QuivrHQ/quivr - name: RAG-Anything description: '"RAG-Anything: All-in-One RAG Framework"' homepage_url: https://github.com/HKUDS/RAG-Anything logo: rag-anything.png repo_url: https://github.com/HKUDS/RAG-Anything - name: ragflow description: RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. homepage_url: https://ragflow.io/ logo: ragflow.png repo_url: https://github.com/infiniflow/ragflow - name: Workflow items: - name: Dify description: Dify is an open-source LLM app development platform, with an intuitive interface for AI workflow, RAG pipeline, agent capabilities, model management, and observability. homepage_url: https://dify.ai logo: dify.png repo_url: https://github.com/langgenius/dify - name: FastGPT description: FastGPT is a knowledge-based platform built on the LLMs, offers a comprehensive suite of out-of-the-box capabilities such as data processing, RAG retrieval, and visual AI workflow orchestration, letting you easily develop and deploy complex question-answering systems without the need for extensive setup or configuration. homepage_url: https://tryfastgpt.ai logo: fastgpt.svg repo_url: https://github.com/labring/FastGPT - name: Haystack description: AI orchestration framework to build customizable, production-ready LLM applications. Best suited for RAG, question answering, semantic search, or conversational agent chatbots. homepage_url: https://haystack.deepset.ai logo: deepset.png repo_url: https://github.com/deepset-ai/haystack - name: Inference description: Turn any computer or edge device into a command center for computer vision projects. homepage_url: https://inference.roboflow.com logo: roboflow.png repo_url: https://github.com/roboflow/inference tags: - computer_vision - name: LangChain description: Build context-aware reasoning applications. homepage_url: https://python.langchain.com/ logo: langchain.svg repo_url: https://github.com/langchain-ai/langchain - name: LlamaIndex description: The leading framework for building LLM-powered agents over your data. homepage_url: https://docs.llamaindex.ai/ logo: llamaindex.svg repo_url: https://github.com/run-llama/llama_index - name: Runtime subcategories: - name: AI Terminal items: - name: aider description: aider is AI pair programming in your terminal homepage_url: https://aider.chat/ logo: aider repo_url: https://github.com/Aider-AI/aider - name: Codex description: Lightweight coding agent that runs in your terminal. homepage_url: https://github.com/openai/codex logo: openai.png repo_url: https://github.com/openai/codex tags: - coding - name: Crush description: The glamourous AI coding agent for your favourite terminal 💘 homepage_url: https://charm.land/ logo: crush repo_url: https://github.com/charmbracelet/crush - name: Gemini CLI description: An open-source AI agent that brings the power of Gemini directly into your terminal. homepage_url: https://github.com/google-gemini/gemini-cli logo: gemini-cli repo_url: https://github.com/google-gemini/gemini-cli - name: OpenCode description: The AI coding agent built for the terminal. homepage_url: https://opencode.ai/ logo: opencode repo_url: https://github.com/sst/opencode - name: Stakpak description: DevOps agent that won't accidentally tweet your AWS credentials 🦀 homepage_url: https://github.com/stakpak/agent logo: stakpak.png repo_url: https://github.com/stakpak/agent - name: AI Agent items: - name: goose description: an open source, extensible AI agent that goes beyond code suggestions - install, execute, edit, and test with any LLM homepage_url: https://block.github.io/goose/ logo: goose repo_url: https://github.com/block/goose - name: Magentic-UI description: A research prototype of a human-centered web agent homepage_url: https://www.microsoft.com/en-us/research/blog/magentic-ui-an-experimental-human-centered-web-agent/ logo: magentic-ui repo_url: https://github.com/microsoft/magentic-ui - name: OpenManus description: OpenManus is an open-source project with no fortress, purely open ground. homepage_url: https://openmanus.github.io/ logo: openmanus.png repo_url: https://github.com/mannaandpoem/OpenManus - name: Tongyi Deep Research description: Tongyi DeepResearch, the Leading Open-source DeepResearch Agent homepage_url: https://tongyi-agent.github.io/blog/introducing-tongyi-deep-research/ logo: tongyi-deep-research repo_url: https://github.com/Alibaba-NLP/DeepResearch - name: Code Agent items: - name: Continue description: Create, share, and use custom AI code assistants with open-source IDE extensions and a hub of models, rules, prompts, docs, and other building blocks. homepage_url: https://docs.continue.dev/ logo: continue.png repo_url: https://github.com/continuedev/continue - name: Open SWE description: An Open-Source Asynchronous Coding Agent. homepage_url: https://swe.langchain.com/ logo: langchain.svg repo_url: https://github.com/langchain-ai/open-swe - name: SWE-agent description: 'SWE-agent takes a GitHub issue and tries to automatically fix it, using your LM of choice. It can also be employed for offensive cybersecurity or competitive coding challenges. [NeurIPS 2024] ' homepage_url: https://swe-agent.com/ logo: swe-agent.svg repo_url: https://github.com/SWE-agent/SWE-agent - name: Tabby description: Self-hosted AI coding assistant. homepage_url: https://tabbyml.com logo: tabby.png repo_url: https://github.com/TabbyML/tabby - name: Evolve Agent items: - name: AgentEvolver description: 'AgentEvolver: Towards Efficient Self-Evolving Agent System' homepage_url: https://modelscope.github.io/AgentEvolver/ logo: agentevolver.png repo_url: https://github.com/modelscope/AgentEvolver - name: EvoAgentX description: '🚀 EvoAgentX: Building a Self-Evolving Ecosystem of AI Agents' homepage_url: https://evoagentx.github.io/EvoAgentX/ logo: evoagentx repo_url: https://github.com/EvoAgentX/EvoAgentX - name: Tool items: - name: Beads description: Beads - A memory upgrade for your coding agent homepage_url: https://github.com/steveyegge/beads logo: default.png repo_url: https://github.com/steveyegge/beads - name: Browser Use description: Make websites accessible for AI agents. homepage_url: https://browser-use.com logo: browser-use.svg repo_url: https://github.com/browser-use/browser-use - name: Graphiti description: Build real-time knowledge graphs for AI agents. homepage_url: https://help.getzep.com/graphiti logo: graphiti.png repo_url: https://github.com/getzep/graphiti - name: Mem0 description: The memory layer for AI agents. homepage_url: https://mem0.ai/research logo: mem0.svg repo_url: https://github.com/mem0ai/mem0 - name: OpenAI CUA description: Computer Using Agent Sample App. homepage_url: https://github.com/openai/openai-cua-sample-app logo: openai.png repo_url: https://github.com/openai/openai-cua-sample-app - name: SidClaw description: The approval and accountability layer for AI agents. Identity → Policy → Approval → Trace. 13 framework integrations. Free during early access. homepage_url: https://sidclaw.com logo: sidclaw.png repo_url: https://github.com/sidclawhq/platform - name: WritBase description: MCP-native task management for AI agent fleets homepage_url: https://github.com/Writbase/writbase logo: default.png repo_url: https://github.com/Writbase/writbase - name: Chatbot items: - name: 5ire description: 5ire is a cross-platform desktop AI assistant, MCP client. Compatible with major service providers, supports local knowledge base and tools via model context protocol servers. homepage_url: https://5ire.app logo: 5ire.png repo_url: https://github.com/nanbingxyz/5ire - name: AnythingLLM description: The all-in-one Desktop & Docker AI application with built-in RAG, AI agents, No-code agent builder, MCP compatibility, and more. homepage_url: https://anythingllm.com/ logo: anythingllm.jpeg repo_url: https://github.com/Mintplex-Labs/anything-llm - name: Chat SDK description: A full-featured, hackable Next.js AI chatbot built by Vercel. homepage_url: https://chat.vercel.ai logo: vercel.png repo_url: https://github.com/vercel/ai-chatbot - name: Chatbot UI description: AI chat for any model. homepage_url: https://JoinTakeoff.com logo: chatbotui.png repo_url: https://github.com/mckaywrigley/chatbot-ui - name: Cherry Studio description: Cherry Studio is a desktop client that supports multiple LLM providers. Support for deepseek-r1. homepage_url: https://cherry-ai.com logo: cherry-studio.png repo_url: https://github.com/CherryHQ/cherry-studio - name: FastChat description: An open platform for training, serving, and evaluating large language models. Release repo for Vicuna and Chatbot Arena. homepage_url: https://github.com/lm-sys/fastchat logo: lmsys.png repo_url: https://github.com/lm-sys/fastchat - name: Gradio description: Build and share delightful machine learning apps, all in Python. homepage_url: https://www.gradio.app logo: gradio.png repo_url: https://github.com/gradio-app/gradio - name: Jan description: Jan is an open-source alternative to ChatGPT that runs 100% offline on your computer. homepage_url: https://jan.ai/ logo: jan.png repo_url: https://github.com/janhq/jan - name: LLM description: Access large language models from the command-line homepage_url: https://llm.datasette.io/ logo: default.png repo_url: https://github.com/simonw/llm - name: Lobe Chat description: Lobe Chat is an open-source, modern-design AI chat framework. Supports Multi AI Providers, Knowledge Base, Multi-Modals, and Thinking. homepage_url: https://chat-preview.lobehub.com logo: lobe-chat.png repo_url: https://github.com/lobehub/lobe-chat - name: NextChat description: 'Light and Fast AI Assistant. Support: Web | iOS | MacOS | Android | Linux | Windows.' homepage_url: https://nextchat.club/ logo: nextchat.png repo_url: https://github.com/ChatGPTNextWeb/NextChat - name: opcode description: A powerful GUI app and Toolkit for Claude Code - Create custom agents, manage interactive Claude Code sessions, run secure background agents, and more. homepage_url: https://opcode.sh/ logo: asterisk.png repo_url: https://github.com/getAsterisk/opcode - name: Open WebUI description: User-friendly AI Interface. Supports Ollama, OpenAI API, and more. homepage_url: https://openwebui.com logo: open-webui.png repo_url: https://github.com/open-webui/open-webui - name: PrivateGPT description: Interact with your documents using the power of GPT, 100% privately, no data leaks. homepage_url: https://privategpt.dev logo: privategpt.png repo_url: https://github.com/zylon-ai/private-gpt - name: Database items: - name: Chroma description: The AI-native open-source embedding database. homepage_url: https://www.trychroma.com/ logo: chroma.png repo_url: https://github.com/chroma-core/chroma - name: Deeplake description: Database for AI. Store Vectors, Images, Texts, Videos, etc. Use with LLMs/LangChain. Store, query, version, & visualize any AI data. Stream data in real-time to PyTorch/TensorFlow. homepage_url: https://activeloop.ai logo: deeplake.png repo_url: https://github.com/activeloopai/deeplake - name: Faiss description: A library for efficient similarity search and clustering of dense vectors. homepage_url: https://faiss.ai logo: meta.png repo_url: https://github.com/facebookresearch/faiss - name: Hindsight description: 'Hindsight: Agent Memory That Learns' homepage_url: https://hindsight.vectorize.io/ logo: hindsight.png repo_url: https://github.com/vectorize-io/hindsight - name: Milvus description: Milvus is a high-performance, cloud-native vector database built for scalable vector ANN search. homepage_url: https://milvus.io logo: milvus.png repo_url: https://github.com/milvus-io/milvus - name: Weaviate description: Weaviate is an open-source vector database that stores both objects and vectors, allowing for the combination of vector search with structured filtering with the fault tolerance and scalability of a cloud-native database. homepage_url: https://weaviate.io/developers/weaviate/ logo: weaviate.png repo_url: https://github.com/weaviate/weaviate - name: Sandbox items: - name: Daytona description: Daytona is a Secure and Elastic Infrastructure for Running AI-Generated Code. homepage_url: https://daytona.io logo: daytona.png repo_url: https://github.com/daytonaio/daytona - name: E2B description: Secure open source cloud runtime for AI apps & AI agents. homepage_url: https://e2b.dev/docs logo: e2b.png repo_url: https://github.com/e2b-dev/E2B - name: Observation items: - name: OpenLLMetry description: Open-source observability for your LLM application, based on OpenTelemetry. homepage_url: https://www.traceloop.com/openllmetry logo: openllmetry.png repo_url: https://github.com/traceloop/openllmetry - name: wandb description: The AI developer platform. Use Weights & Biases to train and fine-tune models, and manage models from experimentation to production. homepage_url: https://wandb.ai logo: wandb.png repo_url: https://github.com/wandb/wandb - name: Training subcategories: - name: Framework items: - name: AXLearn description: An Extensible Deep Learning Library homepage_url: https://github.com/apple/axlearn logo: axlearn repo_url: https://github.com/apple/axlearn - name: Candle description: Minimalist ML framework for Rust. homepage_url: https://github.com/huggingface/candle logo: hf.png repo_url: https://github.com/huggingface/candle - name: ColossalAI description: Making large AI models cheaper, faster and more accessible. homepage_url: https://www.colossalai.org logo: colossalai.png repo_url: https://github.com/hpcaitech/ColossalAI - name: DLRover description: 'DLRover: An Automatic Distributed Deep Learning System.' homepage_url: https://github.com/intelligent-machine-learning/dlrover logo: dlrover.png repo_url: https://github.com/intelligent-machine-learning/dlrover - name: Ludwig description: Low-code framework for building custom LLMs, neural networks, and other AI models. homepage_url: https://ludwig.ai logo: ludwig.png repo_url: https://github.com/ludwig-ai/ludwig - name: MaxText description: A simple, performant and scalable Jax LLM! homepage_url: https://cloud.google.com/tpu/docs/tutorials/LLM/jetstream-maxtext-inference-v6e?hl=en logo: google.png repo_url: https://github.com/google/maxtext - name: MLX description: 'MLX: An array framework for Apple silicon.' homepage_url: https://ml-explore.github.io/mlx/ logo: mlx.png repo_url: https://github.com/ml-explore/mlx - name: FineTune items: - name: Axolotl description: Go ahead and axolotl questions. homepage_url: https://docs.axolotl.ai logo: axolotl.svg repo_url: https://github.com/axolotl-ai-cloud/axolotl - name: EasyLM description: Large language models (LLMs) made easy, EasyLM is a one stop solution for pre-training, finetuning, evaluating and serving LLMs in JAX/Flax. homepage_url: https://github.com/young-geng/EasyLM logo: default.png repo_url: https://github.com/young-geng/EasyLM - name: LLaMa-Factory description: Unified Efficient Fine-Tuning of 100+ LLMs & VLMs (ACL 2024). homepage_url: https://huggingface.co/papers/2403.13372 logo: llamafactory.png repo_url: https://github.com/hiyouga/LLaMA-Factory - name: LMFlow description: An Extensible Toolkit for Finetuning and Inference of Large Foundation Models. Large Models for All. homepage_url: https://optimalscale.github.io/LMFlow/ logo: lmflow.png repo_url: https://github.com/OptimalScale/LMFlow - name: maestro description: 'streamline the fine-tuning process for multimodal models: PaliGemma 2, Florence-2, and Qwen2.5-VL.' homepage_url: https://maestro.roboflow.com logo: roboflow.png repo_url: https://github.com/roboflow/maestro - name: MLX-VLM description: MLX-VLM is a package for inference and fine-tuning of Vision Language Models (VLMs) on your Mac using MLX. homepage_url: https://github.com/Blaizzy/mlx-vlm logo: default.png repo_url: https://github.com/Blaizzy/mlx-vlm - name: Swift description: Use PEFT or Full-parameter to finetune 450+ LLMs and 150+ MLLMs. homepage_url: https://swift.readthedocs.io logo: swift.png repo_url: https://github.com/modelscope/ms-swift - name: torchtune description: PyTorch native post-training library. homepage_url: https://pytorch.org/torchtune/main/ logo: torchtune.png repo_url: https://github.com/pytorch/torchtune - name: Transformer Lab description: 'Open Source Application for Advanced LLM Engineering: interact, train, fine-tune, and evaluate large language models on your own computer.' homepage_url: https://transformerlab.ai/ logo: transformerlab.svg repo_url: https://github.com/transformerlab/transformerlab-app - name: unsloth description: Finetune Llama 3.3, DeepSeek-R1 & Reasoning LLMs 2x faster with 70% less memory! homepage_url: https://unsloth.ai logo: unsloth.png repo_url: https://github.com/unslothai/unsloth - name: RLHF items: - name: OpenRLHF description: An Easy-to-use, Scalable and High-performance RLHF Framework (70B+ PPO Full Tuning & Iterative DPO & LoRA & RingAttention & RFT). homepage_url: https://openrlhf.readthedocs.io/ logo: openRLHF.png repo_url: https://github.com/OpenRLHF/OpenRLHF - name: Self-RLHF description: 'Safe RLHF: Constrained Value Alignment via Safe Reinforcement Learning from Human Feedback.' homepage_url: https://pku-beaver.github.io logo: selfrlhf.png repo_url: https://github.com/PKU-Alignment/safe-rlhf - name: Agentic RL items: - name: AReaL description: Lightning-Fast RL for LLM Reasoning and Agents. Made Simple & Flexible. homepage_url: https://inclusionai.github.io/AReaL/intro.html logo: areal.png repo_url: https://github.com/inclusionAI/AReaL - name: rLLM description: Democratizing Reinforcement Learning for LLMs homepage_url: https://rllm-project.readthedocs.io/en/latest/ logo: rllm repo_url: https://github.com/rllm-org/rllm - name: slime description: slime is an LLM post-training framework for RL Scaling. homepage_url: https://thudm.github.io/slime/ logo: slime repo_url: https://github.com/THUDM/slime - name: verl description: 'verl: Volcano Engine Reinforcement Learning for LLMs' homepage_url: https://verl.readthedocs.io/en/latest/index.html logo: verl repo_url: https://github.com/volcengine/verl - name: Benchmark items: - name: AgentBench description: A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24). homepage_url: https://llmbench.ai logo: agentbench.png repo_url: https://github.com/THUDM/AgentBench - name: ASQI Engineer description: ASQI (AI Solutions Quality Index) Engineer - run containerised AI tests and map to score cards! homepage_url: https://asqi.ai logo: default.png repo_url: https://github.com/asqi-engineer/asqi-engineer - name: LiveBench description: 'LiveBench: A Challenging, Contamination-Free LLM Benchmark.' homepage_url: https://livebench.ai logo: default.png repo_url: https://github.com/livebench/livebench - name: lm-evaluation-harness description: A framework for few-shot evaluation of language models. homepage_url: https://www.eleuther.ai logo: eleutherai.png repo_url: https://github.com/EleutherAI/lm-evaluation-harness - name: LongBench description: LongBench v2 and LongBench (ACL 2024). homepage_url: https://longbench2.github.io logo: longbench.png repo_url: https://github.com/THUDM/LongBench - name: MLE-bench description: MLE-bench is a benchmark for measuring how well AI agents perform at machine learning engineering homepage_url: https://openai.com/index/mle-bench/ logo: mle-bench repo_url: https://github.com/openai/mle-bench/ - name: OpenCompass description: OpenCompass is an LLM evaluation platform, supporting a wide range of models (Llama3, Mistral, InternLM2, GPT-4, LLaMa2, Qwen, GLM, Claude, etc) over 100+ datasets. homepage_url: https://opencompass.org.cn/ logo: opencompass.svg repo_url: https://github.com/open-compass/opencompass - name: opik description: Debug, evaluate, and monitor your LLM applications, RAG systems, and agentic workflows with comprehensive tracing, automated evaluations, and production-ready dashboards. homepage_url: https://www.comet.com/docs/opik/ logo: opik.png repo_url: https://github.com/comet-ml/opik - name: terminal-bench description: A benchmark for LLMs on complicated tasks in the terminal homepage_url: https://www.tbench.ai/ logo: terminal-bench repo_url: https://github.com/laude-institute/terminal-bench - name: Workflow items: - name: Flyte description: Scalable and flexible workflow orchestration platform that seamlessly unifies data, ML and analytics stacks. homepage_url: https://flyte.org logo: flyte.png repo_url: https://github.com/flyteorg/flyte - name: Kubeflow description: Machine Learning Toolkit for Kubernetes. homepage_url: https://www.kubeflow.org/ logo: kubeflow.png repo_url: https://github.com/kubeflow/kubeflow - name: Metaflow description: Build, Deploy and Manage AI/ML Systems. homepage_url: https://metaflow.org logo: metaflow.png repo_url: https://github.com/Netflix/metaflow - name: MLflow description: Open source platform for the machine learning lifecycle. homepage_url: https://mlflow.org logo: mlflow.png repo_url: https://github.com/mlflow/mlflow - name: Polyaxon description: MLOps Tools For Managing & Orchestrating The Machine Learning LifeCycle. homepage_url: https://polyaxon.com logo: polyaxon.png repo_url: https://github.com/polyaxon/polyaxon - name: Ray description: Ray is an AI compute engine. Ray consists of a core distributed runtime and a set of AI Libraries for accelerating ML workloads. homepage_url: https://ray.io logo: ray.png repo_url: https://github.com/ray-project/ray - name: Seldon-Core description: An MLOps framework to package, deploy, monitor and manage thousands of production machine learning models. homepage_url: https://www.seldon.io/tech/products/core/ logo: seldon-core.png repo_url: https://github.com/SeldonIO/seldon-core - name: ZenML description: 'ZenML 🙏: The bridge between ML and Ops. https://zenml.io.' homepage_url: https://zenml.io logo: zenml.png repo_url: https://github.com/zenml-io/zenml ================================================ FILE: website/guide.yml ================================================ # Landscape2 guide # # This file allows defining the content of the landscape guide. # # Reference documentation: https://github.com/cncf/landscape2/blob/main/docs/config/guide.yml categories: - category: "Inference" content: | Inference refers to the phase where trained machine learning models, particularly LLMs (Large Language Models), are used to generate predictions, outputs, or perform tasks based on new input data. It covers all aspects of efficiently serving, routing, executing, and managing models in real-time or batch settings, focusing on scalability, low latency, reliability, and adaptability across different environments and hardware. subcategories: - subcategory: "Inference Engine" content: | Inference Engines are optimized frameworks or platforms designed to execute machine learning models, particularly large language models (LLMs), with an emphasis on high performance, low latency, and scalability. These engines allow users to run inferences across various hardware platforms and deploy models efficiently. They handle tasks such as model loading, serving, optimization, and high-throughput execution, making them essential in AI and machine learning applications. - subcategory: "Inference Platform" content: | Inference Platforms are specialized environments or infrastructures designed for deploying and managing large-scale AI inference tasks, particularly for Generative AI (GenAI) models and large language models (LLMs). These platforms focus on providing efficient resource management, scaling capabilities, and ease of use when serving models in production environments, often integrating Kubernetes for orchestration, auto-provisioning, and container-based deployments. - subcategory: "Middleware" content: | Middleware in the context of AI inference refers to software that acts as an intermediary layer between inference engines and large language models (LLMs) or other AI services to optimize performance and scalability. - subcategory: "AI Gateway" content: | AI Gateways serve as the central entry point for managing and routing requests to various backend services, including LLMs and AI models. These gateways are designed to handle high traffic volumes, manage APIs, and often provide additional features like rate limiting, security, monitoring, and AI-specific capabilities. They act as intermediaries between clients and services, enabling efficient communication, management, and scalability for AI-driven applications. - subcategory: "LLM Router" content: | LLM routers are specialized systems designed to intelligently route, proxy, or switch between multiple large language model (LLM) providers based on factors like cost, latency, quality, or model capabilities. They help developers and organizations optimize API usage across different backends while maintaining a unified, often OpenAI-compatible, interface. Some also integrate guardrails and evaluation mechanisms to ensure safe and efficient AI interactions. - subcategory: "Output" content: | Output refers to tools or frameworks that help generate structured or formatted results from large language models (LLMs). These tools are designed to provide well-organized and easily interpretable outputs, especially for complex text generation tasks. - subcategory: "Simulator" content: | Simulator refers to tools simulating the behavior and performance of inference engines. - subcategory: "Benchmark" content: | Benchmark provide standardized methods for evaluating and comparing the performance, quality, and capabilities of LLMs (Large Language Models) and AI systems. They focus on areas such as inference speed, latency, model accuracy, contamination-free testing, and real-world scenario simulation, helping developers and researchers make informed decisions about model and system selection. - category: "Orchestration" content: | Orchestration in AI involves coordinating and managing the various components, workflows, agents, and services that make up complex AI systems. It ensures that models, tools, APIs, agents, and data sources work seamlessly together, supporting scalability, automation, modularity, and efficient deployment from development to production environments. subcategories: - subcategory: "Application Framework" content: | An Application framework provides a structured environment for building and deploying AI applications. It includes components for user interface design, data management, and integration with external services, enabling developers to create robust and scalable AI solutions. - subcategory: "Agent Framework" content: | An Agent framework is an autonomous or semi-autonomous system powered by large language models, integrating RAG, Function Calling, and MCP to understand user intent, retrieve external knowledge, invoke tools or APIs, and dynamically execute complex tasks with precise responses. - subcategory: "Evolutionary Framework" content: | Evolutionary Frameworks combine large language models (LLMs) with evolutionary algorithms to drive scientific discovery and optimization. These frameworks facilitate the iterative improvement of solutions by simulating natural selection processes, enabling the exploration of complex problem spaces and the generation of innovative ideas or designs. - subcategory: "Workflow" content: | A workflow provides a structured environment for building, connecting, managing, and deploying AI applications, especially those powered by large language models (LLMs) or other machine learning techniques. These frameworks offer modular components such as model integration, retrieval-augmented generation (RAG) pipelines, agent capabilities, prompt management, observability, and workflow orchestration, enabling rapid prototyping and scalable production deployments. - subcategory: "RAG" content: | Retrieval-Augmented Generation (RAG) frameworks combine large language models (LLMs) with external knowledge sources to enhance the accuracy, relevance, and context-awareness of AI-generated content. These frameworks facilitate the integration of vector databases, document stores, and search engines, enabling LLMs to retrieve pertinent information during the generation process. - category: "Runtime" content: | Runtime refers to the infrastructure, environments, and systems that execute AI models, agents, and applications in real-time. It provides the foundation for deploying, running, monitoring, and scaling AI workloads, ensuring they perform reliably across cloud, edge, or on-premise setups. Runtimes often include components like chat frameworks, databases, code assistants, and observability tools. subcategories: - subcategory: "AI Terminal" content: | AI Terminals are interactive command-line interfaces or environments that allow users to engage with AI models, particularly large language models (LLMs), in a conversational manner. These terminals provide a text-based interface for users to input queries, receive responses, and interact with AI systems directly from the command line, often supporting features like multi-turn conversations, context retention, and integration with various LLM providers. - subcategory: "AI Agent" content: | An AI Agent is an autonomous or semi-autonomous system powered by large language models (LLMs), integrating RAG, Function Calling, and MCP to understand user intent, retrieve external knowledge, invoke tools or APIs, and dynamically execute complex tasks with precise responses. - subcategory: "Code Agent" content: | Code Agents are specialized AI agents designed to assist with software development tasks by leveraging large language models (LLMs). They can understand and generate code, debug, test, and document software projects. - subcategory: "Evolve Agent" content: | Evolve Agents are AI systems that combine large language models (LLMs) with evolutionary algorithms to autonomously explore, optimize, and innovate solutions across various domains. These agents iteratively improve their performance by simulating natural selection processes, enabling them to adapt and evolve in response to complex problem spaces and dynamic environments. - subcategory: "Tool" content: | A Tool is an external functional module or API invoked by Agents via Function Calling, extending their capabilities to perform specific computations, retrieve real-time data, or interact with third-party services, enabling dynamic task processing and precise outputs. - subcategory: "Chatbot" content: | Chatbots are platforms and toolkits designed to build, customize, deploy, and interact with AI chat applications. They often support multiple LLM providers, local deployment, plugin systems, RAG integration, and user-friendly UIs, enabling developers and users to easily create private, extensible, and cross-platform AI assistants. - subcategory: "Database" content: | Databases for AI applications, especially vector databases, are specialized systems designed to store, index, and query high-dimensional embeddings and AI-related data like vectors, text, images, and videos. They enable fast similarity search, retrieval-augmented generation (RAG), and integration with machine learning pipelines. They often support various data types, including text, images, and videos, and are optimized for high-performance querying and retrieval. - subcategory: "Sandbox" content: | A Sandbox is a secure and isolated environment that allows users to run, test, and experiment with code, applications, or AI models without affecting the host system. Sandboxes provide controlled access to resources, ensuring safety and preventing unintended interactions with the broader system or network. - subcategory: "Evaluation" content: | Evaluation tools are designed to assess the performance, reliability, and behavior of LLM applications. - subcategory: "Observation" content: | Observation in AI refers to tools and platforms that help monitor, evaluate, and analyze machine learning models, particularly large language models (LLMs), to ensure optimal performance, identify issues, and manage experiments. These tools often focus on real-time monitoring, logging, and visualization, helping teams improve and maintain their AI systems. - category: "Training" content: | Training is the process of teaching machine learning models to perform specific tasks by exposing them to datasets and optimizing their internal parameters. It covers initial model pre-training, fine-tuning on specialized data, alignment with human preferences, and evaluation to ensure high performance and safety. Training is critical for creating capable, customized, and robust AI systems across a variety of applications. subcategories: - subcategory: "Framework" content: | Framework refers to tools that assist in the training of machine learning models, particularly large AI models. These tools help improve the efficiency, scalability, and accessibility of the training process. - subcategory: "FineTune" content: | Fine-tuning frameworks enable developers and researchers to efficiently adapt pre-trained large language models (LLMs) or vision-language models (VLMs) to specialized tasks or datasets. These tools often provide streamlined pipelines for training, parameter-efficient fine-tuning (PEFT), evaluation, and deployment, with support for both single-modal and multimodal models across a wide range of hardware environments. - subcategory: "RLHF" content: | RLHF (Reinforcement Learning from Human Feedback) frameworks facilitate the training of large language models (LLMs) using reinforcement learning techniques guided by human feedback. These tools provide scalable, high-performance solutions for implementing RLHF algorithms such as PPO (Proximal Policy Optimization) and DPO (Direct Preference Optimization), enabling efficient fine-tuning of LLMs to align with human preferences and improve their behavior in real-world applications. - subcategory: "Agentic RL" content: | Agentic RL frameworks combine reinforcement learning techniques with agent-based architectures powered by large language models (LLMs). These frameworks enable the development of autonomous agents that can learn from interactions with their environment, - subcategory: "Benchmark" content: | Benchmark provide standardized methods for evaluating and comparing the performance, quality, and capabilities of LLMs (Large Language Models) and AI systems. They focus on areas such as inference speed, latency, model accuracy, contamination-free testing, and real-world scenario simulation, helping developers and researchers make informed decisions about model and system selection. - subcategory: "Workflow" content: | Workflow tools orchestrate, manage, and automate the lifecycle of machine learning and AI applications, they are designed to streamline complex pipelines, enabling scalability, reliability, and reproducibility of ML workflows across distributed systems. ================================================ FILE: website/logos/baml ================================================ BAML ================================================ FILE: website/settings.yml ================================================ # Landscape2 settings # # This settings file allows customizing some aspects of the landscape. # # Reference documentation: https://github.com/cncf/landscape2/blob/main/docs/config/settings.yml foundation: Awesome-LLMOps url: http://127.0.0.1:8000 colors: color1: "rgba(0, 107, 204, 1)" color2: "rgba(255, 0, 170, 1)" color3: "rgba(96, 149, 214, 1)" color4: "rgba(0, 42, 81, 0.7)" color5: "rgba(1, 107, 204, 0.7)" color6: "rgba(0, 42, 81, 0.7)" color7: "rgba(180, 219, 255, 1)" # featured_items: # - field: maturity # options: # - value: graduated # order: 1 # label: Graduated footer: links: github: "https://github.com/InftyAI/Awesome-LLMOps" homepage: "https://github.com/InftyAI/Awesome-LLMOps" logo: "https://raw.githubusercontent.com/InftyAI/Awesome-LLMOps/refs/heads/main/website/logos/awesome-llmops-dark.svg" grid_items_size: large groups: - name: ALL PROJECTS categories: - GPU - Inference - Orchestration - Runtime - Training - name: GPU categories: - Management - Scheduling - name: Inference categories: - Inference Engine - Inference Platform - AI Gateway - LLM Router - Benchmark - Output - Simulator - name: Orchestration categories: - Workflow - Agent - Tools - name: Runtime categories: - Chatbot - Code Assistant - Database - Development Environment - Evaluation - Observation - name: Training categories: - Framework - FineTune - RLHF - Agentic RL - Benchmark - Workflow header: links: github: "https://github.com/InftyAI/Awesome-LLMOps" logo: "https://raw.githubusercontent.com/InftyAI/Awesome-LLMOps/refs/heads/main/website/logos/awesome-llmops-light.svg" screenshot_width: 1500