this port can then be forwarded to a browser tab like below:
================================================
FILE: .devcontainer/backend-dev/devcontainer.json
================================================
{
"name": "Backend Development",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": "bash .devcontainer/backend-dev/post_create_command.sh",
"postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
"customizations": {
"vscode": {
"extensions": [
"GitHub.copilot",
"ms-python.python",
"esbenp.prettier-vscode"
]
}
}
}
================================================
FILE: .devcontainer/backend-dev/post_create_command.sh
================================================
#!/bin/bash
# allow git usage
git config --global --add safe.directory "*"
# ensure pre-commit is installed
pre-commit install
# create python virtual environment
python3 -m venv .venv
# install python dependencies in /backend
cd backend
pip install -r requirements.txt
cd ..
# install code in editable mode in /oasst-shared
pip install -e ./oasst-shared[dev]
# install code in editable mode in /oasst-data
pip install -e ./oasst-data[dev]
# docker compose up for backend-dev
docker compose up backend-dev --build --attach-dependencies -d
# note: commented out for now, you probably want to manually run this part once in the devcontainer
# run run-local.sh script
# cd scripts/backend-development/
# bash run-local.sh
================================================
FILE: .devcontainer/devcontainer.json
================================================
{
"name": "Generic",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": ".devcontainer/post_create_command.sh",
"customizations": {
"vscode": {
"extensions": [
"GitHub.copilot",
"ms-python.python",
"esbenp.prettier-vscode"
]
}
}
}
================================================
FILE: .devcontainer/frontend-dev/devcontainer.json
================================================
{
"name": "Front-end Development",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": ".devcontainer/frontend-dev/post_create_command.sh",
"customizations": {
"vscode": {
"extensions": ["GitHub.copilot", "esbenp.prettier-vscode"]
}
}
}
================================================
FILE: .devcontainer/frontend-dev/post_create_command.sh
================================================
#!/bin/bash
# ensure pre-commit is installed
pre-commit install
# npm install in /website
cd website
npm install
cd ..
================================================
FILE: .devcontainer/post_create_command.sh
================================================
#!/bin/bash
# do as little as possible in this script to keep the container creation fast.
# for more specific devcontainer use cases use the different devcontainer subfolders.
echo "done"
================================================
FILE: .dockerignore
================================================
**/node_modules
website/.next
================================================
FILE: .gitattributes
================================================
* text=auto eol=lf
*.ipynb linguist-documentation
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "npm"
directory: "/website"
schedule:
interval: "weekly"
================================================
FILE: .github/workflows/build-frontend.yaml
================================================
name: Build Frontend
on:
push:
branches:
- main
paths:
- website/**
pull_request:
paths:
- website/**
workflow_call:
jobs:
build-frontend:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./website
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16.x
cache: "npm"
cache-dependency-path: website/package-lock.json
- run: npm ci
- run: npx prisma generate
- run: npm run build
================================================
FILE: .github/workflows/build-postgres.yaml
================================================
name: Build OASST Postgres image
on:
push:
branches:
- main
paths:
- docker/oasst-postgres/**
jobs:
build-postgres:
uses: ./.github/workflows/docker-build.yaml
with:
image-name: oasst-postgres
context: ./docker/oasst-postgres
dockerfile: docker/oasst-postgres/Dockerfile
build-args: ""
================================================
FILE: .github/workflows/deploy-docs-site.yaml
================================================
name: Deploy docs to gitHub pages
on:
push:
branches:
- main
paths:
- ".github/workflows/deploy-docs-site.yaml"
- "docs/**"
pull_request:
paths:
- ".github/workflows/deploy-docs-site.yaml"
- "docs/**"
jobs:
deploy:
runs-on: ubuntu-22.04
permissions:
contents: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
defaults:
run:
working-directory: docs
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: "18"
- name: Install dependencies
run: yarn install --frozen-lockfile
- name: Build website
run: yarn build
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
if: ${{ github.ref == 'refs/heads/main' }}
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs/build
================================================
FILE: .github/workflows/deploy-to-node.yaml
================================================
name: Deploy to node
on:
workflow_call:
inputs:
stack-name:
required: false
type: string
default: dev
image-tag:
required: false
type: string
default: latest
backend-port:
required: false
type: string
default: 8080
website-port:
required: false
type: string
default: 3000
inference-server-port:
required: false
type: string
default: 8085
jobs:
deploy:
runs-on: ubuntu-latest
environment: ${{ inputs.stack-name }}
env:
WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
WEB_MODERATOR_USERS: ${{ secrets.DEV_WEB_MODERATOR_USERS }}
WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
WEB_GOOGLE_CLIENT_ID: ${{ secrets.DEV_WEB_GOOGLE_CLIENT_ID }}
WEB_GOOGLE_CLIENT_SECRET: ${{ secrets.DEV_WEB_GOOGLE_CLIENT_SECRET }}
WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
WEB_NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
S3_REGION: ${{ secrets.S3_REGION }}
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
INIT_PROMPT_DISABLED_LANGS: ${{ vars.INIT_PROMPT_DISABLED_LANGS }}
MAX_ACTIVE_TREES: ${{ vars.MAX_ACTIVE_TREES }}
MAX_INITIAL_PROMPT_REVIEW: ${{ vars.MAX_INITIAL_PROMPT_REVIEW }}
MAX_TREE_DEPTH: ${{ vars.MAX_TREE_DEPTH }}
MAX_CHILDREN_COUNT: ${{ vars.MAX_CHILDREN_COUNT }}
LONELY_CHILDREN_COUNT: ${{ vars.LONELY_CHILDREN_COUNT }}
P_LONELY_CHILD_EXTENSION: ${{ vars.P_LONELY_CHILD_EXTENSION }}
P_ACTIVATE_BACKLOG_TREE: ${{ vars.P_ACTIVATE_BACKLOG_TREE }}
NUM_REQUIRED_RANKINGS: ${{ vars.NUM_REQUIRED_RANKINGS }}
NUM_REVIEWS_REPLY: ${{ vars.NUM_REVIEWS_REPLY }}
NUM_REVIEWS_INITIAL_PROMPT: ${{ vars.NUM_REVIEWS_INITIAL_PROMPT }}
MIN_ACTIVE_RANKINGS_PER_LANG: ${{ vars.MIN_ACTIVE_RANKINGS_PER_LANG }}
MAX_PROMPT_LOTTERY_WAITING: ${{ vars.MAX_PROMPT_LOTTERY_WAITING }}
GOAL_TREE_SIZE: ${{ vars.GOAL_TREE_SIZE }}
MESSAGE_SIZE_LIMIT: ${{ vars.MESSAGE_SIZE_LIMIT }}
SKIP_TOXICITY_CALCULATION: ${{ vars.SKIP_TOXICITY_CALCULATION }}
STATS_INTERVAL_DAY: ${{ vars.STATS_INTERVAL_DAY }}
STATS_INTERVAL_WEEK: ${{ vars.STATS_INTERVAL_WEEK }}
STATS_INTERVAL_MONTH: ${{ vars.STATS_INTERVAL_MONTH }}
STATS_INTERVAL_TOTAL: ${{ vars.STATS_INTERVAL_TOTAL }}
DISCORD_API_KEY: ${{ secrets.DISCORD_API_KEY }}
DISCORD_CHANNEL_ID: ${{ vars.DISCORD_CHANNEL_ID }}
WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY:
${{ secrets.WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY }}
WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY:
${{ secrets.WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY }}
WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA:
${{ vars.WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA }}
WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN:
${{ vars.WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN }}
LOGURU_LEVEL: ${{ vars.LOGURU_LEVEL }}
MAINTENANCE_MODE: ${{ vars.MAINTENANCE_MODE }}
BACKEND_URL: ${{ vars.BACKEND_URL }}
WEB_NEXT_PUBLIC_BACKEND_URL: ${{ vars.WEB_NEXT_PUBLIC_BACKEND_URL }}
BACKEND_CORS_ORIGINS: ${{ vars.BACKEND_CORS_ORIGINS }}
WEB_INFERENCE_SERVER_HOST: ${{ vars.WEB_INFERENCE_SERVER_HOST }}
WEB_ENABLE_CHAT: ${{ vars.WEB_ENABLE_CHAT }}
WEB_BYE: ${{ vars.WEB_BYE }}
WEB_ENABLE_DRAFTS_WITH_PLUGINS: ${{ vars.WEB_ENABLE_DRAFTS_WITH_PLUGINS }}
WEB_NUM_GENERATED_DRAFTS: ${{ vars.WEB_NUM_GENERATED_DRAFTS }}
WEB_CURRENT_ANNOUNCEMENT: ${{ vars.WEB_CURRENT_ANNOUNCEMENT }}
WEB_INFERENCE_SERVER_API_KEY: ${{secrets.WEB_INFERENCE_SERVER_API_KEY}}
INFERENCE_POSTGRES_PASSWORD: ${{secrets.INFERENCE_POSTGRES_PASSWORD}}
INFERENCE_ALLOW_DEBUG_AUTH: ${{vars.INFERENCE_ALLOW_DEBUG_AUTH}}
INFERENCE_DEBUG_API_KEYS: ${{vars.INFERENCE_DEBUG_API_KEYS}}
INFERENCE_LOG_LEVEL: ${{vars.INFERENCE_LOG_LEVEL}}
INFERENCE_ROOT_TOKEN: ${{secrets.INFERENCE_ROOT_TOKEN}}
INFERENCE_API_ROOT: ${{vars.INFERENCE_API_ROOT}}
INFERENCE_TRUSTED_CLIENT_KEYS: ${{secrets.INFERENCE_TRUSTED_CLIENT_KEYS}}
INFERENCE_AUTH_SALT: ${{secrets.INFERENCE_AUTH_SALT}}
INFERENCE_AUTH_SECRET: ${{secrets.INFERENCE_AUTH_SECRET}}
INFERENCE_AUTH_DISCORD_CLIENT_ID: ${{secrets.INFERENCE_AUTH_DISCORD_CLIENT_ID}}
INFERENCE_AUTH_DISCORD_CLIENT_SECRET: ${{secrets.INFERENCE_AUTH_DISCORD_CLIENT_SECRET}}
INFERENCE_AUTH_GITHUB_CLIENT_ID: ${{secrets.INFERENCE_AUTH_GITHUB_CLIENT_ID}}
INFERENCE_AUTH_GITHUB_CLIENT_SECRET: ${{secrets.INFERENCE_AUTH_GITHUB_CLIENT_SECRET}}
INFERENCE_CORS_ORIGINS: ${{ vars.INFERENCE_CORS_ORIGINS }}
INFERENCE_ALLOWED_MODEL_CONFIG_NAMES:
${{ vars.INFERENCE_ALLOWED_MODEL_CONFIG_NAMES }}
INFERENCE_ASSISTANT_MESSAGE_TIMEOUT:
${{ vars.INFERENCE_ASSISTANT_MESSAGE_TIMEOUT }}
INFERENCE_MESSAGE_QUEUE_EXPIRE: ${{ vars.INFERENCE_MESSAGE_QUEUE_EXPIRE }}
INFERENCE_WORK_QUEUE_MAX_SIZE: ${{ vars.INFERENCE_WORK_QUEUE_MAX_SIZE }}
INFERENCE_ENABLE_SAFETY: ${{ vars.INFERENCE_ENABLE_SAFETY }}
INFERENCE_GUNICORN_WORKERS: ${{ vars.INFERENCE_GUNICORN_WORKERS }}
INFERENCE_CHAT_MAX_MESSAGES: ${{ vars.INFERENCE_CHAT_MAX_MESSAGES }}
INFERENCE_MESSAGE_MAX_LENGTH: ${{ vars.INFERENCE_MESSAGE_MAX_LENGTH }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Run deploy playbook
uses: dawidd6/action-ansible-playbook@v2
with:
# Required, playbook filepath
playbook: deploy-to-node.yaml
# Optional, directory where playbooks live
directory: ansible
# Optional, SSH private key
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
# Optional, literal inventory file contents
inventory: |
[dev]
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
options: |
--extra-vars "stack_name=${{inputs.stack-name}} \
image_tag=${{inputs.image-tag}} \
backend_port=${{inputs.backend-port}} \
website_port=${{inputs.website-port}} \
postgres_password=${{secrets.POSTGRES_PASSWORD}} \
web_api_key=${{secrets.WEB_API_KEY}}"
- name: Run inference deploy playbook
uses: dawidd6/action-ansible-playbook@v2
with:
# Required, playbook filepath
playbook: inference/deploy-server.yaml
# Optional, directory where playbooks live
directory: ansible
# Optional, SSH private key
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
# Optional, literal inventory file contents
inventory: |
[dev]
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
options: |
--extra-vars "stack_name=${{inputs.stack-name}} \
image_tag=${{inputs.image-tag}} \
server_port=${{inputs.inference-server-port}}"
================================================
FILE: .github/workflows/docker-build.yaml
================================================
name: Build
on:
workflow_call:
inputs:
dockerfile:
required: true
type: string
context:
required: true
type: string
image-name:
required: true
type: string
build-args:
required: false
type: string
jobs:
build:
name: Build Images
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.2.1
- name: Login to container registry
uses: docker/login-action@v2.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get base registry
run: |
echo "REGISTRY=ghcr.io/${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Set tag prefix
if: github.ref_name != 'main'
run: |
echo "TAG_PREFIX=${{ github.ref_name }}-" >> $GITHUB_ENV
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4.1.1
with:
images: ${{ env.REGISTRY }}/${{ inputs.image-name }}
tags: |
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
type=sha,prefix=${{ env.TAG_PREFIX }},format=short,enable=${{ github.ref_name != 'main' }}
type=ref,event=tag,enable=${{ github.ref_name != 'main' }}
- name: Build and push Docker image
uses: docker/build-push-action@v3.2.0
with:
file: ${{ inputs.dockerfile }}
context: ${{ inputs.context }}
build-args: ${{ inputs.build-args }}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
================================================
FILE: .github/workflows/pre-commit.yaml
================================================
name: pre-commit
on:
workflow_call:
pull_request_target:
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
# in case of PR, check out the PR's head branch
- uses: actions/checkout@v3
if: github.event_name == 'pull_request_target'
with:
ref: ${{ github.event.pull_request.head.sha }}
# in case of push, check out the main branch
- uses: actions/checkout@v3
if: github.event_name != 'pull_request_target'
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: "**/requirements*.txt"
- uses: pre-commit/action@v3.0.0
- name: Post PR comment on failure
if: failure() && github.event_name == 'pull_request_target'
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
:x: **pre-commit** failed.
Please run `pre-commit run --all-files` locally and commit the changes.
Find more information in the repository's CONTRIBUTING.md
================================================
FILE: .github/workflows/production-deploy.yaml
================================================
name: Deploy to prod
on:
push:
branches:
- production
jobs:
deploy-to-prod:
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: production
image-tag: ${{ vars.PROD_IMAGE_TAG }}
backend-port: 8280
website-port: 3200
inference-server-port: 8285
================================================
FILE: .github/workflows/production2-deploy.yaml
================================================
name: Deploy to prod2
on:
push:
branches:
- production2
jobs:
deploy-to-prod:
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: production2
image-tag: ${{ vars.PROD_IMAGE_TAG }}
backend-port: 8280
website-port: 3200
inference-server-port: 8285
================================================
FILE: .github/workflows/release.yaml
================================================
name: Release
on:
push:
branches:
- main
release:
types:
- released
jobs:
pre-commit:
uses: ./.github/workflows/pre-commit.yaml
build-backend:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-backend
context: .
dockerfile: docker/Dockerfile.backend
build-args: ""
build-backend-worker:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-backend-worker
context: .
dockerfile: docker/Dockerfile.backend-worker
build-args: ""
build-web:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-web
context: .
dockerfile: docker/Dockerfile.website
build-args: ""
build-bot:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-discord-bot
context: .
dockerfile: docker/Dockerfile.discord-bot
build-args: ""
build-inference-server:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-server
context: .
dockerfile: docker/inference/Dockerfile.server
build-args: ""
build-inference-worker-full:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-full
context: .
dockerfile: docker/inference/Dockerfile.worker-full
build-args: ""
build-inference-worker-hf:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-hf
context: .
dockerfile: docker/inference/Dockerfile.worker-hf
build-args: ""
build-inference-worker-standalone:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-standalone
context: .
dockerfile: docker/inference/Dockerfile.worker-standalone
build-args: ""
build-inference-safety:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-safety
context: .
dockerfile: docker/inference/Dockerfile.safety
build-args: ""
deploy-to-node:
needs:
- build-backend
- build-backend-worker
- build-web
- build-bot
- build-inference-server
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }}
image-tag:
${{ github.event_name == 'release' && github.event.release.tag_name ||
'latest' }}
backend-port: ${{ github.event_name == 'release' && '8180' || '8080' }}
website-port: ${{ github.event_name == 'release' && '3100' || '3000' }}
inference-server-port:
${{ github.event_name == 'release' && '8185' || '8085' }}
================================================
FILE: .github/workflows/test-api-contract.yaml
================================================
name: Test API Contract
on:
push:
branches:
- main
paths:
- "oasst-shared/**"
- "backend/**"
- "website/**"
pull_request:
paths:
- "oasst-shared/**"
- "backend/**"
- "website/**"
workflow_call:
jobs:
test-contract:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: "**/requirements*.txt"
- uses: actions/setup-node@v3
with:
node-version: 16
- run: pip install ./oasst-shared[dev]
- run: pip install ./oasst-data[dev]
- run: cd backend && pip install -r requirements.txt
- run: cd website && npm ci
- run: ./scripts/backend-development/start-mock-server.sh
- name: Run Python OasstApiClient contract tests
run: ./scripts/oasst-shared-development/test.sh
- name: Run JavaScript OasstApiClient contract tests
run: ./scripts/frontend-development/run-contract-test.sh
- run: ./scripts/backend-development/stop-mock-server.sh
#- uses: stefanzweifel/git-auto-commit-action@v4
# with:
# file_pattern: "docs/docs/api/openapi.json"
# commit_message:
# update docs/docs/api/openapi.json by run ${{ github.run_id }}
================================================
FILE: .github/workflows/test-e2e.yaml
================================================
name: E2E Tests (Website)
on:
push:
branches:
- main
paths:
- oasst-shared/**
- backend/**
- website/**
pull_request:
paths:
- oasst-shared/**
- backend/**
- website/**
jobs:
test-e2e:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Start website, backend, etc
run: docker compose --profile ci up --build -d
- name: Run Cypress tests
uses: cypress-io/github-action@v5.0.2
with:
browser: chrome
working-directory: website
- uses: actions/upload-artifact@v3
if: failure() # NOTE: screenshots will be generated only if E2E test failed
with:
name: cypress-screenshots
path: website/cypress/screenshots
- uses: actions/upload-artifact@v3
if: always()
with:
name: cypress-videos
path: website/cypress/videos
================================================
FILE: .gitignore
================================================
.venv*
venv
.env
*.pyc
*.swp
*.egg-info
__pycache__
.DS_Store
# Generated files
backend/oasst-openapi.json
backend/openapi.json
# ignore jupyter notebook checkpoints
.ipynb_checkpoints
*.log
*.db
# edit docs using obsidian.md, these files should not appear in the repo
.obsidian/
.pytest_cache/
/docker-compose.override.yml
# JetBrains PyCharm:
.idea/
================================================
FILE: .pre-commit-config.yaml
================================================
# WARNING!
#
# When making changes to auto-formatters used in pre-commit hooks, you are
# likely to cause merge conflicts with main and/or other pull requests.
# Fixing them might revert other people's work. Expect pain!
# To avoid accidental reversions and keep it easy to review, please make sure
# that changes here are in a pull request by themselves, that it consists of
# two commits:
#
# 1. The changes to this file
# 2. Changes made by running `python3 -m pre_commit run --all-files`.
#
# Then each time your pull request is blocked by a merge conflict, do the
# following steps:
#
# git reset HEAD^1 && git checkout -f # discard the change commit
# git rebase main # re-apply other people's changes
# python3 -m pre_commit run --all-files # re-run the rules
# git add . # add the newly changed files
# git commit -m 'apply pre-commit' # commit it
# git push -f # force push back to your branch
#
# Keep in mind you may have to do this a few times, as changes here may impact
# other pull requests. Try to keep it up-to-date so they can go in when it'll
# cause least disruption.
#
# /WARNING!
exclude: build|stubs|^bot/templates/$|openassistant/templates|docs/docs/api/openapi.json|scripts/postprocessing/regex_pii_detector.py
default_language_version:
python: python3
ci:
autofix_prs: true
autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions"
autoupdate_schedule: quarterly
skip: ["next-lint-website"] # list of hook ids to skip only in pre-commit.ci
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: trailing-whitespace
- id: check-ast
- id: check-yaml
# Always check YAML but skip a few YAML files that are auto-generated
# and which break the standard YAML check. The alternative would be to
# skip any unsafe errors (and thus break YAML compatibility) or use
# some other checker that may not work in general.
exclude: ^copilot/.*/addons/.*$
- id: check-json
- id: check-case-conflict
- id: detect-private-key
- id: fix-encoding-pragma
args: [--remove]
- id: forbid-submodules
- id: mixed-line-ending
- id: requirements-txt-fixer
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-byte-order-marker
- id: check-symlinks
- id: check-merge-conflict
- id: check-added-large-files
args: [--maxkb=1024]
- id: end-of-file-fixer
- repo: https://github.com/psf/black
rev: 23.3.0
hooks:
- id: black-jupyter
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort
args: ["--profile", "black", "--filter-files"]
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.263
hooks:
- id: ruff
args: ["--fix"]
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.7.1
hooks:
- id: prettier
args:
[
--prose-wrap=always,
--write,
--ignore-path=./website/.prettierignore,
]
- repo: local
hooks:
- id: next-lint-website
name: Lint website
files: ^website/
exclude: ^website/node_modules/
types_or: [javascript, jsx, ts, tsx]
language: node
pass_filenames: false
entry: website/next-lint.js
================================================
FILE: .python-version
================================================
3.10.8
================================================
FILE: .vscode/extensions.json
================================================
{
"recommendations": [
"ms-vscode-remote.remote-containers",
"ms-vscode-remote.vscode-remote-extensionpack",
"esbenp.prettier-vscode",
"inlang.vs-code-extension"
]
}
================================================
FILE: .vscode/launch.json
================================================
{
"version": "0.2.0",
"configurations": [
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"justMyCode": false
},
{
"name": "Python: Debug Tests",
"type": "python",
"request": "launch",
"program": "${file}",
"purpose": ["debug-test"],
"console": "integratedTerminal",
"justMyCode": true
},
{
"name": "Debug SFT Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_sft.py",
"args": ["--configs", "defaults", "oasst_export_eu", "gpt-neox"],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false"
}
},
{
"name": "Debug RM Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_rm.py",
"args": ["--configs", "defaults_rm", "oasst_export_eu", "debug"],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false",
"CUDA_VISIBLE_DEVICES": "1,2,3,4,5,6,7"
}
},
{
"name": "Debug RM Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_rm.py",
"args": [
"--configs",
"defaults_rm",
"oasst_export_eu",
"pythia-1B",
"--cache_dir",
"/home/ubuntu/data_cache",
"--use_flash_attention",
"true"
],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false",
"CUDA_VISIBLE_DEVICES": "1"
}
},
{
"name": "Debug Patching",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "models/test_patched_gpt_neox.py",
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"CUDA_VISIBLE_DEVICES": "0"
}
},
{
"name": "Debug RLHF",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"module": "accelerate.commands.launch",
"args": [
"--main_process_port",
"29506",
"--config_file",
"configs/accelerate_config.yaml",
"--num_processes",
"5",
"trainer_rl.py",
"--configs",
"defaults",
"defaults_rlhf",
"pythia_rlhf",
"oasst_export_latin_cyrillic_rlhf"
],
"console": "integratedTerminal",
"justMyCode": false,
"env": {
"CUDA_VISIBLE_DEVICES": "1,2,3,4,5",
"OMP_NUM_THREADS": "1"
}
}
]
}
================================================
FILE: .vscode/settings.json
================================================
{
"python.analysis.extraPaths": ["${workspaceFolder}/oasst-shared"],
"prettier.singleQuote": false,
"eslint.workingDirectories": ["./website", "./docs"],
"python.testing.pytestArgs": ["backend/tests", "oasst-shared/tests"],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
}
================================================
FILE: CODEOWNERS
================================================
* @yk @andreaskoepf
/website/ @AbdBarho @notmd @yk @andreaskoepf
/website/src/data/team.json @yk @andreaskoepf @fozziethebeat @AbdBarho @notmd @theblackcat102 @sanagno @olliestanley @andrewm4894
/model/ @theblackcat102 @sanagno @dvruette @andreaskoepf @yk @jordiclive @shahules786
/copilot/ @andreaskoepf @yk
/docs/ @andrewm4894 @olliestanley @andreaskoepf @yk
/.devcontainer/ @andrewm4894 @andreaskoepf @yk
/notebooks/ @andrewm4894 @olliestanley @andreaskoepf @yk
/data @Vechtomov @bitplane @ontocord @olliestanley @sedthh
/safety @SummerSigh @shahules786 @olliestanley
/inference/ @yk @andreaskoepf @olliestanley @AbdBarho
/backend/ @andreaskoepf @melvinebenezer @yk
/oasst-data/ @andreaskoepf @yk @olliestanley
/oasst-shared/ @andreaskoepf @melvinebenezer @yk @olliestanley @AbdBarho
docker-compose.yaml @andreaskoepf @melvinebenezer @yk @olliestanley @AbdBarho
================================================
FILE: CONTRIBUTING.md
================================================
Open-Assistant
:memo: NOTE: OpenAssistant is completed, and the project is now finished. Thank you to everyone who contributed! Check out our blog post for more information. The final published oasst2 dataset can be found on HuggingFace at OpenAssistant/oasst2
# Table of Contents
- [What is Open Assistant?](#what-is-open-assistant)
- [Useful Links](#useful-links)
- [How To Try It Out](#how-to-try-it-out)
- [The Vision](#the-vision)
- [The Plan](#the-plan)
- [How You Can Help](#how-you-can-help)
---
## What is Open Assistant?
Open Assistant is a project meant to give everyone access to a great chat based large language model.
We believe that by doing this we will create a revolution in innovation in language. In the same way that stable-diffusion helped the world make art and images in new ways we hope Open Assistant can help improve the world by improving language itself. # Useful Links - [Data Collection](https://open-assistant.io) - [Chat](https://open-assistant.io/chat) - [Project Documentation](https://projects.laion.ai/Open-Assistant/) ## How To Try It Out ### Chatting with the AI The chat frontend is now live [here](https://open-assistant.io/chat). Log in and start chatting! Please try to react with a thumbs up or down for the assistant's responses when chatting. ### Contributing to Data Collection The data collection frontend is now live [here](https://open-assistant.io/). Log in and start taking on tasks! We want to collect a high volume of quality data. By submitting, ranking, and labelling model prompts and responses you will be directly helping to improve the capabilities of Open Assistant. ### Running the Development Setup Locally (without chat) **You do not need to run the project locally unless you are contributing to the development process. The website link above will take you to the public website where you can use the data collection app and the chat.** If you would like to run the data collection app locally for development, you can set up an entire stack needed to run **Open-Assistant**, including the website, backend, and associated dependent services, with Docker. To start the demo, run this in the root directory of the repository (check [this FAQ](https://projects.laion.ai/Open-Assistant/docs/faq#docker-compose-instead-of-docker-compose) if you have problems): ```sh docker compose --profile ci up --build --attach-dependencies ``` > **Note:** when running on MacOS with an M1 chip you have to use: > `DB_PLATFORM=linux/x86_64 docker compose ...` Then, navigate to `http://localhost:3000` (It may take some time to boot up) and interact with the website. > **Note:** If an issue occurs with the build, please head to the > [FAQ](https://projects.laion.ai/Open-Assistant/docs/faq) and check out the > entries about Docker. > **Note:** When logging in via email, navigate to `http://localhost:1080` to > get the magic email login link. > **Note:** If you would like to run this in a standardized development > environment (a > ["devcontainer"](https://code.visualstudio.com/docs/devcontainers/containers)) > using > [vscode locally](https://code.visualstudio.com/docs/devcontainers/create-dev-container#_create-a-devcontainerjson-file) > or in a web browser using > [GitHub Codespaces](https://github.com/features/codespaces), you can use the > provided [`.devcontainer`](.devcontainer/) folder. ### Running the Development Setup Locally for Chat **You do not need to run the project locally unless you are contributing to the development process. The website link above will take you to the public website where you can use the data collection app and the chat.** **Also note that the local setup is only for development and is not meant to be used as a local chatbot, unless you know what you are doing.** If you _do_ know what you are doing, then see the `inference` folder for getting the inference system up and running, or have a look at `--profile inference` in addition to `--profile ci` in the above command. ## The Vision We are not going to stop at replicating ChatGPT. We want to build the assistant of the future, able to not only write email and cover letters, but do meaningful work, use APIs, dynamically research information, and much more, with the ability to be personalized and extended by anyone. And we want to do this in a way that is open and accessible, which means we must not only build a great assistant, but also make it small and efficient enough to run on consumer hardware. ## The Plan ##### We want to get to an initial MVP as fast as possible, by following the 3-steps outlined in the [InstructGPT paper](https://arxiv.org/abs/2203.02155) 1. Collect high-quality human generated Instruction-Fulfillment samples (prompt + response), goal >50k. We design a crowdsourced process to collect and reviewed prompts. We do not want to train on flooding/toxic/spam/junk/personal information data. We will have a leaderboard to motivate the community that shows progress and the most active users. Swag will be given to the top-contributors. 2. For each of the collected prompts we will sample multiple completions. Completions of one prompt will then be shown randomly to users to rank them from best to worst. Again this should happen crowd-sourced, e.g. we need to deal with unreliable potentially malicious users. At least multiple votes by independent users have to be collected to measure the overall agreement. The gathered ranking-data will be used to train a reward model. 3. Now follows the RLHF training phase based on the prompts and the reward model. We can then take the resulting model and continue with completion sampling step 2 for a next iteration. ### Slide Decks [Vision & Roadmap](https://docs.google.com/presentation/d/1n7IrAOVOqwdYgiYrXc8Sj0He8krn5MVZO_iLkCjTtu0/edit?usp=sharing) [Important Data Structures](https://docs.google.com/presentation/d/1iaX_nxasVWlvPiSNs0cllR9L_1neZq0RJxd6MFEalUY/edit?usp=sharing) ## How You Can Help All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Check out our [contributing guide](CONTRIBUTING.md) to get started. ================================================ FILE: ansible/.gitignore ================================================ dev/ dev-inference/ *.local.yaml ================================================ FILE: ansible/README.md ================================================ To test the ansible playbook on localhost run `ansible-playbook -i test.inventory.ini dev.yaml`.\ In case you're missing the ansible docker dependency install it with `ansible-galaxy collection install community.docker`.\ Point Redis Insights to the Redis database by visiting localhost:8001 in a browser and select "I already have a database" followed by "Connect to a Redis Database".\ For host, port and name fill in `oasst-redis`, `6379` and `redis`. ================================================ FILE: ansible/deploy-to-node.yaml ================================================ # ansible playbook to set up some docker containers - name: Deploy to node hosts: dev gather_facts: true vars: stack_name: "dev" image_tag: latest backend_port: 8080 website_port: 3000 postgres_password: postgres web_api_key: "1234" tasks: # reduces the chance of oom when pulling new images - name: Clear unused images community.docker.docker_prune: images: true - name: Create network community.docker.docker_network: name: "oasst-{{ stack_name }}" state: present driver: bridge - name: Create stack files directory ansible.builtin.file: path: "./{{ stack_name }}" state: directory mode: 0755 - name: Copy redis.conf to managed node ansible.builtin.copy: src: ./redis.conf dest: "./{{ stack_name }}/redis.conf" mode: 0644 - name: Set up Redis community.docker.docker_container: name: "oasst-{{ stack_name }}-redis" image: redis state: started recreate: "{{ (stack_name == 'dev') | bool }}" restart_policy: always network_mode: "oasst-{{ stack_name }}" healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 2s timeout: 2s retries: 10 command: redis-server /usr/local/etc/redis/redis.conf volumes: - "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf" - name: Create volumes for postgres community.docker.docker_volume: name: "oasst-{{ stack_name }}-postgres-{{ item.name }}" state: present loop: - name: backend - name: web - name: Create postgres containers community.docker.docker_container: name: "oasst-{{ stack_name }}-postgres-{{ item.name }}" image: ghcr.io/laion-ai/open-assistant/oasst-postgres platform: linux/x86_64 state: started pull: true recreate: "{{ (stack_name == 'dev') | bool }}" restart_policy: always network_mode: "oasst-{{ stack_name }}" env: POSTGRES_USER: postgres POSTGRES_PASSWORD: "{{ postgres_password }}" POSTGRES_DB: postgres S3_BUCKET_NAME: "{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}" S3_PREFIX: "{{ item.name }}" AWS_ACCESS_KEY_ID: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}" AWS_SECRET_ACCESS_KEY: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}" AWS_DEFAULT_REGION: "{{ lookup('ansible.builtin.env', 'S3_REGION') }}" volumes: - "oasst-{{ stack_name }}-postgres-{{ item.name }}:/var/lib/postgresql/data" healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] interval: 2s timeout: 2s retries: 10 shm_size: 1G loop: - name: backend - name: web - name: Run the oasst oasst-backend community.docker.docker_container: name: "oasst-{{ stack_name }}-backend" image: "ghcr.io/laion-ai/open-assistant/oasst-backend:{{ image_tag }}" platform: linux/x86_64 state: started recreate: true pull: true restart_policy: always network_mode: "oasst-{{ stack_name }}" env: POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend" POSTGRES_PASSWORD: "{{ postgres_password }}" REDIS_HOST: "oasst-{{ stack_name }}-redis" DEBUG_USE_SEED_DATA: "{{ 'true' if stack_name == 'dev' else 'false' }}" DEBUG_ALLOW_SELF_LABELING: "{{ 'true' if stack_name == 'dev' else 'false' }}" MAX_WORKERS: "1" RATE_LIMIT: "{{ 'false' if stack_name == 'dev' else 'true' }}" DEBUG_SKIP_EMBEDDING_COMPUTATION: "true" DEBUG_SKIP_TOXICITY_CALCULATION: "{{ lookup('ansible.builtin.env', 'SKIP_TOXICITY_CALCULATION') | default('true', true) }}" OFFICIAL_WEB_API_KEY: "{{ web_api_key }}" TREE_MANAGER__INIT_PROMPT_DISABLED_LANGS: "{{ lookup('ansible.builtin.env', 'INIT_PROMPT_DISABLED_LANGS') | default('', true) }}" TREE_MANAGER__MAX_ACTIVE_TREES: "{{ lookup('ansible.builtin.env', 'MAX_ACTIVE_TREES') | default('10', true) }}" TREE_MANAGER__MAX_INITIAL_PROMPT_REVIEW: "{{ lookup('ansible.builtin.env', 'MAX_INITIAL_PROMPT_REVIEW') | default('100', true) }}" TREE_MANAGER__MAX_TREE_DEPTH: "{{ lookup('ansible.builtin.env', 'MAX_TREE_DEPTH') | default('5', true) }}" TREE_MANAGER__GOAL_TREE_SIZE: "{{ lookup('ansible.builtin.env', 'GOAL_TREE_SIZE') | default('15', true) }}" TREE_MANAGER__MAX_CHILDREN_COUNT: "{{ lookup('ansible.builtin.env', 'MAX_CHILDREN_COUNT') | default('3', true) }}" TREE_MANAGER__LONELY_CHILDREN_COUNT: "{{ lookup('ansible.builtin.env', 'LONELY_CHILDREN_COUNT') | default('2', true) }}" TREE_MANAGER__P_LONELY_CHILD_EXTENSION: "{{ lookup('ansible.builtin.env', 'P_LONELY_CHILD_EXTENSION') | default('0.75', true) }}" TREE_MANAGER__P_ACTIVATE_BACKLOG_TREE: "{{ lookup('ansible.builtin.env', 'P_ACTIVATE_BACKLOG_TREE') | default('0.1', true) }}" TREE_MANAGER__NUM_REQUIRED_RANKINGS: "{{ lookup('ansible.builtin.env', 'NUM_REQUIRED_RANKINGS') | default('3', true) }}" TREE_MANAGER__NUM_REVIEWS_REPLY: "{{ lookup('ansible.builtin.env', 'NUM_REVIEWS_REPLY') | default('3', true) }}" TREE_MANAGER__NUM_REVIEWS_INITIAL_PROMPT: "{{ lookup('ansible.builtin.env', 'NUM_REVIEWS_INITIAL_PROMPT') | default('3', true) }}" TREE_MANAGER__MIN_ACTIVE_RANKINGS_PER_LANG: "{{ lookup('ansible.builtin.env', 'MIN_ACTIVE_RANKINGS_PER_LANG') | default('0', true) }}" TREE_MANAGER__MAX_PROMPT_LOTTERY_WAITING: "{{ lookup('ansible.builtin.env', 'MAX_PROMPT_LOTTERY_WAITING') | default('250', true) }}" MESSAGE_SIZE_LIMIT: "{{ lookup('ansible.builtin.env', 'MESSAGE_SIZE_LIMIT') | default('2000', true) }}" USER_STATS_INTERVAL_DAY: "{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_DAY') | default('5', true) }}" USER_STATS_INTERVAL_WEEK: "{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_WEEK') | default('15', true) }}" USER_STATS_INTERVAL_MONTH: "{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_MONTH') | default('60', true) }}" USER_STATS_INTERVAL_TOTAL: "{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_TOTAL') | default('240', true) }}" LOGURU_LEVEL: "{{ lookup('ansible.builtin.env', 'LOGURU_LEVEL') | default('INFO', true) }}" BACKEND_CORS_ORIGINS_CSV: "{{ lookup('ansible.builtin.env', 'BACKEND_CORS_ORIGINS') }}" DISCORD_API_KEY: "{{ lookup('ansible.builtin.env', 'DISCORD_API_KEY') }}" DISCORD_CHANNEL_ID: "{{ lookup('ansible.builtin.env', 'DISCORD_CHANNEL_ID') }}" ports: - "{{ backend_port }}:8080" - name: Setup Celery worker community.docker.docker_container: name: "oasst-{{ stack_name }}-celery-worker" image: "ghcr.io/laion-ai/open-assistant/oasst-backend-worker:{{ image_tag }}" state: started pull: true restart_policy: always network_mode: "oasst-{{ stack_name }}" env: CELERY_BROKER_URL: "redis://oasst-{{ stack_name }}-redis:6379/0" CELERY_RESULT_BACKEND: "redis://oasst-{{ stack_name }}-redis:6379/0" POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend" POSTGRES_PASSWORD: "{{ postgres_password }}" REDIS_HOST: "oasst-{{ stack_name }}-redis" MAX_WORKERS: "1" command: celery -A oasst_backend.celery_worker worker -l info - name: Setup Celery worker beat community.docker.docker_container: name: "oasst-{{ stack_name }}-celery-worker-beat" image: "ghcr.io/laion-ai/open-assistant/oasst-backend-worker:{{ image_tag }}" state: started pull: true restart_policy: always network_mode: "oasst-{{ stack_name }}" env: CELERY_BROKER_URL: "redis://oasst-{{ stack_name }}-redis:6379/0" CELERY_RESULT_BACKEND: "redis://oasst-{{ stack_name }}-redis:6379/0" POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend" POSTGRES_PASSWORD: "{{ postgres_password }}" REDIS_HOST: "oasst-{{ stack_name }}-redis" MAX_WORKERS: "1" command: celery -A oasst_backend.celery_worker beat -l INFO - name: Run the oasst oasst-web frontend community.docker.docker_container: name: "oasst-{{ stack_name }}-web" image: "ghcr.io/laion-ai/open-assistant/oasst-web:{{ image_tag }}" state: started recreate: true pull: true restart_policy: always network_mode: "oasst-{{ stack_name }}" env: ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}" MODERATOR_USERS: "{{ lookup('ansible.builtin.env', 'WEB_MODERATOR_USERS') }}" DATABASE_URL: "postgres://postgres:{{ postgres_password }}@oasst-{{ stack_name }}-postgres-web/postgres" DEBUG_LOGIN: "{{ 'true' if stack_name == 'dev' else 'false' }}" DISCORD_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}" DISCORD_CLIENT_SECRET: "{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_SECRET') }}" GOOGLE_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'WEB_GOOGLE_CLIENT_ID') }}" GOOGLE_CLIENT_SECRET: "{{ lookup('ansible.builtin.env', 'WEB_GOOGLE_CLIENT_SECRET') }}" EMAIL_FROM: open-assistant@laion.ai EMAIL_SERVER_HOST: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_HOST') }}" EMAIL_SERVER_PASSWORD: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PASSWORD') }}" EMAIL_SERVER_PORT: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PORT') }}" EMAIL_SERVER_USER: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}" FASTAPI_URL: "{{ lookup('ansible.builtin.env', 'BACKEND_URL') }}" FASTAPI_KEY: "{{ web_api_key }}" NEXTAUTH_SECRET: "{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}" NEXTAUTH_URL: "{{ 'https://open-assistant.io/' if stack_name == 'production' else ('https://web.' + stack_name + '.open-assistant.io/') }}" CLOUDFLARE_CAPTCHA_SITE_KEY: "{{ lookup('ansible.builtin.env', 'WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY') }}" CLOUDFLARE_CAPTCHA_SECRET_KEY: "{{ lookup('ansible.builtin.env', 'WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY') }}" ENABLE_EMAIL_SIGNIN_CAPTCHA: "{{ lookup('ansible.builtin.env', 'WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA') }}" ENABLE_EMAIL_SIGNIN: "{{ lookup('ansible.builtin.env', 'WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN') }}" MAINTENANCE_MODE: "{{ lookup('ansible.builtin.env', 'MAINTENANCE_MODE') }}" INFERENCE_SERVER_HOST: "{{ lookup('ansible.builtin.env', 'WEB_INFERENCE_SERVER_HOST') }}" INFERENCE_SERVER_API_KEY: "{{ lookup('ansible.builtin.env', 'WEB_INFERENCE_SERVER_API_KEY') }}" ENABLE_CHAT: "{{ lookup('ansible.builtin.env', 'WEB_ENABLE_CHAT') }}" BYE: "{{ lookup('ansible.builtin.env', 'WEB_BYE') }}" ENABLE_DRAFTS_WITH_PLUGINS: "{{ lookup('ansible.builtin.env', 'WEB_ENABLE_DRAFTS_WITH_PLUGINS')}}" NUM_GENERATED_DRAFTS: "{{ lookup('ansible.builtin.env', 'WEB_NUM_GENERATED_DRAFTS') }}" CURRENT_ANNOUNCEMENT: "{{ lookup('ansible.builtin.env', 'WEB_CURRENT_ANNOUNCEMENT') }}" ports: - "{{ website_port }}:3000" command: bash wait-for-postgres.sh node server.js ================================================ FILE: ansible/inference/deploy-server.yaml ================================================ # Ansible playbook to setup inference server - name: Deploy inference hosts: dev gather_facts: true vars: stack_name: "dev" image_tag: latest server_port: 8080 tasks: - name: Create network community.docker.docker_network: name: "oasst-{{ stack_name }}" state: present driver: bridge - name: Create stack files directory ansible.builtin.file: path: "./{{ stack_name }}-inference" state: directory mode: 0755 - name: Copy redis.conf to managed node ansible.builtin.copy: src: ./redis.conf dest: "./{{ stack_name }}-inference/redis.conf" mode: 0644 - name: Set up inference Redis community.docker.docker_container: name: "oasst-inference-{{ stack_name }}-redis" image: redis state: started recreate: "{{ (stack_name == 'dev') | bool }}" restart_policy: always network_mode: "oasst-{{ stack_name }}" healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 2s timeout: 2s retries: 10 command: redis-server /usr/local/etc/redis/redis.conf volumes: - "./{{ stack_name }}-inference/redis.conf:/usr/local/etc/redis/redis.conf" - name: Create volumes for inference postgres community.docker.docker_volume: name: "oasst-inference-{{ stack_name }}-postgres" state: present - name: Create postgres containers community.docker.docker_container: name: "oasst-inference-{{ stack_name }}-postgres" image: ghcr.io/laion-ai/open-assistant/oasst-postgres state: started pull: true recreate: "{{ (stack_name == 'dev') | bool }}" restart_policy: always network_mode: "oasst-{{ stack_name }}" env: POSTGRES_USER: postgres POSTGRES_PASSWORD: "{{ lookup('ansible.builtin.env', 'INFERENCE_POSTGRES_PASSWORD') | default('postgres', true) }}" POSTGRES_DB: postgres S3_BUCKET_NAME: "{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}" S3_PREFIX: "inference" AWS_ACCESS_KEY_ID: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}" AWS_SECRET_ACCESS_KEY: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}" AWS_DEFAULT_REGION: "{{ lookup('ansible.builtin.env', 'S3_REGION') }}" volumes: - "oasst-inference-{{ stack_name }}-postgres:/var/lib/postgresql/data" healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] interval: 2s timeout: 2s retries: 10 shm_size: 1G - name: Run the oasst inference-server community.docker.docker_container: name: "oasst-inference-{{ stack_name }}-server" image: "ghcr.io/laion-ai/open-assistant/oasst-inference-server:{{ image_tag }}" state: started recreate: true pull: true restart_policy: always network_mode: "oasst-{{ stack_name }}" env: POSTGRES_HOST: "oasst-inference-{{ stack_name }}-postgres" POSTGRES_PASSWORD: "{{ lookup('ansible.builtin.env', 'INFERENCE_POSTGRES_PASSWORD') | default('postgres', true) }}" REDIS_HOST: "oasst-inference-{{ stack_name }}-redis" LOGURU_LEVEL: "{{ lookup('ansible.builtin.env', 'INFERENCE_LOG_LEVEL') | default('INFO', true) }}" DEBUG_API_KEYS: "{{ lookup('ansible.builtin.env', 'INFERENCE_DEBUG_API_KEYS') | default('', true) | string }}" ALLOW_DEBUG_AUTH: "{{ lookup('ansible.builtin.env', 'INFERENCE_ALLOW_DEBUG_AUTH') | default('False', true) | string }}" ROOT_TOKEN: "{{ lookup('ansible.builtin.env', 'INFERENCE_ROOT_TOKEN') | default('1234', true) }}" API_ROOT: "{{ lookup('ansible.builtin.env', 'INFERENCE_API_ROOT') | default('https://inference.dev.open-assistant.io', true) }}" TRUSTED_CLIENT_KEYS: "{{ lookup('ansible.builtin.env', 'INFERENCE_TRUSTED_CLIENT_KEYS') | default('', true) }}" AUTH_SALT: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_SALT') | default('', true) }}" AUTH_SECRET: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_SECRET') | default('', true) }}" AUTH_DISCORD_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_DISCORD_CLIENT_ID') | default('', true) }}" AUTH_DISCORD_CLIENT_SECRET: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_DISCORD_CLIENT_SECRET') | default('', true) }}" AUTH_GITHUB_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_GITHUB_CLIENT_ID') | default('', true) }}" AUTH_GITHUB_CLIENT_SECRET: "{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_GITHUB_CLIENT_SECRET') | default('', true) }}" INFERENCE_CORS_ORIGINS: "{{ lookup('ansible.builtin.env', 'INFERENCE_CORS_ORIGINS') | default('*', true) }}" ALLOWED_MODEL_CONFIG_NAMES: "{{ lookup('ansible.builtin.env', 'INFERENCE_ALLOWED_MODEL_CONFIG_NAMES') | default('*', true) }}" ASSISTANT_MESSAGE_TIMEOUT: "{{ lookup('ansible.builtin.env', 'INFERENCE_ASSISTANT_MESSAGE_TIMEOUT') | default(120, true) }}" MESSAGE_QUEUE_EXPIRE: "{{ lookup('ansible.builtin.env', 'INFERENCE_MESSAGE_QUEUE_EXPIRE') | default(120, true) }}" WORK_QUEUE_MAX_SIZE: "{{ lookup('ansible.builtin.env', 'INFERENCE_WORK_QUEUE_MAX_SIZE') | default(100, true) }}" ENABLE_SAFETY: "{{ lookup('ansible.builtin.env', 'INFERENCE_ENABLE_SAFETY') | default('False', true) | string }}" GUNICORN_WORKERS: "{{ lookup('ansible.builtin.env', 'INFERENCE_GUNICORN_WORKERS') | default(1, true) }}" CHAT_MAX_MESSAGES: "{{ lookup('ansible.builtin.env', 'INFERENCE_CHAT_MAX_MESSAGES') | default('', true) }}" MESSAGE_MAX_LENGTH: "{{ lookup('ansible.builtin.env', 'INFERENCE_MESSAGE_MAX_LENGTH') | default('', true) }}" PLUGIN_MAX_DEPTH: "{{ lookup('ansible.builtin.env', 'INFERENCE_PLUGIN_MAX_DEPTH') | default(4, true) }}" ports: - "{{ server_port }}:8080" ================================================ FILE: ansible/inference/deploy-worker.yaml ================================================ # Ansible playbook to setup inference worker - name: Deploy inference hosts: dev gather_facts: true vars: stack_name: "dev" image_tag: latest backend_url: "ws://localhost:8000" api_key: "1234" parallelism: 4 tasks: - name: Create network community.docker.docker_network: name: "oasst-worker-{{ stack_name }}" state: present driver: bridge - name: Create stack files directory ansible.builtin.file: path: "./{{ stack_name }}" state: directory mode: 0755 - name: Run the oasst inference worker community.docker.docker_container: name: "oasst-worker-{{ stack_name }}" image: "ghcr.io/laion-ai/open-assistant/oasst-inference-worker-full:{{ image_tag }}" state: started recreate: true pull: true restart_policy: always network_mode: "oasst-worker-{{ stack_name }}" env: BACKEND_URL: "{{ backend_url }}" API_KEY: "{{ api_key }}" PARALLELISM: "{{ parallelism }}" ================================================ FILE: ansible/inference/redis.conf ================================================ maxmemory 1024mb maxmemory-policy allkeys-lru ================================================ FILE: ansible/inference/test.inventory.ini ================================================ [test] dev ansible_connection=local ================================================ FILE: ansible/pgbackrest.conf ================================================ [oasst] pg1-path=/var/lib/postgresql/data [global] repo1-retention-full=3 repo1-type=s3 repo1-path=/oasst-prod repo1-s3-region=us-east-1 repo1-s3-endpoint=s3.amazonaws.com # repo1-s3-bucket=$S3_BUCKET_NAME # repo1-s3-key=$AWS_ACCESS_KEY # repo1-s3-key-secret=$AWS_SECRET_KEY # Force a checkpoint to start backup immediately. start-fast=y # Use delta restore. delta=y # Enable ZSTD compression. compress-type=zst compress-level=6 log-level-console=info log-level-file=debug ================================================ FILE: ansible/redis.conf ================================================ maxmemory 100mb maxmemory-policy allkeys-lru ================================================ FILE: ansible/test.inventory.ini ================================================ [test] dev ansible_connection=local ================================================ FILE: backend/.gitignore ================================================ __pycache__ .env notes.txt venv celerybeat-schedule.* ================================================ FILE: backend/README.md ================================================  # Open-Assistant REST Backend ## Backend Development Setup ### Local Database In root directory, run `docker compose --profile backend-dev up --build --attach-dependencies` to start a database. The default settings are already configured to connect to the database at `localhost:5432`. (See [FAQ](https://projects.laion.ai/Open-Assistant/docs/faq#enable-dockers-buildkit-backend) if you face any docker problems). > **Note:** when running on MacOS with an M1 chip you have to use: > `DB_PLATFORM=linux/x86_64 docker compose ...` Python 3.10 is required. It is recommended to use `pyenv` which will recognise the `.python-version` in the project root directory. ### Python Packages Next, to install all requirements, You can run 1. `pip install -r backend/requirements.txt` 2. `pip install -e ./oasst-shared/.` 3. `pip install -e ./oasst-data/.` 4. `./scripts/backend-development/run-local.sh` to run the backend. This will start the backend server at `http://localhost:8080`. ## REST Server Configuration - Generate a new environment variable file `.env` by coping the content of the [.env.example](.env.example) file. - Update the values of the environment variables in the `.env` file by setting the DATABASE_URI to you local database URI. - Update the rest of the environment variables according to your needs. ## Running the REST Server locally for development Have a look into the main `README.md` file for more information on how to set up the backend for development. Use the scripts within the scripts/backend-development folder to run the BE API locally. ## Alembic To create an Alembic database migration script after sql-models were modified run `alembic revision --autogenerate -m "..."` ("..." is what you did) in the `/backend` directory. Then edit the newly created file. See [here](https://alembic.sqlalchemy.org/en/latest/tutorial.html) for more information. ## API Documentation Once you have successfully started the backend server, you can access the default api docs at `localhost:8080/docs`. If you need to update the exported openapi.json in the docs/ folder you can run below command to `wget` them from the relevant local fastapi endpoint. This will enable anyone to just see API docs via something like [Swagger.io](https://editor.swagger.io/?url=https://raw.githubusercontent.com/LAION-AI/Open-Assistant/main/docs/docs/api/openapi.json) without having to actually set up and run a development backend. ```bash # save openapi.json to docs/docs/api/ wget localhost:8080/api/v1/openapi.json -O docs/docs/api/backend-openapi.json ``` Note: The api docs should be automatically updated by the `test-api-contract.yaml` workflow. (TODO) ## Running Celery Worker(s) for API and periodic tasks Celery workers are used for Huggingface API calls like toxicity and feature extraction. Celery Beat along with worker is used for periodic tasks like user streak update To run APIs locally - update HUGGING_FACE_API_KEY in backend/oasst_backend/config.py with the correct API_KEY - `export DEBUG_SKIP_TOXICITY_CALCULATION=False` and `export DEBUG_SKIP_EMBEDDING_COMPUTATION=False`in `scripts/backend-development/run-local.sh` - run start_worker.sh in backend dir - to see logs , use `tail -f celery.log` and `tail -f celery.beat.log` In CI - set `DEBUG_SKIP_TOXICITY_CALCULATION=False` and `DEBUG_SKIP_EMBEDDING_COMPUTATION=False` in docker-compose.yaml - Two Docker instances are created. One for Beat and other for the worker - Logs can be viewed like other docker instances ## Exporting Data When you have collected some data in the backend database, you can export it using the `export.py` script provided in this directory. This can be run from the command line using an Python environment with the same requirements as the backend itself. The script connects to the database in the same manner as the backend and therefore uses the same environmental variables. A simple usage of the script, to export all English trees which successfully passed the review process, may look like: ```bash python export.py --lang en --export-file output.jsonl ``` There are many options available to filter the data which can be found in the help message of the script: `python export.py --help`. **Why isn't my export working?** Common issues include (WIP): - The messages have not passed the review process yet so the trees are not ready for export. This can be solved by including the `--include-spam` flag. ================================================ FILE: backend/alembic/README ================================================ Generic single-database configuration. ================================================ FILE: backend/alembic/env.py ================================================ from logging.config import fileConfig import sqlmodel from alembic import context from oasst_backend import models # noqa: F401 from sqlalchemy import engine_from_config, pool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. if config.config_file_name is not None: fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata target_metadata = sqlmodel.SQLModel.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc. def run_migrations_offline() -> None: """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation we don't even need a DBAPI to be available. Calls to context.execute() here emit the given string to the script output. """ url = config.get_main_option("sqlalchemy.url") context.configure( url=url, target_metadata=target_metadata, literal_binds=True, dialect_opts={"paramstyle": "named"}, ) with context.begin_transaction(): context.run_migrations() def run_migrations_online() -> None: """Run migrations in 'online' mode. In this scenario we need to create an Engine and associate a connection with the context. """ connectable = engine_from_config( config.get_section(config.config_ini_section), prefix="sqlalchemy.", poolclass=pool.NullPool, ) with connectable.connect() as connection: context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): context.get_context()._ensure_version_table() connection.execute("LOCK TABLE alembic_version IN ACCESS EXCLUSIVE MODE") context.run_migrations() if context.is_offline_mode(): run_migrations_offline() else: run_migrations_online() ================================================ FILE: backend/alembic/script.py.mako ================================================ """${message} Revision ID: ${up_revision} Revises: ${down_revision | comma,n} Create Date: ${create_date} """ from alembic import op import sqlalchemy as sa import sqlmodel ${imports if imports else ""} # revision identifiers, used by Alembic. revision = ${repr(up_revision)} down_revision = ${repr(down_revision)} branch_labels = ${repr(branch_labels)} depends_on = ${repr(depends_on)} def upgrade() -> None: ${upgrades if upgrades else "pass"} def downgrade() -> None: ${downgrades if downgrades else "pass"} ================================================ FILE: backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py ================================================ """first revision Revision ID: 23e5fea252dd Revises: Create Date: 2022-12-12 12:47:28.801354 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "23e5fea252dd" down_revision = None branch_labels = None depends_on = None def upgrade() -> None: op.create_table( "service_client", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("name", sa.String(200), nullable=False), sa.Column("service_admin_email", sa.String(128), nullable=True), sa.Column("api_key", sa.String(300), nullable=False), sa.Column("can_append", sa.Boolean, nullable=False, server_default="true"), sa.Column("can_write", sa.Boolean, nullable=False, server_default="false"), sa.Column("can_delete", sa.Boolean, nullable=False, server_default="false"), sa.Column("can_read", sa.Boolean, nullable=False, server_default="true"), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_service_client_api_key"), "service_client", ["api_key"], unique=True) op.create_table( "labeler", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("display_name", sa.String(96), nullable=False), sa.Column("discord_username", sa.String(96), nullable=True), sa.Column( "created_date", sa.DateTime, nullable=False, server_default=sa.func.current_timestamp(), ), sa.Column("is_enabled", sa.Boolean, nullable=False, server_default="true"), sa.Column("notes", sa.String(10 * 1024), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("discord_username"), ) op.create_table( "prompt", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("labeler_id", sa.Integer, nullable=False), sa.Column("prompt", sa.Text, nullable=False), sa.Column("response", sa.Text, nullable=True), sa.Column("lang", sa.String(32), nullable=True), sa.Column( "created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp(), ), sa.ForeignKeyConstraint( ["labeler_id"], ["labeler.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("prompt_labeler_id"), "prompt", ["labeler_id"], unique=False) def downgrade() -> None: op.drop_index(op.f("prompt_labeler_id"), table_name="prompt") op.drop_table("prompt") op.drop_table("labeler") op.drop_index(op.f("ix_service_client_api_key"), table_name="service_client") op.drop_table("service_client") ================================================ FILE: backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py ================================================ """v1 db structure Revision ID: cd7de470586e Revises: 23e5fea252dd Create Date: 2022-12-15 11:15:32.830225 """ import uuid import sqlalchemy as sa from alembic import op from sqlalchemy.dialects.postgresql import JSONB, UUID # revision identifiers, used by Alembic. revision = "cd7de470586e" down_revision = "23e5fea252dd" branch_labels = None depends_on = None def upgrade() -> None: # remove database objects op.drop_index(op.f("prompt_labeler_id"), table_name="prompt") op.drop_table("prompt") op.drop_table("labeler") op.drop_index(op.f("ix_service_client_api_key"), table_name="service_client") op.drop_table("service_client") # wreate new database structure op.create_table( "api_client", sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")), sa.Column("api_key", sa.String(512), nullable=False), sa.Column("description", sa.String(256), nullable=False), sa.Column("admin_email", sa.String(256), nullable=True), sa.Column("enabled", sa.Boolean, default=True, nullable=False), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_api_client_api_key"), "api_client", ["api_key"], unique=True) op.create_table( "person", sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")), sa.Column("username", sa.String(128), nullable=False), # unique in combination with api_client_id sa.Column("display_name", sa.String(256), nullable=False), # cached last seen display_name sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), sa.Column("api_client_id", UUID(as_uuid=True), nullable=False), sa.PrimaryKeyConstraint("id"), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]), ) op.create_index(op.f("ix_person_username"), "person", ["api_client_id", "username"], unique=True) op.create_table( "person_stats", sa.Column("person_id", UUID(as_uuid=True)), sa.Column("leader_score", sa.Integer, default=0, nullable=False), # determines position on leader board sa.Column("modified_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), sa.Column("reactions", sa.Integer, default=0, nullable=False), # reactions sent by user sa.Column("posts", sa.Integer, default=0, nullable=False), # posts sent by user sa.Column("upvotes", sa.Integer, default=0, nullable=False), # received upvotes (form other users) sa.Column("downvotes", sa.Integer, default=0, nullable=False), # received downvotes (from other users) sa.Column("work_reward", sa.Integer, default=0, nullable=False), # reward for workpackage completions sa.Column("compare_wins", sa.Integer, default=0, nullable=False), # num times user's post won compare tasks sa.Column("compare_losses", sa.Integer, default=0, nullable=False), # num times users's post lost compare tasks sa.PrimaryKeyConstraint("person_id"), sa.ForeignKeyConstraint(["person_id"], ["person.id"]), ) op.create_table( "work_package", sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")), sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), sa.Column("expiry_date", sa.DateTime(), nullable=True), sa.Column("person_id", UUID(as_uuid=True), nullable=True), sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=False), sa.Column("api_client_id", UUID(as_uuid=True), nullable=False), sa.PrimaryKeyConstraint("id"), sa.ForeignKeyConstraint(["person_id"], ["person.id"]), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]), ) op.create_index(op.f("ix_work_package_person_id"), "work_package", ["person_id"], unique=False) op.create_table( "post", sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")), sa.Column("parent_id", UUID(as_uuid=True), nullable=True), # root posts have NULL parent sa.Column("thread_id", UUID(as_uuid=True), nullable=False), # id of thread root sa.Column("workpackage_id", UUID(as_uuid=True), nullable=True), # workpackage id to pass to handler on reply sa.Column("person_id", UUID(as_uuid=True), nullable=True), # sender (recipients are part of payload) sa.Column("api_client_id", UUID(as_uuid=True), nullable=False), sa.Column("role", sa.String(128), nullable=False), # 'assistant', 'user' or something else sa.Column("frontend_post_id", sa.String(200), nullable=False), # unique together with api_client_id sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=True), sa.PrimaryKeyConstraint("id"), sa.ForeignKeyConstraint(["person_id"], ["person.id"]), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]), ) op.create_index(op.f("ix_post_frontend_post_id"), "post", ["api_client_id", "frontend_post_id"], unique=True) op.create_index(op.f("ix_post_thread_id"), "post", ["thread_id"], unique=False) op.create_index(op.f("ix_post_workpackage_id"), "post", ["workpackage_id"], unique=False) op.create_index(op.f("ix_post_person_id"), "post", ["person_id"], unique=False) op.create_table( "post_reaction", sa.Column("post_id", UUID(as_uuid=True), nullable=False), sa.Column("person_id", UUID(as_uuid=True), nullable=False), # sender (recipients are part of payload) sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=False), sa.Column("api_client_id", UUID(as_uuid=True), nullable=False), sa.PrimaryKeyConstraint("post_id", "person_id"), sa.ForeignKeyConstraint(["post_id"], ["post.id"]), sa.ForeignKeyConstraint(["person_id"], ["person.id"]), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]), ) def downgrade() -> None: op.drop_table("post_reaction") op.drop_index("ix_post_person_id") op.drop_index("ix_post_workpackage_id") op.drop_index("ix_post_thread_id") op.drop_index("ix_post_frontend_post_id") op.drop_table("post") op.drop_index("ix_work_package_person_id") op.drop_table("work_package") op.drop_table("person_stats") op.drop_index("ix_person_username") op.drop_table("person") op.drop_index("ix_api_client_api_key") op.drop_table("api_client") op.create_table( "service_client", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("name", sa.String(200), nullable=False), sa.Column("service_admin_email", sa.String(128), nullable=True), sa.Column("api_key", sa.String(300), nullable=False), sa.Column("can_append", sa.Boolean, nullable=False, server_default="true"), sa.Column("can_write", sa.Boolean, nullable=False, server_default="false"), sa.Column("can_delete", sa.Boolean, nullable=False, server_default="false"), sa.Column("can_read", sa.Boolean, nullable=False, server_default="true"), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_service_client_api_key"), "service_client", ["api_key"], unique=True) op.create_table( "labeler", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("display_name", sa.String(96), nullable=False), sa.Column("discord_username", sa.String(96), nullable=True), sa.Column( "created_date", sa.DateTime, nullable=False, server_default=sa.func.current_timestamp(), ), sa.Column("is_enabled", sa.Boolean, nullable=False, server_default="true"), sa.Column("notes", sa.String(10 * 1024), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("discord_username"), ) op.create_table( "prompt", sa.Column("id", sa.Integer, sa.Identity()), sa.Column("labeler_id", sa.Integer, nullable=False), sa.Column("prompt", sa.Text, nullable=False), sa.Column("response", sa.Text, nullable=True), sa.Column("lang", sa.String(32), nullable=True), sa.Column( "created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp(), ), sa.ForeignKeyConstraint( ["labeler_id"], ["labeler.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("prompt_labeler_id"), "prompt", ["labeler_id"], unique=False) ================================================ FILE: backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py ================================================ """add auth_method to person Revision ID: 6368515778c5 Revises: cd7de470586e Create Date: 2022-12-17 17:57:33.022549 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "6368515778c5" down_revision = "cd7de470586e" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("person", sa.Column("auth_method", sa.String(length=128), nullable=True)) op.execute("UPDATE person SET auth_method = 'local'") op.alter_column("person", "auth_method", nullable=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("person", "auth_method") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py ================================================ """add_auth_method_to_ix_person_username Revision ID: 0daec5f8135f Revises: 6368515778c5 Create Date: 2022-12-22 18:35:59.609013 """ import sqlalchemy as sa # noqa: F401 from alembic import op # revision identifiers, used by Alembic. revision = "0daec5f8135f" down_revision = "6368515778c5" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_person_username", table_name="person") op.create_index("ix_person_username", "person", ["api_client_id", "username", "auth_method"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_person_username", table_name="person") op.create_index("ix_person_username", "person", ["api_client_id", "username"], unique=False) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py ================================================ """Adds text labels table. Revision ID: 067c4002f2d9 Revises: 0daec5f8135f Create Date: 2022-12-25 17:05:21.208843 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "067c4002f2d9" down_revision = "0daec5f8135f" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "text_labels", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("post_id", postgresql.UUID(as_uuid=True), nullable=True), sa.Column("labels", postgresql.JSONB(astext_type=sa.Text()), nullable=True), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("text", sqlmodel.sql.sqltypes.AutoString(length=65536), nullable=False), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.ForeignKeyConstraint( ["post_id"], ["post.id"], ), sa.PrimaryKeyConstraint("id"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("text_labels") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py ================================================ """add_journal_table Revision ID: 3358eb6834e6 Revises: 067c4002f2d9 Create Date: 2022-12-27 14:44:59.483868 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "3358eb6834e6" down_revision = "067c4002f2d9" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "journal", sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column( "created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False ), sa.Column( "event_payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False, ), sa.Column("person_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("post_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("event_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.ForeignKeyConstraint( ["person_id"], ["person.id"], ), sa.ForeignKeyConstraint( ["post_id"], ["post.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_journal_person_id"), "journal", ["person_id"], unique=False) op.create_table( "journal_integration", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("last_run", sa.DateTime(), nullable=True), sa.Column("description", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), sa.Column("last_journal_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("last_error", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column("next_run", sa.DateTime(), nullable=True), sa.ForeignKeyConstraint( ["last_journal_id"], ["journal.id"], ), sa.PrimaryKeyConstraint("id", "description"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("journal_integration") op.drop_index(op.f("ix_journal_person_id"), table_name="journal") op.drop_table("journal") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py ================================================ """post ref for work_package Revision ID: d24b37426857 Revises: 3358eb6834e6 Create Date: 2022-12-28 11:42:26.773704 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "d24b37426857" down_revision = "3358eb6834e6" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("post", sa.Column("depth", sa.Integer(), server_default=sa.text("0"), nullable=False)) op.add_column("post", sa.Column("children_count", sa.Integer(), server_default=sa.text("0"), nullable=False)) op.add_column("post_reaction", sa.Column("work_package_id", postgresql.UUID(as_uuid=True), nullable=False)) op.drop_constraint("post_reaction_post_id_fkey", "post_reaction", type_="foreignkey") op.create_foreign_key(None, "post_reaction", "work_package", ["work_package_id"], ["id"]) op.drop_column("post_reaction", "post_id") op.add_column("work_package", sa.Column("done", sa.Boolean(), server_default=sa.text("false"), nullable=False)) op.add_column("work_package", sa.Column("ack", sa.Boolean(), nullable=True)) op.add_column("work_package", sa.Column("frontend_ref_post_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True)) op.add_column("work_package", sa.Column("thread_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) op.add_column("work_package", sa.Column("parent_post_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("work_package", "parent_post_id") op.drop_column("work_package", "thread_id") op.drop_column("work_package", "frontend_ref_post_id") op.drop_column("work_package", "ack") op.drop_column("work_package", "done") op.add_column("post_reaction", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=False)) op.drop_constraint(None, "post_reaction", type_="foreignkey") op.create_foreign_key("post_reaction_post_id_fkey", "post_reaction", "post", ["post_id"], ["id"]) op.drop_column("post_reaction", "work_package_id") op.drop_column("post", "children_count") op.drop_column("post", "depth") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py ================================================ """Added lang column for ISO-639-1 codes Revision ID: ef0b52902560 Revises: d24b37426857 Create Date: 2022-12-28 18:24:21.393973 """ import sqlalchemy as sa import sqlmodel from alembic import op # revision identifiers, used by Alembic. revision = "ef0b52902560" down_revision = "d24b37426857" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "post", sa.Column("lang", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False, default="en-US") ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("post", "lang") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py ================================================ """add collective flag to task Revision ID: 464ec4667aae Revises: ef0b52902560 Create Date: 2022-12-29 21:03:06.841962 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "464ec4667aae" down_revision = "ef0b52902560" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "work_package", sa.Column("collective", sa.Boolean(), server_default=sa.text("false"), nullable=False) ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("work_package", "collective") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py ================================================ """add field trusted api client Revision ID: 73ce3675c1f5 Revises: 464ec4667aae Create Date: 2022-12-30 01:09:06.446020 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "73ce3675c1f5" down_revision = "464ec4667aae" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("api_client", sa.Column("trusted", sa.Boolean(), server_default=sa.text("false"), nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("api_client", "trusted") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py ================================================ """name changes: person->user, post->message, work_package->task Revision ID: abb47e9d145a Revises: 73ce3675c1f5 Create Date: 2022-12-30 20:54:49.880568 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "abb47e9d145a" down_revision = "73ce3675c1f5" branch_labels = None depends_on = None def upgrade() -> None: # clear DB op.execute("DELETE FROM journal;") op.execute("DELETE FROM work_package;") op.execute("DELETE FROM post_reaction;") op.execute("DELETE FROM post;") op.execute("DELETE FROM person_stats;") op.execute("DELETE FROM person;") op.execute("DELETE FROM text_labels;") # ### commands auto generated by Alembic - please adjust! ### op.create_table( "user", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("username", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("auth_method", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("display_name", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_user_username", "user", ["api_client_id", "username", "auth_method"], unique=True) op.create_table( "message", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True), sa.Column("depth", sa.Integer(), server_default=sa.text("0"), nullable=False), sa.Column("children_count", sa.Integer(), server_default=sa.text("0"), nullable=False), sa.Column("parent_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("task_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("role", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("frontend_message_id", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.Column("lang", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index("ix_message_frontend_message_id", "message", ["api_client_id", "frontend_message_id"], unique=True) op.create_index(op.f("ix_message_message_tree_id"), "message", ["message_tree_id"], unique=False) op.create_index(op.f("ix_message_task_id"), "message", ["task_id"], unique=False) op.create_index(op.f("ix_message_user_id"), "message", ["user_id"], unique=False) op.create_table( "task", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("expiry_date", sa.DateTime(), nullable=True), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), sa.Column("done", sa.Boolean(), server_default=sa.text("false"), nullable=False), sa.Column("collective", sa.Boolean(), server_default=sa.text("false"), nullable=False), sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("ack", sa.Boolean(), nullable=True), sa.Column("frontend_message_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.Column("parent_message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_task_user_id"), "task", ["user_id"], unique=False) op.create_table( "user_stats", sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("modified_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("leader_score", sa.Integer(), nullable=False), sa.Column("reactions", sa.Integer(), nullable=False), sa.Column("messages", sa.Integer(), nullable=False), sa.Column("upvotes", sa.Integer(), nullable=False), sa.Column("downvotes", sa.Integer(), nullable=False), sa.Column("task_reward", sa.Integer(), nullable=False), sa.Column("compare_wins", sa.Integer(), nullable=False), sa.Column("compare_losses", sa.Integer(), nullable=False), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("user_id"), ) op.create_table( "message_reaction", sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False), sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.ForeignKeyConstraint( ["api_client_id"], ["api_client.id"], ), sa.ForeignKeyConstraint( ["task_id"], ["task.id"], ), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("task_id", "user_id"), ) op.drop_constraint("text_labels_post_id_fkey", "text_labels", type_="foreignkey") op.drop_constraint("journal_post_id_fkey", "journal", type_="foreignkey") op.drop_constraint("journal_person_id_fkey", "journal", type_="foreignkey") op.drop_table("post_reaction") op.drop_index("ix_post_frontend_post_id", table_name="post") op.drop_index("ix_post_person_id", table_name="post") op.drop_index("ix_post_thread_id", table_name="post") op.drop_index("ix_post_workpackage_id", table_name="post") op.drop_table("post") op.drop_index("ix_work_package_person_id", table_name="work_package") op.drop_table("work_package") op.drop_table("person_stats") op.drop_index("ix_person_username", table_name="person") op.drop_table("person") op.add_column("journal", sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) op.add_column("journal", sa.Column("message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) op.drop_index("ix_journal_person_id", table_name="journal") op.create_index(op.f("ix_journal_user_id"), "journal", ["user_id"], unique=False) op.create_foreign_key(None, "journal", "user", ["user_id"], ["id"]) op.create_foreign_key(None, "journal", "message", ["message_id"], ["id"]) op.drop_column("journal", "person_id") op.drop_column("journal", "post_id") op.add_column("text_labels", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=True)) op.create_foreign_key(None, "text_labels", "message", ["message_id"], ["id"]) op.drop_column("text_labels", "post_id") # ### end Alembic commands ### def downgrade() -> None: # clear DB op.execute("DELETE FROM journal;") op.execute("DELETE FROM message_reaction;") op.execute("DELETE FROM task;") op.execute("DELETE FROM message;") op.execute("DELETE FROM user_stats;") op.execute('DELETE FROM "user";') op.execute("DELETE FROM text_labels;") # ### commands auto generated by Alembic - please adjust! ### op.add_column("text_labels", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=True)) op.drop_constraint("text_labels_message_id_fkey", "text_labels", type_="foreignkey") op.drop_column("text_labels", "message_id") op.add_column("journal", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=True)) op.add_column("journal", sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True)) op.drop_constraint("journal_message_id_fkey", "journal", type_="foreignkey") op.drop_constraint("journal_user_id_fkey", "journal", type_="foreignkey") op.drop_index(op.f("ix_journal_user_id"), table_name="journal") op.create_index("ix_journal_person_id", "journal", ["person_id"], unique=False) op.drop_column("journal", "message_id") op.drop_column("journal", "user_id") op.create_table( "person", sa.Column( "id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False ), sa.Column("username", sa.VARCHAR(length=128), autoincrement=False, nullable=False), sa.Column("display_name", sa.VARCHAR(length=256), autoincrement=False, nullable=False), sa.Column( "created_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("auth_method", sa.VARCHAR(length=128), autoincrement=False, nullable=False), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="person_api_client_id_fkey"), sa.PrimaryKeyConstraint("id", name="person_pkey"), ) op.create_table( "person_stats", sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("leader_score", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column( "modified_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), sa.Column("reactions", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("posts", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("upvotes", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("downvotes", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("work_reward", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("compare_wins", sa.INTEGER(), autoincrement=False, nullable=False), sa.Column("compare_losses", sa.INTEGER(), autoincrement=False, nullable=False), sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="person_stats_person_id_fkey"), sa.PrimaryKeyConstraint("person_id", name="person_stats_pkey"), ) op.create_table( "work_package", sa.Column( "id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False ), sa.Column( "created_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), sa.Column("expiry_date", postgresql.TIMESTAMP(), autoincrement=False, nullable=True), sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("done", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False), sa.Column("ack", sa.BOOLEAN(), autoincrement=False, nullable=True), sa.Column("frontend_ref_post_id", sa.VARCHAR(), autoincrement=False, nullable=True), sa.Column("thread_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("parent_post_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("collective", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="work_package_api_client_id_fkey"), sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="work_package_person_id_fkey"), sa.PrimaryKeyConstraint("id", name="work_package_pkey"), ) op.create_index("ix_work_package_person_id", "work_package", ["person_id"], unique=False) op.create_table( "post", sa.Column( "id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False ), sa.Column("parent_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("thread_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("workpackage_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True), sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("role", sa.VARCHAR(length=128), autoincrement=False, nullable=False), sa.Column("frontend_post_id", sa.VARCHAR(length=200), autoincrement=False, nullable=False), sa.Column( "created_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True), sa.Column("depth", sa.INTEGER(), server_default=sa.text("0"), autoincrement=False, nullable=False), sa.Column("children_count", sa.INTEGER(), server_default=sa.text("0"), autoincrement=False, nullable=False), sa.Column("lang", sa.VARCHAR(length=200), autoincrement=False, nullable=False), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="post_api_client_id_fkey"), sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="post_person_id_fkey"), sa.PrimaryKeyConstraint("id", name="post_pkey"), ) op.create_index("ix_post_workpackage_id", "post", ["workpackage_id"], unique=False) op.create_index("ix_post_thread_id", "post", ["thread_id"], unique=False) op.create_index("ix_post_person_id", "post", ["person_id"], unique=False) op.create_index("ix_post_frontend_post_id", "post", ["api_client_id", "frontend_post_id"], unique=False) op.create_table( "post_reaction", sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column( "created_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.Column("work_package_id", postgresql.UUID(), autoincrement=False, nullable=False), sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="post_reaction_api_client_id_fkey"), sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="post_reaction_person_id_fkey"), sa.ForeignKeyConstraint(["work_package_id"], ["work_package.id"], name="post_reaction_work_package_id_fkey"), ) op.create_index("ix_person_username", "person", ["api_client_id", "username", "auth_method"], unique=False) op.create_foreign_key("text_labels_post_id_fkey", "text_labels", "post", ["post_id"], ["id"]) op.create_foreign_key("journal_person_id_fkey", "journal", "person", ["person_id"], ["id"]) op.create_foreign_key("journal_post_id_fkey", "journal", "post", ["post_id"], ["id"]) op.drop_table("message_reaction") op.drop_table("user_stats") op.drop_index(op.f("ix_task_user_id"), table_name="task") op.drop_table("task") op.drop_index(op.f("ix_message_user_id"), table_name="message") op.drop_index(op.f("ix_message_task_id"), table_name="message") op.drop_index(op.f("ix_message_message_tree_id"), table_name="message") op.drop_index("ix_message_frontend_message_id", table_name="message") op.drop_table("message") op.drop_index("ix_user_username", table_name="user") op.drop_table("user") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py ================================================ """add deleted field to post Revision ID: 8d269bc4fdbd Revises: abb47e9d145a Create Date: 2022-12-31 04:38:41.799206 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "8d269bc4fdbd" down_revision = "abb47e9d145a" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message", sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "deleted") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py ================================================ """added MessageTreeState table Revision ID: d4161e384f83 Revises: 8d269bc4fdbd Create Date: 2023-01-05 11:44:02.630633 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "d4161e384f83" down_revision = "8d269bc4fdbd" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "message_tree_state", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False), sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("goal_tree_size", sa.Integer(), nullable=False), sa.Column("current_num_non_filtered_messages", sa.Integer(), nullable=False), sa.Column("max_depth", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("id"), ) op.create_index( op.f("ix_message_tree_state_message_tree_id"), "message_tree_state", ["message_tree_id"], unique=False ) op.create_index("ix_message_tree_state_tree_id", "message_tree_state", ["message_tree_id"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_message_tree_state_tree_id", table_name="message_tree_state") op.drop_index(op.f("ix_message_tree_state_message_tree_id"), table_name="message_tree_state") op.drop_table("message_tree_state") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py ================================================ """removed date_created and deleted flag from message_tree_state Revision ID: 3b0adfadbef9 Revises: d4161e384f83 Create Date: 2023-01-05 13:46:11.338655 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "3b0adfadbef9" down_revision = "d4161e384f83" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message_tree_state", "deleted") op.drop_column("message_tree_state", "created_date") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "message_tree_state", sa.Column( "created_date", postgresql.TIMESTAMP(), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=False, ), ) op.add_column( "message_tree_state", sa.Column("deleted", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False), ) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py ================================================ """Added user to TextLabels Revision ID: 20cd871f4ec7 Revises: d4161e384f83 Create Date: 2023-01-05 17:45:15.696468 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "20cd871f4ec7" down_revision = "3b0adfadbef9" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("text_labels", sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False)) op.create_foreign_key(None, "text_labels", "user", ["user_id"], ["id"]) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_constraint(None, "text_labels", type_="foreignkey") op.drop_column("text_labels", "user_id") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py ================================================ """added frontend_type to api_client Revision ID: ba61fe17fb6e Revises: 20cd871f4ec7 Create Date: 2023-01-07 12:50:32.195930 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "ba61fe17fb6e" down_revision = "20cd871f4ec7" branch_labels = None depends_on = None def upgrade() -> None: op.add_column("api_client", sa.Column("frontend_type", sa.String(256), nullable=True)) def downgrade() -> None: op.drop_column("api_client", "frontend_type") ================================================ FILE: backend/alembic/versions/2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py ================================================ """added miniLM_embedding column to message Revision ID: 023548d474f7 Revises: ba61fe17fb6e Create Date: 2023-01-08 11:06:25.613290 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "023548d474f7" down_revision = "ba61fe17fb6e" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message", sa.Column("miniLM_embedding", sa.ARRAY(sa.Float()), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "miniLM_embedding") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py ================================================ """embedding for message now in its own table Revision ID: 35bdc1a08bb8 Revises: 023548d474f7 Create Date: 2023-01-08 16:03:48.454207 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "35bdc1a08bb8" down_revision = "023548d474f7" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "message_embedding", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("embedding", sa.ARRAY(sa.Float()), nullable=True), sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), sa.ForeignKeyConstraint( ["message_id"], ["message.id"], ), sa.PrimaryKeyConstraint("message_id", "model"), ) op.drop_column("message", "miniLM_embedding") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "message", sa.Column( "miniLM_embedding", postgresql.ARRAY(postgresql.DOUBLE_PRECISION(precision=53)), autoincrement=False, nullable=True, ), ) op.drop_table("message_embedding") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_08_2128-aac6b2f66006_created_date.py ================================================ """Created date Revision ID: aac6b2f66006 Revises: 35bdc1a08bb8 Create Date: 2023-01-08 21:28:27.342729 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "aac6b2f66006" down_revision = "35bdc1a08bb8" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "message_embedding", sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message_embedding", "created_date") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py ================================================ """MessageToxicity Revision ID: bcc2fe18d214 Revises: 20cd871f4ec7 Create Date: 2023-01-08 22:00:43.297719 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "bcc2fe18d214" down_revision = "846cc08ac79f" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "message_toxicity", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("toxicity", sa.Float(), nullable=True), sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False), sa.ForeignKeyConstraint( ["message_id"], ["message.id"], ), sa.PrimaryKeyConstraint("message_id", "model"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("message_toxicity") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py ================================================ """restructure message_tree_state table Revision ID: 92a367bb9f40 Revises: ba61fe17fb6e Create Date: 2023-01-08 22:08:46.458195 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "92a367bb9f40" down_revision = "aac6b2f66006" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("message_tree_state") op.create_table( "message_tree_state", sa.Column("message_tree_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("goal_tree_size", sa.Integer(), nullable=False), sa.Column("max_depth", sa.Integer(), nullable=False), sa.Column("max_children_count", sa.Integer(), nullable=False), sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("active", sa.Boolean(), nullable=False), sa.Column("accepted_messages", sa.Integer(), nullable=False), sa.ForeignKeyConstraint( ["message_tree_id"], ["message.id"], ), sa.PrimaryKeyConstraint("message_tree_id"), ) op.create_index(op.f("ix_message_tree_state_active"), "message_tree_state", ["active"], unique=False) op.create_index(op.f("ix_message_tree_state_state"), "message_tree_state", ["state"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_message_tree_state_state"), table_name="message_tree_state") op.drop_index(op.f("ix_message_tree_state_active"), table_name="message_tree_state") op.drop_table("message_tree_state") op.create_table( "message_tree_state", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("goal_tree_size", sa.Integer(), nullable=False), sa.Column("current_num_non_filtered_messages", sa.Integer(), nullable=False), sa.Column("max_depth", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("id"), ) op.create_index( op.f("ix_message_tree_state_message_tree_id"), "message_tree_state", ["message_tree_id"], unique=False ) op.create_index("ix_message_tree_state_tree_id", "message_tree_state", ["message_tree_id"], unique=True) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py ================================================ """add review_count & ranking_count to message Revision ID: 05975b274a81 Revises: 92a367bb9f40 Create Date: 2023-01-09 00:47:25.496036 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "05975b274a81" down_revision = "92a367bb9f40" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message", sa.Column("review_count", sa.Integer(), server_default=sa.text("0"), nullable=False)) op.add_column("message", sa.Column("review_result", sa.Boolean(), server_default=sa.text("false"), nullable=False)) op.add_column("message", sa.Column("ranking_count", sa.Integer(), server_default=sa.text("0"), nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "ranking_count") op.drop_column("message", "review_result") op.drop_column("message", "review_count") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py ================================================ """Add enabled, deleted, notes fields to User Revision ID: 846cc08ac79f Revises: aac6b2f66006 Create Date: 2023-01-10 17:33:07.104596 """ import sqlalchemy as sa import sqlmodel from alembic import op # revision identifiers, used by Alembic. revision = "846cc08ac79f" down_revision = "befa42582ea4" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("user", sa.Column("enabled", sa.Boolean(), server_default=sa.text("true"), nullable=False)) op.add_column("user", sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False)) op.add_column( "user", sa.Column("notes", sqlmodel.sql.sqltypes.AutoString(length=1024), server_default="", nullable=False), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("user", "notes") op.drop_column("user", "deleted") op.drop_column("user", "enabled") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py ================================================ """remove accepted_messages from message_tree_state Revision ID: befa42582ea4 Revises: 05975b274a81 Create Date: 2023-01-12 01:19:59.654864 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "befa42582ea4" down_revision = "05975b274a81" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message_tree_state", "accepted_messages") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "message_tree_state", sa.Column("accepted_messages", sa.INTEGER(), autoincrement=False, nullable=False) ) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_14_1509-619255ae9076_add_rank_to_message_table.py ================================================ """add rank to message table Revision ID: 619255ae9076 Revises: bcc2fe18d214 Create Date: 2023-01-14 15:09:03.462482 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "619255ae9076" down_revision = "bcc2fe18d214" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message", sa.Column("rank", sa.Integer(), nullable=True)) op.add_column("message_toxicity", sa.Column("score", sa.Float(), nullable=True)) op.add_column("message_toxicity", sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False)) op.drop_column("message_toxicity", "toxicity") op.add_column("user_stats", sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False)) op.add_column("user_stats", sa.Column("prompts", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("replies_assistant", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("replies_prompter", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("labels_simple", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("labels_full", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("rankings_total", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("rankings_good", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("accepted_prompts", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("accepted_replies_assistant", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("accepted_replies_prompter", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_assistant_ranked_1", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_assistant_ranked_2", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_assistant_ranked_3", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_prompter_ranked_1", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_prompter_ranked_2", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("reply_prompter_ranked_3", sa.Integer(), nullable=False)) op.add_column("user_stats", sa.Column("streak_last_day_date", sa.DateTime(), nullable=True)) op.add_column("user_stats", sa.Column("streak_days", sa.Integer(), nullable=True)) op.drop_column("user_stats", "messages") op.drop_column("user_stats", "upvotes") op.drop_column("user_stats", "task_reward") op.drop_column("user_stats", "compare_wins") op.drop_column("user_stats", "compare_losses") op.drop_column("user_stats", "downvotes") op.drop_column("user_stats", "reactions") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("user_stats", sa.Column("reactions", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("downvotes", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("compare_losses", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("compare_wins", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("task_reward", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("upvotes", sa.INTEGER(), autoincrement=False, nullable=False)) op.add_column("user_stats", sa.Column("messages", sa.INTEGER(), autoincrement=False, nullable=False)) op.drop_column("user_stats", "streak_days") op.drop_column("user_stats", "streak_last_day_date") op.drop_column("user_stats", "reply_prompter_ranked_3") op.drop_column("user_stats", "reply_prompter_ranked_2") op.drop_column("user_stats", "reply_prompter_ranked_1") op.drop_column("user_stats", "reply_assistant_ranked_3") op.drop_column("user_stats", "reply_assistant_ranked_2") op.drop_column("user_stats", "reply_assistant_ranked_1") op.drop_column("user_stats", "accepted_replies_prompter") op.drop_column("user_stats", "accepted_replies_assistant") op.drop_column("user_stats", "accepted_prompts") op.drop_column("user_stats", "rankings_good") op.drop_column("user_stats", "rankings_total") op.drop_column("user_stats", "labels_full") op.drop_column("user_stats", "labels_simple") op.drop_column("user_stats", "replies_prompter") op.drop_column("user_stats", "replies_assistant") op.drop_column("user_stats", "prompts") op.drop_column("user_stats", "time_frame") op.add_column( "message_toxicity", sa.Column("toxicity", postgresql.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True), ) op.drop_column("message_toxicity", "label") op.drop_column("message_toxicity", "score") op.drop_column("message", "rank") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py ================================================ """change user_stats ranking counts Revision ID: 7c98102efbca Revises: 619255ae9076 Create Date: 2023-01-15 00:02:45.622986 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects.postgresql import UUID # revision identifiers, used by Alembic. revision = "7c98102efbca" down_revision = "619255ae9076" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("user_stats") op.create_table( "user_stats", sa.Column("user_id", UUID(as_uuid=True), nullable=False), sa.Column("modified_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False), sa.Column("base_date", sa.DateTime(), nullable=True), sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False), sa.Column("leader_score", sa.Integer(), nullable=False), sa.Column("prompts", sa.Integer(), nullable=False), sa.Column("replies_assistant", sa.Integer(), nullable=False), sa.Column("replies_prompter", sa.Integer(), nullable=False), sa.Column("labels_simple", sa.Integer(), nullable=False), sa.Column("labels_full", sa.Integer(), nullable=False), sa.Column("rankings_total", sa.Integer(), nullable=False), sa.Column("rankings_good", sa.Integer(), nullable=False), sa.Column("accepted_prompts", sa.Integer(), nullable=False), sa.Column("accepted_replies_assistant", sa.Integer(), nullable=False), sa.Column("accepted_replies_prompter", sa.Integer(), nullable=False), sa.Column("reply_ranked_1", sa.Integer(), nullable=False), sa.Column("reply_ranked_2", sa.Integer(), nullable=False), sa.Column("reply_ranked_3", sa.Integer(), nullable=False), sa.Column("streak_last_day_date", sa.DateTime(), nullable=True), sa.Column("streak_days", sa.Integer(), nullable=True), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("user_id", "time_frame"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "user_stats", sa.Column("reply_prompter_ranked_3", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.add_column( "user_stats", sa.Column("reply_assistant_ranked_1", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.add_column( "user_stats", sa.Column("reply_assistant_ranked_2", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.add_column( "user_stats", sa.Column("reply_prompter_ranked_2", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.add_column( "user_stats", sa.Column("reply_prompter_ranked_1", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.add_column( "user_stats", sa.Column("reply_assistant_ranked_3", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False), ) op.drop_column("user_stats", "reply_ranked_3") op.drop_column("user_stats", "reply_ranked_2") op.drop_column("user_stats", "reply_ranked_1") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_15_1139-423557e869e4_add_indices_for_created_date.py ================================================ """add indices for created_date Revision ID: 423557e869e4 Revises: 7c98102efbca Create Date: 2023-01-15 11:39:10.407859 """ from alembic import op # revision identifiers, used by Alembic. revision = "423557e869e4" down_revision = "7c98102efbca" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_index(op.f("ix_message_created_date"), "message", ["created_date"], unique=False) op.create_index(op.f("ix_message_reaction_created_date"), "message_reaction", ["created_date"], unique=False) op.create_index(op.f("ix_text_labels_created_date"), "text_labels", ["created_date"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_text_labels_created_date"), table_name="text_labels") op.drop_index(op.f("ix_message_reaction_created_date"), table_name="message_reaction") op.drop_index(op.f("ix_message_created_date"), table_name="message") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py ================================================ """add rank and indices to user_stats Revision ID: 0964ac95170d Revises: 423557e869e4 Create Date: 2023-01-15 16:54:09.510018 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "0964ac95170d" down_revision = "423557e869e4" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("user_stats", sa.Column("rank", sa.Integer(), nullable=True)) op.create_index( "ix_user_stats__timeframe__rank__user_id", "user_stats", ["time_frame", "rank", "user_id"], unique=True ) op.create_index("ix_user_stats__timeframe__user_id", "user_stats", ["time_frame", "user_id"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_user_stats__timeframe__user_id", table_name="user_stats") op.drop_index("ix_user_stats__timeframe__rank__user_id", table_name="user_stats") op.drop_column("user_stats", "rank") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py ================================================ """switch to timestamp with tz Revision ID: 7f0a28a156f4 Revises: 0964ac95170d Create Date: 2023-01-19 21:53:01.107137 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "7f0a28a156f4" down_revision = "0964ac95170d" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column(table_name="user_stats", column_name="modified_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="user_stats", column_name="base_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="journal_integration", column_name="last_run", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="message_embedding", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="message_reaction", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="message_toxicity", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="message", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="task", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="task", column_name="expiry_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="text_labels", column_name="created_date", type_=sa.DateTime(timezone=True)) op.alter_column(table_name="user", column_name="created_date", type_=sa.DateTime(timezone=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column(table_name="user_stats", column_name="modified_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="user_stats", column_name="base_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="journal_integration", column_name="last_run", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="message_embedding", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="message_reaction", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="message_toxicity", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="message", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="task", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="task", column_name="expiry_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="text_labels", column_name="created_date", type_=sa.DateTime(timezone=False)) op.alter_column(table_name="user", column_name="created_date", type_=sa.DateTime(timezone=False)) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py ================================================ """add ix_user_display_name_id Revision ID: 4f26fec4d204 Revises: 0964ac95170d Create Date: 2023-01-19 22:00:00 """ from alembic import op # revision identifiers, used by Alembic. revision = "4f26fec4d204" down_revision = "7f0a28a156f4" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_index("ix_user_display_name_id", "user", ["display_name", "id"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_user_display_name_id", table_name="user") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py ================================================ """use 'en' instead 'en-US' as default lang Revision ID: 160ac010efcc Revises: 4f26fec4d204 Create Date: 2023-01-20 16:50:00 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "160ac010efcc" down_revision = "4f26fec4d204" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "lang") op.add_column("message", sa.Column("lang", sa.String(length=32), server_default="en", nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "lang") op.add_column("message", sa.Column("lang", sa.VARCHAR(length=200), autoincrement=False, nullable=False)) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py ================================================ """add message_id to message_reaction Revision ID: 8ba17b5f467a Revises: 160ac010efcc Create Date: 2023-01-24 11:34:42.167575 """ import sqlalchemy as sa import sqlmodel from alembic import op # revision identifiers, used by Alembic. revision = "8ba17b5f467a" down_revision = "160ac010efcc" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message_reaction", sa.Column("message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) op.create_index(op.f("ix_message_reaction_message_id"), "message_reaction", ["message_id"], unique=False) op.add_column("text_labels", sa.Column("task_id", sqlmodel.sql.sqltypes.GUID(), nullable=True)) op.create_index(op.f("ix_text_labels_task_id"), "text_labels", ["task_id"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_text_labels_task_id"), table_name="text_labels") op.drop_column("text_labels", "task_id") op.drop_index(op.f("ix_message_reaction_message_id"), table_name="message_reaction") op.drop_column("message_reaction", "message_id") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_24_2256-40ed93df0ed5_add_message_emoji.py ================================================ """add message_emoji Revision ID: 40ed93df0ed5 Revises: 8ba17b5f467a Create Date: 2023-01-24 22:56:28.229408 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "40ed93df0ed5" down_revision = "8ba17b5f467a" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "message_emoji", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column( "created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False ), sa.Column("emoji", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.ForeignKeyConstraint(["message_id"], ["message.id"], ondelete="CASCADE"), sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("message_id", "user_id", "emoji"), ) op.create_index("ix_message_emoji__user_id__message_id", "message_emoji", ["user_id", "message_id"], unique=False) op.add_column("message", sa.Column("emojis", postgresql.JSONB(astext_type=sa.Text()), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "emojis") op.drop_index("ix_message_emoji__user_id__message_id", table_name="message_emoji") op.drop_table("message_emoji") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py ================================================ """add task created date index Revision ID: c84fcd6900dc Revises: 40ed93df0ed5 Create Date: 2023-01-26 18:35:43.061589 """ from alembic import op # revision identifiers, used by Alembic. revision = "c84fcd6900dc" down_revision = "40ed93df0ed5" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_index(op.f("ix_task_created_date"), "task", ["created_date"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_task_created_date"), table_name="task") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py ================================================ """add user.show_on_leaderboard Revision ID: f856bf19d32b Revises: c84fcd6900dc Create Date: 2023-01-27 20:13:56.533374 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "f856bf19d32b" down_revision = "c84fcd6900dc" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "user", sa.Column("show_on_leaderboard", sa.Boolean(), server_default=sa.text("true"), nullable=False) ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("user", "show_on_leaderboard") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py ================================================ """add origin column to message_tree_state Revision ID: 49d8445b4c90 Revises: f856bf19d32b Create Date: 2023-01-28 11:57:45.580027 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "49d8445b4c90" down_revision = "f856bf19d32b" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message", sa.Column("synthetic", sa.Boolean(), server_default=sa.text("false"), nullable=False)) op.add_column("message", sa.Column("model_name", sa.String(length=1024), nullable=True)) op.add_column("message_tree_state", sa.Column("origin", sa.String(length=1024), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message_tree_state", "origin") op.drop_column("message", "model_name") op.drop_column("message", "synthetic") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py ================================================ """move user_streak from user_stats to user table Revision ID: 7b8f0011e0b0 Revises: 8a5feed819ee Create Date: 2023-01-29 12:07:29.379326 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "7b8f0011e0b0" down_revision = "49d8445b4c90" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "user", sa.Column( "streak_last_day_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), autoincrement=False, nullable=True, ), ) op.add_column("user", sa.Column("streak_days", sa.INTEGER(), autoincrement=False, nullable=True)) op.add_column( "user", sa.Column("last_activity_date", sa.DateTime(timezone=True), autoincrement=False, nullable=True) ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("user", "streak_days") op.drop_column("user", "streak_last_day_date") op.drop_column("user", "last_activity_date") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py ================================================ """add tos_acceptance_date to user Revision ID: 55361f323d12 Revises: 7b8f0011e0b0 Create Date: 2023-02-01 00:22:08.280251 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "55361f323d12" down_revision = "f60958968ff8" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("user", sa.Column("tos_acceptance_date", sa.DateTime(timezone=True), nullable=True)) op.drop_column("user_stats", "streak_days") op.drop_column("user_stats", "streak_last_day_date") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column( "user_stats", sa.Column("streak_last_day_date", postgresql.TIMESTAMP(), autoincrement=False, nullable=True) ) op.add_column("user_stats", sa.Column("streak_days", sa.INTEGER(), autoincrement=False, nullable=True)) op.drop_column("user", "tos_acceptance_date") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py ================================================ """add won_prompt_lottery_date to mts Revision ID: f60958968ff8 Revises: 7b8f0011e0b0 Create Date: 2023-02-01 10:10:38.301707 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "f60958968ff8" down_revision = "7b8f0011e0b0" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message_tree_state", sa.Column("won_prompt_lottery_date", sa.DateTime(timezone=True), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message_tree_state", "won_prompt_lottery_date") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py ================================================ """add skip bool & skip_reason to task Revision ID: 9e7ec4a9e3f2 Revises: 7b8f0011e0b0 Create Date: 2023-02-01 21:46:49.971052 """ import sqlalchemy as sa import sqlmodel from alembic import op # revision identifiers, used by Alembic. revision = "9e7ec4a9e3f2" down_revision = "55361f323d12" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("task", sa.Column("skipped", sa.Boolean(), server_default=sa.text("false"), nullable=False)) op.add_column("task", sa.Column("skip_reason", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("task", "skip_reason") op.drop_column("task", "skipped") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py ================================================ """add troll_stats Revision ID: 4d7e0b0ebe84 Revises: 9e7ec4a9e3f2 Create Date: 2023-02-02 15:44:12.647260 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "4d7e0b0ebe84" down_revision = "9e7ec4a9e3f2" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "troll_stats", sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("base_date", sa.DateTime(timezone=True), nullable=True), sa.Column( "modified_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False ), sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False), sa.Column("troll_score", sa.Integer(), nullable=False), sa.Column("rank", sa.Integer(), nullable=True), sa.Column("red_flags", sa.Integer(), nullable=False), sa.Column("upvotes", sa.Integer(), nullable=False), sa.Column("downvotes", sa.Integer(), nullable=False), sa.Column("spam_prompts", sa.Integer(), nullable=False), sa.Column("quality", sa.Float(), nullable=True), sa.Column("humor", sa.Float(), nullable=True), sa.Column("toxicity", sa.Float(), nullable=True), sa.Column("violence", sa.Float(), nullable=True), sa.Column("helpfulness", sa.Float(), nullable=True), sa.Column("spam", sa.Integer(), nullable=False), sa.Column("lang_mismach", sa.Integer(), nullable=False), sa.Column("not_appropriate", sa.Integer(), nullable=False), sa.Column("pii", sa.Integer(), nullable=False), sa.Column("hate_speech", sa.Integer(), nullable=False), sa.Column("sexual_content", sa.Integer(), nullable=False), sa.Column("political_content", sa.Integer(), nullable=False), sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("user_id", "time_frame"), ) op.create_index("ix_troll_stats__timeframe__user_id", "troll_stats", ["time_frame", "user_id"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_troll_stats__timeframe__user_id", table_name="troll_stats") op.drop_table("troll_stats") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_02_1817-8c8241d1f973_add_account_table.py ================================================ """Add Account table Revision ID: 8c8241d1f973 Revises: 4d7e0b0ebe84 Create Date: 2023-01-30 15:10:58.776315 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "8c8241d1f973" down_revision = "4d7e0b0ebe84" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "account", sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False), sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=False), sa.Column("provider", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column("provider_account_id", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.ForeignKeyConstraint(["user_id"], ["user.id"]), sa.PrimaryKeyConstraint("id"), ) op.create_index("provider", "account", ["provider_account_id"], unique=True) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("provider", table_name="account") op.drop_table("account") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py ================================================ """Added new table for flagged messages Revision ID: caee1e8ee0bc Revises: 8c8241d1f973 Create Date: 2023-02-07 19:22:12.696257 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "caee1e8ee0bc" down_revision = "8c8241d1f973" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "flagged_message", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column( "created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False ), sa.Column("processed", sa.Boolean(), nullable=False), sa.ForeignKeyConstraint(["message_id"], ["message.id"], ondelete="CASCADE"), sa.PrimaryKeyConstraint("message_id"), ) op.create_index(op.f("ix_flagged_message_created_date"), "flagged_message", ["created_date"], unique=False) op.create_index(op.f("ix_flagged_message_processed"), "flagged_message", ["processed"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_flagged_message_processed"), table_name="flagged_message") op.drop_index(op.f("ix_flagged_message_created_date"), table_name="flagged_message") op.drop_table("flagged_message") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_11_1030-ba40d055714a_add_cached_stats.py ================================================ """add cached_stats Revision ID: ba40d055714a Revises: caee1e8ee0bc Create Date: 2023-02-11 10:30:21.996198 """ import sqlalchemy as sa import sqlmodel from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "ba40d055714a" down_revision = "caee1e8ee0bc" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "cached_stats", sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), sa.Column( "modified_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False ), sa.Column("stats", postgresql.JSONB(astext_type=sa.Text()), nullable=False), sa.PrimaryKeyConstraint("name"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_table("cached_stats") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py ================================================ """add text_labels message_id index Revision ID: 165b55de5a94 Revises: ba40d055714a Create Date: 2023-02-14 17:56:48.263684 """ from alembic import op # revision identifiers, used by Alembic. revision = "165b55de5a94" down_revision = "ba40d055714a" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_index(op.f("ix_text_labels_message_id"), "text_labels", ["message_id"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index(op.f("ix_text_labels_message_id"), table_name="text_labels") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py ================================================ """message review_result nullable Revision ID: 8cd0c34d0c3c Revises: 165b55de5a94 Create Date: 2023-02-15 17:54:58.029278 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "8cd0c34d0c3c" down_revision = "165b55de5a94" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "message", "review_result", existing_type=sa.BOOLEAN(), nullable=True, server_default=None, existing_server_default=sa.text("false"), ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.alter_column( "message", "review_result", existing_type=sa.BOOLEAN(), nullable=False, server_default=sa.text("false") ) # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py ================================================ """add lang to message_tree_state Revision ID: 9db92d504f64 Revises: 8cd0c34d0c3c Create Date: 2023-02-26 00:52:40.624843 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision = "9db92d504f64" down_revision = "8cd0c34d0c3c" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("message_tree_state", sa.Column("lang", sa.String(length=32), nullable=True)) op.execute( "WITH msg AS (SELECT id, lang FROM message WHERE parent_id is NULL) UPDATE message_tree_state mts SET lang = msg.lang FROM msg WHERE mts.message_tree_id = msg.id" ) op.alter_column("message_tree_state", "lang", nullable=False) op.drop_index("ix_message_tree_state_state", table_name="message_tree_state") op.create_index("ix_message_tree_state__lang__state", "message_tree_state", ["state", "lang"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_index("ix_message_tree_state__lang__state", table_name="message_tree_state") op.create_index("ix_message_tree_state_state", "message_tree_state", ["state"], unique=False) op.drop_column("message_tree_state", "lang") # ### end Alembic commands ### ================================================ FILE: backend/alembic/versions/2023_05_07_2129-1b6e3ae16e9d_add_text_search.py ================================================ """add text search Revision ID: 1b6e3ae16e9d Revises: 9db92d504f64 Create Date: 2023-05-07 21:29:35.545612 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "1b6e3ae16e9d" down_revision = "9db92d504f64" branch_labels = None depends_on = None def upgrade() -> None: op.add_column("message", sa.Column("search_vector", postgresql.TSVECTOR(), nullable=True)) op.create_index("idx_search_vector", "message", ["search_vector"], postgresql_using="gin") def downgrade() -> None: op.drop_index("idx_search_vector", "message") op.drop_column("message", "search_vector") ================================================ FILE: backend/alembic/versions/2023_06_06_1505-c181661eba3a_add_message_revisions.py ================================================ """add_message_revisions Revision ID: c181661eba3a Revises: 1b6e3ae16e9d Create Date: 2023-06-06 15:05:58.079120 """ import sqlalchemy as sa from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision = "c181661eba3a" down_revision = "1b6e3ae16e9d" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table( "message_revision", sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True), sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=True), sa.Column( "created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=True ), sa.ForeignKeyConstraint( ["message_id"], ["message.id"], ), sa.ForeignKeyConstraint( ["user_id"], ["user.id"], ), sa.PrimaryKeyConstraint("id"), ) op.create_index(op.f("ix_message_revision_message_id"), "message_revision", ["message_id"], unique=False) op.add_column("message", sa.Column("edited", sa.Boolean(), server_default=sa.text("false"), nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.drop_column("message", "edited") op.drop_index(op.f("ix_message_revision_message_id"), table_name="message_revision") op.drop_table("message_revision") # ### end Alembic commands ### ================================================ FILE: backend/alembic.ini ================================================ # A generic, single database configuration. [alembic] # path to migration scripts script_location = %(here)s/alembic # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file # for all available tokens file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s # sys.path path, will be prepended to sys.path if present. # defaults to the current working directory. prepend_sys_path = . # timezone to use when rendering the date within the migration file # as well as the filename. # If specified, requires the python-dateutil library that can be # installed by adding `alembic[tz]` to the pip requirements # string value is passed to dateutil.tz.gettz() # leave blank for localtime # timezone = # max length of characters to apply to the # "slug" field # truncate_slug_length = 40 # set to 'true' to run the environment during # the 'revision' command, regardless of autogenerate # revision_environment = false # set to 'true' to allow .pyc and .pyo files without # a source .py file to be detected as revisions in the # versions/ directory # sourceless = false # version location specification; This defaults # to alembic/versions. When using multiple version # directories, initial revisions must be specified with --version-path. # The path separator used here should be the separator specified by "version_path_separator" below. # version_locations = %(here)s/bar:%(here)s/bat:alembic/versions # version path separator; As mentioned above, this is the character used to split # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. # If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. # Valid values for version_path_separator are: # # version_path_separator = : # version_path_separator = ; # version_path_separator = space version_path_separator = os # Use os.pathsep. Default configuration used for new projects. # the output encoding used when revision files # are written from script.py.mako # output_encoding = utf-8 # sqlalchemy.url = postgresql://Our goal is to cover a wide range of topics, languages and tasks. ### **Current Progress** To see the datasets people are currently working on, please refer to **[the spreadsheet](https://docs.google.com/spreadsheets/d/1NYYa6vHiRnk5kwnyYaCT0cBO62--Tm3w4ihdBtp4ISk)**. ### **Repository Structure** - Each dataset is organized into its own folder, which may include notebooks, processing scripts, markdown files and other materials that explain the dataset creation process - The dataset files themselves are stored on Hugging Face - The root `__init__.py` lists the dataset names and corresponding Hugging Face datasets - The final version of each dataset is pushed to the [OpenAssisstant Hugging Face](https://huggingface.co/OpenAssistant) - All data **must** be `UTF-8` encoded to simplify training! ## **Dataset Formats** To simplify the training process, all datasets must be `UTF-8` encoded and stored in either one of these two formats: - parquet with the option `row_group_size=100` and `index=False` - jsonl or jsonl.gz ## **Dataset Types** There are 4 types of datasets that currently accepted: - Instruction - Multi-turn Dialog - Safety - Text-only ### **Instruction dataset** Instruction datasets are designed to align language models with human interactions. These can take the form of question-answer, request-response, task-solution pairs, and so on. The instruction dataset must include the following columns: 1. **INSTRUCTION** (string): Instruction text 2. **RESPONSE** (string): Expected response to the instruction 3. **SOURCE** (string): Original data source short name, e.g. "wikipedia" 4. **METADATA** (JSON string, optional): Any other useful information stored in JSON
For example, NSFW content can be marked as `{"nsfw": true}` ### **Multi-turn dialog dataset** This type of dataset is designed for conversations with multiple continuations. In this format, each conversation is represented as a tree structure, where each node represents a message from the user or the assistant. For instance, Open-Assistant is collecting the data in a similar format ([example](https://github.com/LAION-AI/Open-Assistant/blob/main/model/model_eval/manual/data/en_100_message.jsonl.gz)). The dataset must be a jsonl file with the following schema: ```python { "thread": { "text": "", # Message text "role": "", # Message role: "prompter" or "assistant" "meta": {}, # Message optional metadata, for example, message rank, safety score and so on "replies": [] # A list of message responses, each with the same structure as "thread" }, "source": "", # Source of the conversation "meta": {} # Optional metadata of the conversation } ``` For example: ```json { "thread": { "text": "What is the best programing language in 2023?", "role": "prompter", "meta": { "lang": "en" }, "replies": [ { "text": "It depends on the task that you aiming to solve.", "role": "assistant", "meta": { "rank": 0 }, "replies": [ { "text": "I want to start learning to code", "role": "prompter", "meta": { "rank": 0 }, "replies": [] }, { "text": "I want to make money", "role": "prompter", "meta": { "rank": 1 }, "replies": [] } ] }, { "text": "Python is the best.", "role": "assistant", "meta": { "rank": 1 }, "replies": [] } ] }, "source": "twitter", "meta": { "post_id": "..." } } ``` ### **Safety dataset** For datasets that are intended to be used to train safety models, prosocial format is proposed. The format is given below 1. **USER** (string): the potentially unsafe utterance 2. **RESPONSE** (string, optional): the guiding utterance grounded on rules-of-thumb (rots) 3. **ROTs** (List): the relevant rules-of-thumb for text not labeled as **casual** 4. **SAFETY_LABEL** (string): the final verdict of the context according to safety_annotations: {**casual**, **possibly_needs_caution**, **probably_needs_caution**, **needs_caution**, **needs_intervention**} 5. **EPISODE_DONE** (bool): an indicator of whether it is the end of the dialogue 6. **SOURCE** (string,optional) : the source of the seed text that was used to craft the first utterance of the dialogue: {socialchemistry, sbic, ethics_amt, ethics_reddit} ### **Text-only dataset** For datasets that do not fit any previous types. The text-only dataset must include the following columns: 1. **TEXT** (string) 2. **SOURCE** (string) 3. **METADATA** (JSON string, optional) ## **Dataset Requirements** The dataset must adhere to the following requirements: - Must have a permissive license - Must not contain child sexual abuse materials - Must not contain materials with private individual's personal information (e.g. name, address, phone number, government ID, or medical information) ## **How to Contribute** To add a new dataset to OpenAssistant, follow these steps: 1. **Create an issue**: Create a new [issue](https://github.com/LAION-AI/Open-Assistant/issues/new) and describe your proposal for the new dataset. 2. **Create a dataset on Hugging Face**: Create a dataset on [HuggingFace](https://huggingface.co). See [below](#creating-a-dataset-on-huggingface) for more details. 3. **Make a pull request**: Add a new dataset loading script to this folder and link the issue in the pull request description. For more information, see [below](#making-a-pull-request). ### **Creating a Dataset on Hugging Face** To create a new dataset on Hugging Face, follow these steps: #### 1. Convert your dataset file(s) to the Parquet format using [pandas](https://pandas.pydata.org/) and [pyarrow](https://pypi.org/project/pyarrow/) libraries: ```python import pandas as pd # Create a pandas dataframe from your dataset file(s) df = pd.read_json(...) # or any other way # Save the file in the Parquet format df.to_parquet("dataset.parquet", row_group_size=100, engine="pyarrow", index=False) ``` Make sure the text data in the dataframe is properly encoded as `UTF-8`! #### 2. Install Hugging Face Hub ```bash pip install huggingface_hub ``` #### 3. Log in to Hugging Face Use your [access token](https://huggingface.co/docs/hub/security-tokens) to login: - Via terminal ```bash huggingface-cli login ``` - in Jupyter notebook (currently does not work in [Visual Studio Code](https://github.com/huggingface/huggingface_hub/issues/752)) ```python from huggingface_hub import notebook_login notebook_login() ``` #### 4. Push the Parquet file to Hugging Face using the following code: ```python from datasets import Dataset ds = Dataset.from_parquet("dataset.parquet") ds.push_to_hub("your_huggingface_name/dataset_name") ``` #### 5. Update the Hugging Face `README.md` file Update the `README.md` file of your dataset by visiting this link: https://huggingface.co/datasets/your_huggingface_name/dataset_name/edit/main/README.md (paste your HuggingFace name and dataset) ### **Making a Pull Request** #### 1. Fork this repository #### 2. Create a new branch in your fork #### 3. Add your dataset to the repository - Create a folder with the name of your dataset. - Add files that describe your dataset and its creation, such as a README, notebooks, scrapers, etc. - Add your dataset to the parent `__init__.py` ```python INSTRUCTION_DATASETS = { ..., "dataset_name": "your_huggingface_name/dataset_name" } ``` #### 4. Stage your changes and run the pre-commit hook ```bash pre-commit run ``` #### 5. Submit a pull request - Submit a pull request and include a link to the issue it resolves in the description, for example: `Resolves #123` ================================================ FILE: data/datasets/TSSB-3M/README.md ================================================ # Dataset summary This dataset contains over 3 million isolated single statement bug fixes. Each bug fix is related to a commit in a public Python that does not change more than a single statement 1. The original dataset comes from the [TSSB-3M](https://zenodo.org/record/5845439) dataset 2. By requesting the GitHub api to obtain the commit message, we expand and create a new dataset [TSSB-3M-ext](https://huggingface.co/datasets/zirui3/TSSB-3M-ext) 3. Convert `TSSB-3M-ext` into instruction form to form the [TSSB-3M-instruction](https://huggingface.co/datasets/zirui3/TSSB-3M-instructions) dataset ================================================ FILE: data/datasets/TSSB-3M/generate_dataset.py ================================================ """Convert the source TSSB-3M dataset to instruction data """ import json import random import re from os.path import join from tqdm import tqdm INSTRUCTIONS_LIST = [ "Find the bug in the following code:", "Identify the error in the code snippet provided:", "Spot the issue within the given code segment:", "Locate the problem in the code example below:", "Uncover the malfunction in the following piece of code:", "Detect the flaw in the code provided:", "Pinpoint the glitch in the code sample below:", "Search for the anomaly in the given code:", "Determine the defect within the following code:", "Discover the fault in the code segment provided:", "Trace the irregularity in the code example below:", "Please locate the error in the code provided.", "Can you identify the mistake in this code?", "There seems to be a problem with this code. Can you find it?", "Please investigate the code and locate the bug.", "Please examine the code and find the error.", "Can you pinpoint the issue with this code?", "Please review the code and identify the bug.", "Can you detect the problem with this code?", "Please analyze the code and find the mistake.", "Can you spot the bug in the code provided?", ] RESPONSE_PREFIX_WORDS = [ "The fix of the bug can be laid out as", "The resolution of the error can be portrayed like so", "The solution for the flaw can be summarized as such", "The remedy of the mistake can be captured in this way", "The correction of the fault can be depicted like this", "The patch for the glitch can be articulated as", "The workaround of the defect can be conveyed in this manner", "The troubleshooting of the issue can be explained like this", "The adjustment to the anomaly can be illustrated as follows", "The modification for the irregularity can be exemplified like this", ] def gen_instruction(): idx = random.randint(0, len(INSTRUCTIONS_LIST) - 1) return INSTRUCTIONS_LIST[idx] def gen_response_prefix(): idx = random.randint(0, len(RESPONSE_PREFIX_WORDS) - 1) return RESPONSE_PREFIX_WORDS[idx] TEMPLATE = """User: {} {} Reply: The fixed code is: ``` {} ``` """ # template for pretty output(multiple lines with `User:` & `Reply`) TEMPLATE_COMMIT_MSG = """User: {} {} Reply: {}: {} The fixed code is: ``` {} ``` """ INSTRUCTON_TEMPLATE = """{} {} """ # template for json output(value) RESPONSE_TEMPLATE = """The fixed code is: ``` {} ``` """ RESPONSE_TEMPLATE_COMMIT_MSG = """{}: {} The fixed code is: ``` {} ``` """ def remove_starting_plus_minus(text): if text.startswith("+") or text.startswith("-"): return text[1:] else: return text def remove_extraneous_diff_info(text): pattern = "@@.*@@" return re.sub(pattern, "", text) def clean(text): return remove_extraneous_diff_info(remove_starting_plus_minus(text)) def clean_PII(text): # Remove sign-off messege generated by `git commit --signoff`, eg. "Signed-off-by: user_name