Copy disabled (too large)
Download .txt
Showing preview only (12,416K chars total). Download the full file to get everything.
Repository: LAION-AI/Open-Assistant
Branch: main
Commit: f1e6ed9526f5
Files: 1538
Total size: 11.6 MB
Directory structure:
gitextract_4_dpaklk/
├── .devcontainer/
│ ├── README.md
│ ├── backend-dev/
│ │ ├── devcontainer.json
│ │ └── post_create_command.sh
│ ├── devcontainer.json
│ ├── frontend-dev/
│ │ ├── devcontainer.json
│ │ └── post_create_command.sh
│ └── post_create_command.sh
├── .dockerignore
├── .gitattributes
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── build-frontend.yaml
│ ├── build-postgres.yaml
│ ├── deploy-docs-site.yaml
│ ├── deploy-to-node.yaml
│ ├── docker-build.yaml
│ ├── pre-commit.yaml
│ ├── production-deploy.yaml
│ ├── production2-deploy.yaml
│ ├── release.yaml
│ ├── test-api-contract.yaml
│ └── test-e2e.yaml
├── .gitignore
├── .pre-commit-config.yaml
├── .python-version
├── .vscode/
│ ├── extensions.json
│ ├── launch.json
│ └── settings.json
├── CODEOWNERS
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── ansible/
│ ├── .gitignore
│ ├── README.md
│ ├── deploy-to-node.yaml
│ ├── inference/
│ │ ├── deploy-server.yaml
│ │ ├── deploy-worker.yaml
│ │ ├── redis.conf
│ │ └── test.inventory.ini
│ ├── pgbackrest.conf
│ ├── redis.conf
│ └── test.inventory.ini
├── backend/
│ ├── .gitignore
│ ├── README.md
│ ├── alembic/
│ │ ├── README
│ │ ├── env.py
│ │ ├── script.py.mako
│ │ └── versions/
│ │ ├── 2022_12_15_0000-23e5fea252dd_first_revision.py
│ │ ├── 2022_12_16_0000-cd7de470586e_v1_db_structure.py
│ │ ├── 2022_12_17_2230-6368515778c5_add_auth_method_to_person.py
│ │ ├── 2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py
│ │ ├── 2022_12_25_1705-067c4002f2d9_add_text_labels.py
│ │ ├── 2022_12_27_1444-3358eb6834e6_add_journal_table.py
│ │ ├── 2022_12_28_1142-d24b37426857_post_ref_for_work_package.py
│ │ ├── 2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py
│ │ ├── 2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py
│ │ ├── 2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py
│ │ ├── 2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py
│ │ ├── 2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py
│ │ ├── 2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py
│ │ ├── 2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py
│ │ ├── 2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py
│ │ ├── 2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py
│ │ ├── 2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py
│ │ ├── 2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py
│ │ ├── 2023_01_08_2128-aac6b2f66006_created_date.py
│ │ ├── 2023_01_08_2200-bcc2fe18d214_messagetoxicity.py
│ │ ├── 2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py
│ │ ├── 2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py
│ │ ├── 2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py
│ │ ├── 2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py
│ │ ├── 2023_01_14_1509-619255ae9076_add_rank_to_message_table.py
│ │ ├── 2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py
│ │ ├── 2023_01_15_1139-423557e869e4_add_indices_for_created_date.py
│ │ ├── 2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py
│ │ ├── 2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py
│ │ ├── 2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py
│ │ ├── 2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py
│ │ ├── 2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py
│ │ ├── 2023_01_24_2256-40ed93df0ed5_add_message_emoji.py
│ │ ├── 2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py
│ │ ├── 2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py
│ │ ├── 2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py
│ │ ├── 2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py
│ │ ├── 2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py
│ │ ├── 2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py
│ │ ├── 2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py
│ │ ├── 2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py
│ │ ├── 2023_02_02_1817-8c8241d1f973_add_account_table.py
│ │ ├── 2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py
│ │ ├── 2023_02_11_1030-ba40d055714a_add_cached_stats.py
│ │ ├── 2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py
│ │ ├── 2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py
│ │ ├── 2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py
│ │ ├── 2023_05_07_2129-1b6e3ae16e9d_add_text_search.py
│ │ └── 2023_06_06_1505-c181661eba3a_add_message_revisions.py
│ ├── alembic.ini
│ ├── export.py
│ ├── import.py
│ ├── main.py
│ ├── oasst_backend/
│ │ ├── __init__.py
│ │ ├── api/
│ │ │ ├── __init__.py
│ │ │ ├── deps.py
│ │ │ └── v1/
│ │ │ ├── __init__.py
│ │ │ ├── admin.py
│ │ │ ├── api.py
│ │ │ ├── auth.py
│ │ │ ├── frontend_messages.py
│ │ │ ├── frontend_users.py
│ │ │ ├── hugging_face.py
│ │ │ ├── leaderboards.py
│ │ │ ├── login.py
│ │ │ ├── messages.py
│ │ │ ├── stats.py
│ │ │ ├── tasks.py
│ │ │ ├── text_labels.py
│ │ │ ├── trollboards.py
│ │ │ ├── users.py
│ │ │ └── utils.py
│ │ ├── auth.py
│ │ ├── cached_stats_repository.py
│ │ ├── celery_worker.py
│ │ ├── config.py
│ │ ├── database.py
│ │ ├── journal_writer.py
│ │ ├── models/
│ │ │ ├── __init__.py
│ │ │ ├── api_client.py
│ │ │ ├── cached_stats.py
│ │ │ ├── db_payload.py
│ │ │ ├── flagged_message.py
│ │ │ ├── journal.py
│ │ │ ├── message.py
│ │ │ ├── message_embedding.py
│ │ │ ├── message_emoji.py
│ │ │ ├── message_reaction.py
│ │ │ ├── message_revision.py
│ │ │ ├── message_toxicity.py
│ │ │ ├── message_tree_state.py
│ │ │ ├── payload_column_type.py
│ │ │ ├── task.py
│ │ │ ├── text_labels.py
│ │ │ ├── troll_stats.py
│ │ │ ├── user.py
│ │ │ └── user_stats.py
│ │ ├── prompt_repository.py
│ │ ├── scheduled_tasks.py
│ │ ├── schemas/
│ │ │ ├── __init__.py
│ │ │ ├── hugging_face.py
│ │ │ ├── message_tree.py
│ │ │ └── text_labels.py
│ │ ├── task_repository.py
│ │ ├── tree_manager.py
│ │ ├── user_repository.py
│ │ ├── user_stats_repository.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── database_utils.py
│ │ ├── discord.py
│ │ ├── exported_tree_loading.py
│ │ ├── hugging_face.py
│ │ ├── language_classification.py
│ │ ├── message_tree_topic_modeling.py
│ │ ├── ranking.py
│ │ ├── similarity_functions.py
│ │ ├── topic_model_requirments.txt
│ │ └── tree_export.py
│ ├── requirements.txt
│ ├── requirements_worker.txt
│ ├── rerank.py
│ ├── sql_snippets.md
│ ├── test_data/
│ │ ├── generic/
│ │ │ └── test_generic_data.json
│ │ └── realistic/
│ │ └── realistic_seed_data.json
│ ├── tests/
│ │ ├── __init__.py
│ │ ├── test_settings.py
│ │ └── test_tree_manager_config.py
│ └── update_message_attributes.py
├── copilot/
│ ├── .workspace
│ ├── README.md
│ ├── api/
│ │ └── manifest.yml
│ ├── environments/
│ │ └── staging/
│ │ └── manifest.yml
│ └── web/
│ └── manifest.yml
├── data/
│ ├── __init__.py
│ ├── datasets/
│ │ ├── README.md
│ │ ├── TSSB-3M/
│ │ │ ├── README.md
│ │ │ ├── generate_dataset.py
│ │ │ ├── invalid_commit_messages.tsv
│ │ │ └── load_script.py
│ │ ├── __init__.py
│ │ ├── bart_searchgpt_wiki_nlp_augment/
│ │ │ ├── 1_clean_wikitext.py
│ │ │ ├── 2_wikitext_doc2query.ipynb
│ │ │ ├── 3_10k_bart_trial.ipynb
│ │ │ ├── 3_10k_bart_trial.py
│ │ │ ├── 4_convert_to_oa_format.py
│ │ │ ├── 5_test_downloading_my_dataset.py
│ │ │ ├── README.md
│ │ │ └── requirement.txt
│ │ ├── biostars_qa/
│ │ │ ├── README.md
│ │ │ ├── get_biostars_dataset.py
│ │ │ └── requirements.txt
│ │ ├── cmu_wiki_qa/
│ │ │ ├── README.md
│ │ │ ├── cmu_parser.ipynb
│ │ │ └── requirements.txt
│ │ ├── cocktail_recipes/
│ │ │ └── loading_script.py
│ │ ├── codet_humaneval_mbpp/
│ │ │ ├── HumanEval_and_MBPP_code_gen.ipynb
│ │ │ ├── HumanEval_and_MBPP_test_gen.ipynb
│ │ │ └── README.md
│ │ ├── fa-isna-news/
│ │ │ └── README.md
│ │ ├── fa-wikipedia/
│ │ │ └── README.md
│ │ ├── fd_dialogue/
│ │ │ └── README.md
│ │ ├── grade_school_math_instructions/
│ │ │ └── dataset_creation.ipynb
│ │ ├── gutenberg/
│ │ │ ├── README.md
│ │ │ ├── project_gutenberg_crawler.ipynb
│ │ │ └── requirements.txt
│ │ ├── iapp_wiki_qa_squad/
│ │ │ ├── README.md
│ │ │ ├── iapp_wiki_qa_squad_oa.ipynb
│ │ │ ├── loading_script.py
│ │ │ └── requirements.txt
│ │ ├── instructional_codesearchnet_python/
│ │ │ ├── GenerateOpenAssistantInstructionResponseFormat.ipynb
│ │ │ ├── README.md
│ │ │ └── Summarize_codesearchnet_for_python.ipynb
│ │ ├── logicreference_OA/
│ │ │ ├── README.md
│ │ │ ├── generate_dataset.py
│ │ │ └── requirements.txt
│ │ ├── mt_note_generation/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── mt_note_generation.py
│ │ │ └── prepare.py
│ │ ├── nsfw_selfharm_reddit/
│ │ │ ├── .gitignore
│ │ │ ├── README.md
│ │ │ ├── dataset-cookbook.ipynb
│ │ │ ├── prosocial.ipynb
│ │ │ └── utils/
│ │ │ ├── __init__.py
│ │ │ ├── is_question.py
│ │ │ └── reddit.py
│ │ ├── oa_dolly_15k/
│ │ │ ├── README.md
│ │ │ └── create_dataset.py
│ │ ├── oa_leet10k/
│ │ │ ├── README.md
│ │ │ └── oa_leet10k.ipynb
│ │ ├── oa_stackexchange/
│ │ │ ├── README.md
│ │ │ ├── combine.py
│ │ │ ├── download.py
│ │ │ ├── merge_parquets.py
│ │ │ ├── process.py
│ │ │ ├── stats.py
│ │ │ └── upload.py
│ │ ├── poetry_instruction/
│ │ │ ├── README.md
│ │ │ ├── prepare.py
│ │ │ └── requirements.txt
│ │ ├── prosocial_confessions/
│ │ │ ├── README.md
│ │ │ └── prosocial-confessions.ipynb
│ │ ├── reasoning_bg_oa/
│ │ │ ├── README.MD
│ │ │ └── data_process.py
│ │ ├── reasoning_gsm_qna_oa/
│ │ │ ├── README.MD
│ │ │ └── data_process.py
│ │ ├── recipes/
│ │ │ ├── README.md
│ │ │ └── tasty_recipes.ipynb
│ │ ├── safety_directory/
│ │ │ ├── child_help/
│ │ │ │ └── child_help.py
│ │ │ ├── emergency_infos/
│ │ │ │ ├── wikipedia_emergency_info.js
│ │ │ │ └── wikipedia_emergency_info.json
│ │ │ └── emergency_numbers/
│ │ │ └── emergency_numbers.py
│ │ ├── semantics_ws_qna_oa/
│ │ │ ├── README.MD
│ │ │ ├── data_process.py
│ │ │ └── random_stuff.py
│ │ ├── soda_synthetic_dialogue/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── prepare.py
│ │ │ └── soda_synthetic_dialogue.py
│ │ ├── tatoeba_mt_qna_oa/
│ │ │ ├── README.MD
│ │ │ ├── data_process.py
│ │ │ ├── language_names.py
│ │ │ ├── language_paraphrase.py
│ │ │ └── language_translate.py
│ │ ├── tell_a_joke/
│ │ │ └── tell_a_joke.ipynb
│ │ ├── tlcv2.0_oa/
│ │ │ ├── README.md
│ │ │ └── tlcv2_0_oa.ipynb
│ │ ├── tv_dialogue/
│ │ │ ├── README.md
│ │ │ ├── imsdb.ipynb
│ │ │ ├── public.ipynb
│ │ │ └── requirements.txt
│ │ ├── ubuntu_dialogue_qa/
│ │ │ ├── README.md
│ │ │ ├── requirements.txt
│ │ │ └── ubuntu_parser.ipynb
│ │ ├── youtube_subs_howto100M/
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── prepare.py
│ │ │ ├── requirements.txt
│ │ │ └── youtube_subs_howto100M.py
│ │ └── zhihu-kol/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── convert_parquet.py
│ │ ├── main.py
│ │ ├── requirements.txt
│ │ ├── scrape_by_topic.py
│ │ └── upload_hf.py
│ └── dev-requirements.txt
├── deploy/
│ ├── README.md
│ ├── dev-node/
│ │ └── nginx/
│ │ ├── docker-compose.yaml
│ │ ├── get_cert.sh
│ │ ├── nginx.conf
│ │ └── renew_certs.sh
│ ├── prod-node/
│ │ └── nginx/
│ │ ├── docker-compose.yaml
│ │ ├── get_cert.sh
│ │ ├── nginx.conf
│ │ └── renew_certs.sh
│ └── prod2-node/
│ └── nginx/
│ ├── docker-compose.yaml
│ ├── get_cert.sh
│ ├── nginx.conf
│ └── renew_certs.sh
├── discord-bots/
│ ├── oa-bot-js/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── docker-compose.yml
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── bot.ts
│ │ │ ├── commands/
│ │ │ │ ├── bot.ts
│ │ │ │ ├── chat.ts
│ │ │ │ └── task.ts
│ │ │ ├── events/
│ │ │ │ ├── commands.ts
│ │ │ │ ├── interactions.ts
│ │ │ │ ├── messages.ts
│ │ │ │ └── ready.ts
│ │ │ ├── handlers/
│ │ │ │ ├── commands.ts
│ │ │ │ ├── events.ts
│ │ │ │ └── interactions.ts
│ │ │ ├── index.ts
│ │ │ ├── interactions/
│ │ │ │ ├── model.ts
│ │ │ │ ├── modelselect.ts
│ │ │ │ ├── tasks.ts
│ │ │ │ └── vote.ts
│ │ │ └── modules/
│ │ │ ├── chat.ts
│ │ │ ├── db.ts
│ │ │ ├── inference/
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── open-assistant/
│ │ │ │ ├── interactions/
│ │ │ │ │ ├── info.ts
│ │ │ │ │ ├── init.ts
│ │ │ │ │ ├── label.ts
│ │ │ │ │ ├── lang.ts
│ │ │ │ │ └── task.ts
│ │ │ │ ├── interactions.ts
│ │ │ │ ├── labels.ts
│ │ │ │ ├── langs.ts
│ │ │ │ ├── tasks.ts
│ │ │ │ └── user.ts
│ │ │ └── redis.ts
│ │ └── tsconfig.json
│ └── oa-bot-py/
│ ├── .gitignore
│ ├── README.md
│ ├── bot/
│ │ ├── __init__.py
│ │ ├── __main__.py
│ │ ├── bot.py
│ │ ├── db/
│ │ │ ├── schema.sql
│ │ │ └── schemas.py
│ │ ├── extensions/
│ │ │ ├── __init__.py
│ │ │ ├── guild_settings.py
│ │ │ ├── help.py
│ │ │ ├── hot_reload.py
│ │ │ ├── text_labels.py
│ │ │ ├── user_input_test.py
│ │ │ └── work.py
│ │ ├── messages.py
│ │ ├── settings.py
│ │ └── utils.py
│ ├── message_templates.py
│ ├── requirements.txt
│ └── templates/
│ ├── boot.msg
│ ├── help.msg
│ ├── task_assistant_reply.msg
│ ├── task_initial_prompt.msg
│ ├── task_prompter_reply.msg
│ ├── task_rank_conversation_replies.msg
│ ├── task_rank_initial_prompts.msg
│ ├── task_rate_summary.msg
│ ├── task_summarize_story.msg
│ ├── teaser_assistant_reply.msg
│ ├── teaser_initial_prompt.msg
│ ├── teaser_prompter_reply.msg
│ ├── teaser_rank_conversation_replies.msg
│ ├── teaser_rank_initial_prompts.msg
│ ├── teaser_rate_summary.msg
│ ├── teaser_summarize_story.msg
│ └── welcome.msg
├── docker/
│ ├── Dockerfile.backend
│ ├── Dockerfile.backend-worker
│ ├── Dockerfile.discord-bot
│ ├── Dockerfile.model-training
│ ├── Dockerfile.website
│ ├── grafana/
│ │ ├── README.md
│ │ ├── dashboards/
│ │ │ ├── dashboard.yaml
│ │ │ └── fastapi-backend.json
│ │ └── datasources/
│ │ └── datasource.yml
│ ├── inference/
│ │ ├── Dockerfile.safety
│ │ ├── Dockerfile.server
│ │ ├── Dockerfile.worker
│ │ ├── Dockerfile.worker-full
│ │ ├── Dockerfile.worker-hf
│ │ └── Dockerfile.worker-standalone
│ ├── netdata/
│ │ ├── README.md
│ │ └── go.d/
│ │ ├── postgres.conf
│ │ ├── prometheus.conf
│ │ └── redis.conf
│ ├── oasst-postgres/
│ │ ├── Dockerfile
│ │ └── backup_pg_to_s3.sh
│ └── prometheus/
│ ├── README.md
│ └── prometheus.yml
├── docker-compose.yaml
├── docs/
│ ├── .gitignore
│ ├── .yarnrc.yml
│ ├── README.md
│ ├── babel.config.js
│ ├── blog/
│ │ ├── 2023-02-05-we-need-your-help.mdx
│ │ ├── 2023-02-11-architecture.md
│ │ ├── 2023-02-24-open-assistant-inference-backend-development-hands-on-coding/
│ │ │ └── index.mdx
│ │ ├── 2023-04-06-open-assistant-first-models-are-here/
│ │ │ └── index.mdx
│ │ ├── 2023-04-10-open-assistant-livestream-just-chatting/
│ │ │ └── index.mdx
│ │ ├── 2023-04-15-open-assistant-released/
│ │ │ └── index.mdx
│ │ ├── 2023-04-21-open-assistant-storms-youtube/
│ │ │ └── index.md
│ │ ├── 2023-10-25-open-assistant-is-completed/
│ │ │ └── index.mdx
│ │ └── authors.yml
│ ├── docs/
│ │ ├── api/
│ │ │ ├── backend-openapi.json
│ │ │ └── inference-openapi.json
│ │ ├── architecture/
│ │ │ ├── README.md
│ │ │ └── inference.md
│ │ ├── data/
│ │ │ ├── README.md
│ │ │ ├── augmentation.md
│ │ │ ├── datasets.md
│ │ │ ├── schemas.mdx
│ │ │ └── supervised-datasets.md
│ │ ├── faq.md
│ │ ├── guides/
│ │ │ ├── README.md
│ │ │ ├── data_collection.md
│ │ │ ├── developers.md
│ │ │ ├── examples.md
│ │ │ └── guidelines.md
│ │ ├── intro.md
│ │ ├── plugins/
│ │ │ ├── README.md
│ │ │ ├── details.md
│ │ │ ├── getting-started.md
│ │ │ └── list.md
│ │ ├── presentations/
│ │ │ └── README.md
│ │ ├── research/
│ │ │ ├── README.md
│ │ │ ├── general.md
│ │ │ ├── retrieval.md
│ │ │ └── search-based-qa.md
│ │ └── tasks/
│ │ ├── README.md
│ │ ├── label_assistant_reply.md
│ │ ├── label_prompter_reply.md
│ │ ├── rank_assistant_replies.md
│ │ ├── reply_as_assistant.md
│ │ └── reply_as_user.md
│ ├── docusaurus.config.js
│ ├── package.json
│ ├── sidebars.js
│ ├── src/
│ │ ├── components/
│ │ │ └── HomepageFeatures/
│ │ │ ├── index.tsx
│ │ │ └── styles.module.css
│ │ ├── css/
│ │ │ └── custom.css
│ │ └── pages/
│ │ ├── index.module.css
│ │ └── index.tsx
│ └── static/
│ └── .nojekyll
├── inference/
│ ├── .gitignore
│ ├── README.md
│ ├── full-dev-setup.sh
│ ├── safety/
│ │ ├── README.md
│ │ ├── main.py
│ │ ├── requirements.txt
│ │ ├── safety_main.sh
│ │ └── settings.py
│ ├── server/
│ │ ├── README.md
│ │ ├── alembic/
│ │ │ ├── README
│ │ │ ├── env.py
│ │ │ ├── script.py.mako
│ │ │ └── versions/
│ │ │ ├── .gitinclude
│ │ │ ├── 2023_03_12_1742-7d5be54acd49_initial_revision.py
│ │ │ ├── 2023_03_21_2116-629d5081160f_changed_worker_config_to_worker_info.py
│ │ │ ├── 2023_03_22_2113-78f16015b904_add_refresh_token_table.py
│ │ │ ├── 2023_04_12_2033-f0e18084aae4_add_deleted_field_to_user.py
│ │ │ ├── 2023_04_14_1611-b66fd8f9da1f_add_hidden_field_to_chats.py
│ │ │ ├── 2023_04_24_2130-401eef162771_add_chat_data_opt_out_field.py
│ │ │ ├── 2023_04_29_1739-ea19bbc743f9_add_safe_content_to_message.py
│ │ │ ├── 2023_05_01_2253-5b4211625a9f_added_used_plugin_to_message.py
│ │ │ └── 2023_05_29_1551-5ed411a331f4_add_active_thread_tail_messsage_id_and_.py
│ │ ├── alembic.ini
│ │ ├── export.py
│ │ ├── main.py
│ │ ├── oasst_inference_server/
│ │ │ ├── __init__.py
│ │ │ ├── admin.py
│ │ │ ├── auth.py
│ │ │ ├── chat_repository.py
│ │ │ ├── chat_utils.py
│ │ │ ├── compliance.py
│ │ │ ├── database.py
│ │ │ ├── deps.py
│ │ │ ├── models/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── chat.py
│ │ │ │ ├── user.py
│ │ │ │ └── worker.py
│ │ │ ├── plugin_utils.py
│ │ │ ├── plugins/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── gale_pleaser/
│ │ │ │ │ ├── README.md
│ │ │ │ │ ├── ai-plugin.json
│ │ │ │ │ └── main.py
│ │ │ │ ├── gale_roaster/
│ │ │ │ │ ├── ai-plugin.json
│ │ │ │ │ └── main.py
│ │ │ │ └── web_retriever/
│ │ │ │ ├── ai-plugin.json
│ │ │ │ └── main.py
│ │ │ ├── queueing.py
│ │ │ ├── routes/
│ │ │ │ ├── account.py
│ │ │ │ ├── admin.py
│ │ │ │ ├── auth.py
│ │ │ │ ├── chats.py
│ │ │ │ ├── configs.py
│ │ │ │ └── workers.py
│ │ │ ├── schemas/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── auth.py
│ │ │ │ ├── chat.py
│ │ │ │ └── worker.py
│ │ │ ├── settings.py
│ │ │ ├── user_chat_repository.py
│ │ │ └── worker_utils.py
│ │ ├── requirements.txt
│ │ └── server_main.sh
│ ├── tests/
│ │ └── locust/
│ │ └── locustfile.py
│ ├── text-client/
│ │ ├── __main__.py
│ │ ├── requirements.txt
│ │ └── text_client_utils.py
│ └── worker/
│ ├── README.md
│ ├── __main__.py
│ ├── basic_hf_server.py
│ ├── chat_chain.py
│ ├── chat_chain_prompts.py
│ ├── chat_chain_utils.py
│ ├── download_model.py
│ ├── download_model_hf.py
│ ├── get_model_config_prop.py
│ ├── hf_langchain_inference.py
│ ├── hf_stopping.py
│ ├── hf_streamer.py
│ ├── interface.py
│ ├── openapi_parser.py
│ ├── requirements-hf.txt
│ ├── requirements.txt
│ ├── run_worker_container.sh
│ ├── settings.py
│ ├── utils.py
│ ├── work.py
│ ├── worker_full_main.sh
│ ├── worker_hf_main.sh
│ └── worker_standalone_main.sh
├── inlang.config.js
├── model/
│ ├── .gitignore
│ ├── MESSAGE_AND_TOKEN_FORMAT.md
│ ├── README.md
│ ├── model_eval/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── eval_datasets.py
│ │ ├── eval_rm.py
│ │ ├── manual/
│ │ │ ├── config/
│ │ │ │ ├── default.json
│ │ │ │ ├── default_rallio.json
│ │ │ │ ├── noprefix.json
│ │ │ │ ├── noprefix2.json
│ │ │ │ ├── noprefix2_profile.json
│ │ │ │ ├── synth.json
│ │ │ │ ├── synth2.json
│ │ │ │ ├── synth2_gale_prefix.json
│ │ │ │ ├── synth2_rallio_original.json
│ │ │ │ ├── synth_rallio.json
│ │ │ │ ├── system_profile.json
│ │ │ │ └── system_profile2.json
│ │ │ ├── create_synth_import.py
│ │ │ ├── data/
│ │ │ │ ├── en_100_message.jsonl
│ │ │ │ ├── en_100_text.jsonl
│ │ │ │ ├── en_100_tree.jsonl
│ │ │ │ └── prompt_lottery_en_250_text.jsonl
│ │ │ ├── requirements.txt
│ │ │ ├── sampling_report.py
│ │ │ ├── sampling_reports/
│ │ │ │ ├── 2023-03-01_theblackcat102_pythia-12b-deduped-sft_sampling.json
│ │ │ │ ├── 2023-03-01_theblackcat102_pythia-1b-deduped-sft_sampling.json
│ │ │ │ └── 2023-03-01_theblackcat102_pythia-3b-deduped-sft_sampling.json
│ │ │ └── subsample_dataset.py
│ │ ├── rejection_sampling.py
│ │ ├── sample_results/
│ │ │ ├── comparison.json
│ │ │ ├── rejected_samples.json
│ │ │ ├── selected_samples.json
│ │ │ └── theblackcat102-pythia-12b-deduped-sft.json
│ │ ├── sampling_score.py
│ │ └── utils.py
│ ├── model_training/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── check_dataset_appearances.py
│ │ ├── check_dataset_counts.py
│ │ ├── configs/
│ │ │ ├── accelerate_config.yaml
│ │ │ ├── config.yaml
│ │ │ ├── config_rl.yaml
│ │ │ ├── config_rm.yaml
│ │ │ ├── deepspeed_rl.json
│ │ │ ├── deepspeed_rl_zero3.json
│ │ │ ├── ppo_config.yaml
│ │ │ ├── triton_config_rm.pbtxt
│ │ │ ├── triton_config_sft.pbtxt
│ │ │ ├── zero3_config_falcon.json
│ │ │ ├── zero3_config_pretrain.json
│ │ │ ├── zero3_config_sft.json
│ │ │ ├── zero_config.json
│ │ │ ├── zero_config_pretrain.json
│ │ │ └── zero_config_sft_65b.json
│ │ ├── custom_datasets/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── dialogue_collator.py
│ │ │ ├── extra_rm_datasets.py
│ │ │ ├── formatting.py
│ │ │ ├── instruction.py
│ │ │ ├── oasst_dataset.py
│ │ │ ├── pretrain_datasets.py
│ │ │ ├── prompt_dialogue.py
│ │ │ ├── qa_datasets.py
│ │ │ ├── rank_datasets.py
│ │ │ ├── ranking_collator.py
│ │ │ ├── summarization.py
│ │ │ ├── toxic_conversation.py
│ │ │ ├── translation.py
│ │ │ └── utils.py
│ │ ├── efficiency_utils.py
│ │ ├── metrics.py
│ │ ├── models/
│ │ │ ├── __init__.py
│ │ │ ├── gptj.py
│ │ │ ├── patching.py
│ │ │ ├── patching_falcon.py
│ │ │ ├── patching_llama.py
│ │ │ ├── patching_neox.py
│ │ │ ├── patching_utils.py
│ │ │ ├── peft_modeling.py
│ │ │ ├── prefix_llama.py
│ │ │ ├── reward_model.py
│ │ │ └── rope.py
│ │ ├── tests/
│ │ │ ├── __init__.py
│ │ │ ├── resources/
│ │ │ │ └── data_collator/
│ │ │ │ ├── special_tokens_map.json
│ │ │ │ ├── tokenizer.json
│ │ │ │ └── tokenizer_config.json
│ │ │ ├── test_datasets.py
│ │ │ ├── test_dialogue_data_collator.py
│ │ │ ├── test_formatting.py
│ │ │ ├── test_oasst_dataset.py
│ │ │ ├── test_patched_falcon.py
│ │ │ ├── test_patched_gpt_neox.py
│ │ │ ├── test_patched_llama.py
│ │ │ ├── test_ranking_collator.py
│ │ │ ├── test_rm_loading.py
│ │ │ └── test_utils.py
│ │ ├── to_triton.py
│ │ ├── tools/
│ │ │ ├── augment_oasst.py
│ │ │ ├── check_oasst_export.py
│ │ │ ├── export_model.py
│ │ │ ├── model_chat.py
│ │ │ ├── model_cli.py
│ │ │ └── sample_rm_data.py
│ │ ├── trainer_rl.py
│ │ ├── trainer_rm.py
│ │ ├── trainer_sft.py
│ │ └── utils/
│ │ ├── losses.py
│ │ ├── ppo_utils.py
│ │ ├── utils.py
│ │ └── utils_rl.py
│ ├── pretokenizer/
│ │ ├── README.md
│ │ ├── configs/
│ │ │ └── pretokenize.yaml
│ │ ├── create_hf_tokenizer_config.py
│ │ ├── indexed_dataset.py
│ │ ├── pretokenize.py
│ │ ├── requirements.txt
│ │ └── tokenizer.py
│ └── pyproject.toml
├── notebooks/
│ ├── README.md
│ ├── TSSB-3M-bugs-dataset/
│ │ ├── TSSB-3M-bugs_dataset.ipynb
│ │ └── TSSB-3M-bugs_dataset.md
│ ├── closed-book-qa/
│ │ ├── Closed Book QA Generator.ipynb
│ │ └── README.md
│ ├── data-augmentation/
│ │ ├── README.md
│ │ ├── anthropic/
│ │ │ ├── README.md
│ │ │ ├── safety data-augmentation.ipynb
│ │ │ └── trainer.py
│ │ ├── changemyview-builder/
│ │ │ ├── README.md
│ │ │ └── data_processor.ipynb
│ │ ├── essay-instructions/
│ │ │ ├── README.md
│ │ │ └── essay-instructions.ipynb
│ │ ├── essay-revision/
│ │ │ ├── README.md
│ │ │ └── essay-revision.ipynb
│ │ ├── hippocorpus/
│ │ │ ├── README.md
│ │ │ └── hippocorpus.ipynb
│ │ ├── movie-descriptions/
│ │ │ ├── README.md
│ │ │ └── movie_descriptions.ipynb
│ │ ├── movie-dialogs/
│ │ │ ├── README.md
│ │ │ └── convert-to-instruction-format.ipynb
│ │ ├── stackexchange-builder/
│ │ │ ├── README.md
│ │ │ └── stackexchange-builder.ipynb
│ │ ├── unified-qa/
│ │ │ ├── README.md
│ │ │ └── unified-qa.ipynb
│ │ ├── wikidata-qa/
│ │ │ ├── README.md
│ │ │ ├── requirements.txt
│ │ │ └── wikidata.ipynb
│ │ └── writing-prompt/
│ │ ├── README.md
│ │ └── writing_prompt.ipynb
│ ├── detoxify-evaluation/
│ │ ├── README.md
│ │ └── detoxify-evaluation.ipynb
│ ├── diverse/
│ │ ├── README.md
│ │ └── diverse.ipynb
│ ├── example/
│ │ ├── README.md
│ │ ├── data/
│ │ │ └── data.csv
│ │ ├── example.ipynb
│ │ └── requirements.txt
│ ├── openassistant-oasst1/
│ │ ├── README.md
│ │ └── getting-started.ipynb
│ └── openbugger/
│ ├── README.md
│ └── openbugger_example.ipynb
├── oasst-data/
│ ├── README.md
│ ├── examples/
│ │ ├── clean_dataset.py
│ │ ├── filter_messages.py
│ │ ├── filter_trees.py
│ │ ├── split_dataset.py
│ │ └── tree_to_messages.py
│ ├── oasst_data/
│ │ ├── __init__.py
│ │ ├── reader.py
│ │ ├── schemas.py
│ │ ├── traversal.py
│ │ └── writer.py
│ └── pyproject.toml
├── oasst-shared/
│ ├── README.md
│ ├── oasst_shared/
│ │ ├── __init__.py
│ │ ├── api_client.py
│ │ ├── exceptions/
│ │ │ ├── __init__.py
│ │ │ └── oasst_api_error.py
│ │ ├── model_configs.py
│ │ ├── schemas/
│ │ │ ├── __init__.py
│ │ │ ├── inference.py
│ │ │ └── protocol.py
│ │ └── utils.py
│ ├── pyproject.toml
│ └── tests/
│ ├── __init__.py
│ └── test_oasst_api_client.py
├── pyproject.toml
├── redis.conf
├── safety/
│ └── README.md
├── scripts/
│ ├── backend-development/
│ │ ├── README.md
│ │ ├── run-local-no-limit.sh
│ │ ├── run-local.sh
│ │ ├── start-docker.sh
│ │ ├── start-mock-server.sh
│ │ ├── start-worker.sh
│ │ ├── stop-mock-server.sh
│ │ └── stop-worker.sh
│ ├── data-collection/
│ │ └── twitter/
│ │ ├── README.md
│ │ ├── requirements.txt
│ │ ├── twitter_create_convs.py
│ │ └── twitter_process_json.py
│ ├── data_augment/
│ │ └── data_augment.py
│ ├── discord/
│ │ ├── stats.py
│ │ └── verify-lobby.py
│ ├── frontend-development/
│ │ ├── README.md
│ │ ├── find-missing-locales.py
│ │ ├── run-bot-local.sh
│ │ └── run-contract-test.sh
│ ├── oasst-shared-development/
│ │ └── test.sh
│ ├── postprocessing/
│ │ ├── importance_selection.py
│ │ ├── infogain_selector.py
│ │ ├── ranking_disagreement.py
│ │ ├── rankings.py
│ │ ├── regex_pii_detector.py
│ │ ├── scoring.py
│ │ └── task_schedule.py
│ └── xor-codec/
│ └── xor_codec.py
├── text-frontend/
│ ├── __main__.py
│ ├── auto_main.py
│ └── requirements.txt
└── website/
├── .eslintrc.json
├── .gitignore
├── .nvmrc
├── .prettierignore
├── .prettierrc.json
├── .storybook/
│ ├── decorators.js
│ ├── main.js
│ └── preview.js
├── README.md
├── cypress/
│ ├── README.md
│ ├── components/
│ │ └── Container.cy.tsx
│ ├── contract/
│ │ └── oasst_api_contract_tests.cy.ts
│ ├── e2e/
│ │ ├── auth/
│ │ │ └── signin.cy.ts
│ │ └── tasks/
│ │ ├── label_assistant_reply.cy.ts
│ │ ├── label_initial_prompt.cy.ts
│ │ ├── label_prompter_reply.cy.ts
│ │ ├── no_tasks_available.cy.ts
│ │ └── random.cy.ts
│ ├── fixtures/
│ │ └── example.json
│ ├── support/
│ │ ├── commands.ts
│ │ ├── component-index.html
│ │ ├── component.ts
│ │ ├── e2e.ts
│ │ └── index.ts
│ └── tsconfig.json
├── cypress-visual-screenshots/
│ └── baseline/
│ └── .gitkeep
├── cypress.config.contract.js
├── cypress.config.js
├── docs/
│ └── add_edit_translations.md
├── jest.config.js
├── jest.setup.js
├── next-i18next.config.js
├── next-lint.js
├── next.config.js
├── package.json
├── postcss.config.js
├── prisma/
│ ├── migrations/
│ │ ├── 20230326131923_initial_migration/
│ │ │ └── migration.sql
│ │ ├── 20230805220637_paperack/
│ │ │ └── migration.sql
│ │ └── migration_lock.toml
│ ├── schema.prisma
│ └── seed.ts
├── public/
│ ├── fonts/
│ │ └── lexend.txt
│ ├── locales/
│ │ ├── ar/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── bar/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── bg/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ca/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── cs/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── da/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── de/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── el/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── en/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── eo/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── es/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── eu/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fa/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fi/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fr/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── gl/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── he/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── hi/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── hu/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── id/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── it/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ja/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ko/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── lt/
│ │ │ ├── account.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ms/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── nb-NO/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── nl/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── pl/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── pt-BR/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ro/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ru/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sk/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sl/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sr/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sv/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── swg/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── th/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── tr/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── uk-UA/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── vi/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ └── zh/
│ │ ├── account.json
│ │ ├── chat.json
│ │ ├── common.json
│ │ ├── dashboard.json
│ │ ├── error.json
│ │ ├── index.json
│ │ ├── labelling.json
│ │ ├── leaderboard.json
│ │ ├── message.json
│ │ ├── stats.json
│ │ ├── tasks.json
│ │ └── tos.json
│ ├── manifest.json
│ └── mockServiceWorker.js
├── src/
│ ├── README.md
│ ├── components/
│ │ ├── Account/
│ │ │ ├── UserStats.tsx
│ │ │ └── XPBar.tsx
│ │ ├── AdminArea.tsx
│ │ ├── AnimatedCircles/
│ │ │ ├── AnimatedCircles.tsx
│ │ │ └── index.tsx
│ │ ├── AuthLayout.tsx
│ │ ├── Buttons/
│ │ │ ├── Buttons.stories.tsx
│ │ │ ├── LikertButtons.tsx
│ │ │ ├── Skip.tsx
│ │ │ └── Submit.tsx
│ │ ├── CallToAction.tsx
│ │ ├── Chat/
│ │ │ ├── ChatAssistantDraftPager.tsx
│ │ │ ├── ChatAssistantDraftViewer.tsx
│ │ │ ├── ChatConfig.tsx
│ │ │ ├── ChatConfigDesktop.tsx
│ │ │ ├── ChatConfigForm.tsx
│ │ │ ├── ChatConfigMobile.tsx
│ │ │ ├── ChatConfigSaver.tsx
│ │ │ ├── ChatConfigSummary.tsx
│ │ │ ├── ChatContext.tsx
│ │ │ ├── ChatConversation.tsx
│ │ │ ├── ChatConversationTree.tsx
│ │ │ ├── ChatForm.tsx
│ │ │ ├── ChatInitialDataContext.tsx
│ │ │ ├── ChatInputIconButton.tsx
│ │ │ ├── ChatListBase.tsx
│ │ │ ├── ChatListDesktop.tsx
│ │ │ ├── ChatListItem.tsx
│ │ │ ├── ChatListMobile.tsx
│ │ │ ├── ChatMessageEntry.tsx
│ │ │ ├── ChatSection.tsx
│ │ │ ├── ChatViewSelection.tsx
│ │ │ ├── ChatWarning.tsx
│ │ │ ├── CreateChatButton.tsx
│ │ │ ├── CustomInstructions.tsx
│ │ │ ├── DeletePresetButton.tsx
│ │ │ ├── EncourageMessage.tsx
│ │ │ ├── InferencePoweredBy.tsx
│ │ │ ├── PluginsChooser.tsx
│ │ │ ├── QueueInfoMessage.tsx
│ │ │ ├── SavePresetButton.tsx
│ │ │ ├── WorkParameters.tsx
│ │ │ └── useListChatPagination.ts
│ │ ├── CloudflareCaptcha.tsx
│ │ ├── CollapsableText.tsx
│ │ ├── Container.tsx
│ │ ├── Dashboard/
│ │ │ ├── LeaderboardWidget.tsx
│ │ │ ├── SlimFooter.tsx
│ │ │ ├── TaskOption.tsx
│ │ │ ├── WelcomeCard.tsx
│ │ │ └── index.ts
│ │ ├── DataTable/
│ │ │ ├── DataTable.tsx
│ │ │ ├── DataTableAction.tsx
│ │ │ ├── jsonExpandRowModel.tsx
│ │ │ └── useCursorPagination.ts
│ │ ├── EmptyState.tsx
│ │ ├── Explain.tsx
│ │ ├── Faq.tsx
│ │ ├── Footer.tsx
│ │ ├── Header/
│ │ │ ├── ColorModeToggler.tsx
│ │ │ ├── Header.stories.tsx
│ │ │ ├── Header.tsx
│ │ │ ├── UserMenu.stories.tsx
│ │ │ ├── UserMenu.tsx
│ │ │ ├── UserScore.tsx
│ │ │ └── index.ts
│ │ ├── Hero.tsx
│ │ ├── JsonCard.tsx
│ │ ├── LanguageSelector/
│ │ │ ├── LanguageSelector.tsx
│ │ │ └── index.tsx
│ │ ├── Layout/
│ │ │ └── ChatLayout.tsx
│ │ ├── Layout.tsx
│ │ ├── LeaderboardTable/
│ │ │ ├── LeaderboardTable.tsx
│ │ │ ├── TrollboardTable.tsx
│ │ │ ├── index.tsx
│ │ │ ├── useBoardPagination.ts
│ │ │ ├── useBoardRowProps.ts
│ │ │ └── useFetchBoard.ts
│ │ ├── Loading/
│ │ │ ├── Loading.stories.tsx
│ │ │ ├── LoadingScreen.tsx
│ │ │ └── MessageLoading.tsx
│ │ ├── MarkdownEditor.tsx
│ │ ├── Messages/
│ │ │ ├── AdminMessageTable.tsx
│ │ │ ├── BaseMessageEntry.tsx
│ │ │ ├── LabelFlagGroup.tsx
│ │ │ ├── LabelInputGroup.tsx
│ │ │ ├── LabelPopup.tsx
│ │ │ ├── LabelYesNoGroup.tsx
│ │ │ ├── MessageConversation.stories.tsx
│ │ │ ├── MessageConversation.tsx
│ │ │ ├── MessageCreateDate.tsx
│ │ │ ├── MessageEmojiButton.stories.tsx
│ │ │ ├── MessageEmojiButton.tsx
│ │ │ ├── MessageHistoryTable.tsx
│ │ │ ├── MessageInlineEmojiRow.tsx
│ │ │ ├── MessageSyntheticBadge.tsx
│ │ │ ├── MessageTableEntry.stories.tsx
│ │ │ ├── MessageTableEntry.tsx
│ │ │ ├── MessageTree.tsx
│ │ │ ├── MessageWithChildren.stories.tsx
│ │ │ ├── MessageWithChildren.tsx
│ │ │ ├── PluginUsageDetails.tsx
│ │ │ ├── RenderedCodeblock.tsx
│ │ │ ├── RenderedMarkdown.tsx
│ │ │ ├── ReportPopup.tsx
│ │ │ └── SyntaxHighlighter.tsx
│ │ ├── PolicyCards/
│ │ │ ├── PolicyChapterCard.tsx
│ │ │ └── PolicySectionCard.tsx
│ │ ├── Roadmap.tsx
│ │ ├── RoleSelect.tsx
│ │ ├── Services.tsx
│ │ ├── SideMenu.tsx
│ │ ├── SideMenuLayout.tsx
│ │ ├── Sortable/
│ │ │ ├── Sortable.stories.tsx
│ │ │ ├── Sortable.tsx
│ │ │ └── SortableItem.tsx
│ │ ├── Stats/
│ │ │ ├── Stats.components.tsx
│ │ │ ├── Stats.stories.tsx
│ │ │ ├── Stats.tsx
│ │ │ └── index.ts
│ │ ├── Survey/
│ │ │ ├── LabelLikertGroup.tsx
│ │ │ ├── SurveyCard.tsx
│ │ │ ├── TaskControls.tsx
│ │ │ ├── TrackedTextarea.tsx
│ │ │ └── TwoColumnsWithCards.tsx
│ │ ├── TaskInfo/
│ │ │ └── TaskInfo.tsx
│ │ ├── TaskPage/
│ │ │ └── TaskPage.tsx
│ │ ├── Tasks/
│ │ │ ├── CreateTask.tsx
│ │ │ ├── EvaluateTask.tsx
│ │ │ ├── LabelTask/
│ │ │ │ ├── LabelTask.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── Task/
│ │ │ │ ├── Task.stories.tsx
│ │ │ │ ├── Task.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── TaskHeader/
│ │ │ │ ├── TaskHeader.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── TaskTypes.tsx
│ │ │ └── UnchangedWarning.tsx
│ │ ├── TeamMember.tsx
│ │ ├── ToS.tsx
│ │ ├── ToSWrapper.tsx
│ │ ├── UserAvatar.tsx
│ │ ├── UserDisplayNameCell.tsx
│ │ ├── UserMessageConversation.tsx
│ │ ├── UserTable.tsx
│ │ ├── Vision.tsx
│ │ └── icons/
│ │ ├── Markdown.tsx
│ │ └── MarkdownOff.tsx
│ ├── context/
│ │ └── TaskContext.ts
│ ├── data/
│ │ └── team.json
│ ├── flags.ts
│ ├── hooks/
│ │ ├── auth/
│ │ │ ├── useHasAnyRole.ts
│ │ │ └── useHasRole.ts
│ │ ├── chat/
│ │ │ └── useMessageVote.ts
│ │ ├── env/
│ │ │ └── BrowserEnv.ts
│ │ ├── layout/
│ │ │ └── sidebarItems.ts
│ │ ├── locale/
│ │ │ └── useCurrentLocale.ts
│ │ ├── message/
│ │ │ ├── useDeleteMessage.ts
│ │ │ └── useUndeleteMessage.ts
│ │ ├── tasks/
│ │ │ ├── useCreateReply.ts
│ │ │ ├── useEvaluateReplies.ts
│ │ │ ├── useGenericTaskAPI.tsx
│ │ │ └── useLabelingTask.ts
│ │ └── ui/
│ │ ├── useFallbackRef.ts
│ │ ├── useScrollToElementOnMount.ts
│ │ └── useUserScore.ts
│ ├── lib/
│ │ ├── api.ts
│ │ ├── auth.ts
│ │ ├── captcha.ts
│ │ ├── chat_stream.test.ts
│ │ ├── chat_stream.ts
│ │ ├── constants.ts
│ │ ├── defaultServerSideProps.ts
│ │ ├── display_name_validation.ts
│ │ ├── errors.ts
│ │ ├── i18n.ts
│ │ ├── isChatEnable.ts
│ │ ├── iso6393.ts
│ │ ├── languages.ts
│ │ ├── leaderboard_utilities.ts
│ │ ├── logger.ts
│ │ ├── oasst_api_client.ts
│ │ ├── oasst_client_factory.ts
│ │ ├── oasst_inference_client.ts
│ │ ├── prismadb.ts
│ │ ├── routes.ts
│ │ └── users.ts
│ ├── middleware.ts
│ ├── pages/
│ │ ├── 404.tsx
│ │ ├── 500.tsx
│ │ ├── _app.tsx
│ │ ├── _document.tsx
│ │ ├── about.tsx
│ │ ├── account/
│ │ │ ├── delete.tsx
│ │ │ ├── edit.tsx
│ │ │ ├── index.tsx
│ │ │ └── paperack.tsx
│ │ ├── admin/
│ │ │ ├── edit/
│ │ │ │ └── [id].tsx
│ │ │ ├── index.tsx
│ │ │ ├── manage_user/
│ │ │ │ └── [id].tsx
│ │ │ ├── messages/
│ │ │ │ ├── [id].tsx
│ │ │ │ └── index.tsx
│ │ │ ├── parameters.tsx
│ │ │ ├── status/
│ │ │ │ └── index.tsx
│ │ │ └── trollboard.tsx
│ │ ├── api/
│ │ │ ├── account/
│ │ │ │ ├── delete.ts
│ │ │ │ └── index.ts
│ │ │ ├── admin/
│ │ │ │ ├── delete_message/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── edit_message/
│ │ │ │ │ └── [id].tsx
│ │ │ │ ├── messages/
│ │ │ │ │ ├── [id]/
│ │ │ │ │ │ ├── history.ts
│ │ │ │ │ │ └── tree/
│ │ │ │ │ │ └── index.ts
│ │ │ │ │ └── index.ts
│ │ │ │ ├── parameters.ts
│ │ │ │ ├── set_tree_halted/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── status.ts
│ │ │ │ ├── trollboard.ts
│ │ │ │ ├── undelete_message/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── update_user.ts
│ │ │ │ └── users.ts
│ │ │ ├── auth/
│ │ │ │ └── [...nextauth].ts
│ │ │ ├── available_tasks.ts
│ │ │ ├── chat/
│ │ │ │ ├── assistant_message.ts
│ │ │ │ ├── events.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── message.ts
│ │ │ │ ├── message_eval.ts
│ │ │ │ ├── models.ts
│ │ │ │ ├── plugin_config.ts
│ │ │ │ ├── plugins.ts
│ │ │ │ ├── prompter_message.ts
│ │ │ │ └── vote.ts
│ │ │ ├── config.ts
│ │ │ ├── leaderboard.ts
│ │ │ ├── messages/
│ │ │ │ ├── [id]/
│ │ │ │ │ ├── children.ts
│ │ │ │ │ ├── conversation.ts
│ │ │ │ │ ├── emoji.ts
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── parent.ts
│ │ │ │ │ └── tree.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── user.ts
│ │ │ ├── new_task/
│ │ │ │ └── [task_type].ts
│ │ │ ├── paperack.ts
│ │ │ ├── reject_task.ts
│ │ │ ├── report.ts
│ │ │ ├── set_label.ts
│ │ │ ├── stats/
│ │ │ │ └── cached_stats.ts
│ │ │ ├── tos.ts
│ │ │ ├── update_task.ts
│ │ │ ├── user_stats.ts
│ │ │ ├── username.ts
│ │ │ └── valid_labels.ts
│ │ ├── auth/
│ │ │ ├── signin.tsx
│ │ │ └── verify.tsx
│ │ ├── brb.tsx
│ │ ├── bye.tsx
│ │ ├── chat/
│ │ │ ├── [id].tsx
│ │ │ └── index.tsx
│ │ ├── contributors.tsx
│ │ ├── create/
│ │ │ ├── assistant_reply.tsx
│ │ │ ├── initial_prompt.tsx
│ │ │ └── user_reply.tsx
│ │ ├── dashboard.tsx
│ │ ├── evaluate/
│ │ │ ├── rank_assistant_replies.tsx
│ │ │ ├── rank_initial_prompts.tsx
│ │ │ └── rank_user_replies.tsx
│ │ ├── index.tsx
│ │ ├── label/
│ │ │ ├── label_assistant_reply.tsx
│ │ │ ├── label_initial_prompt.tsx
│ │ │ └── label_prompter_reply.tsx
│ │ ├── leaderboard.tsx
│ │ ├── messages/
│ │ │ ├── [id]/
│ │ │ │ └── index.tsx
│ │ │ └── index.tsx
│ │ ├── privacy-policy.tsx
│ │ ├── stats.tsx
│ │ ├── tasks/
│ │ │ ├── all.tsx
│ │ │ └── random.tsx
│ │ ├── team.tsx
│ │ └── terms-of-service.tsx
│ ├── reportWebVitals.js
│ ├── styles/
│ │ ├── Chakra.tsx
│ │ ├── Theme/
│ │ │ ├── colors.ts
│ │ │ ├── components/
│ │ │ │ ├── Badge.ts
│ │ │ │ ├── Card.ts
│ │ │ │ ├── Container.ts
│ │ │ │ └── Table.ts
│ │ │ └── index.ts
│ │ └── globals.css
│ ├── test-utils/
│ │ └── createMockRouter.ts
│ ├── test_pages/
│ │ ├── README.md
│ │ ├── about.test.tsx
│ │ └── index.test.tsx
│ ├── types/
│ │ ├── Account.ts
│ │ ├── Chat.ts
│ │ ├── Config.ts
│ │ ├── Conversation.ts
│ │ ├── Emoji.ts
│ │ ├── Hooks.ts
│ │ ├── Leaderboard.ts
│ │ ├── Providers.ts
│ │ ├── Stat.ts
│ │ ├── Task.ts
│ │ ├── TaskResponses.ts
│ │ ├── Tasks.ts
│ │ ├── Trollboard.ts
│ │ └── Users.ts
│ └── utils/
│ ├── buildTree.ts
│ └── chat.ts
├── tailwind.config.js
├── tsconfig.json
├── types/
│ ├── env.d.ts
│ ├── i18next.d.ts
│ └── next-auth.d.ts
└── wait-for-postgres.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .devcontainer/README.md
================================================
# devcontainer
## example usage
Below are some example use cases you might want to run from within the
devcontainer (either
[within VSCode locally](https://code.visualstudio.com/docs/devcontainers/create-dev-container#_create-a-devcontainerjson-file)
or in your browser via
[GitHub Codespaces](https://github.com/features/codespaces)).
**Note**: If you want to chose a specific .devcontainer within GitHub codespaces
select "New with options" and you will be able to select any of the pre-defined
devcontainers in this repo.
### Run pre-commit
```bash
# run pre-commit
pre-commit run --all-files
```
A successful run should look something like this:
```
@andrewm4894 ➜ /workspaces/Open-Assistant (devcontainer-improvements) $ pre-commit run --all-files
[INFO] Initializing environment for https://github.com/pre-commit/pre-commit-hooks.
[INFO] Initializing environment for https://github.com/psf/black.
[INFO] Initializing environment for https://github.com/psf/black:.[jupyter].
[INFO] Initializing environment for https://github.com/pycqa/flake8.
[INFO] Initializing environment for https://github.com/pycqa/isort.
[INFO] Initializing environment for https://github.com/pre-commit/mirrors-prettier.
[INFO] Initializing environment for https://github.com/pre-commit/mirrors-prettier:prettier@2.7.1.
[INFO] Initializing environment for local.
[INFO] Installing environment for https://github.com/pre-commit/pre-commit-hooks.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
[INFO] Installing environment for https://github.com/psf/black.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
[INFO] Installing environment for https://github.com/pycqa/flake8.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
[INFO] Installing environment for https://github.com/pycqa/isort.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
[INFO] Installing environment for https://github.com/pre-commit/mirrors-prettier.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
[INFO] Installing environment for local.
[INFO] Once installed this environment will be reused.
[INFO] This may take a few minutes...
trim trailing whitespace.................................................Passed
check python ast.........................................................Passed
check yaml...............................................................Passed
check json...............................................................Passed
check for case conflicts.................................................Passed
detect private key.......................................................Passed
fix python encoding pragma...............................................Passed
forbid submodules....................................(no files to check)Skipped
mixed line ending........................................................Passed
fix requirements.txt.....................................................Passed
check that executables have shebangs.....................................Passed
check that scripts with shebangs are executable..........................Passed
check BOM - deprecated: use fix-byte-order-marker........................Passed
check for broken symlinks............................(no files to check)Skipped
check for merge conflicts................................................Passed
check for added large files..............................................Passed
fix end of files.........................................................Passed
black-jupyter............................................................Passed
flake8...................................................................Passed
isort....................................................................Passed
prettier.................................................................Passed
Lint website.............................................................Passed
```
### Docker compose
```bash
# build the image
docker compose up --build
```
You should see some docker containers being pulled and activated.
Once you see a line like:
```
open-assistant-web-1 | Listening on port 3000 url: http://localhost:3000
```
you should be able to access that port like below:
<img width="640" alt="image" src="https://user-images.githubusercontent.com/2178292/210395676-e9c2aab5-cb54-4ae6-b1eb-ac929fd73607.png">
this port can then be forwarded to a browser tab like below:
<img width="640" alt="image" src="https://user-images.githubusercontent.com/2178292/210396207-1b2e259f-4d5d-475d-b225-91e2bd004071.png">
================================================
FILE: .devcontainer/backend-dev/devcontainer.json
================================================
{
"name": "Backend Development",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": "bash .devcontainer/backend-dev/post_create_command.sh",
"postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
"customizations": {
"vscode": {
"extensions": [
"GitHub.copilot",
"ms-python.python",
"esbenp.prettier-vscode"
]
}
}
}
================================================
FILE: .devcontainer/backend-dev/post_create_command.sh
================================================
#!/bin/bash
# allow git usage
git config --global --add safe.directory "*"
# ensure pre-commit is installed
pre-commit install
# create python virtual environment
python3 -m venv .venv
# install python dependencies in /backend
cd backend
pip install -r requirements.txt
cd ..
# install code in editable mode in /oasst-shared
pip install -e ./oasst-shared[dev]
# install code in editable mode in /oasst-data
pip install -e ./oasst-data[dev]
# docker compose up for backend-dev
docker compose up backend-dev --build --attach-dependencies -d
# note: commented out for now, you probably want to manually run this part once in the devcontainer
# run run-local.sh script
# cd scripts/backend-development/
# bash run-local.sh
================================================
FILE: .devcontainer/devcontainer.json
================================================
{
"name": "Generic",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": ".devcontainer/post_create_command.sh",
"customizations": {
"vscode": {
"extensions": [
"GitHub.copilot",
"ms-python.python",
"esbenp.prettier-vscode"
]
}
}
}
================================================
FILE: .devcontainer/frontend-dev/devcontainer.json
================================================
{
"name": "Front-end Development",
"image": "mcr.microsoft.com/vscode/devcontainers/universal",
"features": {
"ghcr.io/devcontainers-contrib/features/pre-commit:2": {
"version": "latest"
}
},
"postCreateCommand": ".devcontainer/frontend-dev/post_create_command.sh",
"customizations": {
"vscode": {
"extensions": ["GitHub.copilot", "esbenp.prettier-vscode"]
}
}
}
================================================
FILE: .devcontainer/frontend-dev/post_create_command.sh
================================================
#!/bin/bash
# ensure pre-commit is installed
pre-commit install
# npm install in /website
cd website
npm install
cd ..
================================================
FILE: .devcontainer/post_create_command.sh
================================================
#!/bin/bash
# do as little as possible in this script to keep the container creation fast.
# for more specific devcontainer use cases use the different devcontainer subfolders.
echo "done"
================================================
FILE: .dockerignore
================================================
**/node_modules
website/.next
================================================
FILE: .gitattributes
================================================
* text=auto eol=lf
*.ipynb linguist-documentation
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "npm"
directory: "/website"
schedule:
interval: "weekly"
================================================
FILE: .github/workflows/build-frontend.yaml
================================================
name: Build Frontend
on:
push:
branches:
- main
paths:
- website/**
pull_request:
paths:
- website/**
workflow_call:
jobs:
build-frontend:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./website
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16.x
cache: "npm"
cache-dependency-path: website/package-lock.json
- run: npm ci
- run: npx prisma generate
- run: npm run build
================================================
FILE: .github/workflows/build-postgres.yaml
================================================
name: Build OASST Postgres image
on:
push:
branches:
- main
paths:
- docker/oasst-postgres/**
jobs:
build-postgres:
uses: ./.github/workflows/docker-build.yaml
with:
image-name: oasst-postgres
context: ./docker/oasst-postgres
dockerfile: docker/oasst-postgres/Dockerfile
build-args: ""
================================================
FILE: .github/workflows/deploy-docs-site.yaml
================================================
name: Deploy docs to gitHub pages
on:
push:
branches:
- main
paths:
- ".github/workflows/deploy-docs-site.yaml"
- "docs/**"
pull_request:
paths:
- ".github/workflows/deploy-docs-site.yaml"
- "docs/**"
jobs:
deploy:
runs-on: ubuntu-22.04
permissions:
contents: write
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
defaults:
run:
working-directory: docs
steps:
- uses: actions/checkout@v3
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: "18"
- name: Install dependencies
run: yarn install --frozen-lockfile
- name: Build website
run: yarn build
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
if: ${{ github.ref == 'refs/heads/main' }}
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./docs/build
================================================
FILE: .github/workflows/deploy-to-node.yaml
================================================
name: Deploy to node
on:
workflow_call:
inputs:
stack-name:
required: false
type: string
default: dev
image-tag:
required: false
type: string
default: latest
backend-port:
required: false
type: string
default: 8080
website-port:
required: false
type: string
default: 3000
inference-server-port:
required: false
type: string
default: 8085
jobs:
deploy:
runs-on: ubuntu-latest
environment: ${{ inputs.stack-name }}
env:
WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
WEB_MODERATOR_USERS: ${{ secrets.DEV_WEB_MODERATOR_USERS }}
WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
WEB_GOOGLE_CLIENT_ID: ${{ secrets.DEV_WEB_GOOGLE_CLIENT_ID }}
WEB_GOOGLE_CLIENT_SECRET: ${{ secrets.DEV_WEB_GOOGLE_CLIENT_SECRET }}
WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
WEB_NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
S3_REGION: ${{ secrets.S3_REGION }}
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
INIT_PROMPT_DISABLED_LANGS: ${{ vars.INIT_PROMPT_DISABLED_LANGS }}
MAX_ACTIVE_TREES: ${{ vars.MAX_ACTIVE_TREES }}
MAX_INITIAL_PROMPT_REVIEW: ${{ vars.MAX_INITIAL_PROMPT_REVIEW }}
MAX_TREE_DEPTH: ${{ vars.MAX_TREE_DEPTH }}
MAX_CHILDREN_COUNT: ${{ vars.MAX_CHILDREN_COUNT }}
LONELY_CHILDREN_COUNT: ${{ vars.LONELY_CHILDREN_COUNT }}
P_LONELY_CHILD_EXTENSION: ${{ vars.P_LONELY_CHILD_EXTENSION }}
P_ACTIVATE_BACKLOG_TREE: ${{ vars.P_ACTIVATE_BACKLOG_TREE }}
NUM_REQUIRED_RANKINGS: ${{ vars.NUM_REQUIRED_RANKINGS }}
NUM_REVIEWS_REPLY: ${{ vars.NUM_REVIEWS_REPLY }}
NUM_REVIEWS_INITIAL_PROMPT: ${{ vars.NUM_REVIEWS_INITIAL_PROMPT }}
MIN_ACTIVE_RANKINGS_PER_LANG: ${{ vars.MIN_ACTIVE_RANKINGS_PER_LANG }}
MAX_PROMPT_LOTTERY_WAITING: ${{ vars.MAX_PROMPT_LOTTERY_WAITING }}
GOAL_TREE_SIZE: ${{ vars.GOAL_TREE_SIZE }}
MESSAGE_SIZE_LIMIT: ${{ vars.MESSAGE_SIZE_LIMIT }}
SKIP_TOXICITY_CALCULATION: ${{ vars.SKIP_TOXICITY_CALCULATION }}
STATS_INTERVAL_DAY: ${{ vars.STATS_INTERVAL_DAY }}
STATS_INTERVAL_WEEK: ${{ vars.STATS_INTERVAL_WEEK }}
STATS_INTERVAL_MONTH: ${{ vars.STATS_INTERVAL_MONTH }}
STATS_INTERVAL_TOTAL: ${{ vars.STATS_INTERVAL_TOTAL }}
DISCORD_API_KEY: ${{ secrets.DISCORD_API_KEY }}
DISCORD_CHANNEL_ID: ${{ vars.DISCORD_CHANNEL_ID }}
WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY:
${{ secrets.WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY }}
WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY:
${{ secrets.WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY }}
WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA:
${{ vars.WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA }}
WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN:
${{ vars.WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN }}
LOGURU_LEVEL: ${{ vars.LOGURU_LEVEL }}
MAINTENANCE_MODE: ${{ vars.MAINTENANCE_MODE }}
BACKEND_URL: ${{ vars.BACKEND_URL }}
WEB_NEXT_PUBLIC_BACKEND_URL: ${{ vars.WEB_NEXT_PUBLIC_BACKEND_URL }}
BACKEND_CORS_ORIGINS: ${{ vars.BACKEND_CORS_ORIGINS }}
WEB_INFERENCE_SERVER_HOST: ${{ vars.WEB_INFERENCE_SERVER_HOST }}
WEB_ENABLE_CHAT: ${{ vars.WEB_ENABLE_CHAT }}
WEB_BYE: ${{ vars.WEB_BYE }}
WEB_ENABLE_DRAFTS_WITH_PLUGINS: ${{ vars.WEB_ENABLE_DRAFTS_WITH_PLUGINS }}
WEB_NUM_GENERATED_DRAFTS: ${{ vars.WEB_NUM_GENERATED_DRAFTS }}
WEB_CURRENT_ANNOUNCEMENT: ${{ vars.WEB_CURRENT_ANNOUNCEMENT }}
WEB_INFERENCE_SERVER_API_KEY: ${{secrets.WEB_INFERENCE_SERVER_API_KEY}}
INFERENCE_POSTGRES_PASSWORD: ${{secrets.INFERENCE_POSTGRES_PASSWORD}}
INFERENCE_ALLOW_DEBUG_AUTH: ${{vars.INFERENCE_ALLOW_DEBUG_AUTH}}
INFERENCE_DEBUG_API_KEYS: ${{vars.INFERENCE_DEBUG_API_KEYS}}
INFERENCE_LOG_LEVEL: ${{vars.INFERENCE_LOG_LEVEL}}
INFERENCE_ROOT_TOKEN: ${{secrets.INFERENCE_ROOT_TOKEN}}
INFERENCE_API_ROOT: ${{vars.INFERENCE_API_ROOT}}
INFERENCE_TRUSTED_CLIENT_KEYS: ${{secrets.INFERENCE_TRUSTED_CLIENT_KEYS}}
INFERENCE_AUTH_SALT: ${{secrets.INFERENCE_AUTH_SALT}}
INFERENCE_AUTH_SECRET: ${{secrets.INFERENCE_AUTH_SECRET}}
INFERENCE_AUTH_DISCORD_CLIENT_ID: ${{secrets.INFERENCE_AUTH_DISCORD_CLIENT_ID}}
INFERENCE_AUTH_DISCORD_CLIENT_SECRET: ${{secrets.INFERENCE_AUTH_DISCORD_CLIENT_SECRET}}
INFERENCE_AUTH_GITHUB_CLIENT_ID: ${{secrets.INFERENCE_AUTH_GITHUB_CLIENT_ID}}
INFERENCE_AUTH_GITHUB_CLIENT_SECRET: ${{secrets.INFERENCE_AUTH_GITHUB_CLIENT_SECRET}}
INFERENCE_CORS_ORIGINS: ${{ vars.INFERENCE_CORS_ORIGINS }}
INFERENCE_ALLOWED_MODEL_CONFIG_NAMES:
${{ vars.INFERENCE_ALLOWED_MODEL_CONFIG_NAMES }}
INFERENCE_ASSISTANT_MESSAGE_TIMEOUT:
${{ vars.INFERENCE_ASSISTANT_MESSAGE_TIMEOUT }}
INFERENCE_MESSAGE_QUEUE_EXPIRE: ${{ vars.INFERENCE_MESSAGE_QUEUE_EXPIRE }}
INFERENCE_WORK_QUEUE_MAX_SIZE: ${{ vars.INFERENCE_WORK_QUEUE_MAX_SIZE }}
INFERENCE_ENABLE_SAFETY: ${{ vars.INFERENCE_ENABLE_SAFETY }}
INFERENCE_GUNICORN_WORKERS: ${{ vars.INFERENCE_GUNICORN_WORKERS }}
INFERENCE_CHAT_MAX_MESSAGES: ${{ vars.INFERENCE_CHAT_MAX_MESSAGES }}
INFERENCE_MESSAGE_MAX_LENGTH: ${{ vars.INFERENCE_MESSAGE_MAX_LENGTH }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Run deploy playbook
uses: dawidd6/action-ansible-playbook@v2
with:
# Required, playbook filepath
playbook: deploy-to-node.yaml
# Optional, directory where playbooks live
directory: ansible
# Optional, SSH private key
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
# Optional, literal inventory file contents
inventory: |
[dev]
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
options: |
--extra-vars "stack_name=${{inputs.stack-name}} \
image_tag=${{inputs.image-tag}} \
backend_port=${{inputs.backend-port}} \
website_port=${{inputs.website-port}} \
postgres_password=${{secrets.POSTGRES_PASSWORD}} \
web_api_key=${{secrets.WEB_API_KEY}}"
- name: Run inference deploy playbook
uses: dawidd6/action-ansible-playbook@v2
with:
# Required, playbook filepath
playbook: inference/deploy-server.yaml
# Optional, directory where playbooks live
directory: ansible
# Optional, SSH private key
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
# Optional, literal inventory file contents
inventory: |
[dev]
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
options: |
--extra-vars "stack_name=${{inputs.stack-name}} \
image_tag=${{inputs.image-tag}} \
server_port=${{inputs.inference-server-port}}"
================================================
FILE: .github/workflows/docker-build.yaml
================================================
name: Build
on:
workflow_call:
inputs:
dockerfile:
required: true
type: string
context:
required: true
type: string
image-name:
required: true
type: string
build-args:
required: false
type: string
jobs:
build:
name: Build Images
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.2.1
- name: Login to container registry
uses: docker/login-action@v2.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get base registry
run: |
echo "REGISTRY=ghcr.io/${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV
- name: Set tag prefix
if: github.ref_name != 'main'
run: |
echo "TAG_PREFIX=${{ github.ref_name }}-" >> $GITHUB_ENV
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4.1.1
with:
images: ${{ env.REGISTRY }}/${{ inputs.image-name }}
tags: |
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
type=sha,prefix=${{ env.TAG_PREFIX }},format=short,enable=${{ github.ref_name != 'main' }}
type=ref,event=tag,enable=${{ github.ref_name != 'main' }}
- name: Build and push Docker image
uses: docker/build-push-action@v3.2.0
with:
file: ${{ inputs.dockerfile }}
context: ${{ inputs.context }}
build-args: ${{ inputs.build-args }}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
================================================
FILE: .github/workflows/pre-commit.yaml
================================================
name: pre-commit
on:
workflow_call:
pull_request_target:
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
# in case of PR, check out the PR's head branch
- uses: actions/checkout@v3
if: github.event_name == 'pull_request_target'
with:
ref: ${{ github.event.pull_request.head.sha }}
# in case of push, check out the main branch
- uses: actions/checkout@v3
if: github.event_name != 'pull_request_target'
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: "**/requirements*.txt"
- uses: pre-commit/action@v3.0.0
- name: Post PR comment on failure
if: failure() && github.event_name == 'pull_request_target'
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
:x: **pre-commit** failed.
Please run `pre-commit run --all-files` locally and commit the changes.
Find more information in the repository's CONTRIBUTING.md
================================================
FILE: .github/workflows/production-deploy.yaml
================================================
name: Deploy to prod
on:
push:
branches:
- production
jobs:
deploy-to-prod:
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: production
image-tag: ${{ vars.PROD_IMAGE_TAG }}
backend-port: 8280
website-port: 3200
inference-server-port: 8285
================================================
FILE: .github/workflows/production2-deploy.yaml
================================================
name: Deploy to prod2
on:
push:
branches:
- production2
jobs:
deploy-to-prod:
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: production2
image-tag: ${{ vars.PROD_IMAGE_TAG }}
backend-port: 8280
website-port: 3200
inference-server-port: 8285
================================================
FILE: .github/workflows/release.yaml
================================================
name: Release
on:
push:
branches:
- main
release:
types:
- released
jobs:
pre-commit:
uses: ./.github/workflows/pre-commit.yaml
build-backend:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-backend
context: .
dockerfile: docker/Dockerfile.backend
build-args: ""
build-backend-worker:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-backend-worker
context: .
dockerfile: docker/Dockerfile.backend-worker
build-args: ""
build-web:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-web
context: .
dockerfile: docker/Dockerfile.website
build-args: ""
build-bot:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-discord-bot
context: .
dockerfile: docker/Dockerfile.discord-bot
build-args: ""
build-inference-server:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-server
context: .
dockerfile: docker/inference/Dockerfile.server
build-args: ""
build-inference-worker-full:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-full
context: .
dockerfile: docker/inference/Dockerfile.worker-full
build-args: ""
build-inference-worker-hf:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-hf
context: .
dockerfile: docker/inference/Dockerfile.worker-hf
build-args: ""
build-inference-worker-standalone:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-worker-standalone
context: .
dockerfile: docker/inference/Dockerfile.worker-standalone
build-args: ""
build-inference-safety:
uses: ./.github/workflows/docker-build.yaml
needs: pre-commit
with:
image-name: oasst-inference-safety
context: .
dockerfile: docker/inference/Dockerfile.safety
build-args: ""
deploy-to-node:
needs:
- build-backend
- build-backend-worker
- build-web
- build-bot
- build-inference-server
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }}
image-tag:
${{ github.event_name == 'release' && github.event.release.tag_name ||
'latest' }}
backend-port: ${{ github.event_name == 'release' && '8180' || '8080' }}
website-port: ${{ github.event_name == 'release' && '3100' || '3000' }}
inference-server-port:
${{ github.event_name == 'release' && '8185' || '8085' }}
================================================
FILE: .github/workflows/test-api-contract.yaml
================================================
name: Test API Contract
on:
push:
branches:
- main
paths:
- "oasst-shared/**"
- "backend/**"
- "website/**"
pull_request:
paths:
- "oasst-shared/**"
- "backend/**"
- "website/**"
workflow_call:
jobs:
test-contract:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
cache-dependency-path: "**/requirements*.txt"
- uses: actions/setup-node@v3
with:
node-version: 16
- run: pip install ./oasst-shared[dev]
- run: pip install ./oasst-data[dev]
- run: cd backend && pip install -r requirements.txt
- run: cd website && npm ci
- run: ./scripts/backend-development/start-mock-server.sh
- name: Run Python OasstApiClient contract tests
run: ./scripts/oasst-shared-development/test.sh
- name: Run JavaScript OasstApiClient contract tests
run: ./scripts/frontend-development/run-contract-test.sh
- run: ./scripts/backend-development/stop-mock-server.sh
#- uses: stefanzweifel/git-auto-commit-action@v4
# with:
# file_pattern: "docs/docs/api/openapi.json"
# commit_message:
# update docs/docs/api/openapi.json by run ${{ github.run_id }}
================================================
FILE: .github/workflows/test-e2e.yaml
================================================
name: E2E Tests (Website)
on:
push:
branches:
- main
paths:
- oasst-shared/**
- backend/**
- website/**
pull_request:
paths:
- oasst-shared/**
- backend/**
- website/**
jobs:
test-e2e:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Start website, backend, etc
run: docker compose --profile ci up --build -d
- name: Run Cypress tests
uses: cypress-io/github-action@v5.0.2
with:
browser: chrome
working-directory: website
- uses: actions/upload-artifact@v3
if: failure() # NOTE: screenshots will be generated only if E2E test failed
with:
name: cypress-screenshots
path: website/cypress/screenshots
- uses: actions/upload-artifact@v3
if: always()
with:
name: cypress-videos
path: website/cypress/videos
================================================
FILE: .gitignore
================================================
.venv*
venv
.env
*.pyc
*.swp
*.egg-info
__pycache__
.DS_Store
# Generated files
backend/oasst-openapi.json
backend/openapi.json
# ignore jupyter notebook checkpoints
.ipynb_checkpoints
*.log
*.db
# edit docs using obsidian.md, these files should not appear in the repo
.obsidian/
.pytest_cache/
/docker-compose.override.yml
# JetBrains PyCharm:
.idea/
================================================
FILE: .pre-commit-config.yaml
================================================
# WARNING!
#
# When making changes to auto-formatters used in pre-commit hooks, you are
# likely to cause merge conflicts with main and/or other pull requests.
# Fixing them might revert other people's work. Expect pain!
# To avoid accidental reversions and keep it easy to review, please make sure
# that changes here are in a pull request by themselves, that it consists of
# two commits:
#
# 1. The changes to this file
# 2. Changes made by running `python3 -m pre_commit run --all-files`.
#
# Then each time your pull request is blocked by a merge conflict, do the
# following steps:
#
# git reset HEAD^1 && git checkout -f # discard the change commit
# git rebase main # re-apply other people's changes
# python3 -m pre_commit run --all-files # re-run the rules
# git add . # add the newly changed files
# git commit -m 'apply pre-commit' # commit it
# git push -f # force push back to your branch
#
# Keep in mind you may have to do this a few times, as changes here may impact
# other pull requests. Try to keep it up-to-date so they can go in when it'll
# cause least disruption.
#
# /WARNING!
exclude: build|stubs|^bot/templates/$|openassistant/templates|docs/docs/api/openapi.json|scripts/postprocessing/regex_pii_detector.py
default_language_version:
python: python3
ci:
autofix_prs: true
autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions"
autoupdate_schedule: quarterly
skip: ["next-lint-website"] # list of hook ids to skip only in pre-commit.ci
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: trailing-whitespace
- id: check-ast
- id: check-yaml
# Always check YAML but skip a few YAML files that are auto-generated
# and which break the standard YAML check. The alternative would be to
# skip any unsafe errors (and thus break YAML compatibility) or use
# some other checker that may not work in general.
exclude: ^copilot/.*/addons/.*$
- id: check-json
- id: check-case-conflict
- id: detect-private-key
- id: fix-encoding-pragma
args: [--remove]
- id: forbid-submodules
- id: mixed-line-ending
- id: requirements-txt-fixer
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-byte-order-marker
- id: check-symlinks
- id: check-merge-conflict
- id: check-added-large-files
args: [--maxkb=1024]
- id: end-of-file-fixer
- repo: https://github.com/psf/black
rev: 23.3.0
hooks:
- id: black-jupyter
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort
args: ["--profile", "black", "--filter-files"]
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.263
hooks:
- id: ruff
args: ["--fix"]
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.7.1
hooks:
- id: prettier
args:
[
--prose-wrap=always,
--write,
--ignore-path=./website/.prettierignore,
]
- repo: local
hooks:
- id: next-lint-website
name: Lint website
files: ^website/
exclude: ^website/node_modules/
types_or: [javascript, jsx, ts, tsx]
language: node
pass_filenames: false
entry: website/next-lint.js
================================================
FILE: .python-version
================================================
3.10.8
================================================
FILE: .vscode/extensions.json
================================================
{
"recommendations": [
"ms-vscode-remote.remote-containers",
"ms-vscode-remote.vscode-remote-extensionpack",
"esbenp.prettier-vscode",
"inlang.vs-code-extension"
]
}
================================================
FILE: .vscode/launch.json
================================================
{
"version": "0.2.0",
"configurations": [
{
"name": "Python: Current File",
"type": "python",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
"justMyCode": false
},
{
"name": "Python: Debug Tests",
"type": "python",
"request": "launch",
"program": "${file}",
"purpose": ["debug-test"],
"console": "integratedTerminal",
"justMyCode": true
},
{
"name": "Debug SFT Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_sft.py",
"args": ["--configs", "defaults", "oasst_export_eu", "gpt-neox"],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false"
}
},
{
"name": "Debug RM Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_rm.py",
"args": ["--configs", "defaults_rm", "oasst_export_eu", "debug"],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false",
"CUDA_VISIBLE_DEVICES": "1,2,3,4,5,6,7"
}
},
{
"name": "Debug RM Training",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "trainer_rm.py",
"args": [
"--configs",
"defaults_rm",
"oasst_export_eu",
"pythia-1B",
"--cache_dir",
"/home/ubuntu/data_cache",
"--use_flash_attention",
"true"
],
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"TOKENIZERS_PARALLELISM": "false",
"CUDA_VISIBLE_DEVICES": "1"
}
},
{
"name": "Debug Patching",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"program": "models/test_patched_gpt_neox.py",
"console": "integratedTerminal",
"justMyCode": true,
"env": {
"CUDA_VISIBLE_DEVICES": "0"
}
},
{
"name": "Debug RLHF",
"type": "python",
"request": "launch",
"cwd": "${workspaceFolder}/model/model_training",
"module": "accelerate.commands.launch",
"args": [
"--main_process_port",
"29506",
"--config_file",
"configs/accelerate_config.yaml",
"--num_processes",
"5",
"trainer_rl.py",
"--configs",
"defaults",
"defaults_rlhf",
"pythia_rlhf",
"oasst_export_latin_cyrillic_rlhf"
],
"console": "integratedTerminal",
"justMyCode": false,
"env": {
"CUDA_VISIBLE_DEVICES": "1,2,3,4,5",
"OMP_NUM_THREADS": "1"
}
}
]
}
================================================
FILE: .vscode/settings.json
================================================
{
"python.analysis.extraPaths": ["${workspaceFolder}/oasst-shared"],
"prettier.singleQuote": false,
"eslint.workingDirectories": ["./website", "./docs"],
"python.testing.pytestArgs": ["backend/tests", "oasst-shared/tests"],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
}
================================================
FILE: CODEOWNERS
================================================
* @yk @andreaskoepf
/website/ @AbdBarho @notmd @yk @andreaskoepf
/website/src/data/team.json @yk @andreaskoepf @fozziethebeat @AbdBarho @notmd @theblackcat102 @sanagno @olliestanley @andrewm4894
/model/ @theblackcat102 @sanagno @dvruette @andreaskoepf @yk @jordiclive @shahules786
/copilot/ @andreaskoepf @yk
/docs/ @andrewm4894 @olliestanley @andreaskoepf @yk
/.devcontainer/ @andrewm4894 @andreaskoepf @yk
/notebooks/ @andrewm4894 @olliestanley @andreaskoepf @yk
/data @Vechtomov @bitplane @ontocord @olliestanley @sedthh
/safety @SummerSigh @shahules786 @olliestanley
/inference/ @yk @andreaskoepf @olliestanley @AbdBarho
/backend/ @andreaskoepf @melvinebenezer @yk
/oasst-data/ @andreaskoepf @yk @olliestanley
/oasst-shared/ @andreaskoepf @melvinebenezer @yk @olliestanley @AbdBarho
docker-compose.yaml @andreaskoepf @melvinebenezer @yk @olliestanley @AbdBarho
================================================
FILE: CONTRIBUTING.md
================================================
<a href="https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/CONTRIBUTING.md?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en&_x_tr_pto=wapp"></a>
# I’m in! Now what?
[Join the OpenAssistant Contributors Discord Server!](https://ykilcher.com/open-assistant-discord),
this is for work coordination.
[Join the LAION Discord Server!](https://discord.com/invite/mVcgxMPD7e), it has
a dedicated channel and is more public.
[and / or the YK Discord Server](https://ykilcher.com/discord), also has a
dedicated, but not as active, channel.
[Visit the Notion](https://ykilcher.com/open-assistant)
### Taking on Tasks
We have a growing task list of
[issues](https://github.com/LAION-AI/Open-Assistant/issues). Find an issue that
appeals to you and make a comment that you'd like to work on it. Include in your
comment a brief description of how you'll solve the problem and if there are any
open questions you want to discuss. Once a project coordinator has assigned the
issue to you, start working on it.
If the issue is currently unclear but you are interested, please post in Discord
and someone can help clarify the issue in more detail.
**Always Welcome:** Documentation markdowns in `docs/`, docstrings, diagrams of
the system architecture, and other documentation.
### Submitting Work
We're all working on different parts of Open Assistant together. To make
contributions smoothly we recommend the following:
1. [Fork this project repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo)
and clone it to your local machine. (Read more
[About Forks](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks))
1. Before working on any changes, try to
[sync the forked repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)
to keep it up-to-date with the upstream repository.
1. On a
[new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository)
in your fork (aka a "feature branch" and not `main`) work on a small focused
change that only touches on a few files.
1. Run `pre-commit` and make sure all files have formatting fixed. This
simplifies life for reviewers.
1. Package up a small bit of work that solves part of the problem
[into a Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)
and
[send it out for review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review).
[Here](https://github.com/LAION-AI/Open-Assistant/pull/658) is an example PR
for this project to illustrate this flow.
1. If you're lucky, we can merge your change into `main` without any problems.
If there are changes to files you're working on, resolve them by:
1. First try to rebase as suggested
[in these instructions](https://timwise.co.uk/2019/10/14/merge-vs-rebase/#should-you-rebase).
1. If rebasing feels too painful, merge as suggested
[in these instructions](https://timwise.co.uk/2019/10/14/merge-vs-rebase/#should-you-merge).
1. Once you've resolved conflicts (if any), finish the review and
[squash and merge](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits)
your PR (when squashing try to clean up or update the individual commit
messages to be one sensible single one).
1. Merge in your change and move on to a new issue or the second step of your
current issue.
Additionally, if someone is working on an issue that interests you, ask if they
need help on it or would like suggestions on how to approach the issue. If so,
share wildly. If they seem to have a good handle on it, let them work on their
solution until a challenge comes up.
#### Tips
- At any point you can compare your feature branch to the upstream/main of
`LAION-AI/Open-Assistant` by using a URL like this:
https://github.com/LAION-AI/Open-Assistant/compare/main...andrewm4894:Open-Assistant:my-example-feature-branch.
Obviously just replace `andrewm4894` with your own GitHub user name and
`my-example-feature-branch` with whatever you called the feature branch you
are working on, so something like
`https://github.com/LAION-AI/Open-Assistant/compare/main...<your_github_username>:Open-Assistant:<your_branch_name>`.
This will show the changes that would appear in a PR, so you can check this to
make sure only the files you have changed or added will be part of the PR.
- Try not to work on the `main` branch in your fork - ideally you can keep this
as just an updated copy of `main` from `LAION-AI/Open-Assistant`.
- If your feature branch gets messed up, just update the `main` branch in your
fork and create a fresh new clean "feature branch" where you can add your
changes one by one in separate commits or all as a single commit.
### When does a review finish
A review finishes when all blocking comments are addressed and at least one
owning reviewer has approved the PR. Be sure to acknowledge any non-blocking
comments either by making the requested change, explaining why it's not being
addressed now, or filing an issue to handle it later.
## Developer Setup
Work is organized in the
[project board](https://github.com/orgs/LAION-AI/projects/3).
**Anything that is in the `Todo` column and not assigned, is up for grabs.
Meaning we'd be happy for anyone to do these tasks.**
If you want to work on something, assign yourself to it or write a comment that
you want to work on it and what you plan to do.
- There's an [introduction for developers](docs/docs/guides/developers.md) that
gives an overview of the different tools and technologies used in the project.
- To get started with development, if you want to work on the backend, have a
look at `backend/README.md`.
- If you want to work on any frontend, have a look at `website/README.md`.
There is also a minimal implementation of a frontend in the `text-frontend`
folder.
We are using Python 3.10 for the backend.
Check out the
[High-Level Protocol Architecture](https://www.notion.so/High-Level-Protocol-Architecture-6f1fd3551da74213b560ead369f132dc)
### Website
The website is built using Next.js and is in the `website` folder.
### Pre-commit
We are using `pre-commit` to enforce code style and formatting.
Install `pre-commit` from [its website](https://pre-commit.com) and run
`pre-commit install` to install the pre-commit hooks.
In case you haven't done this, have already committed, and CI is failing, you
can run `pre-commit run --all-files` to run the pre-commit hooks on all files.
### Deployment
Upon making a release on GitHub, all docker images are automatically built and
pushed to ghcr.io. The docker images are tagged with the release version and the
`latest` tag. Further, the ansible playbook in `ansible/dev.yaml` is run to
automatically deploy the built release to the dev machine.
### Contribute a Dataset
See
[here](https://github.com/LAION-AI/Open-Assistant/blob/main/data/datasets/README.md)
## Translations
[](https://inlang.com/editor/github.com/LAION-AI/Open-Assistant?ref=badge)
To add translations, you can manually edit the JSON translation files in
`website/public/locales`, use the [inlang](https://inlang.com/) online editor,
or run `npm run inlang:lint` inside `website` to find missing translations.
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
<h1 align="center">
<span>Open-Assistant</span>
<img width="auto" height="50px" src="https://github.com/LAION-AI/Open-Assistant/blob/main/assets/logo_crop.png"/>
</h1>
<blockquote>
<p>:memo: <strong>NOTE</strong>: OpenAssistant is completed, and the project is now finished. Thank you to everyone who contributed! Check out our <a href="https://projects.laion.ai/Open-Assistant/blog/2023/10/25/open-assistant-is-completed">blog post</a> for more information. The final published oasst2 dataset can be found on HuggingFace at <a href="https://huggingface.co/datasets/OpenAssistant/oasst2">OpenAssistant/oasst2</a></p>
</blockquote>
<div align="center">
<a href="https://github.com/LAION-AI/Open-Assistant/stargazers"></a>
<a href="https://laion-ai.github.io/Open-Assistant/"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/build-frontend.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/build-postgres.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/pre-commit.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/test-api-contract.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/test-e2e.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/deploy-docs-site.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/production-deploy.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/actions/workflows/release.yaml"></a>
<a href="https://github.com/LAION-AI/Open-Assistant/releases"></a>
<a href="https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/README.md?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en&_x_tr_pto=wapp"></a>
</div>
# Table of Contents
- [What is Open Assistant?](#what-is-open-assistant)
- [Useful Links](#useful-links)
- [How To Try It Out](#how-to-try-it-out)
- [The Vision](#the-vision)
- [The Plan](#the-plan)
- [How You Can Help](#how-you-can-help)
---
## What is Open Assistant?
<p align="center">
Open Assistant is a project meant to give everyone access to a great chat based
large language model.
</p>
We believe that by doing this we will create a revolution in innovation in
language. In the same way that stable-diffusion helped the world make art and
images in new ways we hope Open Assistant can help improve the world by
improving language itself.
# Useful Links
- [Data Collection](https://open-assistant.io)
- [Chat](https://open-assistant.io/chat)
- [Project Documentation](https://projects.laion.ai/Open-Assistant/)
## How To Try It Out
### Chatting with the AI
The chat frontend is now live [here](https://open-assistant.io/chat). Log in and
start chatting! Please try to react with a thumbs up or down for the assistant's
responses when chatting.
### Contributing to Data Collection
The data collection frontend is now live [here](https://open-assistant.io/). Log
in and start taking on tasks! We want to collect a high volume of quality data.
By submitting, ranking, and labelling model prompts and responses you will be
directly helping to improve the capabilities of Open Assistant.
### Running the Development Setup Locally (without chat)
**You do not need to run the project locally unless you are contributing to the
development process. The website link above will take you to the public website
where you can use the data collection app and the chat.**
If you would like to run the data collection app locally for development, you
can set up an entire stack needed to run **Open-Assistant**, including the
website, backend, and associated dependent services, with Docker.
To start the demo, run this in the root directory of the repository (check
[this FAQ](https://projects.laion.ai/Open-Assistant/docs/faq#docker-compose-instead-of-docker-compose)
if you have problems):
```sh
docker compose --profile ci up --build --attach-dependencies
```
> **Note:** when running on MacOS with an M1 chip you have to use:
> `DB_PLATFORM=linux/x86_64 docker compose ...`
Then, navigate to `http://localhost:3000` (It may take some time to boot up) and
interact with the website.
> **Note:** If an issue occurs with the build, please head to the
> [FAQ](https://projects.laion.ai/Open-Assistant/docs/faq) and check out the
> entries about Docker.
> **Note:** When logging in via email, navigate to `http://localhost:1080` to
> get the magic email login link.
> **Note:** If you would like to run this in a standardized development
> environment (a
> ["devcontainer"](https://code.visualstudio.com/docs/devcontainers/containers))
> using
> [vscode locally](https://code.visualstudio.com/docs/devcontainers/create-dev-container#_create-a-devcontainerjson-file)
> or in a web browser using
> [GitHub Codespaces](https://github.com/features/codespaces), you can use the
> provided [`.devcontainer`](.devcontainer/) folder.
### Running the Development Setup Locally for Chat
**You do not need to run the project locally unless you are contributing to the
development process. The website link above will take you to the public website
where you can use the data collection app and the chat.**
**Also note that the local setup is only for development and is not meant to be
used as a local chatbot, unless you know what you are doing.**
If you _do_ know what you are doing, then see the `inference` folder for getting
the inference system up and running, or have a look at `--profile inference` in
addition to `--profile ci` in the above command.
## The Vision
We are not going to stop at replicating ChatGPT. We want to build the assistant
of the future, able to not only write email and cover letters, but do meaningful
work, use APIs, dynamically research information, and much more, with the
ability to be personalized and extended by anyone. And we want to do this in a
way that is open and accessible, which means we must not only build a great
assistant, but also make it small and efficient enough to run on consumer
hardware.
## The Plan
##### We want to get to an initial MVP as fast as possible, by following the 3-steps outlined in the [InstructGPT paper](https://arxiv.org/abs/2203.02155)
1. Collect high-quality human generated Instruction-Fulfillment samples
(prompt + response), goal >50k. We design a crowdsourced process to collect
and reviewed prompts. We do not want to train on
flooding/toxic/spam/junk/personal information data. We will have a
leaderboard to motivate the community that shows progress and the most active
users. Swag will be given to the top-contributors.
2. For each of the collected prompts we will sample multiple completions.
Completions of one prompt will then be shown randomly to users to rank them
from best to worst. Again this should happen crowd-sourced, e.g. we need to
deal with unreliable potentially malicious users. At least multiple votes by
independent users have to be collected to measure the overall agreement. The
gathered ranking-data will be used to train a reward model.
3. Now follows the RLHF training phase based on the prompts and the reward
model.
We can then take the resulting model and continue with completion sampling step
2 for a next iteration.
### Slide Decks
[Vision & Roadmap](https://docs.google.com/presentation/d/1n7IrAOVOqwdYgiYrXc8Sj0He8krn5MVZO_iLkCjTtu0/edit?usp=sharing)
[Important Data Structures](https://docs.google.com/presentation/d/1iaX_nxasVWlvPiSNs0cllR9L_1neZq0RJxd6MFEalUY/edit?usp=sharing)
## How You Can Help
All open source projects begin with people like you. Open source is the belief
that if we collaborate we can together gift our knowledge and technology to the
world for the benefit of humanity.
Check out our [contributing guide](CONTRIBUTING.md) to get started.
================================================
FILE: ansible/.gitignore
================================================
dev/
dev-inference/
*.local.yaml
================================================
FILE: ansible/README.md
================================================
To test the ansible playbook on localhost run
`ansible-playbook -i test.inventory.ini dev.yaml`.\
In case you're missing the ansible docker dependency install it with `ansible-galaxy collection install community.docker`.\
Point Redis Insights to the Redis database by visiting localhost:8001 in a
browser and select "I already have a database" followed by "Connect to a Redis
Database".\
For host, port and name fill in `oasst-redis`, `6379` and `redis`.
================================================
FILE: ansible/deploy-to-node.yaml
================================================
# ansible playbook to set up some docker containers
- name: Deploy to node
hosts: dev
gather_facts: true
vars:
stack_name: "dev"
image_tag: latest
backend_port: 8080
website_port: 3000
postgres_password: postgres
web_api_key: "1234"
tasks:
# reduces the chance of oom when pulling new images
- name: Clear unused images
community.docker.docker_prune:
images: true
- name: Create network
community.docker.docker_network:
name: "oasst-{{ stack_name }}"
state: present
driver: bridge
- name: Create stack files directory
ansible.builtin.file:
path: "./{{ stack_name }}"
state: directory
mode: 0755
- name: Copy redis.conf to managed node
ansible.builtin.copy:
src: ./redis.conf
dest: "./{{ stack_name }}/redis.conf"
mode: 0644
- name: Set up Redis
community.docker.docker_container:
name: "oasst-{{ stack_name }}-redis"
image: redis
state: started
recreate: "{{ (stack_name == 'dev') | bool }}"
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
healthcheck:
test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
interval: 2s
timeout: 2s
retries: 10
command: redis-server /usr/local/etc/redis/redis.conf
volumes:
- "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf"
- name: Create volumes for postgres
community.docker.docker_volume:
name: "oasst-{{ stack_name }}-postgres-{{ item.name }}"
state: present
loop:
- name: backend
- name: web
- name: Create postgres containers
community.docker.docker_container:
name: "oasst-{{ stack_name }}-postgres-{{ item.name }}"
image: ghcr.io/laion-ai/open-assistant/oasst-postgres
platform: linux/x86_64
state: started
pull: true
recreate: "{{ (stack_name == 'dev') | bool }}"
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: "{{ postgres_password }}"
POSTGRES_DB: postgres
S3_BUCKET_NAME:
"{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}"
S3_PREFIX: "{{ item.name }}"
AWS_ACCESS_KEY_ID:
"{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}"
AWS_SECRET_ACCESS_KEY:
"{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}"
AWS_DEFAULT_REGION: "{{ lookup('ansible.builtin.env', 'S3_REGION') }}"
volumes:
- "oasst-{{ stack_name }}-postgres-{{ item.name
}}:/var/lib/postgresql/data"
healthcheck:
test: ["CMD", "pg_isready", "-U", "postgres"]
interval: 2s
timeout: 2s
retries: 10
shm_size: 1G
loop:
- name: backend
- name: web
- name: Run the oasst oasst-backend
community.docker.docker_container:
name: "oasst-{{ stack_name }}-backend"
image: "ghcr.io/laion-ai/open-assistant/oasst-backend:{{ image_tag }}"
platform: linux/x86_64
state: started
recreate: true
pull: true
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend"
POSTGRES_PASSWORD: "{{ postgres_password }}"
REDIS_HOST: "oasst-{{ stack_name }}-redis"
DEBUG_USE_SEED_DATA:
"{{ 'true' if stack_name == 'dev' else 'false' }}"
DEBUG_ALLOW_SELF_LABELING:
"{{ 'true' if stack_name == 'dev' else 'false' }}"
MAX_WORKERS: "1"
RATE_LIMIT: "{{ 'false' if stack_name == 'dev' else 'true' }}"
DEBUG_SKIP_EMBEDDING_COMPUTATION: "true"
DEBUG_SKIP_TOXICITY_CALCULATION:
"{{ lookup('ansible.builtin.env', 'SKIP_TOXICITY_CALCULATION') |
default('true', true) }}"
OFFICIAL_WEB_API_KEY: "{{ web_api_key }}"
TREE_MANAGER__INIT_PROMPT_DISABLED_LANGS:
"{{ lookup('ansible.builtin.env', 'INIT_PROMPT_DISABLED_LANGS') |
default('', true) }}"
TREE_MANAGER__MAX_ACTIVE_TREES:
"{{ lookup('ansible.builtin.env', 'MAX_ACTIVE_TREES') |
default('10', true) }}"
TREE_MANAGER__MAX_INITIAL_PROMPT_REVIEW:
"{{ lookup('ansible.builtin.env', 'MAX_INITIAL_PROMPT_REVIEW') |
default('100', true) }}"
TREE_MANAGER__MAX_TREE_DEPTH:
"{{ lookup('ansible.builtin.env', 'MAX_TREE_DEPTH') | default('5',
true) }}"
TREE_MANAGER__GOAL_TREE_SIZE:
"{{ lookup('ansible.builtin.env', 'GOAL_TREE_SIZE') | default('15',
true) }}"
TREE_MANAGER__MAX_CHILDREN_COUNT:
"{{ lookup('ansible.builtin.env', 'MAX_CHILDREN_COUNT') |
default('3', true) }}"
TREE_MANAGER__LONELY_CHILDREN_COUNT:
"{{ lookup('ansible.builtin.env', 'LONELY_CHILDREN_COUNT') |
default('2', true) }}"
TREE_MANAGER__P_LONELY_CHILD_EXTENSION:
"{{ lookup('ansible.builtin.env', 'P_LONELY_CHILD_EXTENSION') |
default('0.75', true) }}"
TREE_MANAGER__P_ACTIVATE_BACKLOG_TREE:
"{{ lookup('ansible.builtin.env', 'P_ACTIVATE_BACKLOG_TREE') |
default('0.1', true) }}"
TREE_MANAGER__NUM_REQUIRED_RANKINGS:
"{{ lookup('ansible.builtin.env', 'NUM_REQUIRED_RANKINGS') |
default('3', true) }}"
TREE_MANAGER__NUM_REVIEWS_REPLY:
"{{ lookup('ansible.builtin.env', 'NUM_REVIEWS_REPLY') |
default('3', true) }}"
TREE_MANAGER__NUM_REVIEWS_INITIAL_PROMPT:
"{{ lookup('ansible.builtin.env', 'NUM_REVIEWS_INITIAL_PROMPT') |
default('3', true) }}"
TREE_MANAGER__MIN_ACTIVE_RANKINGS_PER_LANG:
"{{ lookup('ansible.builtin.env', 'MIN_ACTIVE_RANKINGS_PER_LANG') |
default('0', true) }}"
TREE_MANAGER__MAX_PROMPT_LOTTERY_WAITING:
"{{ lookup('ansible.builtin.env', 'MAX_PROMPT_LOTTERY_WAITING') |
default('250', true) }}"
MESSAGE_SIZE_LIMIT:
"{{ lookup('ansible.builtin.env', 'MESSAGE_SIZE_LIMIT') |
default('2000', true) }}"
USER_STATS_INTERVAL_DAY:
"{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_DAY') |
default('5', true) }}"
USER_STATS_INTERVAL_WEEK:
"{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_WEEK') |
default('15', true) }}"
USER_STATS_INTERVAL_MONTH:
"{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_MONTH') |
default('60', true) }}"
USER_STATS_INTERVAL_TOTAL:
"{{ lookup('ansible.builtin.env', 'STATS_INTERVAL_TOTAL') |
default('240', true) }}"
LOGURU_LEVEL:
"{{ lookup('ansible.builtin.env', 'LOGURU_LEVEL') | default('INFO',
true) }}"
BACKEND_CORS_ORIGINS_CSV:
"{{ lookup('ansible.builtin.env', 'BACKEND_CORS_ORIGINS') }}"
DISCORD_API_KEY:
"{{ lookup('ansible.builtin.env', 'DISCORD_API_KEY') }}"
DISCORD_CHANNEL_ID:
"{{ lookup('ansible.builtin.env', 'DISCORD_CHANNEL_ID') }}"
ports:
- "{{ backend_port }}:8080"
- name: Setup Celery worker
community.docker.docker_container:
name: "oasst-{{ stack_name }}-celery-worker"
image:
"ghcr.io/laion-ai/open-assistant/oasst-backend-worker:{{ image_tag }}"
state: started
pull: true
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
CELERY_BROKER_URL: "redis://oasst-{{ stack_name }}-redis:6379/0"
CELERY_RESULT_BACKEND: "redis://oasst-{{ stack_name }}-redis:6379/0"
POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend"
POSTGRES_PASSWORD: "{{ postgres_password }}"
REDIS_HOST: "oasst-{{ stack_name }}-redis"
MAX_WORKERS: "1"
command: celery -A oasst_backend.celery_worker worker -l info
- name: Setup Celery worker beat
community.docker.docker_container:
name: "oasst-{{ stack_name }}-celery-worker-beat"
image:
"ghcr.io/laion-ai/open-assistant/oasst-backend-worker:{{ image_tag }}"
state: started
pull: true
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
CELERY_BROKER_URL: "redis://oasst-{{ stack_name }}-redis:6379/0"
CELERY_RESULT_BACKEND: "redis://oasst-{{ stack_name }}-redis:6379/0"
POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend"
POSTGRES_PASSWORD: "{{ postgres_password }}"
REDIS_HOST: "oasst-{{ stack_name }}-redis"
MAX_WORKERS: "1"
command: celery -A oasst_backend.celery_worker beat -l INFO
- name: Run the oasst oasst-web frontend
community.docker.docker_container:
name: "oasst-{{ stack_name }}-web"
image: "ghcr.io/laion-ai/open-assistant/oasst-web:{{ image_tag }}"
state: started
recreate: true
pull: true
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}"
MODERATOR_USERS:
"{{ lookup('ansible.builtin.env', 'WEB_MODERATOR_USERS') }}"
DATABASE_URL:
"postgres://postgres:{{ postgres_password }}@oasst-{{ stack_name
}}-postgres-web/postgres"
DEBUG_LOGIN: "{{ 'true' if stack_name == 'dev' else 'false' }}"
DISCORD_CLIENT_ID:
"{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}"
DISCORD_CLIENT_SECRET:
"{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_SECRET') }}"
GOOGLE_CLIENT_ID:
"{{ lookup('ansible.builtin.env', 'WEB_GOOGLE_CLIENT_ID') }}"
GOOGLE_CLIENT_SECRET:
"{{ lookup('ansible.builtin.env', 'WEB_GOOGLE_CLIENT_SECRET') }}"
EMAIL_FROM: open-assistant@laion.ai
EMAIL_SERVER_HOST:
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_HOST') }}"
EMAIL_SERVER_PASSWORD:
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PASSWORD') }}"
EMAIL_SERVER_PORT:
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PORT') }}"
EMAIL_SERVER_USER:
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}"
FASTAPI_URL: "{{ lookup('ansible.builtin.env', 'BACKEND_URL') }}"
FASTAPI_KEY: "{{ web_api_key }}"
NEXTAUTH_SECRET:
"{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}"
NEXTAUTH_URL:
"{{ 'https://open-assistant.io/' if stack_name == 'production' else
('https://web.' + stack_name + '.open-assistant.io/') }}"
CLOUDFLARE_CAPTCHA_SITE_KEY:
"{{ lookup('ansible.builtin.env',
'WEB_NEXT_PUBLIC_CLOUDFLARE_CAPTCHA_SITE_KEY') }}"
CLOUDFLARE_CAPTCHA_SECRET_KEY:
"{{ lookup('ansible.builtin.env',
'WEB_CLOUDFLARE_CAPTCHA_SECRET_KEY') }}"
ENABLE_EMAIL_SIGNIN_CAPTCHA:
"{{ lookup('ansible.builtin.env',
'WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN_CAPTCHA') }}"
ENABLE_EMAIL_SIGNIN:
"{{ lookup('ansible.builtin.env',
'WEB_NEXT_PUBLIC_ENABLE_EMAIL_SIGNIN') }}"
MAINTENANCE_MODE:
"{{ lookup('ansible.builtin.env', 'MAINTENANCE_MODE') }}"
INFERENCE_SERVER_HOST:
"{{ lookup('ansible.builtin.env', 'WEB_INFERENCE_SERVER_HOST') }}"
INFERENCE_SERVER_API_KEY:
"{{ lookup('ansible.builtin.env', 'WEB_INFERENCE_SERVER_API_KEY') }}"
ENABLE_CHAT: "{{ lookup('ansible.builtin.env', 'WEB_ENABLE_CHAT') }}"
BYE: "{{ lookup('ansible.builtin.env', 'WEB_BYE') }}"
ENABLE_DRAFTS_WITH_PLUGINS:
"{{ lookup('ansible.builtin.env',
'WEB_ENABLE_DRAFTS_WITH_PLUGINS')}}"
NUM_GENERATED_DRAFTS:
"{{ lookup('ansible.builtin.env', 'WEB_NUM_GENERATED_DRAFTS') }}"
CURRENT_ANNOUNCEMENT:
"{{ lookup('ansible.builtin.env', 'WEB_CURRENT_ANNOUNCEMENT') }}"
ports:
- "{{ website_port }}:3000"
command: bash wait-for-postgres.sh node server.js
================================================
FILE: ansible/inference/deploy-server.yaml
================================================
# Ansible playbook to setup inference server
- name: Deploy inference
hosts: dev
gather_facts: true
vars:
stack_name: "dev"
image_tag: latest
server_port: 8080
tasks:
- name: Create network
community.docker.docker_network:
name: "oasst-{{ stack_name }}"
state: present
driver: bridge
- name: Create stack files directory
ansible.builtin.file:
path: "./{{ stack_name }}-inference"
state: directory
mode: 0755
- name: Copy redis.conf to managed node
ansible.builtin.copy:
src: ./redis.conf
dest: "./{{ stack_name }}-inference/redis.conf"
mode: 0644
- name: Set up inference Redis
community.docker.docker_container:
name: "oasst-inference-{{ stack_name }}-redis"
image: redis
state: started
recreate: "{{ (stack_name == 'dev') | bool }}"
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
healthcheck:
test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
interval: 2s
timeout: 2s
retries: 10
command: redis-server /usr/local/etc/redis/redis.conf
volumes:
- "./{{ stack_name
}}-inference/redis.conf:/usr/local/etc/redis/redis.conf"
- name: Create volumes for inference postgres
community.docker.docker_volume:
name: "oasst-inference-{{ stack_name }}-postgres"
state: present
- name: Create postgres containers
community.docker.docker_container:
name: "oasst-inference-{{ stack_name }}-postgres"
image: ghcr.io/laion-ai/open-assistant/oasst-postgres
state: started
pull: true
recreate: "{{ (stack_name == 'dev') | bool }}"
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD:
"{{ lookup('ansible.builtin.env', 'INFERENCE_POSTGRES_PASSWORD') |
default('postgres', true) }}"
POSTGRES_DB: postgres
S3_BUCKET_NAME:
"{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}"
S3_PREFIX: "inference"
AWS_ACCESS_KEY_ID:
"{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}"
AWS_SECRET_ACCESS_KEY:
"{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}"
AWS_DEFAULT_REGION: "{{ lookup('ansible.builtin.env', 'S3_REGION') }}"
volumes:
- "oasst-inference-{{ stack_name }}-postgres:/var/lib/postgresql/data"
healthcheck:
test: ["CMD", "pg_isready", "-U", "postgres"]
interval: 2s
timeout: 2s
retries: 10
shm_size: 1G
- name: Run the oasst inference-server
community.docker.docker_container:
name: "oasst-inference-{{ stack_name }}-server"
image:
"ghcr.io/laion-ai/open-assistant/oasst-inference-server:{{ image_tag
}}"
state: started
recreate: true
pull: true
restart_policy: always
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_HOST: "oasst-inference-{{ stack_name }}-postgres"
POSTGRES_PASSWORD:
"{{ lookup('ansible.builtin.env', 'INFERENCE_POSTGRES_PASSWORD') |
default('postgres', true) }}"
REDIS_HOST: "oasst-inference-{{ stack_name }}-redis"
LOGURU_LEVEL:
"{{ lookup('ansible.builtin.env', 'INFERENCE_LOG_LEVEL') |
default('INFO', true) }}"
DEBUG_API_KEYS:
"{{ lookup('ansible.builtin.env', 'INFERENCE_DEBUG_API_KEYS') |
default('', true) | string }}"
ALLOW_DEBUG_AUTH:
"{{ lookup('ansible.builtin.env', 'INFERENCE_ALLOW_DEBUG_AUTH') |
default('False', true) | string }}"
ROOT_TOKEN:
"{{ lookup('ansible.builtin.env', 'INFERENCE_ROOT_TOKEN') |
default('1234', true) }}"
API_ROOT:
"{{ lookup('ansible.builtin.env', 'INFERENCE_API_ROOT') |
default('https://inference.dev.open-assistant.io', true) }}"
TRUSTED_CLIENT_KEYS:
"{{ lookup('ansible.builtin.env', 'INFERENCE_TRUSTED_CLIENT_KEYS') |
default('', true) }}"
AUTH_SALT:
"{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_SALT') |
default('', true) }}"
AUTH_SECRET:
"{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_SECRET') |
default('', true) }}"
AUTH_DISCORD_CLIENT_ID:
"{{ lookup('ansible.builtin.env',
'INFERENCE_AUTH_DISCORD_CLIENT_ID') | default('', true) }}"
AUTH_DISCORD_CLIENT_SECRET:
"{{ lookup('ansible.builtin.env',
'INFERENCE_AUTH_DISCORD_CLIENT_SECRET') | default('', true) }}"
AUTH_GITHUB_CLIENT_ID:
"{{ lookup('ansible.builtin.env', 'INFERENCE_AUTH_GITHUB_CLIENT_ID')
| default('', true) }}"
AUTH_GITHUB_CLIENT_SECRET:
"{{ lookup('ansible.builtin.env',
'INFERENCE_AUTH_GITHUB_CLIENT_SECRET') | default('', true) }}"
INFERENCE_CORS_ORIGINS:
"{{ lookup('ansible.builtin.env', 'INFERENCE_CORS_ORIGINS') |
default('*', true) }}"
ALLOWED_MODEL_CONFIG_NAMES:
"{{ lookup('ansible.builtin.env',
'INFERENCE_ALLOWED_MODEL_CONFIG_NAMES') | default('*', true) }}"
ASSISTANT_MESSAGE_TIMEOUT:
"{{ lookup('ansible.builtin.env',
'INFERENCE_ASSISTANT_MESSAGE_TIMEOUT') | default(120, true) }}"
MESSAGE_QUEUE_EXPIRE:
"{{ lookup('ansible.builtin.env', 'INFERENCE_MESSAGE_QUEUE_EXPIRE')
| default(120, true) }}"
WORK_QUEUE_MAX_SIZE:
"{{ lookup('ansible.builtin.env', 'INFERENCE_WORK_QUEUE_MAX_SIZE') |
default(100, true) }}"
ENABLE_SAFETY:
"{{ lookup('ansible.builtin.env', 'INFERENCE_ENABLE_SAFETY') |
default('False', true) | string }}"
GUNICORN_WORKERS:
"{{ lookup('ansible.builtin.env', 'INFERENCE_GUNICORN_WORKERS') |
default(1, true) }}"
CHAT_MAX_MESSAGES:
"{{ lookup('ansible.builtin.env', 'INFERENCE_CHAT_MAX_MESSAGES') |
default('', true) }}"
MESSAGE_MAX_LENGTH:
"{{ lookup('ansible.builtin.env', 'INFERENCE_MESSAGE_MAX_LENGTH') |
default('', true) }}"
PLUGIN_MAX_DEPTH:
"{{ lookup('ansible.builtin.env', 'INFERENCE_PLUGIN_MAX_DEPTH') |
default(4, true) }}"
ports:
- "{{ server_port }}:8080"
================================================
FILE: ansible/inference/deploy-worker.yaml
================================================
# Ansible playbook to setup inference worker
- name: Deploy inference
hosts: dev
gather_facts: true
vars:
stack_name: "dev"
image_tag: latest
backend_url: "ws://localhost:8000"
api_key: "1234"
parallelism: 4
tasks:
- name: Create network
community.docker.docker_network:
name: "oasst-worker-{{ stack_name }}"
state: present
driver: bridge
- name: Create stack files directory
ansible.builtin.file:
path: "./{{ stack_name }}"
state: directory
mode: 0755
- name: Run the oasst inference worker
community.docker.docker_container:
name: "oasst-worker-{{ stack_name }}"
image:
"ghcr.io/laion-ai/open-assistant/oasst-inference-worker-full:{{
image_tag }}"
state: started
recreate: true
pull: true
restart_policy: always
network_mode: "oasst-worker-{{ stack_name }}"
env:
BACKEND_URL: "{{ backend_url }}"
API_KEY: "{{ api_key }}"
PARALLELISM: "{{ parallelism }}"
================================================
FILE: ansible/inference/redis.conf
================================================
maxmemory 1024mb
maxmemory-policy allkeys-lru
================================================
FILE: ansible/inference/test.inventory.ini
================================================
[test]
dev ansible_connection=local
================================================
FILE: ansible/pgbackrest.conf
================================================
[oasst]
pg1-path=/var/lib/postgresql/data
[global]
repo1-retention-full=3
repo1-type=s3
repo1-path=/oasst-prod
repo1-s3-region=us-east-1
repo1-s3-endpoint=s3.amazonaws.com
# repo1-s3-bucket=$S3_BUCKET_NAME
# repo1-s3-key=$AWS_ACCESS_KEY
# repo1-s3-key-secret=$AWS_SECRET_KEY
# Force a checkpoint to start backup immediately.
start-fast=y
# Use delta restore.
delta=y
# Enable ZSTD compression.
compress-type=zst
compress-level=6
log-level-console=info
log-level-file=debug
================================================
FILE: ansible/redis.conf
================================================
maxmemory 100mb
maxmemory-policy allkeys-lru
================================================
FILE: ansible/test.inventory.ini
================================================
[test]
dev ansible_connection=local
================================================
FILE: backend/.gitignore
================================================
__pycache__
.env
notes.txt
venv
celerybeat-schedule.*
================================================
FILE: backend/README.md
================================================
<a href="https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/backend/README.md?_x_tr_sl=auto&_x_tr_tl=en&_x_tr_hl=en&_x_tr_pto=wapp"></a>
# Open-Assistant REST Backend
## Backend Development Setup
### Local Database
In root directory, run
`docker compose --profile backend-dev up --build --attach-dependencies` to start
a database. The default settings are already configured to connect to the
database at `localhost:5432`. (See
[FAQ](https://projects.laion.ai/Open-Assistant/docs/faq#enable-dockers-buildkit-backend)
if you face any docker problems).
> **Note:** when running on MacOS with an M1 chip you have to use:
> `DB_PLATFORM=linux/x86_64 docker compose ...`
Python 3.10 is required. It is recommended to use `pyenv` which will recognise
the `.python-version` in the project root directory.
### Python Packages
Next, to install all requirements, You can run
1. `pip install -r backend/requirements.txt`
2. `pip install -e ./oasst-shared/.`
3. `pip install -e ./oasst-data/.`
4. `./scripts/backend-development/run-local.sh` to run the backend. This will
start the backend server at `http://localhost:8080`.
## REST Server Configuration
- Generate a new environment variable file `.env` by coping the content of the
[.env.example](.env.example) file.
- Update the values of the environment variables in the `.env` file by setting
the DATABASE_URI to you local database URI.
- Update the rest of the environment variables according to your needs.
## Running the REST Server locally for development
Have a look into the main `README.md` file for more information on how to set up
the backend for development. Use the scripts within the
scripts/backend-development folder to run the BE API locally.
## Alembic
To create an Alembic database migration script after sql-models were modified
run `alembic revision --autogenerate -m "..."` ("..." is what you did) in the
`/backend` directory. Then edit the newly created file. See
[here](https://alembic.sqlalchemy.org/en/latest/tutorial.html) for more
information.
## API Documentation
Once you have successfully started the backend server, you can access the
default api docs at `localhost:8080/docs`. If you need to update the exported
openapi.json in the docs/ folder you can run below command to `wget` them from
the relevant local fastapi endpoint. This will enable anyone to just see API
docs via something like
[Swagger.io](https://editor.swagger.io/?url=https://raw.githubusercontent.com/LAION-AI/Open-Assistant/main/docs/docs/api/openapi.json)
without having to actually set up and run a development backend.
```bash
# save openapi.json to docs/docs/api/
wget localhost:8080/api/v1/openapi.json -O docs/docs/api/backend-openapi.json
```
Note: The api docs should be automatically updated by the
`test-api-contract.yaml` workflow. (TODO)
## Running Celery Worker(s) for API and periodic tasks
Celery workers are used for Huggingface API calls like toxicity and feature
extraction. Celery Beat along with worker is used for periodic tasks like user
streak update
To run APIs locally
- update HUGGING_FACE_API_KEY in backend/oasst_backend/config.py with the
correct API_KEY
- `export DEBUG_SKIP_TOXICITY_CALCULATION=False` and
`export DEBUG_SKIP_EMBEDDING_COMPUTATION=False`in
`scripts/backend-development/run-local.sh`
- run start_worker.sh in backend dir
- to see logs , use `tail -f celery.log` and `tail -f celery.beat.log`
In CI
- set `DEBUG_SKIP_TOXICITY_CALCULATION=False` and
`DEBUG_SKIP_EMBEDDING_COMPUTATION=False` in docker-compose.yaml
- Two Docker instances are created. One for Beat and other for the worker
- Logs can be viewed like other docker instances
## Exporting Data
When you have collected some data in the backend database, you can export it
using the `export.py` script provided in this directory. This can be run from
the command line using an Python environment with the same requirements as the
backend itself. The script connects to the database in the same manner as the
backend and therefore uses the same environmental variables.
A simple usage of the script, to export all English trees which successfully
passed the review process, may look like:
```bash
python export.py --lang en --export-file output.jsonl
```
There are many options available to filter the data which can be found in the
help message of the script: `python export.py --help`.
**Why isn't my export working?**
Common issues include (WIP):
- The messages have not passed the review process yet so the trees are not ready
for export. This can be solved by including the `--include-spam` flag.
================================================
FILE: backend/alembic/README
================================================
Generic single-database configuration.
================================================
FILE: backend/alembic/env.py
================================================
from logging.config import fileConfig
import sqlmodel
from alembic import context
from oasst_backend import models # noqa: F401
from sqlalchemy import engine_from_config, pool
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = sqlmodel.SQLModel.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.get_context()._ensure_version_table()
connection.execute("LOCK TABLE alembic_version IN ACCESS EXCLUSIVE MODE")
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
================================================
FILE: backend/alembic/script.py.mako
================================================
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
import sqlmodel
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
================================================
FILE: backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py
================================================
"""first revision
Revision ID: 23e5fea252dd
Revises:
Create Date: 2022-12-12 12:47:28.801354
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "23e5fea252dd"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"service_client",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("service_admin_email", sa.String(128), nullable=True),
sa.Column("api_key", sa.String(300), nullable=False),
sa.Column("can_append", sa.Boolean, nullable=False, server_default="true"),
sa.Column("can_write", sa.Boolean, nullable=False, server_default="false"),
sa.Column("can_delete", sa.Boolean, nullable=False, server_default="false"),
sa.Column("can_read", sa.Boolean, nullable=False, server_default="true"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_service_client_api_key"), "service_client", ["api_key"], unique=True)
op.create_table(
"labeler",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("display_name", sa.String(96), nullable=False),
sa.Column("discord_username", sa.String(96), nullable=True),
sa.Column(
"created_date",
sa.DateTime,
nullable=False,
server_default=sa.func.current_timestamp(),
),
sa.Column("is_enabled", sa.Boolean, nullable=False, server_default="true"),
sa.Column("notes", sa.String(10 * 1024), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("discord_username"),
)
op.create_table(
"prompt",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("labeler_id", sa.Integer, nullable=False),
sa.Column("prompt", sa.Text, nullable=False),
sa.Column("response", sa.Text, nullable=True),
sa.Column("lang", sa.String(32), nullable=True),
sa.Column(
"created_date",
sa.DateTime(),
nullable=False,
server_default=sa.func.current_timestamp(),
),
sa.ForeignKeyConstraint(
["labeler_id"],
["labeler.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("prompt_labeler_id"), "prompt", ["labeler_id"], unique=False)
def downgrade() -> None:
op.drop_index(op.f("prompt_labeler_id"), table_name="prompt")
op.drop_table("prompt")
op.drop_table("labeler")
op.drop_index(op.f("ix_service_client_api_key"), table_name="service_client")
op.drop_table("service_client")
================================================
FILE: backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py
================================================
"""v1 db structure
Revision ID: cd7de470586e
Revises: 23e5fea252dd
Create Date: 2022-12-15 11:15:32.830225
"""
import uuid
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import JSONB, UUID
# revision identifiers, used by Alembic.
revision = "cd7de470586e"
down_revision = "23e5fea252dd"
branch_labels = None
depends_on = None
def upgrade() -> None:
# remove database objects
op.drop_index(op.f("prompt_labeler_id"), table_name="prompt")
op.drop_table("prompt")
op.drop_table("labeler")
op.drop_index(op.f("ix_service_client_api_key"), table_name="service_client")
op.drop_table("service_client")
# wreate new database structure
op.create_table(
"api_client",
sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")),
sa.Column("api_key", sa.String(512), nullable=False),
sa.Column("description", sa.String(256), nullable=False),
sa.Column("admin_email", sa.String(256), nullable=True),
sa.Column("enabled", sa.Boolean, default=True, nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_api_client_api_key"), "api_client", ["api_key"], unique=True)
op.create_table(
"person",
sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")),
sa.Column("username", sa.String(128), nullable=False), # unique in combination with api_client_id
sa.Column("display_name", sa.String(256), nullable=False), # cached last seen display_name
sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]),
)
op.create_index(op.f("ix_person_username"), "person", ["api_client_id", "username"], unique=True)
op.create_table(
"person_stats",
sa.Column("person_id", UUID(as_uuid=True)),
sa.Column("leader_score", sa.Integer, default=0, nullable=False), # determines position on leader board
sa.Column("modified_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.Column("reactions", sa.Integer, default=0, nullable=False), # reactions sent by user
sa.Column("posts", sa.Integer, default=0, nullable=False), # posts sent by user
sa.Column("upvotes", sa.Integer, default=0, nullable=False), # received upvotes (form other users)
sa.Column("downvotes", sa.Integer, default=0, nullable=False), # received downvotes (from other users)
sa.Column("work_reward", sa.Integer, default=0, nullable=False), # reward for workpackage completions
sa.Column("compare_wins", sa.Integer, default=0, nullable=False), # num times user's post won compare tasks
sa.Column("compare_losses", sa.Integer, default=0, nullable=False), # num times users's post lost compare tasks
sa.PrimaryKeyConstraint("person_id"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"]),
)
op.create_table(
"work_package",
sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")),
sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.Column("expiry_date", sa.DateTime(), nullable=True),
sa.Column("person_id", UUID(as_uuid=True), nullable=True),
sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid
sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"]),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]),
)
op.create_index(op.f("ix_work_package_person_id"), "work_package", ["person_id"], unique=False)
op.create_table(
"post",
sa.Column("id", UUID(as_uuid=True), default=uuid.uuid4, server_default=sa.text("gen_random_uuid()")),
sa.Column("parent_id", UUID(as_uuid=True), nullable=True), # root posts have NULL parent
sa.Column("thread_id", UUID(as_uuid=True), nullable=False), # id of thread root
sa.Column("workpackage_id", UUID(as_uuid=True), nullable=True), # workpackage id to pass to handler on reply
sa.Column("person_id", UUID(as_uuid=True), nullable=True), # sender (recipients are part of payload)
sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
sa.Column("role", sa.String(128), nullable=False), # 'assistant', 'user' or something else
sa.Column("frontend_post_id", sa.String(200), nullable=False), # unique together with api_client_id
sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid
sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"]),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]),
)
op.create_index(op.f("ix_post_frontend_post_id"), "post", ["api_client_id", "frontend_post_id"], unique=True)
op.create_index(op.f("ix_post_thread_id"), "post", ["thread_id"], unique=False)
op.create_index(op.f("ix_post_workpackage_id"), "post", ["workpackage_id"], unique=False)
op.create_index(op.f("ix_post_person_id"), "post", ["person_id"], unique=False)
op.create_table(
"post_reaction",
sa.Column("post_id", UUID(as_uuid=True), nullable=False),
sa.Column("person_id", UUID(as_uuid=True), nullable=False), # sender (recipients are part of payload)
sa.Column("created_date", sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.Column("payload_type", sa.String(200), nullable=False), # deserialization hint & dbg aid
sa.Column("payload", JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("api_client_id", UUID(as_uuid=True), nullable=False),
sa.PrimaryKeyConstraint("post_id", "person_id"),
sa.ForeignKeyConstraint(["post_id"], ["post.id"]),
sa.ForeignKeyConstraint(["person_id"], ["person.id"]),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"]),
)
def downgrade() -> None:
op.drop_table("post_reaction")
op.drop_index("ix_post_person_id")
op.drop_index("ix_post_workpackage_id")
op.drop_index("ix_post_thread_id")
op.drop_index("ix_post_frontend_post_id")
op.drop_table("post")
op.drop_index("ix_work_package_person_id")
op.drop_table("work_package")
op.drop_table("person_stats")
op.drop_index("ix_person_username")
op.drop_table("person")
op.drop_index("ix_api_client_api_key")
op.drop_table("api_client")
op.create_table(
"service_client",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("service_admin_email", sa.String(128), nullable=True),
sa.Column("api_key", sa.String(300), nullable=False),
sa.Column("can_append", sa.Boolean, nullable=False, server_default="true"),
sa.Column("can_write", sa.Boolean, nullable=False, server_default="false"),
sa.Column("can_delete", sa.Boolean, nullable=False, server_default="false"),
sa.Column("can_read", sa.Boolean, nullable=False, server_default="true"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_service_client_api_key"), "service_client", ["api_key"], unique=True)
op.create_table(
"labeler",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("display_name", sa.String(96), nullable=False),
sa.Column("discord_username", sa.String(96), nullable=True),
sa.Column(
"created_date",
sa.DateTime,
nullable=False,
server_default=sa.func.current_timestamp(),
),
sa.Column("is_enabled", sa.Boolean, nullable=False, server_default="true"),
sa.Column("notes", sa.String(10 * 1024), nullable=True),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("discord_username"),
)
op.create_table(
"prompt",
sa.Column("id", sa.Integer, sa.Identity()),
sa.Column("labeler_id", sa.Integer, nullable=False),
sa.Column("prompt", sa.Text, nullable=False),
sa.Column("response", sa.Text, nullable=True),
sa.Column("lang", sa.String(32), nullable=True),
sa.Column(
"created_date",
sa.DateTime(),
nullable=False,
server_default=sa.func.current_timestamp(),
),
sa.ForeignKeyConstraint(
["labeler_id"],
["labeler.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("prompt_labeler_id"), "prompt", ["labeler_id"], unique=False)
================================================
FILE: backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py
================================================
"""add auth_method to person
Revision ID: 6368515778c5
Revises: cd7de470586e
Create Date: 2022-12-17 17:57:33.022549
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "6368515778c5"
down_revision = "cd7de470586e"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("person", sa.Column("auth_method", sa.String(length=128), nullable=True))
op.execute("UPDATE person SET auth_method = 'local'")
op.alter_column("person", "auth_method", nullable=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("person", "auth_method")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py
================================================
"""add_auth_method_to_ix_person_username
Revision ID: 0daec5f8135f
Revises: 6368515778c5
Create Date: 2022-12-22 18:35:59.609013
"""
import sqlalchemy as sa # noqa: F401
from alembic import op
# revision identifiers, used by Alembic.
revision = "0daec5f8135f"
down_revision = "6368515778c5"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_person_username", table_name="person")
op.create_index("ix_person_username", "person", ["api_client_id", "username", "auth_method"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_person_username", table_name="person")
op.create_index("ix_person_username", "person", ["api_client_id", "username"], unique=False)
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py
================================================
"""Adds text labels table.
Revision ID: 067c4002f2d9
Revises: 0daec5f8135f
Create Date: 2022-12-25 17:05:21.208843
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "067c4002f2d9"
down_revision = "0daec5f8135f"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"text_labels",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("post_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column("labels", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("text", sqlmodel.sql.sqltypes.AutoString(length=65536), nullable=False),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.ForeignKeyConstraint(
["post_id"],
["post.id"],
),
sa.PrimaryKeyConstraint("id"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("text_labels")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py
================================================
"""add_journal_table
Revision ID: 3358eb6834e6
Revises: 067c4002f2d9
Create Date: 2022-12-27 14:44:59.483868
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "3358eb6834e6"
down_revision = "067c4002f2d9"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"journal",
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column(
"created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column(
"event_payload",
postgresql.JSONB(astext_type=sa.Text()),
nullable=False,
),
sa.Column("person_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("post_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("event_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.ForeignKeyConstraint(
["person_id"],
["person.id"],
),
sa.ForeignKeyConstraint(
["post_id"],
["post.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_journal_person_id"), "journal", ["person_id"], unique=False)
op.create_table(
"journal_integration",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("last_run", sa.DateTime(), nullable=True),
sa.Column("description", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False),
sa.Column("last_journal_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("last_error", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column("next_run", sa.DateTime(), nullable=True),
sa.ForeignKeyConstraint(
["last_journal_id"],
["journal.id"],
),
sa.PrimaryKeyConstraint("id", "description"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("journal_integration")
op.drop_index(op.f("ix_journal_person_id"), table_name="journal")
op.drop_table("journal")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py
================================================
"""post ref for work_package
Revision ID: d24b37426857
Revises: 3358eb6834e6
Create Date: 2022-12-28 11:42:26.773704
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "d24b37426857"
down_revision = "3358eb6834e6"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("post", sa.Column("depth", sa.Integer(), server_default=sa.text("0"), nullable=False))
op.add_column("post", sa.Column("children_count", sa.Integer(), server_default=sa.text("0"), nullable=False))
op.add_column("post_reaction", sa.Column("work_package_id", postgresql.UUID(as_uuid=True), nullable=False))
op.drop_constraint("post_reaction_post_id_fkey", "post_reaction", type_="foreignkey")
op.create_foreign_key(None, "post_reaction", "work_package", ["work_package_id"], ["id"])
op.drop_column("post_reaction", "post_id")
op.add_column("work_package", sa.Column("done", sa.Boolean(), server_default=sa.text("false"), nullable=False))
op.add_column("work_package", sa.Column("ack", sa.Boolean(), nullable=True))
op.add_column("work_package", sa.Column("frontend_ref_post_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True))
op.add_column("work_package", sa.Column("thread_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
op.add_column("work_package", sa.Column("parent_post_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("work_package", "parent_post_id")
op.drop_column("work_package", "thread_id")
op.drop_column("work_package", "frontend_ref_post_id")
op.drop_column("work_package", "ack")
op.drop_column("work_package", "done")
op.add_column("post_reaction", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=False))
op.drop_constraint(None, "post_reaction", type_="foreignkey")
op.create_foreign_key("post_reaction_post_id_fkey", "post_reaction", "post", ["post_id"], ["id"])
op.drop_column("post_reaction", "work_package_id")
op.drop_column("post", "children_count")
op.drop_column("post", "depth")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py
================================================
"""Added lang column for ISO-639-1 codes
Revision ID: ef0b52902560
Revises: d24b37426857
Create Date: 2022-12-28 18:24:21.393973
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "ef0b52902560"
down_revision = "d24b37426857"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"post", sa.Column("lang", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False, default="en-US")
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("post", "lang")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py
================================================
"""add collective flag to task
Revision ID: 464ec4667aae
Revises: ef0b52902560
Create Date: 2022-12-29 21:03:06.841962
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "464ec4667aae"
down_revision = "ef0b52902560"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"work_package", sa.Column("collective", sa.Boolean(), server_default=sa.text("false"), nullable=False)
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("work_package", "collective")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py
================================================
"""add field trusted api client
Revision ID: 73ce3675c1f5
Revises: 464ec4667aae
Create Date: 2022-12-30 01:09:06.446020
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "73ce3675c1f5"
down_revision = "464ec4667aae"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("api_client", sa.Column("trusted", sa.Boolean(), server_default=sa.text("false"), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("api_client", "trusted")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py
================================================
"""name changes: person->user, post->message, work_package->task
Revision ID: abb47e9d145a
Revises: 73ce3675c1f5
Create Date: 2022-12-30 20:54:49.880568
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "abb47e9d145a"
down_revision = "73ce3675c1f5"
branch_labels = None
depends_on = None
def upgrade() -> None:
# clear DB
op.execute("DELETE FROM journal;")
op.execute("DELETE FROM work_package;")
op.execute("DELETE FROM post_reaction;")
op.execute("DELETE FROM post;")
op.execute("DELETE FROM person_stats;")
op.execute("DELETE FROM person;")
op.execute("DELETE FROM text_labels;")
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"user",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("username", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("auth_method", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("display_name", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_user_username", "user", ["api_client_id", "username", "auth_method"], unique=True)
op.create_table(
"message",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("depth", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("children_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("parent_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("task_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("role", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("frontend_message_id", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.Column("lang", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_message_frontend_message_id", "message", ["api_client_id", "frontend_message_id"], unique=True)
op.create_index(op.f("ix_message_message_tree_id"), "message", ["message_tree_id"], unique=False)
op.create_index(op.f("ix_message_task_id"), "message", ["task_id"], unique=False)
op.create_index(op.f("ix_message_user_id"), "message", ["user_id"], unique=False)
op.create_table(
"task",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("expiry_date", sa.DateTime(), nullable=True),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("done", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("collective", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("ack", sa.Boolean(), nullable=True),
sa.Column("frontend_message_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.Column("parent_message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_task_user_id"), "task", ["user_id"], unique=False)
op.create_table(
"user_stats",
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("modified_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("leader_score", sa.Integer(), nullable=False),
sa.Column("reactions", sa.Integer(), nullable=False),
sa.Column("messages", sa.Integer(), nullable=False),
sa.Column("upvotes", sa.Integer(), nullable=False),
sa.Column("downvotes", sa.Integer(), nullable=False),
sa.Column("task_reward", sa.Integer(), nullable=False),
sa.Column("compare_wins", sa.Integer(), nullable=False),
sa.Column("compare_losses", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("user_id"),
)
op.create_table(
"message_reaction",
sa.Column("task_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.Column("payload_type", sqlmodel.sql.sqltypes.AutoString(length=200), nullable=False),
sa.Column("api_client_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.ForeignKeyConstraint(
["api_client_id"],
["api_client.id"],
),
sa.ForeignKeyConstraint(
["task_id"],
["task.id"],
),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("task_id", "user_id"),
)
op.drop_constraint("text_labels_post_id_fkey", "text_labels", type_="foreignkey")
op.drop_constraint("journal_post_id_fkey", "journal", type_="foreignkey")
op.drop_constraint("journal_person_id_fkey", "journal", type_="foreignkey")
op.drop_table("post_reaction")
op.drop_index("ix_post_frontend_post_id", table_name="post")
op.drop_index("ix_post_person_id", table_name="post")
op.drop_index("ix_post_thread_id", table_name="post")
op.drop_index("ix_post_workpackage_id", table_name="post")
op.drop_table("post")
op.drop_index("ix_work_package_person_id", table_name="work_package")
op.drop_table("work_package")
op.drop_table("person_stats")
op.drop_index("ix_person_username", table_name="person")
op.drop_table("person")
op.add_column("journal", sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
op.add_column("journal", sa.Column("message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
op.drop_index("ix_journal_person_id", table_name="journal")
op.create_index(op.f("ix_journal_user_id"), "journal", ["user_id"], unique=False)
op.create_foreign_key(None, "journal", "user", ["user_id"], ["id"])
op.create_foreign_key(None, "journal", "message", ["message_id"], ["id"])
op.drop_column("journal", "person_id")
op.drop_column("journal", "post_id")
op.add_column("text_labels", sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=True))
op.create_foreign_key(None, "text_labels", "message", ["message_id"], ["id"])
op.drop_column("text_labels", "post_id")
# ### end Alembic commands ###
def downgrade() -> None:
# clear DB
op.execute("DELETE FROM journal;")
op.execute("DELETE FROM message_reaction;")
op.execute("DELETE FROM task;")
op.execute("DELETE FROM message;")
op.execute("DELETE FROM user_stats;")
op.execute('DELETE FROM "user";')
op.execute("DELETE FROM text_labels;")
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("text_labels", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=True))
op.drop_constraint("text_labels_message_id_fkey", "text_labels", type_="foreignkey")
op.drop_column("text_labels", "message_id")
op.add_column("journal", sa.Column("post_id", postgresql.UUID(), autoincrement=False, nullable=True))
op.add_column("journal", sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True))
op.drop_constraint("journal_message_id_fkey", "journal", type_="foreignkey")
op.drop_constraint("journal_user_id_fkey", "journal", type_="foreignkey")
op.drop_index(op.f("ix_journal_user_id"), table_name="journal")
op.create_index("ix_journal_person_id", "journal", ["person_id"], unique=False)
op.drop_column("journal", "message_id")
op.drop_column("journal", "user_id")
op.create_table(
"person",
sa.Column(
"id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False
),
sa.Column("username", sa.VARCHAR(length=128), autoincrement=False, nullable=False),
sa.Column("display_name", sa.VARCHAR(length=256), autoincrement=False, nullable=False),
sa.Column(
"created_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("auth_method", sa.VARCHAR(length=128), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="person_api_client_id_fkey"),
sa.PrimaryKeyConstraint("id", name="person_pkey"),
)
op.create_table(
"person_stats",
sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("leader_score", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column(
"modified_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
sa.Column("reactions", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("posts", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("upvotes", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("downvotes", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("work_reward", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("compare_wins", sa.INTEGER(), autoincrement=False, nullable=False),
sa.Column("compare_losses", sa.INTEGER(), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="person_stats_person_id_fkey"),
sa.PrimaryKeyConstraint("person_id", name="person_stats_pkey"),
)
op.create_table(
"work_package",
sa.Column(
"id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False
),
sa.Column(
"created_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
sa.Column("expiry_date", postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("done", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False),
sa.Column("ack", sa.BOOLEAN(), autoincrement=False, nullable=True),
sa.Column("frontend_ref_post_id", sa.VARCHAR(), autoincrement=False, nullable=True),
sa.Column("thread_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("parent_post_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("collective", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="work_package_api_client_id_fkey"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="work_package_person_id_fkey"),
sa.PrimaryKeyConstraint("id", name="work_package_pkey"),
)
op.create_index("ix_work_package_person_id", "work_package", ["person_id"], unique=False)
op.create_table(
"post",
sa.Column(
"id", postgresql.UUID(), server_default=sa.text("gen_random_uuid()"), autoincrement=False, nullable=False
),
sa.Column("parent_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("thread_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("workpackage_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=True),
sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("role", sa.VARCHAR(length=128), autoincrement=False, nullable=False),
sa.Column("frontend_post_id", sa.VARCHAR(length=200), autoincrement=False, nullable=False),
sa.Column(
"created_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True),
sa.Column("depth", sa.INTEGER(), server_default=sa.text("0"), autoincrement=False, nullable=False),
sa.Column("children_count", sa.INTEGER(), server_default=sa.text("0"), autoincrement=False, nullable=False),
sa.Column("lang", sa.VARCHAR(length=200), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="post_api_client_id_fkey"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="post_person_id_fkey"),
sa.PrimaryKeyConstraint("id", name="post_pkey"),
)
op.create_index("ix_post_workpackage_id", "post", ["workpackage_id"], unique=False)
op.create_index("ix_post_thread_id", "post", ["thread_id"], unique=False)
op.create_index("ix_post_person_id", "post", ["person_id"], unique=False)
op.create_index("ix_post_frontend_post_id", "post", ["api_client_id", "frontend_post_id"], unique=False)
op.create_table(
"post_reaction",
sa.Column("person_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column(
"created_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
sa.Column("payload_type", sa.VARCHAR(length=200), autoincrement=False, nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False),
sa.Column("api_client_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.Column("work_package_id", postgresql.UUID(), autoincrement=False, nullable=False),
sa.ForeignKeyConstraint(["api_client_id"], ["api_client.id"], name="post_reaction_api_client_id_fkey"),
sa.ForeignKeyConstraint(["person_id"], ["person.id"], name="post_reaction_person_id_fkey"),
sa.ForeignKeyConstraint(["work_package_id"], ["work_package.id"], name="post_reaction_work_package_id_fkey"),
)
op.create_index("ix_person_username", "person", ["api_client_id", "username", "auth_method"], unique=False)
op.create_foreign_key("text_labels_post_id_fkey", "text_labels", "post", ["post_id"], ["id"])
op.create_foreign_key("journal_person_id_fkey", "journal", "person", ["person_id"], ["id"])
op.create_foreign_key("journal_post_id_fkey", "journal", "post", ["post_id"], ["id"])
op.drop_table("message_reaction")
op.drop_table("user_stats")
op.drop_index(op.f("ix_task_user_id"), table_name="task")
op.drop_table("task")
op.drop_index(op.f("ix_message_user_id"), table_name="message")
op.drop_index(op.f("ix_message_task_id"), table_name="message")
op.drop_index(op.f("ix_message_message_tree_id"), table_name="message")
op.drop_index("ix_message_frontend_message_id", table_name="message")
op.drop_table("message")
op.drop_index("ix_user_username", table_name="user")
op.drop_table("user")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py
================================================
"""add deleted field to post
Revision ID: 8d269bc4fdbd
Revises: abb47e9d145a
Create Date: 2022-12-31 04:38:41.799206
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "8d269bc4fdbd"
down_revision = "abb47e9d145a"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message", sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "deleted")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py
================================================
"""added MessageTreeState table
Revision ID: d4161e384f83
Revises: 8d269bc4fdbd
Create Date: 2023-01-05 11:44:02.630633
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "d4161e384f83"
down_revision = "8d269bc4fdbd"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_tree_state",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("goal_tree_size", sa.Integer(), nullable=False),
sa.Column("current_num_non_filtered_messages", sa.Integer(), nullable=False),
sa.Column("max_depth", sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_message_tree_state_message_tree_id"), "message_tree_state", ["message_tree_id"], unique=False
)
op.create_index("ix_message_tree_state_tree_id", "message_tree_state", ["message_tree_id"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_message_tree_state_tree_id", table_name="message_tree_state")
op.drop_index(op.f("ix_message_tree_state_message_tree_id"), table_name="message_tree_state")
op.drop_table("message_tree_state")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py
================================================
"""removed date_created and deleted flag from message_tree_state
Revision ID: 3b0adfadbef9
Revises: d4161e384f83
Create Date: 2023-01-05 13:46:11.338655
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "3b0adfadbef9"
down_revision = "d4161e384f83"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message_tree_state", "deleted")
op.drop_column("message_tree_state", "created_date")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"message_tree_state",
sa.Column(
"created_date",
postgresql.TIMESTAMP(),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=False,
),
)
op.add_column(
"message_tree_state",
sa.Column("deleted", sa.BOOLEAN(), server_default=sa.text("false"), autoincrement=False, nullable=False),
)
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py
================================================
"""Added user to TextLabels
Revision ID: 20cd871f4ec7
Revises: d4161e384f83
Create Date: 2023-01-05 17:45:15.696468
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "20cd871f4ec7"
down_revision = "3b0adfadbef9"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("text_labels", sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False))
op.create_foreign_key(None, "text_labels", "user", ["user_id"], ["id"])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_constraint(None, "text_labels", type_="foreignkey")
op.drop_column("text_labels", "user_id")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py
================================================
"""added frontend_type to api_client
Revision ID: ba61fe17fb6e
Revises: 20cd871f4ec7
Create Date: 2023-01-07 12:50:32.195930
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "ba61fe17fb6e"
down_revision = "20cd871f4ec7"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("api_client", sa.Column("frontend_type", sa.String(256), nullable=True))
def downgrade() -> None:
op.drop_column("api_client", "frontend_type")
================================================
FILE: backend/alembic/versions/2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py
================================================
"""added miniLM_embedding column to message
Revision ID: 023548d474f7
Revises: ba61fe17fb6e
Create Date: 2023-01-08 11:06:25.613290
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "023548d474f7"
down_revision = "ba61fe17fb6e"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message", sa.Column("miniLM_embedding", sa.ARRAY(sa.Float()), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "miniLM_embedding")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py
================================================
"""embedding for message now in its own table
Revision ID: 35bdc1a08bb8
Revises: 023548d474f7
Create Date: 2023-01-08 16:03:48.454207
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "35bdc1a08bb8"
down_revision = "023548d474f7"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_embedding",
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("embedding", sa.ARRAY(sa.Float()), nullable=True),
sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False),
sa.ForeignKeyConstraint(
["message_id"],
["message.id"],
),
sa.PrimaryKeyConstraint("message_id", "model"),
)
op.drop_column("message", "miniLM_embedding")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"message",
sa.Column(
"miniLM_embedding",
postgresql.ARRAY(postgresql.DOUBLE_PRECISION(precision=53)),
autoincrement=False,
nullable=True,
),
)
op.drop_table("message_embedding")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_08_2128-aac6b2f66006_created_date.py
================================================
"""Created date
Revision ID: aac6b2f66006
Revises: 35bdc1a08bb8
Create Date: 2023-01-08 21:28:27.342729
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "aac6b2f66006"
down_revision = "35bdc1a08bb8"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"message_embedding",
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message_embedding", "created_date")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py
================================================
"""MessageToxicity
Revision ID: bcc2fe18d214
Revises: 20cd871f4ec7
Create Date: 2023-01-08 22:00:43.297719
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "bcc2fe18d214"
down_revision = "846cc08ac79f"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_toxicity",
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("toxicity", sa.Float(), nullable=True),
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False),
sa.ForeignKeyConstraint(
["message_id"],
["message.id"],
),
sa.PrimaryKeyConstraint("message_id", "model"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("message_toxicity")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py
================================================
"""restructure message_tree_state table
Revision ID: 92a367bb9f40
Revises: ba61fe17fb6e
Create Date: 2023-01-08 22:08:46.458195
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "92a367bb9f40"
down_revision = "aac6b2f66006"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("message_tree_state")
op.create_table(
"message_tree_state",
sa.Column("message_tree_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("goal_tree_size", sa.Integer(), nullable=False),
sa.Column("max_depth", sa.Integer(), nullable=False),
sa.Column("max_children_count", sa.Integer(), nullable=False),
sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("active", sa.Boolean(), nullable=False),
sa.Column("accepted_messages", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(
["message_tree_id"],
["message.id"],
),
sa.PrimaryKeyConstraint("message_tree_id"),
)
op.create_index(op.f("ix_message_tree_state_active"), "message_tree_state", ["active"], unique=False)
op.create_index(op.f("ix_message_tree_state_state"), "message_tree_state", ["state"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_message_tree_state_state"), table_name="message_tree_state")
op.drop_index(op.f("ix_message_tree_state_active"), table_name="message_tree_state")
op.drop_table("message_tree_state")
op.create_table(
"message_tree_state",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("message_tree_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("state", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("goal_tree_size", sa.Integer(), nullable=False),
sa.Column("current_num_non_filtered_messages", sa.Integer(), nullable=False),
sa.Column("max_depth", sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
op.f("ix_message_tree_state_message_tree_id"), "message_tree_state", ["message_tree_id"], unique=False
)
op.create_index("ix_message_tree_state_tree_id", "message_tree_state", ["message_tree_id"], unique=True)
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py
================================================
"""add review_count & ranking_count to message
Revision ID: 05975b274a81
Revises: 92a367bb9f40
Create Date: 2023-01-09 00:47:25.496036
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "05975b274a81"
down_revision = "92a367bb9f40"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message", sa.Column("review_count", sa.Integer(), server_default=sa.text("0"), nullable=False))
op.add_column("message", sa.Column("review_result", sa.Boolean(), server_default=sa.text("false"), nullable=False))
op.add_column("message", sa.Column("ranking_count", sa.Integer(), server_default=sa.text("0"), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "ranking_count")
op.drop_column("message", "review_result")
op.drop_column("message", "review_count")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py
================================================
"""Add enabled, deleted, notes fields to User
Revision ID: 846cc08ac79f
Revises: aac6b2f66006
Create Date: 2023-01-10 17:33:07.104596
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "846cc08ac79f"
down_revision = "befa42582ea4"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user", sa.Column("enabled", sa.Boolean(), server_default=sa.text("true"), nullable=False))
op.add_column("user", sa.Column("deleted", sa.Boolean(), server_default=sa.text("false"), nullable=False))
op.add_column(
"user",
sa.Column("notes", sqlmodel.sql.sqltypes.AutoString(length=1024), server_default="", nullable=False),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("user", "notes")
op.drop_column("user", "deleted")
op.drop_column("user", "enabled")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py
================================================
"""remove accepted_messages from message_tree_state
Revision ID: befa42582ea4
Revises: 05975b274a81
Create Date: 2023-01-12 01:19:59.654864
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "befa42582ea4"
down_revision = "05975b274a81"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message_tree_state", "accepted_messages")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"message_tree_state", sa.Column("accepted_messages", sa.INTEGER(), autoincrement=False, nullable=False)
)
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_14_1509-619255ae9076_add_rank_to_message_table.py
================================================
"""add rank to message table
Revision ID: 619255ae9076
Revises: bcc2fe18d214
Create Date: 2023-01-14 15:09:03.462482
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "619255ae9076"
down_revision = "bcc2fe18d214"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message", sa.Column("rank", sa.Integer(), nullable=True))
op.add_column("message_toxicity", sa.Column("score", sa.Float(), nullable=True))
op.add_column("message_toxicity", sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False))
op.drop_column("message_toxicity", "toxicity")
op.add_column("user_stats", sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False))
op.add_column("user_stats", sa.Column("prompts", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("replies_assistant", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("replies_prompter", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("labels_simple", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("labels_full", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("rankings_total", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("rankings_good", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("accepted_prompts", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("accepted_replies_assistant", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("accepted_replies_prompter", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_assistant_ranked_1", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_assistant_ranked_2", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_assistant_ranked_3", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_prompter_ranked_1", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_prompter_ranked_2", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("reply_prompter_ranked_3", sa.Integer(), nullable=False))
op.add_column("user_stats", sa.Column("streak_last_day_date", sa.DateTime(), nullable=True))
op.add_column("user_stats", sa.Column("streak_days", sa.Integer(), nullable=True))
op.drop_column("user_stats", "messages")
op.drop_column("user_stats", "upvotes")
op.drop_column("user_stats", "task_reward")
op.drop_column("user_stats", "compare_wins")
op.drop_column("user_stats", "compare_losses")
op.drop_column("user_stats", "downvotes")
op.drop_column("user_stats", "reactions")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user_stats", sa.Column("reactions", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("downvotes", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("compare_losses", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("compare_wins", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("task_reward", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("upvotes", sa.INTEGER(), autoincrement=False, nullable=False))
op.add_column("user_stats", sa.Column("messages", sa.INTEGER(), autoincrement=False, nullable=False))
op.drop_column("user_stats", "streak_days")
op.drop_column("user_stats", "streak_last_day_date")
op.drop_column("user_stats", "reply_prompter_ranked_3")
op.drop_column("user_stats", "reply_prompter_ranked_2")
op.drop_column("user_stats", "reply_prompter_ranked_1")
op.drop_column("user_stats", "reply_assistant_ranked_3")
op.drop_column("user_stats", "reply_assistant_ranked_2")
op.drop_column("user_stats", "reply_assistant_ranked_1")
op.drop_column("user_stats", "accepted_replies_prompter")
op.drop_column("user_stats", "accepted_replies_assistant")
op.drop_column("user_stats", "accepted_prompts")
op.drop_column("user_stats", "rankings_good")
op.drop_column("user_stats", "rankings_total")
op.drop_column("user_stats", "labels_full")
op.drop_column("user_stats", "labels_simple")
op.drop_column("user_stats", "replies_prompter")
op.drop_column("user_stats", "replies_assistant")
op.drop_column("user_stats", "prompts")
op.drop_column("user_stats", "time_frame")
op.add_column(
"message_toxicity",
sa.Column("toxicity", postgresql.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True),
)
op.drop_column("message_toxicity", "label")
op.drop_column("message_toxicity", "score")
op.drop_column("message", "rank")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py
================================================
"""change user_stats ranking counts
Revision ID: 7c98102efbca
Revises: 619255ae9076
Create Date: 2023-01-15 00:02:45.622986
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects.postgresql import UUID
# revision identifiers, used by Alembic.
revision = "7c98102efbca"
down_revision = "619255ae9076"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("user_stats")
op.create_table(
"user_stats",
sa.Column("user_id", UUID(as_uuid=True), nullable=False),
sa.Column("modified_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("base_date", sa.DateTime(), nullable=True),
sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("leader_score", sa.Integer(), nullable=False),
sa.Column("prompts", sa.Integer(), nullable=False),
sa.Column("replies_assistant", sa.Integer(), nullable=False),
sa.Column("replies_prompter", sa.Integer(), nullable=False),
sa.Column("labels_simple", sa.Integer(), nullable=False),
sa.Column("labels_full", sa.Integer(), nullable=False),
sa.Column("rankings_total", sa.Integer(), nullable=False),
sa.Column("rankings_good", sa.Integer(), nullable=False),
sa.Column("accepted_prompts", sa.Integer(), nullable=False),
sa.Column("accepted_replies_assistant", sa.Integer(), nullable=False),
sa.Column("accepted_replies_prompter", sa.Integer(), nullable=False),
sa.Column("reply_ranked_1", sa.Integer(), nullable=False),
sa.Column("reply_ranked_2", sa.Integer(), nullable=False),
sa.Column("reply_ranked_3", sa.Integer(), nullable=False),
sa.Column("streak_last_day_date", sa.DateTime(), nullable=True),
sa.Column("streak_days", sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("user_id", "time_frame"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"user_stats",
sa.Column("reply_prompter_ranked_3", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.add_column(
"user_stats",
sa.Column("reply_assistant_ranked_1", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.add_column(
"user_stats",
sa.Column("reply_assistant_ranked_2", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.add_column(
"user_stats",
sa.Column("reply_prompter_ranked_2", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.add_column(
"user_stats",
sa.Column("reply_prompter_ranked_1", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.add_column(
"user_stats",
sa.Column("reply_assistant_ranked_3", sa.INTEGER(), server_default="0", autoincrement=False, nullable=False),
)
op.drop_column("user_stats", "reply_ranked_3")
op.drop_column("user_stats", "reply_ranked_2")
op.drop_column("user_stats", "reply_ranked_1")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_15_1139-423557e869e4_add_indices_for_created_date.py
================================================
"""add indices for created_date
Revision ID: 423557e869e4
Revises: 7c98102efbca
Create Date: 2023-01-15 11:39:10.407859
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "423557e869e4"
down_revision = "7c98102efbca"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f("ix_message_created_date"), "message", ["created_date"], unique=False)
op.create_index(op.f("ix_message_reaction_created_date"), "message_reaction", ["created_date"], unique=False)
op.create_index(op.f("ix_text_labels_created_date"), "text_labels", ["created_date"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_text_labels_created_date"), table_name="text_labels")
op.drop_index(op.f("ix_message_reaction_created_date"), table_name="message_reaction")
op.drop_index(op.f("ix_message_created_date"), table_name="message")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py
================================================
"""add rank and indices to user_stats
Revision ID: 0964ac95170d
Revises: 423557e869e4
Create Date: 2023-01-15 16:54:09.510018
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "0964ac95170d"
down_revision = "423557e869e4"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user_stats", sa.Column("rank", sa.Integer(), nullable=True))
op.create_index(
"ix_user_stats__timeframe__rank__user_id", "user_stats", ["time_frame", "rank", "user_id"], unique=True
)
op.create_index("ix_user_stats__timeframe__user_id", "user_stats", ["time_frame", "user_id"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_user_stats__timeframe__user_id", table_name="user_stats")
op.drop_index("ix_user_stats__timeframe__rank__user_id", table_name="user_stats")
op.drop_column("user_stats", "rank")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py
================================================
"""switch to timestamp with tz
Revision ID: 7f0a28a156f4
Revises: 0964ac95170d
Create Date: 2023-01-19 21:53:01.107137
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "7f0a28a156f4"
down_revision = "0964ac95170d"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(table_name="user_stats", column_name="modified_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="user_stats", column_name="base_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="journal_integration", column_name="last_run", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="message_embedding", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="message_reaction", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="message_toxicity", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="message", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="task", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="task", column_name="expiry_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="text_labels", column_name="created_date", type_=sa.DateTime(timezone=True))
op.alter_column(table_name="user", column_name="created_date", type_=sa.DateTime(timezone=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(table_name="user_stats", column_name="modified_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="user_stats", column_name="base_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="journal_integration", column_name="last_run", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="message_embedding", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="message_reaction", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="message_toxicity", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="message", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="task", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="task", column_name="expiry_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="text_labels", column_name="created_date", type_=sa.DateTime(timezone=False))
op.alter_column(table_name="user", column_name="created_date", type_=sa.DateTime(timezone=False))
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py
================================================
"""add ix_user_display_name_id
Revision ID: 4f26fec4d204
Revises: 0964ac95170d
Create Date: 2023-01-19 22:00:00
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "4f26fec4d204"
down_revision = "7f0a28a156f4"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index("ix_user_display_name_id", "user", ["display_name", "id"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_user_display_name_id", table_name="user")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py
================================================
"""use 'en' instead 'en-US' as default lang
Revision ID: 160ac010efcc
Revises: 4f26fec4d204
Create Date: 2023-01-20 16:50:00
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "160ac010efcc"
down_revision = "4f26fec4d204"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "lang")
op.add_column("message", sa.Column("lang", sa.String(length=32), server_default="en", nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "lang")
op.add_column("message", sa.Column("lang", sa.VARCHAR(length=200), autoincrement=False, nullable=False))
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py
================================================
"""add message_id to message_reaction
Revision ID: 8ba17b5f467a
Revises: 160ac010efcc
Create Date: 2023-01-24 11:34:42.167575
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "8ba17b5f467a"
down_revision = "160ac010efcc"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message_reaction", sa.Column("message_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
op.create_index(op.f("ix_message_reaction_message_id"), "message_reaction", ["message_id"], unique=False)
op.add_column("text_labels", sa.Column("task_id", sqlmodel.sql.sqltypes.GUID(), nullable=True))
op.create_index(op.f("ix_text_labels_task_id"), "text_labels", ["task_id"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_text_labels_task_id"), table_name="text_labels")
op.drop_column("text_labels", "task_id")
op.drop_index(op.f("ix_message_reaction_message_id"), table_name="message_reaction")
op.drop_column("message_reaction", "message_id")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_24_2256-40ed93df0ed5_add_message_emoji.py
================================================
"""add message_emoji
Revision ID: 40ed93df0ed5
Revises: 8ba17b5f467a
Create Date: 2023-01-24 22:56:28.229408
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "40ed93df0ed5"
down_revision = "8ba17b5f467a"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_emoji",
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column(
"created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column("emoji", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["message.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("message_id", "user_id", "emoji"),
)
op.create_index("ix_message_emoji__user_id__message_id", "message_emoji", ["user_id", "message_id"], unique=False)
op.add_column("message", sa.Column("emojis", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "emojis")
op.drop_index("ix_message_emoji__user_id__message_id", table_name="message_emoji")
op.drop_table("message_emoji")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py
================================================
"""add task created date index
Revision ID: c84fcd6900dc
Revises: 40ed93df0ed5
Create Date: 2023-01-26 18:35:43.061589
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "c84fcd6900dc"
down_revision = "40ed93df0ed5"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f("ix_task_created_date"), "task", ["created_date"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_task_created_date"), table_name="task")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py
================================================
"""add user.show_on_leaderboard
Revision ID: f856bf19d32b
Revises: c84fcd6900dc
Create Date: 2023-01-27 20:13:56.533374
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "f856bf19d32b"
down_revision = "c84fcd6900dc"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"user", sa.Column("show_on_leaderboard", sa.Boolean(), server_default=sa.text("true"), nullable=False)
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("user", "show_on_leaderboard")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py
================================================
"""add origin column to message_tree_state
Revision ID: 49d8445b4c90
Revises: f856bf19d32b
Create Date: 2023-01-28 11:57:45.580027
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "49d8445b4c90"
down_revision = "f856bf19d32b"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message", sa.Column("synthetic", sa.Boolean(), server_default=sa.text("false"), nullable=False))
op.add_column("message", sa.Column("model_name", sa.String(length=1024), nullable=True))
op.add_column("message_tree_state", sa.Column("origin", sa.String(length=1024), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message_tree_state", "origin")
op.drop_column("message", "model_name")
op.drop_column("message", "synthetic")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py
================================================
"""move user_streak from user_stats to user table
Revision ID: 7b8f0011e0b0
Revises: 8a5feed819ee
Create Date: 2023-01-29 12:07:29.379326
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "7b8f0011e0b0"
down_revision = "49d8445b4c90"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"user",
sa.Column(
"streak_last_day_date",
sa.DateTime(timezone=True),
server_default=sa.text("CURRENT_TIMESTAMP"),
autoincrement=False,
nullable=True,
),
)
op.add_column("user", sa.Column("streak_days", sa.INTEGER(), autoincrement=False, nullable=True))
op.add_column(
"user", sa.Column("last_activity_date", sa.DateTime(timezone=True), autoincrement=False, nullable=True)
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("user", "streak_days")
op.drop_column("user", "streak_last_day_date")
op.drop_column("user", "last_activity_date")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py
================================================
"""add tos_acceptance_date to user
Revision ID: 55361f323d12
Revises: 7b8f0011e0b0
Create Date: 2023-02-01 00:22:08.280251
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "55361f323d12"
down_revision = "f60958968ff8"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("user", sa.Column("tos_acceptance_date", sa.DateTime(timezone=True), nullable=True))
op.drop_column("user_stats", "streak_days")
op.drop_column("user_stats", "streak_last_day_date")
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"user_stats", sa.Column("streak_last_day_date", postgresql.TIMESTAMP(), autoincrement=False, nullable=True)
)
op.add_column("user_stats", sa.Column("streak_days", sa.INTEGER(), autoincrement=False, nullable=True))
op.drop_column("user", "tos_acceptance_date")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py
================================================
"""add won_prompt_lottery_date to mts
Revision ID: f60958968ff8
Revises: 7b8f0011e0b0
Create Date: 2023-02-01 10:10:38.301707
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "f60958968ff8"
down_revision = "7b8f0011e0b0"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message_tree_state", sa.Column("won_prompt_lottery_date", sa.DateTime(timezone=True), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message_tree_state", "won_prompt_lottery_date")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py
================================================
"""add skip bool & skip_reason to task
Revision ID: 9e7ec4a9e3f2
Revises: 7b8f0011e0b0
Create Date: 2023-02-01 21:46:49.971052
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
# revision identifiers, used by Alembic.
revision = "9e7ec4a9e3f2"
down_revision = "55361f323d12"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("task", sa.Column("skipped", sa.Boolean(), server_default=sa.text("false"), nullable=False))
op.add_column("task", sa.Column("skip_reason", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("task", "skip_reason")
op.drop_column("task", "skipped")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py
================================================
"""add troll_stats
Revision ID: 4d7e0b0ebe84
Revises: 9e7ec4a9e3f2
Create Date: 2023-02-02 15:44:12.647260
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "4d7e0b0ebe84"
down_revision = "9e7ec4a9e3f2"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"troll_stats",
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("base_date", sa.DateTime(timezone=True), nullable=True),
sa.Column(
"modified_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
sa.Column("troll_score", sa.Integer(), nullable=False),
sa.Column("rank", sa.Integer(), nullable=True),
sa.Column("red_flags", sa.Integer(), nullable=False),
sa.Column("upvotes", sa.Integer(), nullable=False),
sa.Column("downvotes", sa.Integer(), nullable=False),
sa.Column("spam_prompts", sa.Integer(), nullable=False),
sa.Column("quality", sa.Float(), nullable=True),
sa.Column("humor", sa.Float(), nullable=True),
sa.Column("toxicity", sa.Float(), nullable=True),
sa.Column("violence", sa.Float(), nullable=True),
sa.Column("helpfulness", sa.Float(), nullable=True),
sa.Column("spam", sa.Integer(), nullable=False),
sa.Column("lang_mismach", sa.Integer(), nullable=False),
sa.Column("not_appropriate", sa.Integer(), nullable=False),
sa.Column("pii", sa.Integer(), nullable=False),
sa.Column("hate_speech", sa.Integer(), nullable=False),
sa.Column("sexual_content", sa.Integer(), nullable=False),
sa.Column("political_content", sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("user_id", "time_frame"),
)
op.create_index("ix_troll_stats__timeframe__user_id", "troll_stats", ["time_frame", "user_id"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_troll_stats__timeframe__user_id", table_name="troll_stats")
op.drop_table("troll_stats")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_02_1817-8c8241d1f973_add_account_table.py
================================================
"""Add Account table
Revision ID: 8c8241d1f973
Revises: 4d7e0b0ebe84
Create Date: 2023-01-30 15:10:58.776315
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "8c8241d1f973"
down_revision = "4d7e0b0ebe84"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"account",
sa.Column("id", postgresql.UUID(as_uuid=True), server_default=sa.text("gen_random_uuid()"), nullable=False),
sa.Column("user_id", sqlmodel.sql.sqltypes.GUID(), nullable=False),
sa.Column("provider", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column("provider_account_id", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.ForeignKeyConstraint(["user_id"], ["user.id"]),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("provider", "account", ["provider_account_id"], unique=True)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("provider", table_name="account")
op.drop_table("account")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py
================================================
"""Added new table for flagged messages
Revision ID: caee1e8ee0bc
Revises: 8c8241d1f973
Create Date: 2023-02-07 19:22:12.696257
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "caee1e8ee0bc"
down_revision = "8c8241d1f973"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"flagged_message",
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column(
"created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column("processed", sa.Boolean(), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["message.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("message_id"),
)
op.create_index(op.f("ix_flagged_message_created_date"), "flagged_message", ["created_date"], unique=False)
op.create_index(op.f("ix_flagged_message_processed"), "flagged_message", ["processed"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_flagged_message_processed"), table_name="flagged_message")
op.drop_index(op.f("ix_flagged_message_created_date"), table_name="flagged_message")
op.drop_table("flagged_message")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_11_1030-ba40d055714a_add_cached_stats.py
================================================
"""add cached_stats
Revision ID: ba40d055714a
Revises: caee1e8ee0bc
Create Date: 2023-02-11 10:30:21.996198
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "ba40d055714a"
down_revision = "caee1e8ee0bc"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"cached_stats",
sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.Column(
"modified_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column("stats", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.PrimaryKeyConstraint("name"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table("cached_stats")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py
================================================
"""add text_labels message_id index
Revision ID: 165b55de5a94
Revises: ba40d055714a
Create Date: 2023-02-14 17:56:48.263684
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "165b55de5a94"
down_revision = "ba40d055714a"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_index(op.f("ix_text_labels_message_id"), "text_labels", ["message_id"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f("ix_text_labels_message_id"), table_name="text_labels")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py
================================================
"""message review_result nullable
Revision ID: 8cd0c34d0c3c
Revises: 165b55de5a94
Create Date: 2023-02-15 17:54:58.029278
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "8cd0c34d0c3c"
down_revision = "165b55de5a94"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"message",
"review_result",
existing_type=sa.BOOLEAN(),
nullable=True,
server_default=None,
existing_server_default=sa.text("false"),
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column(
"message", "review_result", existing_type=sa.BOOLEAN(), nullable=False, server_default=sa.text("false")
)
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py
================================================
"""add lang to message_tree_state
Revision ID: 9db92d504f64
Revises: 8cd0c34d0c3c
Create Date: 2023-02-26 00:52:40.624843
"""
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = "9db92d504f64"
down_revision = "8cd0c34d0c3c"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.add_column("message_tree_state", sa.Column("lang", sa.String(length=32), nullable=True))
op.execute(
"WITH msg AS (SELECT id, lang FROM message WHERE parent_id is NULL) UPDATE message_tree_state mts SET lang = msg.lang FROM msg WHERE mts.message_tree_id = msg.id"
)
op.alter_column("message_tree_state", "lang", nullable=False)
op.drop_index("ix_message_tree_state_state", table_name="message_tree_state")
op.create_index("ix_message_tree_state__lang__state", "message_tree_state", ["state", "lang"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_message_tree_state__lang__state", table_name="message_tree_state")
op.create_index("ix_message_tree_state_state", "message_tree_state", ["state"], unique=False)
op.drop_column("message_tree_state", "lang")
# ### end Alembic commands ###
================================================
FILE: backend/alembic/versions/2023_05_07_2129-1b6e3ae16e9d_add_text_search.py
================================================
"""add text search
Revision ID: 1b6e3ae16e9d
Revises: 9db92d504f64
Create Date: 2023-05-07 21:29:35.545612
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "1b6e3ae16e9d"
down_revision = "9db92d504f64"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("message", sa.Column("search_vector", postgresql.TSVECTOR(), nullable=True))
op.create_index("idx_search_vector", "message", ["search_vector"], postgresql_using="gin")
def downgrade() -> None:
op.drop_index("idx_search_vector", "message")
op.drop_column("message", "search_vector")
================================================
FILE: backend/alembic/versions/2023_06_06_1505-c181661eba3a_add_message_revisions.py
================================================
"""add_message_revisions
Revision ID: c181661eba3a
Revises: 1b6e3ae16e9d
Create Date: 2023-06-06 15:05:58.079120
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "c181661eba3a"
down_revision = "1b6e3ae16e9d"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_revision",
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=True),
sa.Column(
"created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=True
),
sa.ForeignKeyConstraint(
["message_id"],
["message.id"],
),
sa.ForeignKeyConstraint(
["user_id"],
["user.id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(op.f("ix_message_revision_message_id"), "message_revision", ["message_id"], unique=False)
op.add_column("message", sa.Column("edited", sa.Boolean(), server_default=sa.text("false"), nullable=False))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "edited")
op.drop_index(op.f("ix_message_revision_message_id"), table_name="message_revision")
op.drop_table("message_revision")
# ### end Alembic commands ###
================================================
FILE: backend/alembic.ini
================================================
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python-dateutil library that can be
# installed by adding `alembic[tz]` to the pip requirements
# string value is passed to dateutil.tz.gettz()
# l
gitextract_4_dpaklk/
├── .devcontainer/
│ ├── README.md
│ ├── backend-dev/
│ │ ├── devcontainer.json
│ │ └── post_create_command.sh
│ ├── devcontainer.json
│ ├── frontend-dev/
│ │ ├── devcontainer.json
│ │ └── post_create_command.sh
│ └── post_create_command.sh
├── .dockerignore
├── .gitattributes
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── build-frontend.yaml
│ ├── build-postgres.yaml
│ ├── deploy-docs-site.yaml
│ ├── deploy-to-node.yaml
│ ├── docker-build.yaml
│ ├── pre-commit.yaml
│ ├── production-deploy.yaml
│ ├── production2-deploy.yaml
│ ├── release.yaml
│ ├── test-api-contract.yaml
│ └── test-e2e.yaml
├── .gitignore
├── .pre-commit-config.yaml
├── .python-version
├── .vscode/
│ ├── extensions.json
│ ├── launch.json
│ └── settings.json
├── CODEOWNERS
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── ansible/
│ ├── .gitignore
│ ├── README.md
│ ├── deploy-to-node.yaml
│ ├── inference/
│ │ ├── deploy-server.yaml
│ │ ├── deploy-worker.yaml
│ │ ├── redis.conf
│ │ └── test.inventory.ini
│ ├── pgbackrest.conf
│ ├── redis.conf
│ └── test.inventory.ini
├── backend/
│ ├── .gitignore
│ ├── README.md
│ ├── alembic/
│ │ ├── README
│ │ ├── env.py
│ │ ├── script.py.mako
│ │ └── versions/
│ │ ├── 2022_12_15_0000-23e5fea252dd_first_revision.py
│ │ ├── 2022_12_16_0000-cd7de470586e_v1_db_structure.py
│ │ ├── 2022_12_17_2230-6368515778c5_add_auth_method_to_person.py
│ │ ├── 2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py
│ │ ├── 2022_12_25_1705-067c4002f2d9_add_text_labels.py
│ │ ├── 2022_12_27_1444-3358eb6834e6_add_journal_table.py
│ │ ├── 2022_12_28_1142-d24b37426857_post_ref_for_work_package.py
│ │ ├── 2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py
│ │ ├── 2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py
│ │ ├── 2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py
│ │ ├── 2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py
│ │ ├── 2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py
│ │ ├── 2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py
│ │ ├── 2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py
│ │ ├── 2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py
│ │ ├── 2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py
│ │ ├── 2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py
│ │ ├── 2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py
│ │ ├── 2023_01_08_2128-aac6b2f66006_created_date.py
│ │ ├── 2023_01_08_2200-bcc2fe18d214_messagetoxicity.py
│ │ ├── 2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py
│ │ ├── 2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py
│ │ ├── 2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py
│ │ ├── 2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py
│ │ ├── 2023_01_14_1509-619255ae9076_add_rank_to_message_table.py
│ │ ├── 2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py
│ │ ├── 2023_01_15_1139-423557e869e4_add_indices_for_created_date.py
│ │ ├── 2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py
│ │ ├── 2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py
│ │ ├── 2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py
│ │ ├── 2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py
│ │ ├── 2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py
│ │ ├── 2023_01_24_2256-40ed93df0ed5_add_message_emoji.py
│ │ ├── 2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py
│ │ ├── 2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py
│ │ ├── 2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py
│ │ ├── 2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py
│ │ ├── 2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py
│ │ ├── 2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py
│ │ ├── 2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py
│ │ ├── 2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py
│ │ ├── 2023_02_02_1817-8c8241d1f973_add_account_table.py
│ │ ├── 2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py
│ │ ├── 2023_02_11_1030-ba40d055714a_add_cached_stats.py
│ │ ├── 2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py
│ │ ├── 2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py
│ │ ├── 2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py
│ │ ├── 2023_05_07_2129-1b6e3ae16e9d_add_text_search.py
│ │ └── 2023_06_06_1505-c181661eba3a_add_message_revisions.py
│ ├── alembic.ini
│ ├── export.py
│ ├── import.py
│ ├── main.py
│ ├── oasst_backend/
│ │ ├── __init__.py
│ │ ├── api/
│ │ │ ├── __init__.py
│ │ │ ├── deps.py
│ │ │ └── v1/
│ │ │ ├── __init__.py
│ │ │ ├── admin.py
│ │ │ ├── api.py
│ │ │ ├── auth.py
│ │ │ ├── frontend_messages.py
│ │ │ ├── frontend_users.py
│ │ │ ├── hugging_face.py
│ │ │ ├── leaderboards.py
│ │ │ ├── login.py
│ │ │ ├── messages.py
│ │ │ ├── stats.py
│ │ │ ├── tasks.py
│ │ │ ├── text_labels.py
│ │ │ ├── trollboards.py
│ │ │ ├── users.py
│ │ │ └── utils.py
│ │ ├── auth.py
│ │ ├── cached_stats_repository.py
│ │ ├── celery_worker.py
│ │ ├── config.py
│ │ ├── database.py
│ │ ├── journal_writer.py
│ │ ├── models/
│ │ │ ├── __init__.py
│ │ │ ├── api_client.py
│ │ │ ├── cached_stats.py
│ │ │ ├── db_payload.py
│ │ │ ├── flagged_message.py
│ │ │ ├── journal.py
│ │ │ ├── message.py
│ │ │ ├── message_embedding.py
│ │ │ ├── message_emoji.py
│ │ │ ├── message_reaction.py
│ │ │ ├── message_revision.py
│ │ │ ├── message_toxicity.py
│ │ │ ├── message_tree_state.py
│ │ │ ├── payload_column_type.py
│ │ │ ├── task.py
│ │ │ ├── text_labels.py
│ │ │ ├── troll_stats.py
│ │ │ ├── user.py
│ │ │ └── user_stats.py
│ │ ├── prompt_repository.py
│ │ ├── scheduled_tasks.py
│ │ ├── schemas/
│ │ │ ├── __init__.py
│ │ │ ├── hugging_face.py
│ │ │ ├── message_tree.py
│ │ │ └── text_labels.py
│ │ ├── task_repository.py
│ │ ├── tree_manager.py
│ │ ├── user_repository.py
│ │ ├── user_stats_repository.py
│ │ └── utils/
│ │ ├── __init__.py
│ │ ├── database_utils.py
│ │ ├── discord.py
│ │ ├── exported_tree_loading.py
│ │ ├── hugging_face.py
│ │ ├── language_classification.py
│ │ ├── message_tree_topic_modeling.py
│ │ ├── ranking.py
│ │ ├── similarity_functions.py
│ │ ├── topic_model_requirments.txt
│ │ └── tree_export.py
│ ├── requirements.txt
│ ├── requirements_worker.txt
│ ├── rerank.py
│ ├── sql_snippets.md
│ ├── test_data/
│ │ ├── generic/
│ │ │ └── test_generic_data.json
│ │ └── realistic/
│ │ └── realistic_seed_data.json
│ ├── tests/
│ │ ├── __init__.py
│ │ ├── test_settings.py
│ │ └── test_tree_manager_config.py
│ └── update_message_attributes.py
├── copilot/
│ ├── .workspace
│ ├── README.md
│ ├── api/
│ │ └── manifest.yml
│ ├── environments/
│ │ └── staging/
│ │ └── manifest.yml
│ └── web/
│ └── manifest.yml
├── data/
│ ├── __init__.py
│ ├── datasets/
│ │ ├── README.md
│ │ ├── TSSB-3M/
│ │ │ ├── README.md
│ │ │ ├── generate_dataset.py
│ │ │ ├── invalid_commit_messages.tsv
│ │ │ └── load_script.py
│ │ ├── __init__.py
│ │ ├── bart_searchgpt_wiki_nlp_augment/
│ │ │ ├── 1_clean_wikitext.py
│ │ │ ├── 2_wikitext_doc2query.ipynb
│ │ │ ├── 3_10k_bart_trial.ipynb
│ │ │ ├── 3_10k_bart_trial.py
│ │ │ ├── 4_convert_to_oa_format.py
│ │ │ ├── 5_test_downloading_my_dataset.py
│ │ │ ├── README.md
│ │ │ └── requirement.txt
│ │ ├── biostars_qa/
│ │ │ ├── README.md
│ │ │ ├── get_biostars_dataset.py
│ │ │ └── requirements.txt
│ │ ├── cmu_wiki_qa/
│ │ │ ├── README.md
│ │ │ ├── cmu_parser.ipynb
│ │ │ └── requirements.txt
│ │ ├── cocktail_recipes/
│ │ │ └── loading_script.py
│ │ ├── codet_humaneval_mbpp/
│ │ │ ├── HumanEval_and_MBPP_code_gen.ipynb
│ │ │ ├── HumanEval_and_MBPP_test_gen.ipynb
│ │ │ └── README.md
│ │ ├── fa-isna-news/
│ │ │ └── README.md
│ │ ├── fa-wikipedia/
│ │ │ └── README.md
│ │ ├── fd_dialogue/
│ │ │ └── README.md
│ │ ├── grade_school_math_instructions/
│ │ │ └── dataset_creation.ipynb
│ │ ├── gutenberg/
│ │ │ ├── README.md
│ │ │ ├── project_gutenberg_crawler.ipynb
│ │ │ └── requirements.txt
│ │ ├── iapp_wiki_qa_squad/
│ │ │ ├── README.md
│ │ │ ├── iapp_wiki_qa_squad_oa.ipynb
│ │ │ ├── loading_script.py
│ │ │ └── requirements.txt
│ │ ├── instructional_codesearchnet_python/
│ │ │ ├── GenerateOpenAssistantInstructionResponseFormat.ipynb
│ │ │ ├── README.md
│ │ │ └── Summarize_codesearchnet_for_python.ipynb
│ │ ├── logicreference_OA/
│ │ │ ├── README.md
│ │ │ ├── generate_dataset.py
│ │ │ └── requirements.txt
│ │ ├── mt_note_generation/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── mt_note_generation.py
│ │ │ └── prepare.py
│ │ ├── nsfw_selfharm_reddit/
│ │ │ ├── .gitignore
│ │ │ ├── README.md
│ │ │ ├── dataset-cookbook.ipynb
│ │ │ ├── prosocial.ipynb
│ │ │ └── utils/
│ │ │ ├── __init__.py
│ │ │ ├── is_question.py
│ │ │ └── reddit.py
│ │ ├── oa_dolly_15k/
│ │ │ ├── README.md
│ │ │ └── create_dataset.py
│ │ ├── oa_leet10k/
│ │ │ ├── README.md
│ │ │ └── oa_leet10k.ipynb
│ │ ├── oa_stackexchange/
│ │ │ ├── README.md
│ │ │ ├── combine.py
│ │ │ ├── download.py
│ │ │ ├── merge_parquets.py
│ │ │ ├── process.py
│ │ │ ├── stats.py
│ │ │ └── upload.py
│ │ ├── poetry_instruction/
│ │ │ ├── README.md
│ │ │ ├── prepare.py
│ │ │ └── requirements.txt
│ │ ├── prosocial_confessions/
│ │ │ ├── README.md
│ │ │ └── prosocial-confessions.ipynb
│ │ ├── reasoning_bg_oa/
│ │ │ ├── README.MD
│ │ │ └── data_process.py
│ │ ├── reasoning_gsm_qna_oa/
│ │ │ ├── README.MD
│ │ │ └── data_process.py
│ │ ├── recipes/
│ │ │ ├── README.md
│ │ │ └── tasty_recipes.ipynb
│ │ ├── safety_directory/
│ │ │ ├── child_help/
│ │ │ │ └── child_help.py
│ │ │ ├── emergency_infos/
│ │ │ │ ├── wikipedia_emergency_info.js
│ │ │ │ └── wikipedia_emergency_info.json
│ │ │ └── emergency_numbers/
│ │ │ └── emergency_numbers.py
│ │ ├── semantics_ws_qna_oa/
│ │ │ ├── README.MD
│ │ │ ├── data_process.py
│ │ │ └── random_stuff.py
│ │ ├── soda_synthetic_dialogue/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── prepare.py
│ │ │ └── soda_synthetic_dialogue.py
│ │ ├── tatoeba_mt_qna_oa/
│ │ │ ├── README.MD
│ │ │ ├── data_process.py
│ │ │ ├── language_names.py
│ │ │ ├── language_paraphrase.py
│ │ │ └── language_translate.py
│ │ ├── tell_a_joke/
│ │ │ └── tell_a_joke.ipynb
│ │ ├── tlcv2.0_oa/
│ │ │ ├── README.md
│ │ │ └── tlcv2_0_oa.ipynb
│ │ ├── tv_dialogue/
│ │ │ ├── README.md
│ │ │ ├── imsdb.ipynb
│ │ │ ├── public.ipynb
│ │ │ └── requirements.txt
│ │ ├── ubuntu_dialogue_qa/
│ │ │ ├── README.md
│ │ │ ├── requirements.txt
│ │ │ └── ubuntu_parser.ipynb
│ │ ├── youtube_subs_howto100M/
│ │ │ ├── __init__.py
│ │ │ ├── hub.py
│ │ │ ├── prepare.py
│ │ │ ├── requirements.txt
│ │ │ └── youtube_subs_howto100M.py
│ │ └── zhihu-kol/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── convert_parquet.py
│ │ ├── main.py
│ │ ├── requirements.txt
│ │ ├── scrape_by_topic.py
│ │ └── upload_hf.py
│ └── dev-requirements.txt
├── deploy/
│ ├── README.md
│ ├── dev-node/
│ │ └── nginx/
│ │ ├── docker-compose.yaml
│ │ ├── get_cert.sh
│ │ ├── nginx.conf
│ │ └── renew_certs.sh
│ ├── prod-node/
│ │ └── nginx/
│ │ ├── docker-compose.yaml
│ │ ├── get_cert.sh
│ │ ├── nginx.conf
│ │ └── renew_certs.sh
│ └── prod2-node/
│ └── nginx/
│ ├── docker-compose.yaml
│ ├── get_cert.sh
│ ├── nginx.conf
│ └── renew_certs.sh
├── discord-bots/
│ ├── oa-bot-js/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── docker-compose.yml
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── bot.ts
│ │ │ ├── commands/
│ │ │ │ ├── bot.ts
│ │ │ │ ├── chat.ts
│ │ │ │ └── task.ts
│ │ │ ├── events/
│ │ │ │ ├── commands.ts
│ │ │ │ ├── interactions.ts
│ │ │ │ ├── messages.ts
│ │ │ │ └── ready.ts
│ │ │ ├── handlers/
│ │ │ │ ├── commands.ts
│ │ │ │ ├── events.ts
│ │ │ │ └── interactions.ts
│ │ │ ├── index.ts
│ │ │ ├── interactions/
│ │ │ │ ├── model.ts
│ │ │ │ ├── modelselect.ts
│ │ │ │ ├── tasks.ts
│ │ │ │ └── vote.ts
│ │ │ └── modules/
│ │ │ ├── chat.ts
│ │ │ ├── db.ts
│ │ │ ├── inference/
│ │ │ │ ├── client.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── types.ts
│ │ │ ├── open-assistant/
│ │ │ │ ├── interactions/
│ │ │ │ │ ├── info.ts
│ │ │ │ │ ├── init.ts
│ │ │ │ │ ├── label.ts
│ │ │ │ │ ├── lang.ts
│ │ │ │ │ └── task.ts
│ │ │ │ ├── interactions.ts
│ │ │ │ ├── labels.ts
│ │ │ │ ├── langs.ts
│ │ │ │ ├── tasks.ts
│ │ │ │ └── user.ts
│ │ │ └── redis.ts
│ │ └── tsconfig.json
│ └── oa-bot-py/
│ ├── .gitignore
│ ├── README.md
│ ├── bot/
│ │ ├── __init__.py
│ │ ├── __main__.py
│ │ ├── bot.py
│ │ ├── db/
│ │ │ ├── schema.sql
│ │ │ └── schemas.py
│ │ ├── extensions/
│ │ │ ├── __init__.py
│ │ │ ├── guild_settings.py
│ │ │ ├── help.py
│ │ │ ├── hot_reload.py
│ │ │ ├── text_labels.py
│ │ │ ├── user_input_test.py
│ │ │ └── work.py
│ │ ├── messages.py
│ │ ├── settings.py
│ │ └── utils.py
│ ├── message_templates.py
│ ├── requirements.txt
│ └── templates/
│ ├── boot.msg
│ ├── help.msg
│ ├── task_assistant_reply.msg
│ ├── task_initial_prompt.msg
│ ├── task_prompter_reply.msg
│ ├── task_rank_conversation_replies.msg
│ ├── task_rank_initial_prompts.msg
│ ├── task_rate_summary.msg
│ ├── task_summarize_story.msg
│ ├── teaser_assistant_reply.msg
│ ├── teaser_initial_prompt.msg
│ ├── teaser_prompter_reply.msg
│ ├── teaser_rank_conversation_replies.msg
│ ├── teaser_rank_initial_prompts.msg
│ ├── teaser_rate_summary.msg
│ ├── teaser_summarize_story.msg
│ └── welcome.msg
├── docker/
│ ├── Dockerfile.backend
│ ├── Dockerfile.backend-worker
│ ├── Dockerfile.discord-bot
│ ├── Dockerfile.model-training
│ ├── Dockerfile.website
│ ├── grafana/
│ │ ├── README.md
│ │ ├── dashboards/
│ │ │ ├── dashboard.yaml
│ │ │ └── fastapi-backend.json
│ │ └── datasources/
│ │ └── datasource.yml
│ ├── inference/
│ │ ├── Dockerfile.safety
│ │ ├── Dockerfile.server
│ │ ├── Dockerfile.worker
│ │ ├── Dockerfile.worker-full
│ │ ├── Dockerfile.worker-hf
│ │ └── Dockerfile.worker-standalone
│ ├── netdata/
│ │ ├── README.md
│ │ └── go.d/
│ │ ├── postgres.conf
│ │ ├── prometheus.conf
│ │ └── redis.conf
│ ├── oasst-postgres/
│ │ ├── Dockerfile
│ │ └── backup_pg_to_s3.sh
│ └── prometheus/
│ ├── README.md
│ └── prometheus.yml
├── docker-compose.yaml
├── docs/
│ ├── .gitignore
│ ├── .yarnrc.yml
│ ├── README.md
│ ├── babel.config.js
│ ├── blog/
│ │ ├── 2023-02-05-we-need-your-help.mdx
│ │ ├── 2023-02-11-architecture.md
│ │ ├── 2023-02-24-open-assistant-inference-backend-development-hands-on-coding/
│ │ │ └── index.mdx
│ │ ├── 2023-04-06-open-assistant-first-models-are-here/
│ │ │ └── index.mdx
│ │ ├── 2023-04-10-open-assistant-livestream-just-chatting/
│ │ │ └── index.mdx
│ │ ├── 2023-04-15-open-assistant-released/
│ │ │ └── index.mdx
│ │ ├── 2023-04-21-open-assistant-storms-youtube/
│ │ │ └── index.md
│ │ ├── 2023-10-25-open-assistant-is-completed/
│ │ │ └── index.mdx
│ │ └── authors.yml
│ ├── docs/
│ │ ├── api/
│ │ │ ├── backend-openapi.json
│ │ │ └── inference-openapi.json
│ │ ├── architecture/
│ │ │ ├── README.md
│ │ │ └── inference.md
│ │ ├── data/
│ │ │ ├── README.md
│ │ │ ├── augmentation.md
│ │ │ ├── datasets.md
│ │ │ ├── schemas.mdx
│ │ │ └── supervised-datasets.md
│ │ ├── faq.md
│ │ ├── guides/
│ │ │ ├── README.md
│ │ │ ├── data_collection.md
│ │ │ ├── developers.md
│ │ │ ├── examples.md
│ │ │ └── guidelines.md
│ │ ├── intro.md
│ │ ├── plugins/
│ │ │ ├── README.md
│ │ │ ├── details.md
│ │ │ ├── getting-started.md
│ │ │ └── list.md
│ │ ├── presentations/
│ │ │ └── README.md
│ │ ├── research/
│ │ │ ├── README.md
│ │ │ ├── general.md
│ │ │ ├── retrieval.md
│ │ │ └── search-based-qa.md
│ │ └── tasks/
│ │ ├── README.md
│ │ ├── label_assistant_reply.md
│ │ ├── label_prompter_reply.md
│ │ ├── rank_assistant_replies.md
│ │ ├── reply_as_assistant.md
│ │ └── reply_as_user.md
│ ├── docusaurus.config.js
│ ├── package.json
│ ├── sidebars.js
│ ├── src/
│ │ ├── components/
│ │ │ └── HomepageFeatures/
│ │ │ ├── index.tsx
│ │ │ └── styles.module.css
│ │ ├── css/
│ │ │ └── custom.css
│ │ └── pages/
│ │ ├── index.module.css
│ │ └── index.tsx
│ └── static/
│ └── .nojekyll
├── inference/
│ ├── .gitignore
│ ├── README.md
│ ├── full-dev-setup.sh
│ ├── safety/
│ │ ├── README.md
│ │ ├── main.py
│ │ ├── requirements.txt
│ │ ├── safety_main.sh
│ │ └── settings.py
│ ├── server/
│ │ ├── README.md
│ │ ├── alembic/
│ │ │ ├── README
│ │ │ ├── env.py
│ │ │ ├── script.py.mako
│ │ │ └── versions/
│ │ │ ├── .gitinclude
│ │ │ ├── 2023_03_12_1742-7d5be54acd49_initial_revision.py
│ │ │ ├── 2023_03_21_2116-629d5081160f_changed_worker_config_to_worker_info.py
│ │ │ ├── 2023_03_22_2113-78f16015b904_add_refresh_token_table.py
│ │ │ ├── 2023_04_12_2033-f0e18084aae4_add_deleted_field_to_user.py
│ │ │ ├── 2023_04_14_1611-b66fd8f9da1f_add_hidden_field_to_chats.py
│ │ │ ├── 2023_04_24_2130-401eef162771_add_chat_data_opt_out_field.py
│ │ │ ├── 2023_04_29_1739-ea19bbc743f9_add_safe_content_to_message.py
│ │ │ ├── 2023_05_01_2253-5b4211625a9f_added_used_plugin_to_message.py
│ │ │ └── 2023_05_29_1551-5ed411a331f4_add_active_thread_tail_messsage_id_and_.py
│ │ ├── alembic.ini
│ │ ├── export.py
│ │ ├── main.py
│ │ ├── oasst_inference_server/
│ │ │ ├── __init__.py
│ │ │ ├── admin.py
│ │ │ ├── auth.py
│ │ │ ├── chat_repository.py
│ │ │ ├── chat_utils.py
│ │ │ ├── compliance.py
│ │ │ ├── database.py
│ │ │ ├── deps.py
│ │ │ ├── models/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── chat.py
│ │ │ │ ├── user.py
│ │ │ │ └── worker.py
│ │ │ ├── plugin_utils.py
│ │ │ ├── plugins/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── gale_pleaser/
│ │ │ │ │ ├── README.md
│ │ │ │ │ ├── ai-plugin.json
│ │ │ │ │ └── main.py
│ │ │ │ ├── gale_roaster/
│ │ │ │ │ ├── ai-plugin.json
│ │ │ │ │ └── main.py
│ │ │ │ └── web_retriever/
│ │ │ │ ├── ai-plugin.json
│ │ │ │ └── main.py
│ │ │ ├── queueing.py
│ │ │ ├── routes/
│ │ │ │ ├── account.py
│ │ │ │ ├── admin.py
│ │ │ │ ├── auth.py
│ │ │ │ ├── chats.py
│ │ │ │ ├── configs.py
│ │ │ │ └── workers.py
│ │ │ ├── schemas/
│ │ │ │ ├── __init__.py
│ │ │ │ ├── auth.py
│ │ │ │ ├── chat.py
│ │ │ │ └── worker.py
│ │ │ ├── settings.py
│ │ │ ├── user_chat_repository.py
│ │ │ └── worker_utils.py
│ │ ├── requirements.txt
│ │ └── server_main.sh
│ ├── tests/
│ │ └── locust/
│ │ └── locustfile.py
│ ├── text-client/
│ │ ├── __main__.py
│ │ ├── requirements.txt
│ │ └── text_client_utils.py
│ └── worker/
│ ├── README.md
│ ├── __main__.py
│ ├── basic_hf_server.py
│ ├── chat_chain.py
│ ├── chat_chain_prompts.py
│ ├── chat_chain_utils.py
│ ├── download_model.py
│ ├── download_model_hf.py
│ ├── get_model_config_prop.py
│ ├── hf_langchain_inference.py
│ ├── hf_stopping.py
│ ├── hf_streamer.py
│ ├── interface.py
│ ├── openapi_parser.py
│ ├── requirements-hf.txt
│ ├── requirements.txt
│ ├── run_worker_container.sh
│ ├── settings.py
│ ├── utils.py
│ ├── work.py
│ ├── worker_full_main.sh
│ ├── worker_hf_main.sh
│ └── worker_standalone_main.sh
├── inlang.config.js
├── model/
│ ├── .gitignore
│ ├── MESSAGE_AND_TOKEN_FORMAT.md
│ ├── README.md
│ ├── model_eval/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── eval_datasets.py
│ │ ├── eval_rm.py
│ │ ├── manual/
│ │ │ ├── config/
│ │ │ │ ├── default.json
│ │ │ │ ├── default_rallio.json
│ │ │ │ ├── noprefix.json
│ │ │ │ ├── noprefix2.json
│ │ │ │ ├── noprefix2_profile.json
│ │ │ │ ├── synth.json
│ │ │ │ ├── synth2.json
│ │ │ │ ├── synth2_gale_prefix.json
│ │ │ │ ├── synth2_rallio_original.json
│ │ │ │ ├── synth_rallio.json
│ │ │ │ ├── system_profile.json
│ │ │ │ └── system_profile2.json
│ │ │ ├── create_synth_import.py
│ │ │ ├── data/
│ │ │ │ ├── en_100_message.jsonl
│ │ │ │ ├── en_100_text.jsonl
│ │ │ │ ├── en_100_tree.jsonl
│ │ │ │ └── prompt_lottery_en_250_text.jsonl
│ │ │ ├── requirements.txt
│ │ │ ├── sampling_report.py
│ │ │ ├── sampling_reports/
│ │ │ │ ├── 2023-03-01_theblackcat102_pythia-12b-deduped-sft_sampling.json
│ │ │ │ ├── 2023-03-01_theblackcat102_pythia-1b-deduped-sft_sampling.json
│ │ │ │ └── 2023-03-01_theblackcat102_pythia-3b-deduped-sft_sampling.json
│ │ │ └── subsample_dataset.py
│ │ ├── rejection_sampling.py
│ │ ├── sample_results/
│ │ │ ├── comparison.json
│ │ │ ├── rejected_samples.json
│ │ │ ├── selected_samples.json
│ │ │ └── theblackcat102-pythia-12b-deduped-sft.json
│ │ ├── sampling_score.py
│ │ └── utils.py
│ ├── model_training/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── check_dataset_appearances.py
│ │ ├── check_dataset_counts.py
│ │ ├── configs/
│ │ │ ├── accelerate_config.yaml
│ │ │ ├── config.yaml
│ │ │ ├── config_rl.yaml
│ │ │ ├── config_rm.yaml
│ │ │ ├── deepspeed_rl.json
│ │ │ ├── deepspeed_rl_zero3.json
│ │ │ ├── ppo_config.yaml
│ │ │ ├── triton_config_rm.pbtxt
│ │ │ ├── triton_config_sft.pbtxt
│ │ │ ├── zero3_config_falcon.json
│ │ │ ├── zero3_config_pretrain.json
│ │ │ ├── zero3_config_sft.json
│ │ │ ├── zero_config.json
│ │ │ ├── zero_config_pretrain.json
│ │ │ └── zero_config_sft_65b.json
│ │ ├── custom_datasets/
│ │ │ ├── README.md
│ │ │ ├── __init__.py
│ │ │ ├── dialogue_collator.py
│ │ │ ├── extra_rm_datasets.py
│ │ │ ├── formatting.py
│ │ │ ├── instruction.py
│ │ │ ├── oasst_dataset.py
│ │ │ ├── pretrain_datasets.py
│ │ │ ├── prompt_dialogue.py
│ │ │ ├── qa_datasets.py
│ │ │ ├── rank_datasets.py
│ │ │ ├── ranking_collator.py
│ │ │ ├── summarization.py
│ │ │ ├── toxic_conversation.py
│ │ │ ├── translation.py
│ │ │ └── utils.py
│ │ ├── efficiency_utils.py
│ │ ├── metrics.py
│ │ ├── models/
│ │ │ ├── __init__.py
│ │ │ ├── gptj.py
│ │ │ ├── patching.py
│ │ │ ├── patching_falcon.py
│ │ │ ├── patching_llama.py
│ │ │ ├── patching_neox.py
│ │ │ ├── patching_utils.py
│ │ │ ├── peft_modeling.py
│ │ │ ├── prefix_llama.py
│ │ │ ├── reward_model.py
│ │ │ └── rope.py
│ │ ├── tests/
│ │ │ ├── __init__.py
│ │ │ ├── resources/
│ │ │ │ └── data_collator/
│ │ │ │ ├── special_tokens_map.json
│ │ │ │ ├── tokenizer.json
│ │ │ │ └── tokenizer_config.json
│ │ │ ├── test_datasets.py
│ │ │ ├── test_dialogue_data_collator.py
│ │ │ ├── test_formatting.py
│ │ │ ├── test_oasst_dataset.py
│ │ │ ├── test_patched_falcon.py
│ │ │ ├── test_patched_gpt_neox.py
│ │ │ ├── test_patched_llama.py
│ │ │ ├── test_ranking_collator.py
│ │ │ ├── test_rm_loading.py
│ │ │ └── test_utils.py
│ │ ├── to_triton.py
│ │ ├── tools/
│ │ │ ├── augment_oasst.py
│ │ │ ├── check_oasst_export.py
│ │ │ ├── export_model.py
│ │ │ ├── model_chat.py
│ │ │ ├── model_cli.py
│ │ │ └── sample_rm_data.py
│ │ ├── trainer_rl.py
│ │ ├── trainer_rm.py
│ │ ├── trainer_sft.py
│ │ └── utils/
│ │ ├── losses.py
│ │ ├── ppo_utils.py
│ │ ├── utils.py
│ │ └── utils_rl.py
│ ├── pretokenizer/
│ │ ├── README.md
│ │ ├── configs/
│ │ │ └── pretokenize.yaml
│ │ ├── create_hf_tokenizer_config.py
│ │ ├── indexed_dataset.py
│ │ ├── pretokenize.py
│ │ ├── requirements.txt
│ │ └── tokenizer.py
│ └── pyproject.toml
├── notebooks/
│ ├── README.md
│ ├── TSSB-3M-bugs-dataset/
│ │ ├── TSSB-3M-bugs_dataset.ipynb
│ │ └── TSSB-3M-bugs_dataset.md
│ ├── closed-book-qa/
│ │ ├── Closed Book QA Generator.ipynb
│ │ └── README.md
│ ├── data-augmentation/
│ │ ├── README.md
│ │ ├── anthropic/
│ │ │ ├── README.md
│ │ │ ├── safety data-augmentation.ipynb
│ │ │ └── trainer.py
│ │ ├── changemyview-builder/
│ │ │ ├── README.md
│ │ │ └── data_processor.ipynb
│ │ ├── essay-instructions/
│ │ │ ├── README.md
│ │ │ └── essay-instructions.ipynb
│ │ ├── essay-revision/
│ │ │ ├── README.md
│ │ │ └── essay-revision.ipynb
│ │ ├── hippocorpus/
│ │ │ ├── README.md
│ │ │ └── hippocorpus.ipynb
│ │ ├── movie-descriptions/
│ │ │ ├── README.md
│ │ │ └── movie_descriptions.ipynb
│ │ ├── movie-dialogs/
│ │ │ ├── README.md
│ │ │ └── convert-to-instruction-format.ipynb
│ │ ├── stackexchange-builder/
│ │ │ ├── README.md
│ │ │ └── stackexchange-builder.ipynb
│ │ ├── unified-qa/
│ │ │ ├── README.md
│ │ │ └── unified-qa.ipynb
│ │ ├── wikidata-qa/
│ │ │ ├── README.md
│ │ │ ├── requirements.txt
│ │ │ └── wikidata.ipynb
│ │ └── writing-prompt/
│ │ ├── README.md
│ │ └── writing_prompt.ipynb
│ ├── detoxify-evaluation/
│ │ ├── README.md
│ │ └── detoxify-evaluation.ipynb
│ ├── diverse/
│ │ ├── README.md
│ │ └── diverse.ipynb
│ ├── example/
│ │ ├── README.md
│ │ ├── data/
│ │ │ └── data.csv
│ │ ├── example.ipynb
│ │ └── requirements.txt
│ ├── openassistant-oasst1/
│ │ ├── README.md
│ │ └── getting-started.ipynb
│ └── openbugger/
│ ├── README.md
│ └── openbugger_example.ipynb
├── oasst-data/
│ ├── README.md
│ ├── examples/
│ │ ├── clean_dataset.py
│ │ ├── filter_messages.py
│ │ ├── filter_trees.py
│ │ ├── split_dataset.py
│ │ └── tree_to_messages.py
│ ├── oasst_data/
│ │ ├── __init__.py
│ │ ├── reader.py
│ │ ├── schemas.py
│ │ ├── traversal.py
│ │ └── writer.py
│ └── pyproject.toml
├── oasst-shared/
│ ├── README.md
│ ├── oasst_shared/
│ │ ├── __init__.py
│ │ ├── api_client.py
│ │ ├── exceptions/
│ │ │ ├── __init__.py
│ │ │ └── oasst_api_error.py
│ │ ├── model_configs.py
│ │ ├── schemas/
│ │ │ ├── __init__.py
│ │ │ ├── inference.py
│ │ │ └── protocol.py
│ │ └── utils.py
│ ├── pyproject.toml
│ └── tests/
│ ├── __init__.py
│ └── test_oasst_api_client.py
├── pyproject.toml
├── redis.conf
├── safety/
│ └── README.md
├── scripts/
│ ├── backend-development/
│ │ ├── README.md
│ │ ├── run-local-no-limit.sh
│ │ ├── run-local.sh
│ │ ├── start-docker.sh
│ │ ├── start-mock-server.sh
│ │ ├── start-worker.sh
│ │ ├── stop-mock-server.sh
│ │ └── stop-worker.sh
│ ├── data-collection/
│ │ └── twitter/
│ │ ├── README.md
│ │ ├── requirements.txt
│ │ ├── twitter_create_convs.py
│ │ └── twitter_process_json.py
│ ├── data_augment/
│ │ └── data_augment.py
│ ├── discord/
│ │ ├── stats.py
│ │ └── verify-lobby.py
│ ├── frontend-development/
│ │ ├── README.md
│ │ ├── find-missing-locales.py
│ │ ├── run-bot-local.sh
│ │ └── run-contract-test.sh
│ ├── oasst-shared-development/
│ │ └── test.sh
│ ├── postprocessing/
│ │ ├── importance_selection.py
│ │ ├── infogain_selector.py
│ │ ├── ranking_disagreement.py
│ │ ├── rankings.py
│ │ ├── regex_pii_detector.py
│ │ ├── scoring.py
│ │ └── task_schedule.py
│ └── xor-codec/
│ └── xor_codec.py
├── text-frontend/
│ ├── __main__.py
│ ├── auto_main.py
│ └── requirements.txt
└── website/
├── .eslintrc.json
├── .gitignore
├── .nvmrc
├── .prettierignore
├── .prettierrc.json
├── .storybook/
│ ├── decorators.js
│ ├── main.js
│ └── preview.js
├── README.md
├── cypress/
│ ├── README.md
│ ├── components/
│ │ └── Container.cy.tsx
│ ├── contract/
│ │ └── oasst_api_contract_tests.cy.ts
│ ├── e2e/
│ │ ├── auth/
│ │ │ └── signin.cy.ts
│ │ └── tasks/
│ │ ├── label_assistant_reply.cy.ts
│ │ ├── label_initial_prompt.cy.ts
│ │ ├── label_prompter_reply.cy.ts
│ │ ├── no_tasks_available.cy.ts
│ │ └── random.cy.ts
│ ├── fixtures/
│ │ └── example.json
│ ├── support/
│ │ ├── commands.ts
│ │ ├── component-index.html
│ │ ├── component.ts
│ │ ├── e2e.ts
│ │ └── index.ts
│ └── tsconfig.json
├── cypress-visual-screenshots/
│ └── baseline/
│ └── .gitkeep
├── cypress.config.contract.js
├── cypress.config.js
├── docs/
│ └── add_edit_translations.md
├── jest.config.js
├── jest.setup.js
├── next-i18next.config.js
├── next-lint.js
├── next.config.js
├── package.json
├── postcss.config.js
├── prisma/
│ ├── migrations/
│ │ ├── 20230326131923_initial_migration/
│ │ │ └── migration.sql
│ │ ├── 20230805220637_paperack/
│ │ │ └── migration.sql
│ │ └── migration_lock.toml
│ ├── schema.prisma
│ └── seed.ts
├── public/
│ ├── fonts/
│ │ └── lexend.txt
│ ├── locales/
│ │ ├── ar/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── bar/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── bg/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ca/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── cs/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── da/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── de/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── el/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── en/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── eo/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── es/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── eu/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fa/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fi/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── fr/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── gl/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── he/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── hi/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── hu/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── id/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── it/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ja/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ko/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── lt/
│ │ │ ├── account.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ms/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── nb-NO/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── nl/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── pl/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── pt-BR/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ro/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── ru/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sk/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sl/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sr/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── sv/
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── swg/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── th/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── tr/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── uk-UA/
│ │ │ ├── account.json
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ ├── vi/
│ │ │ ├── chat.json
│ │ │ ├── common.json
│ │ │ ├── dashboard.json
│ │ │ ├── error.json
│ │ │ ├── index.json
│ │ │ ├── labelling.json
│ │ │ ├── leaderboard.json
│ │ │ ├── message.json
│ │ │ ├── stats.json
│ │ │ ├── tasks.json
│ │ │ └── tos.json
│ │ └── zh/
│ │ ├── account.json
│ │ ├── chat.json
│ │ ├── common.json
│ │ ├── dashboard.json
│ │ ├── error.json
│ │ ├── index.json
│ │ ├── labelling.json
│ │ ├── leaderboard.json
│ │ ├── message.json
│ │ ├── stats.json
│ │ ├── tasks.json
│ │ └── tos.json
│ ├── manifest.json
│ └── mockServiceWorker.js
├── src/
│ ├── README.md
│ ├── components/
│ │ ├── Account/
│ │ │ ├── UserStats.tsx
│ │ │ └── XPBar.tsx
│ │ ├── AdminArea.tsx
│ │ ├── AnimatedCircles/
│ │ │ ├── AnimatedCircles.tsx
│ │ │ └── index.tsx
│ │ ├── AuthLayout.tsx
│ │ ├── Buttons/
│ │ │ ├── Buttons.stories.tsx
│ │ │ ├── LikertButtons.tsx
│ │ │ ├── Skip.tsx
│ │ │ └── Submit.tsx
│ │ ├── CallToAction.tsx
│ │ ├── Chat/
│ │ │ ├── ChatAssistantDraftPager.tsx
│ │ │ ├── ChatAssistantDraftViewer.tsx
│ │ │ ├── ChatConfig.tsx
│ │ │ ├── ChatConfigDesktop.tsx
│ │ │ ├── ChatConfigForm.tsx
│ │ │ ├── ChatConfigMobile.tsx
│ │ │ ├── ChatConfigSaver.tsx
│ │ │ ├── ChatConfigSummary.tsx
│ │ │ ├── ChatContext.tsx
│ │ │ ├── ChatConversation.tsx
│ │ │ ├── ChatConversationTree.tsx
│ │ │ ├── ChatForm.tsx
│ │ │ ├── ChatInitialDataContext.tsx
│ │ │ ├── ChatInputIconButton.tsx
│ │ │ ├── ChatListBase.tsx
│ │ │ ├── ChatListDesktop.tsx
│ │ │ ├── ChatListItem.tsx
│ │ │ ├── ChatListMobile.tsx
│ │ │ ├── ChatMessageEntry.tsx
│ │ │ ├── ChatSection.tsx
│ │ │ ├── ChatViewSelection.tsx
│ │ │ ├── ChatWarning.tsx
│ │ │ ├── CreateChatButton.tsx
│ │ │ ├── CustomInstructions.tsx
│ │ │ ├── DeletePresetButton.tsx
│ │ │ ├── EncourageMessage.tsx
│ │ │ ├── InferencePoweredBy.tsx
│ │ │ ├── PluginsChooser.tsx
│ │ │ ├── QueueInfoMessage.tsx
│ │ │ ├── SavePresetButton.tsx
│ │ │ ├── WorkParameters.tsx
│ │ │ └── useListChatPagination.ts
│ │ ├── CloudflareCaptcha.tsx
│ │ ├── CollapsableText.tsx
│ │ ├── Container.tsx
│ │ ├── Dashboard/
│ │ │ ├── LeaderboardWidget.tsx
│ │ │ ├── SlimFooter.tsx
│ │ │ ├── TaskOption.tsx
│ │ │ ├── WelcomeCard.tsx
│ │ │ └── index.ts
│ │ ├── DataTable/
│ │ │ ├── DataTable.tsx
│ │ │ ├── DataTableAction.tsx
│ │ │ ├── jsonExpandRowModel.tsx
│ │ │ └── useCursorPagination.ts
│ │ ├── EmptyState.tsx
│ │ ├── Explain.tsx
│ │ ├── Faq.tsx
│ │ ├── Footer.tsx
│ │ ├── Header/
│ │ │ ├── ColorModeToggler.tsx
│ │ │ ├── Header.stories.tsx
│ │ │ ├── Header.tsx
│ │ │ ├── UserMenu.stories.tsx
│ │ │ ├── UserMenu.tsx
│ │ │ ├── UserScore.tsx
│ │ │ └── index.ts
│ │ ├── Hero.tsx
│ │ ├── JsonCard.tsx
│ │ ├── LanguageSelector/
│ │ │ ├── LanguageSelector.tsx
│ │ │ └── index.tsx
│ │ ├── Layout/
│ │ │ └── ChatLayout.tsx
│ │ ├── Layout.tsx
│ │ ├── LeaderboardTable/
│ │ │ ├── LeaderboardTable.tsx
│ │ │ ├── TrollboardTable.tsx
│ │ │ ├── index.tsx
│ │ │ ├── useBoardPagination.ts
│ │ │ ├── useBoardRowProps.ts
│ │ │ └── useFetchBoard.ts
│ │ ├── Loading/
│ │ │ ├── Loading.stories.tsx
│ │ │ ├── LoadingScreen.tsx
│ │ │ └── MessageLoading.tsx
│ │ ├── MarkdownEditor.tsx
│ │ ├── Messages/
│ │ │ ├── AdminMessageTable.tsx
│ │ │ ├── BaseMessageEntry.tsx
│ │ │ ├── LabelFlagGroup.tsx
│ │ │ ├── LabelInputGroup.tsx
│ │ │ ├── LabelPopup.tsx
│ │ │ ├── LabelYesNoGroup.tsx
│ │ │ ├── MessageConversation.stories.tsx
│ │ │ ├── MessageConversation.tsx
│ │ │ ├── MessageCreateDate.tsx
│ │ │ ├── MessageEmojiButton.stories.tsx
│ │ │ ├── MessageEmojiButton.tsx
│ │ │ ├── MessageHistoryTable.tsx
│ │ │ ├── MessageInlineEmojiRow.tsx
│ │ │ ├── MessageSyntheticBadge.tsx
│ │ │ ├── MessageTableEntry.stories.tsx
│ │ │ ├── MessageTableEntry.tsx
│ │ │ ├── MessageTree.tsx
│ │ │ ├── MessageWithChildren.stories.tsx
│ │ │ ├── MessageWithChildren.tsx
│ │ │ ├── PluginUsageDetails.tsx
│ │ │ ├── RenderedCodeblock.tsx
│ │ │ ├── RenderedMarkdown.tsx
│ │ │ ├── ReportPopup.tsx
│ │ │ └── SyntaxHighlighter.tsx
│ │ ├── PolicyCards/
│ │ │ ├── PolicyChapterCard.tsx
│ │ │ └── PolicySectionCard.tsx
│ │ ├── Roadmap.tsx
│ │ ├── RoleSelect.tsx
│ │ ├── Services.tsx
│ │ ├── SideMenu.tsx
│ │ ├── SideMenuLayout.tsx
│ │ ├── Sortable/
│ │ │ ├── Sortable.stories.tsx
│ │ │ ├── Sortable.tsx
│ │ │ └── SortableItem.tsx
│ │ ├── Stats/
│ │ │ ├── Stats.components.tsx
│ │ │ ├── Stats.stories.tsx
│ │ │ ├── Stats.tsx
│ │ │ └── index.ts
│ │ ├── Survey/
│ │ │ ├── LabelLikertGroup.tsx
│ │ │ ├── SurveyCard.tsx
│ │ │ ├── TaskControls.tsx
│ │ │ ├── TrackedTextarea.tsx
│ │ │ └── TwoColumnsWithCards.tsx
│ │ ├── TaskInfo/
│ │ │ └── TaskInfo.tsx
│ │ ├── TaskPage/
│ │ │ └── TaskPage.tsx
│ │ ├── Tasks/
│ │ │ ├── CreateTask.tsx
│ │ │ ├── EvaluateTask.tsx
│ │ │ ├── LabelTask/
│ │ │ │ ├── LabelTask.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── Task/
│ │ │ │ ├── Task.stories.tsx
│ │ │ │ ├── Task.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── TaskHeader/
│ │ │ │ ├── TaskHeader.tsx
│ │ │ │ └── index.tsx
│ │ │ ├── TaskTypes.tsx
│ │ │ └── UnchangedWarning.tsx
│ │ ├── TeamMember.tsx
│ │ ├── ToS.tsx
│ │ ├── ToSWrapper.tsx
│ │ ├── UserAvatar.tsx
│ │ ├── UserDisplayNameCell.tsx
│ │ ├── UserMessageConversation.tsx
│ │ ├── UserTable.tsx
│ │ ├── Vision.tsx
│ │ └── icons/
│ │ ├── Markdown.tsx
│ │ └── MarkdownOff.tsx
│ ├── context/
│ │ └── TaskContext.ts
│ ├── data/
│ │ └── team.json
│ ├── flags.ts
│ ├── hooks/
│ │ ├── auth/
│ │ │ ├── useHasAnyRole.ts
│ │ │ └── useHasRole.ts
│ │ ├── chat/
│ │ │ └── useMessageVote.ts
│ │ ├── env/
│ │ │ └── BrowserEnv.ts
│ │ ├── layout/
│ │ │ └── sidebarItems.ts
│ │ ├── locale/
│ │ │ └── useCurrentLocale.ts
│ │ ├── message/
│ │ │ ├── useDeleteMessage.ts
│ │ │ └── useUndeleteMessage.ts
│ │ ├── tasks/
│ │ │ ├── useCreateReply.ts
│ │ │ ├── useEvaluateReplies.ts
│ │ │ ├── useGenericTaskAPI.tsx
│ │ │ └── useLabelingTask.ts
│ │ └── ui/
│ │ ├── useFallbackRef.ts
│ │ ├── useScrollToElementOnMount.ts
│ │ └── useUserScore.ts
│ ├── lib/
│ │ ├── api.ts
│ │ ├── auth.ts
│ │ ├── captcha.ts
│ │ ├── chat_stream.test.ts
│ │ ├── chat_stream.ts
│ │ ├── constants.ts
│ │ ├── defaultServerSideProps.ts
│ │ ├── display_name_validation.ts
│ │ ├── errors.ts
│ │ ├── i18n.ts
│ │ ├── isChatEnable.ts
│ │ ├── iso6393.ts
│ │ ├── languages.ts
│ │ ├── leaderboard_utilities.ts
│ │ ├── logger.ts
│ │ ├── oasst_api_client.ts
│ │ ├── oasst_client_factory.ts
│ │ ├── oasst_inference_client.ts
│ │ ├── prismadb.ts
│ │ ├── routes.ts
│ │ └── users.ts
│ ├── middleware.ts
│ ├── pages/
│ │ ├── 404.tsx
│ │ ├── 500.tsx
│ │ ├── _app.tsx
│ │ ├── _document.tsx
│ │ ├── about.tsx
│ │ ├── account/
│ │ │ ├── delete.tsx
│ │ │ ├── edit.tsx
│ │ │ ├── index.tsx
│ │ │ └── paperack.tsx
│ │ ├── admin/
│ │ │ ├── edit/
│ │ │ │ └── [id].tsx
│ │ │ ├── index.tsx
│ │ │ ├── manage_user/
│ │ │ │ └── [id].tsx
│ │ │ ├── messages/
│ │ │ │ ├── [id].tsx
│ │ │ │ └── index.tsx
│ │ │ ├── parameters.tsx
│ │ │ ├── status/
│ │ │ │ └── index.tsx
│ │ │ └── trollboard.tsx
│ │ ├── api/
│ │ │ ├── account/
│ │ │ │ ├── delete.ts
│ │ │ │ └── index.ts
│ │ │ ├── admin/
│ │ │ │ ├── delete_message/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── edit_message/
│ │ │ │ │ └── [id].tsx
│ │ │ │ ├── messages/
│ │ │ │ │ ├── [id]/
│ │ │ │ │ │ ├── history.ts
│ │ │ │ │ │ └── tree/
│ │ │ │ │ │ └── index.ts
│ │ │ │ │ └── index.ts
│ │ │ │ ├── parameters.ts
│ │ │ │ ├── set_tree_halted/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── status.ts
│ │ │ │ ├── trollboard.ts
│ │ │ │ ├── undelete_message/
│ │ │ │ │ └── [id].ts
│ │ │ │ ├── update_user.ts
│ │ │ │ └── users.ts
│ │ │ ├── auth/
│ │ │ │ └── [...nextauth].ts
│ │ │ ├── available_tasks.ts
│ │ │ ├── chat/
│ │ │ │ ├── assistant_message.ts
│ │ │ │ ├── events.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── message.ts
│ │ │ │ ├── message_eval.ts
│ │ │ │ ├── models.ts
│ │ │ │ ├── plugin_config.ts
│ │ │ │ ├── plugins.ts
│ │ │ │ ├── prompter_message.ts
│ │ │ │ └── vote.ts
│ │ │ ├── config.ts
│ │ │ ├── leaderboard.ts
│ │ │ ├── messages/
│ │ │ │ ├── [id]/
│ │ │ │ │ ├── children.ts
│ │ │ │ │ ├── conversation.ts
│ │ │ │ │ ├── emoji.ts
│ │ │ │ │ ├── index.ts
│ │ │ │ │ ├── parent.ts
│ │ │ │ │ └── tree.ts
│ │ │ │ ├── index.ts
│ │ │ │ └── user.ts
│ │ │ ├── new_task/
│ │ │ │ └── [task_type].ts
│ │ │ ├── paperack.ts
│ │ │ ├── reject_task.ts
│ │ │ ├── report.ts
│ │ │ ├── set_label.ts
│ │ │ ├── stats/
│ │ │ │ └── cached_stats.ts
│ │ │ ├── tos.ts
│ │ │ ├── update_task.ts
│ │ │ ├── user_stats.ts
│ │ │ ├── username.ts
│ │ │ └── valid_labels.ts
│ │ ├── auth/
│ │ │ ├── signin.tsx
│ │ │ └── verify.tsx
│ │ ├── brb.tsx
│ │ ├── bye.tsx
│ │ ├── chat/
│ │ │ ├── [id].tsx
│ │ │ └── index.tsx
│ │ ├── contributors.tsx
│ │ ├── create/
│ │ │ ├── assistant_reply.tsx
│ │ │ ├── initial_prompt.tsx
│ │ │ └── user_reply.tsx
│ │ ├── dashboard.tsx
│ │ ├── evaluate/
│ │ │ ├── rank_assistant_replies.tsx
│ │ │ ├── rank_initial_prompts.tsx
│ │ │ └── rank_user_replies.tsx
│ │ ├── index.tsx
│ │ ├── label/
│ │ │ ├── label_assistant_reply.tsx
│ │ │ ├── label_initial_prompt.tsx
│ │ │ └── label_prompter_reply.tsx
│ │ ├── leaderboard.tsx
│ │ ├── messages/
│ │ │ ├── [id]/
│ │ │ │ └── index.tsx
│ │ │ └── index.tsx
│ │ ├── privacy-policy.tsx
│ │ ├── stats.tsx
│ │ ├── tasks/
│ │ │ ├── all.tsx
│ │ │ └── random.tsx
│ │ ├── team.tsx
│ │ └── terms-of-service.tsx
│ ├── reportWebVitals.js
│ ├── styles/
│ │ ├── Chakra.tsx
│ │ ├── Theme/
│ │ │ ├── colors.ts
│ │ │ ├── components/
│ │ │ │ ├── Badge.ts
│ │ │ │ ├── Card.ts
│ │ │ │ ├── Container.ts
│ │ │ │ └── Table.ts
│ │ │ └── index.ts
│ │ └── globals.css
│ ├── test-utils/
│ │ └── createMockRouter.ts
│ ├── test_pages/
│ │ ├── README.md
│ │ ├── about.test.tsx
│ │ └── index.test.tsx
│ ├── types/
│ │ ├── Account.ts
│ │ ├── Chat.ts
│ │ ├── Config.ts
│ │ ├── Conversation.ts
│ │ ├── Emoji.ts
│ │ ├── Hooks.ts
│ │ ├── Leaderboard.ts
│ │ ├── Providers.ts
│ │ ├── Stat.ts
│ │ ├── Task.ts
│ │ ├── TaskResponses.ts
│ │ ├── Tasks.ts
│ │ ├── Trollboard.ts
│ │ └── Users.ts
│ └── utils/
│ ├── buildTree.ts
│ └── chat.ts
├── tailwind.config.js
├── tsconfig.json
├── types/
│ ├── env.d.ts
│ ├── i18next.d.ts
│ └── next-auth.d.ts
└── wait-for-postgres.sh
Showing preview only (225K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (2537 symbols across 465 files)
FILE: backend/alembic/env.py
function run_migrations_offline (line 29) | def run_migrations_offline() -> None:
function run_migrations_online (line 53) | def run_migrations_online() -> None:
FILE: backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 72) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py
function upgrade (line 21) | def upgrade() -> None:
function downgrade (line 121) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 25) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 43) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 69) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 36) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 27) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 177) | def downgrade() -> None:
FILE: backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 41) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 22) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 37) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_08_2128-aac6b2f66006_created_date.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 27) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 37) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 44) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 30) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_14_1509-619255ae9076_add_rank_to_message_table.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 55) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 54) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_15_1139-423557e869e4_add_indices_for_created_date.py
function upgrade (line 17) | def upgrade() -> None:
function downgrade (line 25) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 28) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 34) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py
function upgrade (line 17) | def upgrade() -> None:
function downgrade (line 23) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 25) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 28) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_24_2256-40ed93df0ed5_add_message_emoji.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 39) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py
function upgrade (line 17) | def upgrade() -> None:
function downgrade (line 23) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 37) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 27) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 55) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_02_1817-8c8241d1f973_add_account_table.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 35) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 36) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_11_1030-ba40d055714a_add_cached_stats.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 34) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py
function upgrade (line 17) | def upgrade() -> None:
function downgrade (line 23) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 31) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 30) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_05_07_2129-1b6e3ae16e9d_add_text_search.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 23) | def downgrade() -> None:
FILE: backend/alembic/versions/2023_06_06_1505-c181661eba3a_add_message_revisions.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 45) | def downgrade() -> None:
FILE: backend/export.py
function fetch_tree_ids (line 26) | def fetch_tree_ids(
function fetch_tree_messages (line 62) | def fetch_tree_messages(
function get_events_for_messages (line 95) | def get_events_for_messages(db: Session, message_ids: list[UUID]) -> dic...
function fetch_tree_messages_and_avg_labels (line 128) | def fetch_tree_messages_and_avg_labels(
function export_trees (line 169) | def export_trees(
function validate_args (line 306) | def validate_args(args):
function parse_args (line 319) | def parse_args():
function main (line 402) | def main():
FILE: backend/import.py
class Importer (line 26) | class Importer:
method __init__ (line 27) | def __init__(self, db: Session, origin: str, model_name: Optional[str]...
method fetch_message (line 47) | def fetch_message(self, message_id: UUID) -> Message:
method fetch_message_tree_state (line 50) | def fetch_message_tree_state(self, message_tree_id: UUID) -> MessageTr...
method import_message (line 53) | def import_message(
method import_tree (line 83) | def import_tree(
function import_file (line 104) | def import_file(
function parse_args (line 155) | def parse_args():
function main (line 180) | def main():
FILE: backend/main.py
function oasst_exception_handler (line 40) | async def oasst_exception_handler(request: fastapi.Request, ex: OasstErr...
function unhandled_exception_handler (line 53) | async def unhandled_exception_handler(request: fastapi.Request, ex: Exce...
function alembic_upgrade (line 74) | def alembic_upgrade():
function create_official_web_api_client (line 89) | def create_official_web_api_client():
function enable_prom_metrics (line 107) | async def enable_prom_metrics():
function connect_redis (line 114) | async def connect_redis():
function create_seed_data (line 138) | def create_seed_data(session: Session):
function ensure_tree_states (line 231) | def ensure_tree_states():
function update_leader_board_day (line 246) | def update_leader_board_day(session: Session) -> None:
function update_leader_board_week (line 257) | def update_leader_board_week(session: Session) -> None:
function update_leader_board_month (line 268) | def update_leader_board_month(session: Session) -> None:
function update_leader_board_total (line 279) | def update_leader_board_total(session: Session) -> None:
function cronjob_delete_expired_tasks (line 290) | def cronjob_delete_expired_tasks(session: Session) -> None:
function update_cached_stats (line 298) | def update_cached_stats(session: Session) -> None:
function get_openapi_schema (line 309) | def get_openapi_schema():
function retry_scoring_failed_message_trees (line 313) | def retry_scoring_failed_message_trees():
function main (line 327) | def main():
FILE: backend/oasst_backend/api/deps.py
function get_db (line 18) | def get_db() -> Generator:
function get_api_key (line 31) | def get_api_key(
class FrontendUserId (line 41) | class FrontendUserId(NamedTuple):
function get_frontend_user_id (line 46) | def get_frontend_user_id(
function create_api_client (line 63) | def create_api_client(
function api_auth (line 92) | def api_auth(
function get_api_client (line 108) | def get_api_client(
function get_trusted_api_client (line 115) | def get_trusted_api_client(
function get_root_token (line 129) | def get_root_token(bearer_token: HTTPAuthorizationCredentials = Security...
function user_identifier (line 141) | async def user_identifier(request: Request) -> str:
class UserRateLimiter (line 153) | class UserRateLimiter(RateLimiter):
method __init__ (line 154) | def __init__(
method __call__ (line 159) | async def __call__(self, request: Request, response: Response, api_key...
class UserTaskTypeRateLimiter (line 175) | class UserTaskTypeRateLimiter(RateLimiter):
method __init__ (line 180) | def __init__(
method __call__ (line 192) | async def __call__(self, request: Request, response: Response, api_key...
class APIClientRateLimiter (line 213) | class APIClientRateLimiter(RateLimiter):
method __init__ (line 214) | def __init__(
method __call__ (line 224) | async def __call__(self, request: Request, response: Response, api_key...
FILE: backend/oasst_backend/api/v1/admin.py
class CreateApiClientRequest (line 23) | class CreateApiClientRequest(pydantic.BaseModel):
function create_api_client (line 31) | async def create_api_client(
function get_backend_settings_full (line 49) | async def get_backend_settings_full(api_client: ApiClient = Depends(deps...
class PublicSettings (line 56) | class PublicSettings(pydantic.BaseModel):
function get_backend_settings_public (line 74) | async def get_backend_settings_public(api_client: ApiClient = Depends(de...
class PurgeResultModel (line 78) | class PurgeResultModel(pydantic.BaseModel):
function purge_user (line 86) | async def purge_user(
function purge_user_messages (line 125) | async def purge_user_messages(
class FlaggedMessageResponse (line 171) | class FlaggedMessageResponse(pydantic.BaseModel):
class FlaggedMessagePage (line 177) | class FlaggedMessagePage(PageResult):
function get_flagged_messages_cursor (line 182) | def get_flagged_messages_cursor(
function get_flagged_messages (line 254) | async def get_flagged_messages(
function process_flagged_messages (line 268) | async def process_flagged_messages(
class MergeUsersRequest (line 281) | class MergeUsersRequest(pydantic.BaseModel):
function merge_users (line 288) | def merge_users(
FILE: backend/oasst_backend/api/v1/auth.py
class TokenData (line 16) | class TokenData(BaseModel):
function get_current_user (line 24) | async def get_current_user(token: str = Security(oauth2_scheme)):
function auth_check (line 44) | async def auth_check(token_data: TokenData = Depends(get_current_user)):
FILE: backend/oasst_backend/api/v1/frontend_messages.py
function get_message_by_frontend_id (line 15) | def get_message_by_frontend_id(
function get_conv_by_frontend_id (line 27) | def get_conv_by_frontend_id(
function get_tree_by_frontend_id (line 41) | def get_tree_by_frontend_id(
function get_children_by_frontend_id (line 61) | def get_children_by_frontend_id(
function get_descendants_by_frontend_id (line 74) | def get_descendants_by_frontend_id(
function get_longest_conv_by_frontend_id (line 88) | def get_longest_conv_by_frontend_id(
function get_max_children_by_frontend_id (line 102) | def get_max_children_by_frontend_id(
FILE: backend/oasst_backend/api/v1/frontend_users.py
function get_users_ordered_by_username (line 20) | def get_users_ordered_by_username(
function query_frontend_user (line 47) | def query_frontend_user(
function create_frontend_user (line 63) | def create_frontend_user(
function query_frontend_user_messages (line 94) | def query_frontend_user_messages(
function query_frontend_user_messages_cursor (line 128) | def query_frontend_user_messages_cursor(
function mark_frontend_user_messages_deleted (line 159) | def mark_frontend_user_messages_deleted(
FILE: backend/oasst_backend/api/v1/hugging_face.py
function get_text_toxicity (line 13) | async def get_text_toxicity(
FILE: backend/oasst_backend/api/v1/leaderboards.py
function get_leaderboard (line 17) | def get_leaderboard(
function update_leaderboard_time_frame (line 34) | def update_leaderboard_time_frame(
function update_leaderboards_all (line 44) | def update_leaderboards_all(
FILE: backend/oasst_backend/api/v1/login.py
function login_discord (line 16) | def login_discord(request: Request):
function callback_discord (line 23) | async def callback_discord(
function get_callback_uri (line 65) | def get_callback_uri(request: Request):
FILE: backend/oasst_backend/api/v1/messages.py
function query_messages (line 23) | def query_messages(
function get_messages_cursor (line 60) | def get_messages_cursor(
function get_message (line 150) | def get_message(
function get_conv (line 166) | def get_conv(
function get_tree (line 183) | def get_tree(
function get_message_tree_state (line 204) | def get_message_tree_state(
function put_message_tree_state (line 226) | def put_message_tree_state(
function get_children (line 252) | def get_children(
function get_descendants (line 268) | def get_descendants(
function get_longest_conv (line 285) | def get_longest_conv(
function get_max_children (line 302) | def get_max_children(
function mark_message_deleted (line 319) | def mark_message_deleted(
function undelete_message (line 331) | def undelete_message(
function edit_message (line 343) | def edit_message(
function get_revision_history (line 358) | def get_revision_history(
function post_message_emoji (line 374) | def post_message_emoji(
FILE: backend/oasst_backend/api/v1/stats.py
function get_message_stats (line 16) | def get_message_stats(
function get_tree_manager__state_counts (line 25) | def get_tree_manager__state_counts(
function get_tree_manager__message_counts (line 35) | def get_tree_manager__message_counts(
function get_tree_manager__stats (line 46) | def get_tree_manager__stats(
function get_cached_stats (line 56) | def get_cached_stats(
function get_cached_stats_all (line 67) | def get_cached_stats_all(
function update_cached_stats (line 77) | def update_cached_stats(
FILE: backend/oasst_backend/api/v1/tasks.py
function request_task (line 57) | def request_task(
function tasks_availability (line 85) | def tasks_availability(
function tasks_acknowledge (line 107) | def tasks_acknowledge(
function tasks_acknowledge_failure (line 137) | def tasks_acknowledge_failure(
function tasks_interaction (line 160) | async def tasks_interaction(
function close_collective_task (line 190) | def close_collective_task(
FILE: backend/oasst_backend/api/v1/text_labels.py
function label_text (line 23) | def label_text(
function get_valid_lables (line 52) | def get_valid_lables(
function get_report_lables (line 79) | def get_report_lables() -> ValidLabelsResponse:
FILE: backend/oasst_backend/api/v1/trollboards.py
function get_trollboard (line 14) | def get_trollboard(
FILE: backend/oasst_backend/api/v1/users.py
function get_users_ordered_by_username (line 22) | def get_users_ordered_by_username(
function get_users_ordered_by_display_name (line 51) | def get_users_ordered_by_display_name(
function get_users_cursor (line 80) | def get_users_cursor(
function get_user (line 174) | def get_user(
function update_user (line 189) | def update_user(
function delete_user (line 207) | def delete_user(
function query_user_messages (line 221) | def query_user_messages(
function query_user_messages_cursor (line 255) | def query_user_messages_cursor(
function mark_user_messages_deleted (line 284) | def mark_user_messages_deleted(
function query_user_stats (line 296) | def query_user_stats(
function query_user_stats_timeframe (line 306) | def query_user_stats_timeframe(
function query_user_stats_timeframe_window (line 317) | def query_user_stats_timeframe_window(
FILE: backend/oasst_backend/api/v1/utils.py
function prepare_message (line 8) | def prepare_message(m: Message) -> protocol.Message:
function prepare_message_list (line 34) | def prepare_message_list(messages: list[Message]) -> list[protocol.Messa...
function prepare_conversation_message (line 38) | def prepare_conversation_message(message: Message) -> protocol.Conversat...
function prepare_conversation_message_list (line 53) | def prepare_conversation_message_list(messages: list[Message]) -> list[p...
function prepare_conversation (line 57) | def prepare_conversation(messages: list[Message]) -> protocol.Conversation:
function prepare_tree (line 61) | def prepare_tree(tree: list[Message], tree_id: UUID) -> protocol.Message...
function prepare_message_revision (line 69) | def prepare_message_revision(revision: MessageRevision) -> protocol.Mess...
function prepare_message_revision_list (line 80) | def prepare_message_revision_list(revisions: list[MessageRevision]) -> l...
FILE: backend/oasst_backend/auth.py
function create_access_token (line 10) | def create_access_token(data: dict) -> str:
function get_account_from_discord_id (line 23) | def get_account_from_discord_id(db: Session, discord_id: str) -> Optiona...
FILE: backend/oasst_backend/cached_stats_repository.py
function row_to_dict (line 9) | def row_to_dict(r) -> dict:
class CachedStatsRepository (line 13) | class CachedStatsRepository:
method __init__ (line 14) | def __init__(self, db: Session):
method qry_human_messages_by_lang (line 17) | def qry_human_messages_by_lang(self) -> dict[str, int]:
method qry_human_messages_by_role (line 25) | def qry_human_messages_by_role(self) -> dict[str, int]:
method qry_message_trees_by_state (line 33) | def qry_message_trees_by_state(self) -> dict[str, int]:
method qry_message_trees_states_by_lang (line 39) | def qry_message_trees_states_by_lang(self) -> list:
method qry_users_accepted_tos (line 51) | def qry_users_accepted_tos(self) -> dict[str, int]:
method update_all_cached_stats (line 56) | def update_all_cached_stats(self):
method _insert_cached_stats (line 72) | def _insert_cached_stats(self, name: CachedStatsName, stats: dict | li...
method get_stats (line 82) | def get_stats(self, name: CachedStatsName) -> CachedStatsResponse:
method get_stats_all (line 88) | def get_stats_all(self) -> AllCachedStatsResponse:
FILE: backend/oasst_backend/config.py
class TreeManagerConfiguration (line 8) | class TreeManagerConfiguration(BaseModel):
method init_prompt_disabled_langs_list (line 155) | def init_prompt_disabled_langs_list(self) -> list[str]:
class Settings (line 159) | class Settings(BaseSettings):
method assemble_db_connection (line 217) | def assemble_db_connection(cls, v: Optional[str], values: Dict[str, An...
method assemble_cors_origins (line 233) | def assemble_cors_origins(cls, v: Optional[List[str]], values: Dict[st...
method validate_user_stats_intervals (line 257) | def validate_user_stats_intervals(cls, v: int):
class Config (line 280) | class Config:
FILE: backend/oasst_backend/journal_writer.py
class JournalEventType (line 13) | class JournalEventType(str, enum.Enum):
class JournalEvent (line 23) | class JournalEvent(BaseModel):
class TextReplyEvent (line 32) | class TextReplyEvent(JournalEvent):
class RatingEvent (line 39) | class RatingEvent(JournalEvent):
class RankingEvent (line 45) | class RankingEvent(JournalEvent):
class JournalWriter (line 50) | class JournalWriter:
method __init__ (line 51) | def __init__(self, db: Session, api_client: ApiClient, user: User):
method log_text_reply (line 57) | def log_text_reply(self, task: Task, message_id: Optional[UUID], role:...
method log_rating (line 66) | def log_rating(self, task: Task, message_id: Optional[UUID], rating: i...
method log_ranking (line 75) | def log_ranking(self, task: Task, message_id: Optional[UUID], ranking:...
method log (line 85) | def log(
FILE: backend/oasst_backend/models/api_client.py
class ApiClient (line 10) | class ApiClient(SQLModel, table=True):
FILE: backend/oasst_backend/models/cached_stats.py
class CachedStats (line 8) | class CachedStats(SQLModel, table=True):
FILE: backend/oasst_backend/models/db_payload.py
class TaskPayload (line 10) | class TaskPayload(BaseModel):
class SummarizationStoryPayload (line 15) | class SummarizationStoryPayload(TaskPayload):
class RateSummaryPayload (line 21) | class RateSummaryPayload(TaskPayload):
class InitialPromptPayload (line 29) | class InitialPromptPayload(TaskPayload):
class PrompterReplyPayload (line 35) | class PrompterReplyPayload(TaskPayload):
class AssistantReplyPayload (line 42) | class AssistantReplyPayload(TaskPayload):
class MessagePayload (line 48) | class MessagePayload(BaseModel):
class ReactionPayload (line 53) | class ReactionPayload(BaseModel):
class RatingReactionPayload (line 58) | class RatingReactionPayload(ReactionPayload):
class RankingReactionPayload (line 64) | class RankingReactionPayload(ReactionPayload):
class RankConversationRepliesPayload (line 74) | class RankConversationRepliesPayload(TaskPayload):
class RankInitialPromptsPayload (line 83) | class RankInitialPromptsPayload(TaskPayload):
class RankPrompterRepliesPayload (line 91) | class RankPrompterRepliesPayload(RankConversationRepliesPayload):
class RankAssistantRepliesPayload (line 98) | class RankAssistantRepliesPayload(RankConversationRepliesPayload):
class LabelInitialPromptPayload (line 105) | class LabelInitialPromptPayload(TaskPayload):
class LabelConversationReplyPayload (line 117) | class LabelConversationReplyPayload(TaskPayload):
class LabelPrompterReplyPayload (line 132) | class LabelPrompterReplyPayload(LabelConversationReplyPayload):
class LabelAssistantReplyPayload (line 139) | class LabelAssistantReplyPayload(LabelConversationReplyPayload):
FILE: backend/oasst_backend/models/flagged_message.py
class FlaggedMessage (line 10) | class FlaggedMessage(SQLModel, table=True):
FILE: backend/oasst_backend/models/journal.py
function generate_time_uuid (line 12) | def generate_time_uuid(node=None, clock_seq=None):
class Journal (line 26) | class Journal(SQLModel, table=True):
class JournalIntegration (line 43) | class JournalIntegration(SQLModel, table=True):
FILE: backend/oasst_backend/models/message.py
class Message (line 18) | class Message(SQLModel, table=True):
method __new__ (line 25) | def __new__(cls, *args: Any, **kwargs: Any):
method ensure_is_message (line 77) | def ensure_is_message(self) -> None:
method has_emoji (line 81) | def has_emoji(self, emoji_code: str) -> bool:
method has_user_emoji (line 84) | def has_user_emoji(self, emoji_code: str) -> bool:
method text (line 88) | def text(self) -> str:
method user_emojis (line 93) | def user_emojis(self) -> str:
method user_is_author (line 97) | def user_is_author(self) -> str:
method user (line 101) | def user(self) -> User:
FILE: backend/oasst_backend/models/message_embedding.py
class MessageEmbedding (line 10) | class MessageEmbedding(SQLModel, table=True):
FILE: backend/oasst_backend/models/message_emoji.py
class MessageEmoji (line 10) | class MessageEmoji(SQLModel, table=True):
FILE: backend/oasst_backend/models/message_reaction.py
class MessageReaction (line 12) | class MessageReaction(SQLModel, table=True):
FILE: backend/oasst_backend/models/message_revision.py
class MessageRevision (line 14) | class MessageRevision(SQLModel, table=True):
FILE: backend/oasst_backend/models/message_toxicity.py
class MessageToxicity (line 10) | class MessageToxicity(SQLModel, table=True):
FILE: backend/oasst_backend/models/message_tree_state.py
class State (line 11) | class State(str, Enum):
class MessageTreeState (line 75) | class MessageTreeState(SQLModel, table=True):
FILE: backend/oasst_backend/models/payload_column_type.py
function payload_type (line 16) | def payload_type(cls: Type[P]) -> Type[P]:
class PayloadContainer (line 21) | class PayloadContainer(BaseModel):
method __init__ (line 25) | def __init__(self, **v):
method check_payload (line 36) | def check_payload(cls, v: BaseModel, values: dict[str, Any]) -> BaseMo...
class Config (line 40) | class Config:
function payload_column_type (line 47) | def payload_column_type(pydantic_type):
FILE: backend/oasst_backend/models/task.py
class Task (line 14) | class Task(SQLModel, table=True):
method expired (line 42) | def expired(self) -> bool:
FILE: backend/oasst_backend/models/text_labels.py
class TextLabels (line 10) | class TextLabels(SQLModel, table=True):
FILE: backend/oasst_backend/models/troll_stats.py
class TrollStats (line 10) | class TrollStats(SQLModel, table=True):
method compute_troll_score (line 47) | def compute_troll_score(self) -> int:
FILE: backend/oasst_backend/models/user.py
class User (line 11) | class User(SQLModel, table=True):
method to_protocol_frontend_user (line 47) | def to_protocol_frontend_user(self):
class Account (line 65) | class Account(SQLModel, table=True):
FILE: backend/oasst_backend/models/user_stats.py
class UserStatsTimeFrame (line 11) | class UserStatsTimeFrame(str, Enum):
class UserStats (line 18) | class UserStats(SQLModel, table=True):
method compute_leader_score (line 54) | def compute_leader_score(self) -> int:
FILE: backend/oasst_backend/prompt_repository.py
class PromptRepository (line 59) | class PromptRepository:
method __init__ (line 60) | def __init__(
method ensure_user_is_enabled (line 93) | def ensure_user_is_enabled(self):
method fetch_message_by_frontend_message_id (line 107) | def fetch_message_by_frontend_message_id(self, frontend_message_id: st...
method insert_message (line 123) | def insert_message(
method insert_revision (line 167) | def insert_revision(
method _validate_task (line 183) | def _validate_task(
method fetch_tree_state (line 211) | def fetch_tree_state(self, message_tree_id: UUID) -> MessageTreeState:
method store_text_reply (line 215) | def store_text_reply(
method revise_message (line 336) | def revise_message(self, message_id: UUID, new_content: str):
method store_rating (line 366) | def store_rating(self, rating: protocol_schema.MessageRating) -> Messa...
method store_ranking (line 396) | def store_ranking(self, ranking: protocol_schema.MessageRanking) -> tu...
method insert_toxicity (line 468) | def insert_toxicity(self, message_id: UUID, model: str, score: float, ...
method insert_message_embedding (line 486) | def insert_message_embedding(self, message_id: UUID, model: str, embed...
method insert_reaction (line 506) | def insert_reaction(
method store_text_labels (line 524) | def store_text_labels(self, text_labels: protocol_schema.TextLabels) -...
method fetch_random_message_tree (line 640) | def fetch_random_message_tree(
method fetch_random_conversation (line 664) | def fetch_random_conversation(
method fetch_random_initial_prompts (line 706) | def fetch_random_initial_prompts(self, size: int = 5):
method fetch_message_tree (line 710) | def fetch_message_tree(
method check_users_recent_replies_for_duplicates (line 723) | def check_users_recent_replies_for_duplicates(self, text: str) -> bool:
method fetch_user_message_trees (line 747) | def fetch_user_message_trees(
method fetch_multiple_random_replies (line 757) | def fetch_multiple_random_replies(self, max_size: int = 5, message_rol...
method fetch_message (line 791) | def fetch_message(self, message_id: UUID, fail_if_missing: bool = True...
method fetch_non_task_text_labels (line 801) | def fetch_non_task_text_labels(self, message_id: UUID, user_id: UUID) ...
method fetch_message_text_labels (line 810) | def fetch_message_text_labels(self, message_id: UUID, user_id: Optiona...
method fetch_message_revision_history (line 816) | def fetch_message_revision_history(self, message_id: UUID) -> list[Mes...
method trace_conversation (line 827) | def trace_conversation(messages: list[Message] | dict[UUID, Message], ...
method fetch_message_conversation (line 855) | def fetch_message_conversation(self, message: Message | UUID) -> list[...
method fetch_tree_from_message (line 865) | def fetch_tree_from_message(
method fetch_message_children (line 879) | def fetch_message_children(
method fetch_message_siblings (line 899) | def fetch_message_siblings(
method trace_descendants (line 923) | def trace_descendants(root: Message, messages: list[Message]) -> list[...
method fetch_message_descendants (line 935) | def fetch_message_descendants(self, message: Message | UUID, max_depth...
method fetch_longest_conversation (line 954) | def fetch_longest_conversation(self, message: Message | UUID) -> list[...
method fetch_message_with_max_children (line 959) | def fetch_message_with_max_children(self, message: Message | UUID) -> ...
method _add_user_emojis_all (line 964) | def _add_user_emojis_all(self, qry: Query, include_user: bool = False)...
method query_messages_ordered_by_created_date (line 1010) | def query_messages_ordered_by_created_date(
method update_children_counts (line 1111) | def update_children_counts(self, message_tree_id: UUID):
method mark_messages_deleted (line 1126) | def mark_messages_deleted(self, messages: Message | UUID | list[Messag...
method undelete_deleted_message (line 1155) | def undelete_deleted_message(self, message: Message | UUID):
method get_stats (line 1185) | def get_stats(self) -> SystemStats:
method skip_task (line 1223) | def skip_task(self, task_id: UUID, reason: Optional[str]):
method handle_message_emoji (line 1244) | def handle_message_emoji(
method fetch_flagged_messages (line 1341) | def fetch_flagged_messages(self, max_count: Optional[int]) -> list[Fla...
method fetch_flagged_messages_by_created_date (line 1348) | def fetch_flagged_messages_by_created_date(
method process_flagged_message (line 1395) | def process_flagged_message(self, message_id: UUID) -> FlaggedMessage:
FILE: backend/oasst_backend/scheduled_tasks.py
function useHFApi (line 20) | async def useHFApi(text, url, model_name):
function toxicity (line 27) | def toxicity(text, message_id, api_client):
function hf_feature_extraction (line 50) | def hf_feature_extraction(text, message_id, api_client):
function update_search_vectors (line 70) | def update_search_vectors(batch_size: int) -> None:
function periodic_user_streak_reset (line 94) | def periodic_user_streak_reset() -> None:
FILE: backend/oasst_backend/schemas/hugging_face.py
class ToxicityClassification (line 4) | class ToxicityClassification(BaseModel):
FILE: backend/oasst_backend/schemas/message_tree.py
class MessageTreeStateResponse (line 7) | class MessageTreeStateResponse(BaseModel):
FILE: backend/oasst_backend/schemas/text_labels.py
class ValidLabelsResponse (line 5) | class ValidLabelsResponse(BaseModel):
FILE: backend/oasst_backend/task_repository.py
function validate_frontend_message_id (line 19) | def validate_frontend_message_id(message_id: str) -> None:
function delete_expired_tasks (line 29) | def delete_expired_tasks(session: Session) -> int:
class TaskRepository (line 36) | class TaskRepository:
method __init__ (line 37) | def __init__(
method store_task (line 50) | def store_task(
method bind_frontend_message_id (line 149) | def bind_frontend_message_id(self, task_id: UUID, frontend_message_id:...
method close_task (line 171) | def close_task(self, frontend_message_id: str, allow_personal_tasks: b...
method insert_task (line 193) | def insert_task(
method fetch_task_by_frontend_message_id (line 218) | def fetch_task_by_frontend_message_id(self, message_id: str) -> Task:
method fetch_task_by_id (line 227) | def fetch_task_by_id(self, task_id: UUID) -> Task:
method fetch_recent_reply_tasks (line 231) | def fetch_recent_reply_tasks(
method delete_expired (line 250) | def delete_expired(self) -> int:
method fetch_pending_tasks_of_user (line 253) | def fetch_pending_tasks_of_user(
FILE: backend/oasst_backend/tree_manager.py
class TaskType (line 42) | class TaskType(Enum):
class TaskRole (line 51) | class TaskRole(Enum):
class TreeStateStats (line 57) | class TreeStateStats(pydantic.BaseModel):
class ActiveTreeSizeRow (line 70) | class ActiveTreeSizeRow(pydantic.BaseModel):
method remaining_messages (line 77) | def remaining_messages(self) -> int:
class Config (line 80) | class Config:
class ExtendibleParentRow (line 84) | class ExtendibleParentRow(pydantic.BaseModel):
class Config (line 91) | class Config:
class IncompleteRankingsRow (line 95) | class IncompleteRankingsRow(pydantic.BaseModel):
class Config (line 102) | class Config:
class TreeMessageCountStats (line 106) | class TreeMessageCountStats(pydantic.BaseModel):
method completed (line 116) | def completed(self) -> int:
class TreeManagerStats (line 120) | class TreeManagerStats(pydantic.BaseModel):
function halt_prompts_of_disabled_users (line 125) | def halt_prompts_of_disabled_users(db: Session):
class TreeManager (line 152) | class TreeManager:
method __init__ (line 153) | def __init__(
method _random_task_selection (line 163) | def _random_task_selection(
method _determine_task_availability_internal (line 209) | def _determine_task_availability_internal(
method _prompt_lottery (line 251) | def _prompt_lottery(self, lang: str, max_activate: int = 1) -> int:
method _auto_moderation (line 348) | def _auto_moderation(self, lang: str) -> None:
method determine_task_availability (line 373) | def determine_task_availability(self, lang: str) -> dict[protocol_sche...
method _get_label_descriptions (line 400) | def _get_label_descriptions(valid_labels: list[TextLabels]) -> list[pr...
method next_task (line 408) | def next_task(
method handle_interaction (line 724) | async def handle_interaction(self, interaction: protocol_schema.AnyInt...
method _enter_state (line 828) | def _enter_state(self, mts: MessageTreeState, state: message_tree_stat...
method enter_low_grade_state (line 854) | def enter_low_grade_state(self, message_tree_id: UUID) -> None:
method check_condition_for_prompt_lottery (line 859) | def check_condition_for_prompt_lottery(self, message_tree_id: UUID) ->...
method check_condition_for_ranking_state (line 876) | def check_condition_for_ranking_state(self, message_tree_id: UUID) -> ...
method check_condition_for_scoring_state (line 900) | def check_condition_for_scoring_state(self, message_tree_id: UUID) -> ...
method ranked_pairs_update (line 927) | def ranked_pairs_update(self, rankings: list[MessageReaction]) -> int:
method update_message_ranks (line 973) | def update_message_ranks(
method activate_backlog_tree (line 1000) | def activate_backlog_tree(self, lang: str) -> MessageTreeState:
method _calculate_acceptance (line 1026) | def _calculate_acceptance(self, labels: list[TextLabels]):
method _query_need_review (line 1034) | def _query_need_review(
method query_prompts_need_review (line 1086) | def query_prompts_need_review(self, lang: str) -> list[Message]:
method query_replies_need_review (line 1095) | def query_replies_need_review(self, lang: str) -> list[Message]:
method query_incomplete_rankings (line 1138) | def query_incomplete_rankings(self, lang: str, user_filter: bool = Tru...
method query_extendible_parents (line 1200) | def query_extendible_parents(self, lang: str) -> tuple[list[Extendible...
method query_extendible_trees (line 1241) | def query_extendible_trees(self, lang: str) -> list[ActiveTreeSizeRow]:
method query_tree_size (line 1260) | def query_tree_size(self, message_tree_id: UUID) -> ActiveTreeSizeRow:
method query_misssing_tree_states (line 1289) | def query_misssing_tree_states(self) -> list[Tuple[UUID, str]]:
method query_tree_ranking_results (line 1323) | def query_tree_ranking_results(
method ensure_tree_states (line 1350) | def ensure_tree_states(self) -> None:
method query_num_growing_trees (line 1403) | def query_num_growing_trees(self, lang: str) -> int:
method query_prompt_lottery_waiting (line 1416) | def query_prompt_lottery_waiting(self, lang: str) -> int:
method query_num_active_trees (line 1422) | def query_num_active_trees(
method query_reviews_for_message (line 1437) | def query_reviews_for_message(self, message_id: UUID) -> list[TextLabe...
method query_moderation_bad_messages (line 1446) | def query_moderation_bad_messages(self, lang: str) -> list[Message]:
method _insert_tree_state (line 1478) | def _insert_tree_state(
method _insert_default_state (line 1502) | def _insert_default_state(
method tree_counts_by_state (line 1525) | def tree_counts_by_state(self, lang: str = None, only_active: bool = F...
method tree_counts_by_state_stats (line 1540) | def tree_counts_by_state_stats(self, lang: str = None, only_active: bo...
method tree_message_count_stats (line 1556) | def tree_message_count_stats(self, only_active: bool = True) -> list[T...
method stats (line 1578) | def stats(self) -> TreeManagerStats:
method get_user_messages_by_tree (line 1584) | def get_user_messages_by_tree(
method _purge_message_internal (line 1619) | def _purge_message_internal(self, message_id: UUID) -> None:
method purge_message_tree (line 1664) | def purge_message_tree(self, message_tree_id: UUID) -> None:
method _reactivate_tree (line 1678) | def _reactivate_tree(self, mts: MessageTreeState):
method purge_user_messages (line 1692) | def purge_user_messages(
method purge_user (line 1752) | def purge_user(self, user_id: UUID, ban: bool = True) -> None:
method retry_scoring_failed_message_trees (line 1772) | def retry_scoring_failed_message_trees(self):
method halt_tree (line 1786) | def halt_tree(self, message_id: UUID, halt: bool = True) -> MessageTre...
FILE: backend/oasst_backend/user_repository.py
class UserRepository (line 17) | class UserRepository:
method __init__ (line 18) | def __init__(self, db: Session, api_client: ApiClient):
method get_user (line 22) | def get_user(self, id: UUID, api_client_id: Optional[UUID] = None) -> ...
method query_frontend_user (line 50) | def query_frontend_user(
method update_user (line 72) | def update_user(
method mark_user_deleted (line 110) | def mark_user_deleted(self, id: UUID) -> None:
method _lookup_user_tx (line 137) | def _lookup_user_tx(
method lookup_client_user (line 174) | def lookup_client_user(self, client_user: protocol_schema.User, create...
method lookup_system_user (line 196) | def lookup_system_user(self, username: str, create_missing: bool = Tru...
method query_users_ordered_by_username (line 204) | def query_users_ordered_by_username(
method query_users_ordered_by_display_name (line 264) | def query_users_ordered_by_display_name(
method update_user_last_activity (line 336) | def update_user_last_activity(self, user: User, update_streak: bool = ...
method merge_users (line 352) | def merge_users(self, destination_user_id: UUID, source_user_ids: list...
FILE: backend/oasst_backend/user_stats_repository.py
function get_thresholds (line 42) | def get_thresholds(baseline: int = 3, alpha: float = 1.1521, max_level: ...
function _create_user_score (line 51) | def _create_user_score(r, highlighted_user_id: UUID | None) -> UserScore:
function _create_troll_score (line 74) | def _create_troll_score(r, highlighted_user_id: UUID | None) -> TrollScore:
class UserStatsRepository (line 95) | class UserStatsRepository:
method __init__ (line 96) | def __init__(self, session: Session):
method get_leaderboard (line 99) | def get_leaderboard(
method get_leaderboard_user_window (line 133) | def get_leaderboard_user_window(
method get_user_stats_all_time_frames (line 175) | def get_user_stats_all_time_frames(self, user_id: UUID) -> dict[str, U...
method get_trollboard (line 200) | def get_trollboard(
method query_total_prompts_per_user (line 239) | def query_total_prompts_per_user(
method query_replies_by_role_per_user (line 252) | def query_replies_by_role_per_user(
method query_labels_by_mode_per_user (line 265) | def query_labels_by_mode_per_user(
method query_rankings_per_user (line 276) | def query_rankings_per_user(self, reference_time: Optional[datetime] =...
method query_ranking_result_users (line 285) | def query_ranking_result_users(self, rank: int = 0, reference_time: Op...
method _update_stats_internal (line 300) | def _update_stats_internal(self, time_frame: UserStatsTimeFrame, base_...
method query_message_emoji_counts_per_user (line 411) | def query_message_emoji_counts_per_user(self, reference_time: Optional...
method query_spam_prompts_per_user (line 425) | def query_spam_prompts_per_user(self, reference_time: Optional[datetim...
method query_labels_per_user (line 439) | def query_labels_per_user(self, reference_time: Optional[datetime] = N...
method _update_troll_stats_internal (line 475) | def _update_troll_stats_internal(self, time_frame: UserStatsTimeFrame,...
method update_leader_ranks (line 547) | def update_leader_ranks(self, time_frame: UserStatsTimeFrame = None):
method update_troll_ranks (line 584) | def update_troll_ranks(self, time_frame: UserStatsTimeFrame = None):
method update_stats_time_frame (line 605) | def update_stats_time_frame(
method update_stats (line 619) | def update_stats(self, *, time_frame: UserStatsTimeFrame):
method update_multiple_time_frames (line 640) | def update_multiple_time_frames(self, time_frames: list[UserStatsTimeF...
method update_all_time_frames (line 645) | def update_all_time_frames(self):
FILE: backend/oasst_backend/utils/database_utils.py
function db_lang_to_postgres_ts_lang (line 50) | def db_lang_to_postgres_ts_lang(db_lang: str) -> str:
class CommitMode (line 55) | class CommitMode(IntEnum):
function managed_tx_method (line 73) | def managed_tx_method(auto_commit: CommitMode = CommitMode.COMMIT, num_r...
function async_managed_tx_method (line 125) | def async_managed_tx_method(
function default_session_factory (line 179) | def default_session_factory() -> Session:
function managed_tx_function (line 183) | def managed_tx_function(
function async_managed_tx_function (line 244) | def async_managed_tx_function(
FILE: backend/oasst_backend/utils/discord.py
function send_new_report_message (line 12) | def send_new_report_message(message_details: dict, label_text: str, user...
FILE: backend/oasst_backend/utils/exported_tree_loading.py
function load_jsonl (line 8) | def load_jsonl(filepaths):
function separate_qa_helper (line 17) | def separate_qa_helper(node, depth, msg_dict):
function store_qa_data_separate (line 29) | def store_qa_data_separate(trees, data):
function group_qa_helper (line 39) | def group_qa_helper(node, depth, msg_pairs):
function store_qa_data_paired (line 52) | def store_qa_data_paired(trees, data: List):
function load_data (line 62) | def load_data(filepaths: List[str], paired=False):
FILE: backend/oasst_backend/utils/hugging_face.py
class HfUrl (line 10) | class HfUrl(str, Enum):
class HfClassificationModel (line 15) | class HfClassificationModel(str, Enum):
class HfEmbeddingModel (line 19) | class HfEmbeddingModel(str, Enum):
class HuggingFaceAPI (line 23) | class HuggingFaceAPI:
method __init__ (line 26) | def __init__(
method post (line 39) | async def post(self, input: str, wait_for_model: bool = True) -> Any:
FILE: backend/oasst_backend/utils/language_classification.py
function load_and_split (line 12) | def load_and_split(foldername, num_words):
function build_and_train_pipeline (line 29) | def build_and_train_pipeline(x_train, y_train):
function benchmark (line 45) | def benchmark(clf, x_test, y_test, langmap):
function main (line 56) | def main(foldername, modelname, num_words):
function load (line 75) | def load(modelname):
function save_model (line 81) | def save_model(model, idx_to_name, num_words, modelname):
function inference_voter (line 91) | def inference_voter(model, text):
FILE: backend/oasst_backend/utils/message_tree_topic_modeling.py
function argument_parsing (line 12) | def argument_parsing():
function load_topic_model (line 31) | def load_topic_model(args):
function fit_topic_model (line 47) | def fit_topic_model(topic_model, data, embeddings, key="query"):
function get_topic_info (line 52) | def get_topic_info(topic_model):
function reduce_topics (line 56) | def reduce_topics(topic_model, data, nr_topics, key="query"):
function get_representative_docs (line 61) | def get_representative_docs(topic_model):
function reduce_outliers (line 65) | def reduce_outliers(topic_model, data, topics, probs, key="query", strat...
function compute_hierarchical_topic_tree (line 77) | def compute_hierarchical_topic_tree(topic_model, data, key="query"):
FILE: backend/oasst_backend/utils/ranking.py
function head_to_head_votes (line 6) | def head_to_head_votes(ranks: List[List[int]]):
function cycle_detect (line 24) | def cycle_detect(pairs):
function get_winner (line 54) | def get_winner(pairs):
function get_ranking (line 67) | def get_ranking(pairs):
function ranked_pairs (line 82) | def ranked_pairs(ranks: List[List[int]]):
FILE: backend/oasst_backend/utils/similarity_functions.py
function embed_data (line 15) | def embed_data(
function cos_sim (line 60) | def cos_sim(a: Tensor, b: Tensor):
function cos_sim_torch (line 82) | def cos_sim_torch(embs_a: Tensor, embs_b: Tensor) -> Tensor:
function gaussian_kernel_torch (line 103) | def gaussian_kernel_torch(embs_a, embs_b, sigma=1.0):
function compute_cos_sim_kernel (line 126) | def compute_cos_sim_kernel(embs, threshold=0.65, kernel_type="cosine"):
function k_hop_message_passing (line 139) | def k_hop_message_passing(A, node_features, k):
function k_hop_message_passing_sparse (line 165) | def k_hop_message_passing_sparse(A, node_features, k):
FILE: backend/oasst_backend/utils/tree_export.py
function prepare_export_message_node (line 26) | def prepare_export_message_node(
function build_export_tree (line 85) | def build_export_tree(
function smart_open (line 117) | def smart_open(filename: str = None) -> TextIO:
function write_trees_to_file (line 130) | def write_trees_to_file(filename: str | None, trees: list[ExportMessageT...
function write_messages_to_file (line 147) | def write_messages_to_file(
FILE: backend/rerank.py
function update_tree_ranking (line 19) | def update_tree_ranking(tm: TreeManager, message_tree_id: UUID) -> int:
function parse_args (line 32) | def parse_args():
function main (line 39) | def main():
FILE: backend/tests/test_settings.py
function test_create_default_settings (line 4) | def test_create_default_settings():
function test_construct_db_uri_from_dict (line 11) | def test_construct_db_uri_from_dict():
function test_connection_string (line 27) | def test_connection_string():
function test_task_expiry_time (line 37) | def test_task_expiry_time():
FILE: backend/tests/test_tree_manager_config.py
function test_tree_manager_config (line 4) | def test_tree_manager_config():
FILE: backend/update_message_attributes.py
function get_messageids_without_toxicity (line 10) | def get_messageids_without_toxicity():
function get_messageids_without_embedding (line 28) | def get_messageids_without_embedding():
function find_and_update_embeddings (line 46) | def find_and_update_embeddings(message_ids):
function find_and_update_toxicity (line 64) | def find_and_update_toxicity(message_ids):
function main (line 82) | def main():
FILE: data/datasets/TSSB-3M/generate_dataset.py
function gen_instruction (line 50) | def gen_instruction():
function gen_response_prefix (line 55) | def gen_response_prefix():
function remove_starting_plus_minus (line 103) | def remove_starting_plus_minus(text):
function remove_extraneous_diff_info (line 110) | def remove_extraneous_diff_info(text):
function clean (line 115) | def clean(text):
function clean_PII (line 119) | def clean_PII(text):
function is_invaid_commit_msg (line 135) | def is_invaid_commit_msg(text):
function clean_commit_msg (line 140) | def clean_commit_msg(text):
function create (line 174) | def create(input_file, output_file, output_json=True):
FILE: data/datasets/bart_searchgpt_wiki_nlp_augment/1_clean_wikitext.py
function memory_and_speed_test (line 12) | def memory_and_speed_test():
function remove_empty_lines (line 30) | def remove_empty_lines(article: str) -> str:
function extract_main_content (line 34) | def extract_main_content(article: str) -> (str, int):
function remove_all_parentesis (line 47) | def remove_all_parentesis(article: str) -> str:
FILE: data/datasets/bart_searchgpt_wiki_nlp_augment/3_10k_bart_trial.py
function num_tokens_from_string (line 8) | def num_tokens_from_string(string: str) -> int:
FILE: data/datasets/biostars_qa/get_biostars_dataset.py
function get_biostars_dataset (line 11) | def get_biostars_dataset(start_idx=9557161, accept_threshold=1000000, sl...
function extract_accepted_data (line 69) | def extract_accepted_data(folder="biostars", merged_json_file=None):
FILE: data/datasets/logicreference_OA/generate_dataset.py
function create_string_feature (line 40) | def create_string_feature(values):
function generate_t5_split (line 57) | def generate_t5_split(path, file_name, examples):
function main (line 67) | def main(_):
FILE: data/datasets/mt_note_generation/hub.py
class OpenAssistantConfig (line 7) | class OpenAssistantConfig(datasets.BuilderConfig):
FILE: data/datasets/mt_note_generation/mt_note_generation.py
class MTNoteGenerationDataset (line 57) | class MTNoteGenerationDataset(datasets.GeneratorBasedBuilder):
method _info (line 74) | def _info(self) -> datasets.DatasetInfo:
method _split_generators (line 83) | def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]:
method _generate_examples (line 111) | def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]:
FILE: data/datasets/mt_note_generation/prepare.py
function preprocess (line 16) | def preprocess(mt_dataset):
function is_chief_complaint (line 28) | def is_chief_complaint(section):
function get_conversations (line 32) | def get_conversations(dataset):
function main (line 62) | def main(output_dir: str = "data"):
FILE: data/datasets/nsfw_selfharm_reddit/utils/__init__.py
function save_to_huggingface (line 9) | def save_to_huggingface(df, name):
FILE: data/datasets/nsfw_selfharm_reddit/utils/is_question.py
class IsQuestion (line 4) | class IsQuestion:
method __init__ (line 5) | def __init__(self):
method __call__ (line 14) | def __call__(self, text: str) -> bool:
FILE: data/datasets/nsfw_selfharm_reddit/utils/reddit.py
function init_praw_reddit (line 13) | def init_praw_reddit(client_id: str | None = None, client_secret: str | ...
function scrap_subreddit (line 28) | def scrap_subreddit(subreddit: str, reddit) -> pd.DataFrame | None:
function get_comments (line 63) | def get_comments(post_ids: list, reddit: praw.Reddit):
FILE: data/datasets/oa_dolly_15k/create_dataset.py
function download_data (line 11) | def download_data(url: str, destination: str):
function build_dataset (line 19) | def build_dataset(data_file: str, include_context: bool = True) -> Dataset:
function to_oa_format (line 29) | def to_oa_format(data: dict, include_context: bool = True) -> dict:
function main (line 45) | def main():
FILE: data/datasets/oa_stackexchange/download.py
function get_all_filenames (line 24) | def get_all_filenames():
function download_url (line 45) | def download_url(dataset_name: str, url: str):
function download_all (line 60) | def download_all():
FILE: data/datasets/oa_stackexchange/merge_parquets.py
function stream_to_parquet (line 10) | def stream_to_parquet(path: Path, tables: Iterable[pa.Table]) -> None:
function stream_from_parquet (line 23) | def stream_from_parquet(path: Path) -> Iterable[pa.Table]:
function stream_from_parquets (line 29) | def stream_from_parquets(paths: Iterable[Path]) -> Iterable[pa.Table]:
function coalesce (line 37) | def coalesce(items: Iterable[T], max_size: int, sizer: Callable[[T], int...
function coalesce_parquets (line 52) | def coalesce_parquets(paths: Iterable[Path], outpath: Path, max_size: in...
function merge_parquet_dir (line 62) | def merge_parquet_dir(path: str, outpath: Path) -> None:
FILE: data/datasets/oa_stackexchange/process.py
function main (line 26) | def main():
function list_cached_datasets (line 32) | def list_cached_datasets():
function process_dataset (line 39) | def process_dataset(dataset):
function parse_and_convert (line 52) | def parse_and_convert(path: str, source: str):
function parse_row (line 88) | def parse_row(element):
function convert_to_oa (line 101) | def convert_to_oa(all, source):
function convert_tags (line 127) | def convert_tags(raw):
function create_metadata (line 131) | def create_metadata(row):
function save_parquet (line 139) | def save_parquet(df, dataset):
function remove_emojis (line 154) | def remove_emojis(string):
function to_markdown (line 170) | def to_markdown(text):
FILE: data/datasets/oa_stackexchange/stats.py
function rows (line 7) | def rows(topic):
FILE: data/datasets/oa_stackexchange/upload.py
function upload_hf (line 11) | def upload_hf():
FILE: data/datasets/reasoning_bg_oa/data_process.py
class QnA (line 12) | class QnA:
function create_qna (line 20) | def create_qna(row):
FILE: data/datasets/reasoning_gsm_qna_oa/data_process.py
function qna_wrapper (line 30) | def qna_wrapper(source, random_list_python, random_list_answer):
class QnA (line 49) | class QnA:
FILE: data/datasets/semantics_ws_qna_oa/data_process.py
function qna_wrapper (line 13) | def qna_wrapper():
class QnA (line 41) | class QnA:
FILE: data/datasets/semantics_ws_qna_oa/random_stuff.py
function qna_random_magic (line 411) | def qna_random_magic(lang, word1, word2, con_type, score_percent, random...
FILE: data/datasets/soda_synthetic_dialogue/hub.py
class OpenAssistantConfig (line 7) | class OpenAssistantConfig(datasets.BuilderConfig):
FILE: data/datasets/soda_synthetic_dialogue/prepare.py
function main (line 77) | def main(output_dir: str = "data"):
FILE: data/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py
class SODASyntheticDialogueDataset (line 49) | class SODASyntheticDialogueDataset(datasets.GeneratorBasedBuilder):
method _info (line 66) | def _info(self) -> datasets.DatasetInfo:
method _split_generators (line 75) | def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]:
method _generate_examples (line 96) | def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]:
FILE: data/datasets/tatoeba_mt_qna_oa/data_process.py
class DataProcess (line 16) | class DataProcess:
method randomize_text (line 21) | def randomize_text(self, text, original_lang=None, target_lang=None):
method convert_code (line 41) | def convert_code(self, code):
method language_name (line 46) | def language_name(self, lang1, lang2):
class QnA (line 75) | class QnA:
function create_qna (line 83) | def create_qna(row):
FILE: data/datasets/youtube_subs_howto100M/hub.py
class OpenAssistantConfig (line 7) | class OpenAssistantConfig(datasets.BuilderConfig):
FILE: data/datasets/youtube_subs_howto100M/prepare.py
function get_video_ids (line 18) | def get_video_ids(raw_file: str, video_id_pattern: str) -> List[str]:
function get_title (line 35) | def get_title(video_id):
function generate_instruction (line 50) | def generate_instruction(title: str) -> str:
function get_subs (line 57) | def get_subs(video_id, languages=["en"]):
function main (line 69) | def main(output_dir: str = "data"):
FILE: data/datasets/youtube_subs_howto100M/youtube_subs_howto100M.py
class YouTubeSubsHowTo100MDataset (line 53) | class YouTubeSubsHowTo100MDataset(datasets.GeneratorBasedBuilder):
method _info (line 70) | def _info(self) -> datasets.DatasetInfo:
method _split_generators (line 79) | def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]:
method _generate_examples (line 98) | def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]:
FILE: data/datasets/zhihu-kol/convert_parquet.py
function reformat_csv_to_openassistant (line 6) | def reformat_csv_to_openassistant(df: pd.DataFrame) -> pd.DataFrame:
FILE: data/datasets/zhihu-kol/main.py
function get_uid_by_url_token (line 13) | def get_uid_by_url_token(url_token: str) -> str:
function get_user_answers (line 50) | def get_user_answers(url_token: str, max_count: int = 100000) -> pd.Data...
function get_answer_content (line 126) | def get_answer_content(qid: str, aid) -> str:
function reformat_csv_to_openassistant (line 158) | def reformat_csv_to_openassistant(df: pd.DataFrame) -> pd.DataFrame:
function save_answers_to_csv (line 189) | def save_answers_to_csv(url_token: str, csv_path: str, max_count: int = ...
FILE: data/datasets/zhihu-kol/scrape_by_topic.py
class Content_Data (line 19) | class Content_Data:
function get_answer_content (line 29) | def get_answer_content(qid: int, aid: int, question_str: str) -> str:
function get_all_href (line 90) | def get_all_href(page: Union[Page, Locator]) -> List[str]:
function scrape_people_roundtable (line 110) | def scrape_people_roundtable():
function end_to_end_auto_scrape (line 147) | def end_to_end_auto_scrape():
FILE: discord-bots/oa-bot-js/src/commands/bot.ts
method execute (line 16) | async execute(interaction, client, commands, commandType) {
FILE: discord-bots/oa-bot-js/src/commands/chat.ts
method execute (line 61) | async execute(interaction, client, commands, commandType, options) {
FILE: discord-bots/oa-bot-js/src/commands/task.ts
method execute (line 13) | async execute(interaction, client, commands, commandType) {
FILE: discord-bots/oa-bot-js/src/events/commands.ts
method execute (line 20) | async execute(interaction, client) {
FILE: discord-bots/oa-bot-js/src/events/interactions.ts
method execute (line 6) | async execute(interaction, client) {
FILE: discord-bots/oa-bot-js/src/events/messages.ts
method execute (line 34) | async execute(message, client) {
FILE: discord-bots/oa-bot-js/src/events/ready.ts
method execute (line 7) | async execute(client) {
FILE: discord-bots/oa-bot-js/src/handlers/commands.ts
function commandHandler (line 11) | async function commandHandler(client) {
FILE: discord-bots/oa-bot-js/src/handlers/events.ts
function eventHandler (line 8) | async function eventHandler(client: Client) {
FILE: discord-bots/oa-bot-js/src/handlers/interactions.ts
function interactionsHandler (line 8) | async function interactionsHandler(client) {
FILE: discord-bots/oa-bot-js/src/interactions/model.ts
method execute (line 15) | async execute(interaction, client, userId) {
FILE: discord-bots/oa-bot-js/src/interactions/modelselect.ts
method execute (line 15) | async execute(interaction, client) {
FILE: discord-bots/oa-bot-js/src/interactions/tasks.ts
method execute (line 41) | async execute(
FILE: discord-bots/oa-bot-js/src/interactions/vote.ts
method execute (line 15) | async execute(interaction, client, messageId, voteType) {
FILE: discord-bots/oa-bot-js/src/modules/chat.ts
function chat (line 5) | async function chat(
function huggingface (line 45) | async function huggingface(model, input) {
FILE: discord-bots/oa-bot-js/src/modules/db.ts
class Database (line 8) | class Database {
method constructor (line 9) | constructor() {
method get (line 14) | get(table, key) {
method set (line 18) | set(table, key, value) {
method delete (line 23) | delete(key) {
FILE: discord-bots/oa-bot-js/src/modules/inference/client.ts
class OasstInferenceClient (line 12) | class OasstInferenceClient {
method constructor (line 14) | constructor(private readonly clientToken: string) {}
method request (line 16) | async request<T = unknown>(path: string, init?: AxiosRequestConfig) {
method inference_login (line 30) | inference_login() {
method get_my_chats (line 34) | get_my_chats(): Promise<ChatItem[]> {
method create_chat (line 38) | async create_chat(): Promise<ChatItem> {
method get_chat (line 61) | get_chat(chat_id: string): Promise<ChatItem> {
method get_message (line 65) | get_message(chat_id: string, message_id: string): Promise<InferenceMes...
method post_prompter_message (line 69) | post_prompter_message({
method post_assistant_message (line 79) | post_assistant_message({
method stream_events (line 89) | stream_events({
method vote (line 109) | vote({
method get_models (line 124) | get_models() {
FILE: discord-bots/oa-bot-js/src/modules/inference/types.ts
type TrustedClient (line 1) | interface TrustedClient {
type InferenceToken (line 9) | interface InferenceToken {
type InferenceTokens (line 14) | interface InferenceTokens {
type ChatItem (line 19) | interface ChatItem {
type InferenceMessage (line 29) | interface InferenceMessage {
type GetChatsResponse (line 47) | interface GetChatsResponse {
type InferenceEventMessage (line 52) | interface InferenceEventMessage {
type InferenceEventError (line 56) | interface InferenceEventError {
type InferenceEventToken (line 62) | interface InferenceEventToken {
type InferenceEventPending (line 67) | interface InferenceEventPending {
type InferenceEvent (line 73) | type InferenceEvent =
type ModelInfo (line 79) | type ModelInfo = {
type ModelParameterConfig (line 85) | type ModelParameterConfig = {
type SamplingParameters (line 91) | interface SamplingParameters {
type ChatConfigForm (line 100) | interface ChatConfigForm extends SamplingParameters {
type InferencePostPrompterMessageParams (line 104) | interface InferencePostPrompterMessageParams {
type InferencePostAssistantMessageParams (line 110) | interface InferencePostAssistantMessageParams {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/interactions/info.ts
function infoInteraction (line 10) | async function infoInteraction(translation, interaction, lang) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/interactions/init.ts
function initInteraction (line 9) | async function initInteraction(interaction, translation, lang) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/interactions/label.ts
function labelInteraction (line 18) | async function labelInteraction(
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/interactions/lang.ts
function langInteraction (line 8) | async function langInteraction(interaction) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/interactions/task.ts
function taskInteraction (line 14) | async function taskInteraction(
function sendErr (line 136) | async function sendErr(err: string) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/labels.ts
function getLabel (line 1) | async function getLabel(translation, previousTask: string, task) {
function getFlags (line 41) | async function getFlags(translation, task) {
function labelText (line 66) | function labelText(label, translation) {
function getLabels (line 114) | async function getLabels(task) {
function formatLabel (line 151) | function formatLabel(label: string, type1?: boolean) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/langs.ts
function getTranlation (line 1) | async function getTranlation(lang: string) {
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/tasks.ts
function saveTask (line 11) | async function saveTask(task, lang, user, answer) {
function formatTaskType (line 26) | function formatTaskType(type: string) {
function submitTask (line 38) | async function submitTask(
FILE: discord-bots/oa-bot-js/src/modules/open-assistant/user.ts
function getUserLang (line 5) | async function getUserLang(userId: string) {
function setUserLang (line 14) | async function setUserLang(userId: string, lang: string) {
FILE: discord-bots/oa-bot-py/bot/bot.py
function on_starting (line 27) | async def on_starting(event: hikari.StartingEvent):
function on_stopping (line 48) | async def on_stopping(event: hikari.StoppingEvent):
function _send_error_embed (line 54) | async def _send_error_embed(
function on_error (line 69) | async def on_error(event: lightbulb.CommandErrorEvent) -> None:
FILE: discord-bots/oa-bot-py/bot/db/schema.sql
type guild_settings (line 2) | CREATE TABLE IF NOT EXISTS guild_settings (
FILE: discord-bots/oa-bot-py/bot/db/schemas.py
class GuildSettings (line 8) | class GuildSettings(BaseModel):
method parse_obj (line 15) | def parse_obj(cls, obj: Row) -> "GuildSettings":
method from_db (line 20) | async def from_db(cls, conn: Connection, guild_id: int) -> t.Optional[...
FILE: discord-bots/oa-bot-py/bot/extensions/guild_settings.py
function settings (line 18) | async def settings(_: lightbulb.SlashContext) -> None:
function get (line 27) | async def get(ctx: lightbulb.SlashContext) -> None:
function log_channel (line 60) | async def log_channel(ctx: lightbulb.SlashContext) -> None:
function load (line 97) | def load(bot: lightbulb.BotApp):
function unload (line 102) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/extensions/help.py
function help_command (line 16) | async def help_command(ctx: lightbulb.Context) -> None:
function tutorial (line 31) | async def tutorial(ctx: lightbulb.Context) -> None:
function load (line 36) | def load(bot: lightbulb.BotApp):
function unload (line 41) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/extensions/hot_reload.py
function _get_extensions (line 16) | def _get_extensions() -> list[str]:
function _plugin_autocomplete (line 23) | async def _plugin_autocomplete(option: hikari.CommandInteractionOption, ...
function reload (line 42) | async def reload(ctx: lightbulb.SlashContext):
function load (line 56) | def load(bot: lightbulb.BotApp):
function unload (line 61) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/extensions/text_labels.py
function clamp (line 21) | def clamp(num: float) -> float:
class LabelModal (line 26) | class LabelModal(miru.Modal):
method __init__ (line 29) | def __init__(self, label: str, content: str, *args: t.Any, **kwargs: t...
method callback (line 42) | async def callback(self, context: miru.ModalContext) -> None:
class LabelSelect (line 79) | class LabelSelect(miru.View):
method __init__ (line 90) | def __init__(self, content: str, *args: t.Any, **kwargs: t.Any):
method label_select (line 135) | async def label_select(self, select: miru.Select, ctx: miru.ViewContex...
function label_message_text (line 148) | async def label_message_text(ctx: lightbulb.MessageContext):
function load (line 172) | def load(bot: lightbulb.BotApp):
function unload (line 177) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/extensions/user_input_test.py
function task_thread (line 31) | async def task_thread(ctx: lightbulb.SlashContext):
function task_dm (line 75) | async def task_dm(ctx: lightbulb.Context):
class TaskModal (line 101) | class TaskModal(miru.Modal):
method callback (line 112) | async def callback(self, context: miru.ModalContext) -> None:
class ModalView (line 116) | class ModalView(miru.View):
method __init__ (line 119) | def __init__(self, modal_title: str, task: str, *args: t.Any, **kwargs...
method modal_button (line 125) | async def modal_button(self, button: miru.Button, ctx: miru.ViewContex...
function task_modal (line 142) | async def task_modal(ctx: lightbulb.SlashContext):
class RatingView (line 158) | class RatingView(miru.View):
method __init__ (line 161) | def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
method _close_if_all_pressed (line 165) | def _close_if_all_pressed(self) -> None:
method button_1 (line 170) | async def button_1(self, button: miru.Button, ctx: miru.ViewContext) -...
method button_2 (line 177) | async def button_2(self, button: miru.Button, ctx: miru.ViewContext) -...
method button_3 (line 184) | async def button_3(self, button: miru.Button, ctx: miru.ViewContext) -...
method button_4 (line 191) | async def button_4(self, button: miru.Button, ctx: miru.ViewContext) -...
method button_5 (line 198) | async def button_5(self, button: miru.Button, ctx: miru.ViewContext) -...
method reset_button (line 205) | async def reset_button(self, button: miru.Button, ctx: miru.ViewContex...
class SelectRating (line 210) | class SelectRating(miru.View):
method select (line 242) | async def select(self, select: miru.Select, ctx: miru.ViewContext) -> ...
function rating_task (line 249) | async def rating_task(ctx: lightbulb.SlashContext):
function load (line 293) | def load(bot: lightbulb.BotApp):
function unload (line 298) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/extensions/work.py
class _TaskHandler (line 42) | class _TaskHandler(t.Generic[_Task_contra]):
method __init__ (line 45) | def __init__(self, ctx: lightbulb.Context, task: _Task_contra) -> None:
method get_task_messages (line 58) | def get_task_messages(task: _Task_contra) -> list[str]:
method send (line 62) | async def send(self) -> t.Literal["accept", "next", "cancel"] | None:
method handle (line 82) | async def handle(self) -> None:
method notify (line 125) | async def notify(self, content: str, event: hikari.DMMessageCreateEven...
method confirm_user_input (line 129) | async def confirm_user_input(self, content: str) -> bool:
method check_user_input (line 133) | def check_user_input(self, content: str) -> bool:
method cancel (line 137) | async def cancel(self, reason: str = "not specified") -> None:
class _RankingTaskHandler (line 153) | class _RankingTaskHandler(_TaskHandler[_Ranking_contra]):
method notify (line 156) | async def notify(self, content: str, event: hikari.DMMessageCreateEven...
class RankAssistantRepliesHandler (line 186) | class RankAssistantRepliesHandler(_RankingTaskHandler[protocol_schema.Ra...
method get_task_messages (line 188) | def get_task_messages(task: protocol_schema.RankAssistantRepliesTask) ...
method check_user_input (line 191) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 196) | async def confirm_user_input(self, content: str) -> bool:
class RankInitialPromptHandler (line 207) | class RankInitialPromptHandler(_RankingTaskHandler[protocol_schema.RankI...
method __init__ (line 208) | def __init__(self, ctx: lightbulb.Context, task: protocol_schema.RankI...
method get_task_messages (line 212) | def get_task_messages(task: protocol_schema.RankInitialPromptsTask) ->...
method check_user_input (line 215) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 220) | async def confirm_user_input(self, content: str) -> bool:
class RankPrompterReplyHandler (line 231) | class RankPrompterReplyHandler(_RankingTaskHandler[protocol_schema.RankP...
method get_task_messages (line 233) | def get_task_messages(task: protocol_schema.RankPrompterRepliesTask) -...
method check_user_input (line 236) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 241) | async def confirm_user_input(self, content: str) -> bool:
class RankConversationReplyHandler (line 252) | class RankConversationReplyHandler(_RankingTaskHandler[protocol_schema.R...
method get_task_messages (line 254) | def get_task_messages(task: protocol_schema.RankConversationRepliesTas...
method check_user_input (line 257) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 262) | async def confirm_user_input(self, content: str) -> bool:
class InitialPromptHandler (line 273) | class InitialPromptHandler(_TaskHandler[protocol_schema.InitialPromptTas...
method get_task_messages (line 275) | def get_task_messages(task: protocol_schema.InitialPromptTask) -> list...
method check_user_input (line 278) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 281) | async def confirm_user_input(self, content: str) -> bool:
class PrompterReplyHandler (line 290) | class PrompterReplyHandler(_TaskHandler[protocol_schema.PrompterReplyTas...
method get_task_messages (line 292) | def get_task_messages(task: protocol_schema.PrompterReplyTask) -> list...
method check_user_input (line 295) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 298) | async def confirm_user_input(self, content: str) -> bool:
class AssistantReplyHandler (line 307) | class AssistantReplyHandler(_TaskHandler[protocol_schema.AssistantReplyT...
method get_task_messages (line 309) | def get_task_messages(task: protocol_schema.AssistantReplyTask) -> lis...
method check_user_input (line 312) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 315) | async def confirm_user_input(self, content: str) -> bool:
class _LabelConversationReplyHandler (line 327) | class _LabelConversationReplyHandler(_TaskHandler[_Label_contra]):
method check_user_input (line 328) | def check_user_input(self, content: str) -> bool:
method confirm_user_input (line 336) | async def confirm_user_input(self, content: str) -> bool:
class LabelAssistantReplyHandler (line 345) | class LabelAssistantReplyHandler(_LabelConversationReplyHandler[protocol...
method get_task_messages (line 347) | def get_task_messages(task: protocol_schema.LabelAssistantReplyTask) -...
class LabelPrompterReplyHandler (line 351) | class LabelPrompterReplyHandler(_LabelConversationReplyHandler[protocol_...
method get_task_messages (line 353) | def get_task_messages(task: protocol_schema.LabelPrompterReplyTask) ->...
function work2 (line 364) | async def work2(ctx: lightbulb.Context) -> None:
class TaskAcceptView (line 444) | class TaskAcceptView(miru.View):
method accept_button (line 453) | async def accept_button(self, button: miru.Button, ctx: miru.ViewConte...
method next_button (line 460) | async def next_button(self, button: miru.Button, ctx: miru.ViewContext...
method cancel_button (line 467) | async def cancel_button(self, button: miru.Button, ctx: miru.ViewConte...
method on_timeout (line 473) | async def on_timeout(self) -> None:
class YesNoView (line 478) | class YesNoView(miru.View):
method yes_button (line 487) | async def yes_button(self, button: miru.Button, ctx: miru.ViewContext)...
method no_button (line 493) | async def no_button(self, button: miru.Button, ctx: miru.ViewContext) ...
method on_timeout (line 498) | async def on_timeout(self) -> None:
function load (line 503) | def load(bot: lightbulb.BotApp):
function unload (line 508) | def unload(bot: lightbulb.BotApp):
FILE: discord-bots/oa-bot-py/bot/messages.py
function _h1 (line 23) | def _h1(text: str) -> str:
function _h2 (line 27) | def _h2(text: str) -> str:
function _h3 (line 31) | def _h3(text: str) -> str:
function _writing_prompt (line 35) | def _writing_prompt(text: str) -> str:
function _ranking_prompt (line 39) | def _ranking_prompt(text: str) -> str:
function _label_prompt (line 43) | def _label_prompt(text: str, mandatory_label: list[str] | None, valid_la...
function _response_prompt (line 50) | def _response_prompt(text: str) -> str:
function _summarize_prompt (line 54) | def _summarize_prompt(text: str) -> str:
function _user (line 58) | def _user(text: str | None) -> str:
function _assistant (line 64) | def _assistant(text: str | None) -> str:
function _make_ordered_list (line 70) | def _make_ordered_list(items: list[protocol_schema.ConversationMessage])...
function _ordered_list (line 74) | def _ordered_list(items: list[protocol_schema.ConversationMessage]) -> str:
function _conversation (line 78) | def _conversation(conv: protocol_schema.Conversation) -> list[str]:
function _li (line 95) | def _li(text: str) -> str:
function initial_prompt_messages (line 104) | def initial_prompt_messages(task: protocol_schema.InitialPromptTask) -> ...
function rank_initial_prompts_messages (line 117) | def rank_initial_prompts_messages(task: protocol_schema.RankInitialPromp...
function rank_prompter_reply_messages (line 132) | def rank_prompter_reply_messages(task: protocol_schema.RankPrompterRepli...
function rank_assistant_reply_message (line 149) | def rank_assistant_reply_message(task: protocol_schema.RankAssistantRepl...
function rank_conversation_reply_messages (line 165) | def rank_conversation_reply_messages(task: protocol_schema.RankConversat...
function label_initial_prompt_message (line 180) | def label_initial_prompt_message(task: protocol_schema.LabelInitialPromp...
function label_prompter_reply_messages (line 193) | def label_prompter_reply_messages(task: protocol_schema.LabelPrompterRep...
function label_assistant_reply_messages (line 211) | def label_assistant_reply_messages(task: protocol_schema.LabelAssistantR...
function prompter_reply_messages (line 230) | def prompter_reply_messages(task: protocol_schema.PrompterReplyTask) -> ...
function assistant_reply_messages (line 254) | def assistant_reply_messages(task: protocol_schema.AssistantReplyTask) -...
function confirm_text_response_message (line 269) | def confirm_text_response_message(content: str) -> str:
function confirm_ranking_response_message (line 277) | def confirm_ranking_response_message(content: str, items: list[protocol_...
function help_message (line 289) | def help_message(can_manage_guild: bool, is_dev: bool) -> str:
function tutorial_message (line 335) | def tutorial_message() -> str:
function confirm_label_response_message (line 343) | def confirm_label_response_message(content: str) -> str:
function task_complete_embed (line 359) | def task_complete_embed(task: protocol_schema.Task, mention: str) -> hik...
function invalid_user_input_embed (line 374) | def invalid_user_input_embed(error_message: str) -> hikari.Embed:
function plain_embed (line 383) | def plain_embed(text: str) -> hikari.Embed:
FILE: discord-bots/oa-bot-py/bot/settings.py
class Settings (line 5) | class Settings(BaseSettings):
class Config (line 15) | class Config(BaseSettings.Config):
FILE: discord-bots/oa-bot-py/bot/utils.py
function format_time (line 8) | def format_time(dt: datetime, fmt: t.Literal["t", "T", "D", "f", "F", "R...
function mention (line 27) | def mention(
FILE: discord-bots/oa-bot-py/message_templates.py
class MessageTemplates (line 8) | class MessageTemplates:
method __init__ (line 11) | def __init__(self, template_dir: str = "./templates"):
method render (line 17) | def render(self, template_name: str, **kwargs: typing.Any):
FILE: docs/src/components/HomepageFeatures/index.tsx
type FeatureItem (line 5) | type FeatureItem = {
function Feature (line 35) | function Feature({ title, Svg, description }: FeatureItem) {
function HomepageFeatures (line 49) | function HomepageFeatures(): JSX.Element {
FILE: docs/src/pages/index.tsx
function HomepageHeader (line 10) | function HomepageHeader() {
function Home (line 30) | function Home(): JSX.Element {
FILE: inference/safety/main.py
function log_exceptions (line 21) | async def log_exceptions(request: fastapi.Request, call_next):
function load_pipeline (line 36) | async def load_pipeline():
function async_predict (line 46) | async def async_predict(pipeline: Blade2Blade, inputs: str):
function safety (line 52) | async def safety(request: inference.SafetyRequest):
function health (line 61) | async def health():
FILE: inference/safety/settings.py
class Settings (line 4) | class Settings(pydantic.BaseSettings):
FILE: inference/server/alembic/env.py
function run_migrations_offline (line 32) | def run_migrations_offline() -> None:
function do_run_migrations (line 56) | def do_run_migrations(connection):
function run_async_migrations (line 65) | async def run_async_migrations() -> None:
FILE: inference/server/alembic/versions/2023_03_12_1742-7d5be54acd49_initial_revision.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 145) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_03_21_2116-629d5081160f_changed_worker_config_to_worker_info.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 26) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_03_22_2113-78f16015b904_add_refresh_token_table.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 36) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_04_12_2033-f0e18084aae4_add_deleted_field_to_user.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_04_14_1611-b66fd8f9da1f_add_hidden_field_to_chats.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_04_24_2130-401eef162771_add_chat_data_opt_out_field.py
function upgrade (line 18) | def upgrade() -> None:
function downgrade (line 24) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_04_29_1739-ea19bbc743f9_add_safe_content_to_message.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 28) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_05_01_2253-5b4211625a9f_added_used_plugin_to_message.py
function upgrade (line 19) | def upgrade() -> None:
function downgrade (line 25) | def downgrade() -> None:
FILE: inference/server/alembic/versions/2023_05_29_1551-5ed411a331f4_add_active_thread_tail_messsage_id_and_.py
function upgrade (line 20) | def upgrade() -> None:
function downgrade (line 49) | def downgrade() -> None:
FILE: inference/server/export.py
function smart_open (line 32) | def smart_open(filename: str = None) -> TextIO:
function maybe_anonymize (line 45) | def maybe_anonymize(anonymizer: Anonymizer | None, collection: str, key:...
function prepare_export_events (line 52) | def prepare_export_events(
function prepare_export_message_tree (line 80) | def prepare_export_message_tree(
function prepare_export_message_node (line 105) | def prepare_export_message_node(
function write_messages_to_file (line 137) | def write_messages_to_file(
function fetch_eligible_chats (line 169) | async def fetch_eligible_chats(session_generator, filters: list[Any]) ->...
function export_chats (line 185) | def export_chats(
function parse_date (line 205) | def parse_date(date_str: str) -> dt.date:
function parse_args (line 209) | def parse_args() -> argparse.Namespace:
function create_filters (line 264) | def create_filters(args: argparse.Namespace) -> list[Any]:
function main (line 283) | def main():
FILE: inference/server/main.py
function log_exceptions (line 36) | async def log_exceptions(request: fastapi.Request, call_next):
function enable_prom_metrics (line 47) | async def enable_prom_metrics():
function log_inference_protocol_version (line 52) | async def log_inference_protocol_version():
function terminate_server (line 56) | def terminate_server(signum, frame):
function alembic_upgrade (line 62) | async def alembic_upgrade():
function setup_rate_limiter (line 89) | async def setup_rate_limiter():
function maybe_add_debug_api_keys (line 110) | async def maybe_add_debug_api_keys():
function welcome_message (line 148) | async def welcome_message():
FILE: inference/server/oasst_inference_server/admin.py
function delete_user_from_db (line 9) | async def delete_user_from_db(session: database.AsyncSession, user_id: s...
FILE: inference/server/oasst_inference_server/auth.py
function get_user_id_from_trusted_client_token (line 26) | def get_user_id_from_trusted_client_token(trusted_client_token: str) -> ...
function get_user_id_from_auth_token (line 33) | def get_user_id_from_auth_token(token: str) -> str:
function get_current_user_id (line 57) | def get_current_user_id(
function create_access_token (line 67) | def create_access_token(user_id: str) -> str:
function create_refresh_token (line 77) | async def create_refresh_token(user_id: str) -> str:
function refresh_access_token (line 89) | async def refresh_access_token(refresh_token: str) -> str:
function derive_key (line 111) | def derive_key() -> bytes:
function build_payload (line 123) | def build_payload(user_id: str, token_type: str, expire_minutes: int) ->...
function store_refresh_token (line 138) | async def store_refresh_token(token: bytes, user_id: str) -> None:
function query_refresh_token (line 148) | async def query_refresh_token(token: bytes) -> models.DbRefreshToken | N...
function validate_access_token (line 159) | def validate_access_token(payload: dict) -> None:
function validate_refresh_token (line 175) | def validate_refresh_token(payload: dict, compare_user_id: str) -> None:
FILE: inference/server/oasst_inference_server/chat_repository.py
class ChatRepository (line 14) | class ChatRepository(pydantic.BaseModel):
class Config (line 19) | class Config:
method get_assistant_message_by_id (line 22) | async def get_assistant_message_by_id(self, message_id: str) -> models...
method get_prompter_message_by_id (line 31) | async def get_prompter_message_by_id(self, message_id: str) -> models....
method start_work (line 40) | async def start_work(
method reset_work (line 73) | async def reset_work(self, message_id: str) -> models.DbMessage:
method abort_work (line 90) | async def abort_work(self, message_id: str, reason: str) -> models.DbM...
method complete_work (line 102) | async def complete_work(
FILE: inference/server/oasst_inference_server/chat_utils.py
function get_model_config (line 5) | def get_model_config(model_config_name: str) -> model_configs.ModelConfig:
FILE: inference/server/oasst_inference_server/compliance.py
function find_compliance_work_request_message (line 16) | async def find_compliance_work_request_message(
function should_do_compliance_check (line 38) | async def should_do_compliance_check(session: database.AsyncSession, wor...
function run_compliance_check (line 55) | async def run_compliance_check(websocket: fastapi.WebSocket, worker_id: ...
function maybe_do_compliance_check (line 123) | async def maybe_do_compliance_check(websocket, worker_id, worker_config,...
function compute_worker_compliance_score (line 137) | async def compute_worker_compliance_score(worker_id: str) -> float:
FILE: inference/server/oasst_inference_server/database.py
function default_json_serializer (line 16) | def default_json_serializer(obj):
function custom_json_serializer (line 23) | def custom_json_serializer(obj):
function custom_json_deserializer (line 27) | def custom_json_deserializer(s):
function make_engine (line 55) | def make_engine():
function get_async_session (line 71) | async def get_async_session(autoflush=True):
function alembic_upgrade (line 77) | def alembic_upgrade(connection):
FILE: inference/server/oasst_inference_server/deps.py
function make_redis_client (line 15) | def make_redis_client():
function create_session (line 25) | async def create_session():
function manual_create_session (line 31) | async def manual_create_session(autoflush=True):
function create_chat_repository (line 36) | async def create_chat_repository(session: AsyncSession = Depends(create_...
function create_user_chat_repository (line 41) | async def create_user_chat_repository(
function manual_chat_repository (line 50) | async def manual_chat_repository():
function manual_user_chat_repository (line 56) | async def manual_user_chat_repository(user_id: str):
function user_identifier (line 61) | async def user_identifier(request: fastapi.Request) -> str:
class UserRateLimiter (line 71) | class UserRateLimiter(RateLimiter):
method __init__ (line 72) | def __init__(
FILE: inference/server/oasst_inference_server/models/chat.py
class DbMessage (line 11) | class DbMessage(SQLModel, table=True):
method has_finished (line 44) | def has_finished(self) -> bool:
method has_started (line 52) | def has_started(self) -> bool:
method to_read (line 57) | def to_read(self) -> inference.MessageRead:
class DbChat (line 77) | class DbChat(SQLModel, table=True):
method to_list_read (line 94) | def to_list_read(self) -> chat_schema.ChatListRead:
method to_read (line 104) | def to_read(self) -> chat_schema.ChatRead:
method get_msg_dict (line 116) | def get_msg_dict(self) -> dict[str, DbMessage]:
class DbReport (line 120) | class DbReport(SQLModel, table=True):
method to_read (line 129) | def to_read(self) -> inference.Report:
class DbMessageEval (line 133) | class DbMessageEval(SQLModel, table=True):
FILE: inference/server/oasst_inference_server/models/user.py
class DbUser (line 7) | class DbUser(SQLModel, table=True):
class DbRefreshToken (line 21) | class DbRefreshToken(SQLModel, table=True):
FILE: inference/server/oasst_inference_server/models/worker.py
class WorkerEventType (line 14) | class WorkerEventType(str, enum.Enum):
class DbWorkerComplianceCheck (line 18) | class DbWorkerComplianceCheck(SQLModel, table=True):
class DbWorkerEvent (line 33) | class DbWorkerEvent(SQLModel, table=True):
class DbWorker (line 44) | class DbWorker(SQLModel, table=True):
method in_compliance_check (line 58) | def in_compliance_check(self) -> bool:
FILE: inference/server/oasst_inference_server/plugin_utils.py
function attempt_fetch_plugin (line 12) | async def attempt_fetch_plugin(session: aiohttp.ClientSession, url: str,...
function fetch_plugin (line 44) | async def fetch_plugin(url: str, retries: int = 3, timeout: float = 5.0)...
FILE: inference/server/oasst_inference_server/plugins/gale_pleaser/main.py
function api_icon (line 23) | async def api_icon():
function api_ai_plugin (line 31) | async def api_ai_plugin(request: Request):
function comfort_user (line 44) | async def comfort_user(
function custom_openapi (line 64) | def custom_openapi():
FILE: inference/server/oasst_inference_server/plugins/gale_roaster/main.py
function roast_user (line 22) | async def roast_user(
function api_ai_plugin (line 37) | async def api_ai_plugin(request: Request):
function custom_openapi (line 47) | def custom_openapi():
FILE: inference/server/oasst_inference_server/plugins/web_retriever/main.py
function extract_image_links (line 41) | def extract_image_links(text: str):
function detect_content_type (line 47) | def detect_content_type(content: bytes) -> str:
function limit_image_count (line 68) | def limit_image_count(images, max_chars=300):
function truncate_paragraphs (line 89) | def truncate_paragraphs(paragraphs, max_length):
function get_url_content (line 110) | async def get_url_content(url: str = Query(..., description="url to fetc...
function api_icon (line 224) | async def api_icon():
function api_ai_plugin (line 229) | async def api_ai_plugin(request: Request):
function custom_openapi (line 241) | def custom_openapi():
FILE: inference/server/oasst_inference_server/queueing.py
class QueueFullException (line 5) | class QueueFullException(Exception):
class RedisQueue (line 9) | class RedisQueue:
method __init__ (line 10) | def __init__(
method enqueue (line 26) | async def enqueue(self, value: str, enforce_max_size: bool = True) -> ...
method dequeue (line 40) | async def dequeue(self, timeout: int = 1) -> str | None:
method set_expire (line 46) | async def set_expire(self, timeout: int) -> None:
method get_enq_counter (line 49) | async def get_enq_counter(self) -> int:
method get_deq_counter (line 56) | async def get_deq_counter(self) -> int:
method get_length (line 63) | async def get_length(self) -> int:
function get_pos_value (line 67) | async def get_pos_value(redis_client: redis.Redis, message_id: str) -> int:
function message_queue (line 74) | def message_queue(redis_client: redis.Redis, message_id: str) -> RedisQu...
function work_queue (line 78) | def work_queue(redis_client: redis.Redis, worker_compat_hash: str) -> Re...
function compliance_queue (line 91) | def compliance_queue(redis_client: redis.Redis, worker_id: str) -> Redis...
FILE: inference/server/oasst_inference_server/routes/account.py
function handle_account_deletion (line 12) | async def handle_account_deletion(
FILE: inference/server/oasst_inference_server/routes/admin.py
function get_bearer_token (line 15) | def get_bearer_token(
function get_root_token (line 26) | def get_root_token(token: str = Depends(get_bearer_token)) -> str:
function create_worker (line 37) | async def create_worker(
function list_workers (line 52) | async def list_workers(
function delete_worker (line 62) | async def delete_worker(
function revoke_refresh_tokens (line 76) | async def revoke_refresh_tokens(
function delete_user (line 93) | async def delete_user(
FILE: inference/server/oasst_inference_server/routes/auth.py
function register_oauth_providers (line 23) | def register_oauth_providers():
function check_user_auth (line 69) | async def check_user_auth(user_id: str = Depends(auth.get_current_user_i...
function get_available_auth_providers (line 74) | async def get_available_auth_providers():
function refresh_token (line 81) | async def refresh_token(refresh_token: str = Security(auth.refresh_schem...
function login_discord (line 87) | async def login_discord(request: Request):
function callback_discord (line 93) | async def callback_discord(
function login_github (line 114) | async def login_github(request: Request):
function callback_github (line 120) | async def callback_github(
function login_google (line 141) | async def login_google(request: Request):
function callback_google (line 147) | async def callback_google(
function get_or_create_user (line 168) | async def get_or_create_user(
function query_user (line 182) | async def query_user(db: database.AsyncSession, provider: str, provider_...
function create_tokens (line 194) | async def create_tokens(user: models.DbUser) -> protocol.TokenPair:
function login_debug (line 207) | async def login_debug(username: str, state: str = r"{}"):
function callback_debug (line 214) | async def callback_debug(code: str, db: database.AsyncSession = Depends(...
function login_trusted (line 242) | async def login_trusted(
FILE: inference/server/oasst_inference_server/routes/chats.py
function list_chats (line 22) | async def list_chats(
function create_chat (line 69) | async def create_chat(
function get_chat (line 80) | async def get_chat(
function delete_chat (line 90) | async def delete_chat(
function create_prompter_message (line 99) | async def create_prompter_message(
function create_assistant_message (line 131) | async def create_assistant_message(
function get_message (line 182) | async def get_message(
function message_events (line 194) | async def message_events(
function handle_create_vote (line 305) | async def handle_create_vote(
function handle_create_message_eval (line 320) | async def handle_create_message_eval(
function handle_create_report (line 336) | async def handle_create_report(
function handle_update_chat (line 353) | async def handle_update_chat(
function handle_hide_all_chats (line 373) | async def handle_hide_all_chats(
FILE: inference/server/oasst_inference_server/routes/configs.py
class ParameterConfig (line 25) | class ParameterConfig(pydantic.BaseModel):
class ModelConfigInfo (line 31) | class ModelConfigInfo(pydantic.BaseModel):
function get_model_configs (line 119) | async def get_model_configs() -> list[ModelConfigInfo]:
function get_plugin_config (line 131) | async def get_plugin_config(plugin: inference.PluginEntry) -> inference....
function get_builtin_plugins (line 142) | async def get_builtin_plugins() -> list[inference.PluginEntry]:
FILE: inference/server/oasst_inference_server/routes/workers.py
class WorkerDisconnectException (line 15) | class WorkerDisconnectException(Exception):
method __init__ (line 16) | def __init__(self):
class WorkerError (line 34) | class WorkerError(Exception):
method __init__ (line 35) | def __init__(
function add_worker_connect_event (line 46) | async def add_worker_connect_event(
class WorkRequestContainer (line 60) | class WorkRequestContainer(pydantic.BaseModel):
class Config (line 66) | class Config:
class WorkRequestNotFound (line 73) | class WorkRequestNotFound(Exception):
method __init__ (line 74) | def __init__(self, request_id: str):
function get_work_request_container (line 79) | def get_work_request_container(work_request_map: WorkRequestContainerMap...
function handle_worker (line 89) | async def handle_worker(
function list_worker_sessions (line 276) | async def list_worker_sessions() -> list[worker_utils.WorkerSession]:
function clear_worker_sessions (line 291) | async def clear_worker_sessions():
function initiate_work_for_message (line 303) | async def initiate_work_for_message(
function handle_token_response (line 334) | async def handle_token_response(
function handle_plugin_intermediate_response (line 347) | async def handle_plugin_intermediate_response(
function handle_generated_text_response (line 360) | async def handle_generated_text_response(
function abort_message (line 387) | async def abort_message(message_id: str, error: str):
function handle_error_response (line 398) | async def handle_error_response(
function handle_general_error_response (line 411) | async def handle_general_error_response(
function handle_safe_prompt_response (line 417) | async def handle_safe_prompt_response(
function handle_timeout (line 438) | async def handle_timeout(message: inference.MessageRead):
FILE: inference/server/oasst_inference_server/schemas/auth.py
class TrustedClient (line 8) | class TrustedClient(pydantic.BaseModel):
class TrustedClientToken (line 16) | class TrustedClientToken(pydantic.BaseModel):
method parse (line 20) | def parse(token: str):
FILE: inference/server/oasst_inference_server/schemas/chat.py
class CreatePrompterMessageRequest (line 8) | class CreatePrompterMessageRequest(pydantic.BaseModel):
class CreateAssistantMessageRequest (line 13) | class CreateAssistantMessageRequest(pydantic.BaseModel):
class PendingResponseEvent (line 24) | class PendingResponseEvent(pydantic.BaseModel):
class TokenResponseEvent (line 30) | class TokenResponseEvent(pydantic.BaseModel):
class ErrorResponseEvent (line 35) | class ErrorResponseEvent(pydantic.BaseModel):
class MessageResponseEvent (line 41) | class MessageResponseEvent(pydantic.BaseModel):
class SafePromptResponseEvent (line 46) | class SafePromptResponseEvent(pydantic.BaseModel):
class PluginIntermediateResponseEvent (line 52) | class PluginIntermediateResponseEvent(pydantic.BaseModel):
class VoteRequest (line 73) | class VoteRequest(pydantic.BaseModel):
class MessageEvalRequest (line 77) | class MessageEvalRequest(pydantic.BaseModel):
class ReportRequest (line 81) | class ReportRequest(pydantic.BaseModel):
class CreateChatRequest (line 86) | class CreateChatRequest(pydantic.BaseModel):
class ChatListRead (line 90) | class ChatListRead(pydantic.BaseModel):
class ChatRead (line 100) | class ChatRead(ChatListRead):
class ListChatsResponse (line 104) | class ListChatsResponse(pydantic.BaseModel):
class MessageCancelledException (line 110) | class MessageCancelledException(Exception):
method __init__ (line 111) | def __init__(self, message_id: str):
class MessageTimeoutException (line 116) | class MessageTimeoutException(Exception):
method __init__ (line 117) | def __init__(self, message: inference.MessageRead):
class ChatUpdateRequest (line 122) | class ChatUpdateRequest(pydantic.BaseModel):
FILE: inference/server/oasst_inference_server/schemas/worker.py
class CreateWorkerRequest (line 4) | class CreateWorkerRequest(pydantic.BaseModel):
class WorkerRead (line 9) | class WorkerRead(pydantic.BaseModel):
class Config (line 15) | class Config:
FILE: inference/server/oasst_inference_server/settings.py
function split_keys_string (line 6) | def split_keys_string(keys: str | None):
class Settings (line 12) | class Settings(pydantic.BaseSettings):
method allowed_worker_compat_hashes_list (line 32) | def allowed_worker_compat_hashes_list(self) -> list[str]:
method allowed_model_config_names_list (line 38) | def allowed_model_config_names_list(self) -> list[str]:
method assemble_db_connection (line 55) | def assemble_db_connection(cls, v: str | None, values: dict[str, Any])...
method debug_api_keys_list (line 76) | def debug_api_keys_list(self) -> list[str]:
method trusted_api_keys_list (line 82) | def trusted_api_keys_list(self) -> list[str]:
method inference_cors_origins_list (line 127) | def inference_cors_origins_list(self) -> list[str]:
FILE: inference/server/oasst_inference_server/user_chat_repository.py
class UserChatRepository (line 11) | class UserChatRepository(pydantic.BaseModel):
class Config (line 17) | class Config:
method get_chats (line 20) | async def get_chats(
method get_chat_by_id (line 46) | async def get_chat_by_id(self, chat_id: str, include_messages: bool = ...
method get_message_by_id (line 61) | async def get_message_by_id(self, chat_id: str, message_id: str) -> mo...
method create_chat (line 79) | async def create_chat(self) -> models.DbChat:
method delete_chat (line 91) | async def delete_chat(self, chat_id: str) -> models.DbChat:
method add_prompter_message (line 114) | async def add_prompter_message(self, chat_id: str, parent_id: str | No...
method initiate_assistant_message (line 168) | async def initiate_assistant_message(
method update_score (line 241) | async def update_score(self, message_id: str, score: int) -> models.Db...
method add_message_eval (line 261) | async def add_message_eval(self, message_id: str, inferior_message_ids...
method add_report (line 280) | async def add_report(self, message_id: str, reason: str, report_type: ...
method update_chat (line 299) | async def update_chat(
method hide_all_chats (line 328) | async def hide_all_chats(self) -> None:
FILE: inference/server/oasst_inference_server/worker_utils.py
class WorkerSessionStatus (line 14) | class WorkerSessionStatus(str, enum.Enum):
class WorkerSession (line 20) | class WorkerSession(pydantic.BaseModel):
function get_api_key (line 31) | def get_api_key(api_key: str = api_key_header) -> str:
function get_protocol_version (line 43) | def get_protocol_version(protocol_version: str = protocol_version_header...
function get_worker_id (line 53) | async def get_worker_id(
function get_worker (line 70) | async def get_worker(
function send_worker_request (line 79) | async def send_worker_request(
function receive_worker_response (line 86) | async def receive_worker_response(
function receive_worker_info (line 92) | async def receive_worker_info(
function store_worker_session (line 98) | async def store_worker_session(worker_session: WorkerSession):
function delete_worker_session (line 102) | async def delete_worker_session(worker_session_id: str):
function build_work_request (line 107) | async def build_work_request(
FILE: inference/tests/locust/locustfile.py
class ChatUser (line 13) | class ChatUser(HttpUser):
method chat (line 21) | def chat(self):
FILE: inference/text-client/__main__.py
function main (line 13) | def main(backend_url: str = "http://127.0.0.1:8000", model_config_name="...
FILE: inference/text-client/text_client_utils.py
class DebugClient (line 8) | class DebugClient:
method __init__ (line 9) | def __init__(self, backend_url, http_client=requests):
method login (line 15) | def login(self, username):
method create_chat (line 22) | def create_chat(self):
method get_available_models (line 33) | def get_available_models(self):
method send_message (line 41) | def send_message(self, message, model_config_name):
FILE: inference/worker/__main__.py
function terminate_worker (line 18) | def terminate_worker(signum, frame):
function main (line 23) | def main():
FILE: inference/worker/basic_hf_server.py
function log_exceptions (line 39) | async def log_exceptions(request: fastapi.Request, call_next):
function model_thread (line 53) | def model_thread():
function load_models (line 127) | def load_models():
function start_model_thread (line 178) | async def start_model_thread():
function welcome_message (line 185) | async def welcome_message():
function generate (line 193) | async def generate(
function health (line 216) | async def health():
FILE: inference/worker/chat_chain.py
class PromptedLLM (line 44) | class PromptedLLM:
method __init__ (line 49) | def __init__(
method call (line 72) | def call(self, prompt: str) -> tuple[str, str]:
function handle_plugin_usage (line 104) | def handle_plugin_usage(
function handle_standard_usage (line 309) | def handle_standard_usage(
function build_memory (line 345) | def build_memory(work_request: inference.WorkRequest) -> ConversationBuf...
function handle_conversation (line 363) | def handle_conversation(
FILE: inference/worker/chat_chain_utils.py
function similarity (line 32) | def similarity(ts1: str, ts2: str) -> float:
function extract_tool_and_input (line 70) | def extract_tool_and_input(llm_output: str, ai_prefix: str) -> tuple[str...
function truncate_str (line 93) | def truncate_str(output: str, max_length: int = 1024) -> str:
function prepare_json (line 107) | def prepare_json(json_str: str) -> str:
function select_tool (line 164) | def select_tool(tool_name: str, tools: list[Tool]) -> Tool | None:
function use_tool (line 179) | def use_tool(tool_name: str, tool_input: str, tools: list[Tool]) -> str:
class RequestsForLLM (line 190) | class RequestsForLLM:
method run (line 191) | def run(self, params: str, url: str, param_location: str, type: str, p...
method run_request (line 194) | def run_request(self, params: str, url: str, param_location: str, type...
method process_response (line 223) | def process_response(self, res: requests.Response) -> str:
function compose_tools_from_plugin (line 237) | def compose_tools_from_plugin(plugin: inference.PluginEntry | None) -> t...
function prepare_prompt (line 322) | def prepare_prompt(
FILE: inference/worker/download_model.py
function terminate (line 8) | def terminate(signum, frame):
FILE: inference/worker/download_model_hf.py
function terminate (line 9) | def terminate(signum, frame):
FILE: inference/worker/hf_langchain_inference.py
class HFInference (line 6) | class HFInference(LLM):
method _llm_type (line 20) | def _llm_type(self) -> str:
method _call (line 23) | def _call(self, prompt: str, stop: list[str] | None = None) -> str:
FILE: inference/worker/hf_stopping.py
class SequenceStoppingCriteria (line 6) | class SequenceStoppingCriteria(StoppingCriteria):
method __init__ (line 9) | def __init__(
method __call__ (line 22) | def __call__(
FILE: inference/worker/hf_streamer.py
class Printer (line 7) | class Printer(typing.Protocol):
method __call__ (line 8) | def __call__(self, value: int) -> None:
function _unpack (line 12) | def _unpack(value):
class HFStreamer (line 21) | class HFStreamer(transformers.generation.streamers.BaseStreamer):
method __init__ (line 22) | def __init__(self, input_ids, printer: Printer):
method put (line 26) | def put(self, value):
method end (line 35) | def end(self):
FILE: inference/worker/interface.py
class GenerateStreamParameters (line 7) | class GenerateStreamParameters(pydantic.BaseModel):
method from_work_parameters (line 21) | def from_work_parameters(params: inference.WorkParameters) -> "Generat...
class GenerateStreamRequest (line 35) | class GenerateStreamRequest(pydantic.BaseModel):
class Token (line 40) | class Token(pydantic.BaseModel):
method __len__ (line 45) | def __len__(self) -> int:
method to_token_response (line 48) | def to_token_response(self, request_id: str) -> inference.TokenResponse:
class StreamDetails (line 57) | class StreamDetails(pydantic.BaseModel):
class GenerateStreamResponse (line 63) | class GenerateStreamResponse(pydantic.BaseModel):
method is_end (line 70) | def is_end(self) -> bool:
method is_error (line 74) | def is_error(self) -> bool:
FILE: inference/worker/openapi_parser.py
function fetch_openapi_spec (line 10) | def fetch_openapi_spec(url):
function get_plugin_config (line 30) | def get_plugin_config(url: str) -> inference.PluginConfig | None:
function resolve_schema_reference (line 43) | def resolve_schema_reference(ref: str, openapi_dict: dict):
function parse_plugin_endpoint (line 57) | def parse_plugin_endpoint(
function get_plugin_endpoints (line 124) | def get_plugin_endpoints(api_url: str, openapi_dict: dict) -> list[infer...
function prepare_plugin_for_llm (line 141) | def prepare_plugin_for_llm(plugin_url: str) -> inference.PluginConfig | ...
FILE: inference/worker/settings.py
class Settings (line 4) | class Settings(pydantic.BaseSettings):
FILE: inference/worker/utils.py
class TokenBuffer (line 37) | class TokenBuffer:
method __init__ (line 46) | def __init__(self, stop_sequences: list[str]) -> None:
method add (line 53) | def add(self, token: interface.Token):
method finish (line 69) | def finish(self, reason: Literal["length", "eos_token", "stop_sequence...
function get_max_input_length (line 90) | def get_max_input_length(worker_config: inference.WorkerConfig, plugin_u...
function get_tokens_until (line 98) | def get_tokens_until(tokens: list[int], target: list[int]) -> list[int]:
function truncate_prompt (line 108) | def truncate_prompt(
function wait_for_inference_server (line 169) | def wait_for_inference_server(http: "HttpClient", timeout: int = 600):
function text_to_events (line 187) | def text_to_events(
function lorem_events (line 220) | def lorem_events(seed):
function send_response (line 228) | def send_response(
class HttpClient (line 237) | class HttpClient(pydantic.BaseModel):
method auth (line 246) | def auth(self):
method _maybe_add_bearer_token (line 252) | def _maybe_add_bearer_token(self, headers: dict[str, str] | None):
method get (line 259) | def get(self, path: str, **kwargs):
method post (line 263) | def post(self, path: str, **kwargs):
function get_inference_server_stream_events (line 268) | def get_inference_server_stream_events(
FILE: inference/worker/work.py
function make_prompt_and_parameters (line 24) | def make_prompt_and_parameters(
function prepare_safe_prompt (line 74) | def prepare_safe_prompt(prompt: str, label: str, rots: str) -> str:
function is_safety_triggered (line 82) | def is_safety_triggered(safety_label: str, safety_level: int) -> bool:
function parse_safety_response (line 90) | def parse_safety_response(safety_opinion: str) -> tuple[str, str]:
function handle_work_request (line 98) | def handle_work_request(
function get_safety_server_response (line 229) | def get_safety_server_response(request: inference.SafetyRequest) -> infe...
function perform_oom_test (line 242) | def perform_oom_test(tokenizer: transformers.PreTrainedTokenizer):
FILE: inlang.config.js
function defineConfig (line 1) | async function defineConfig(env) {
FILE: model/model_eval/eval_datasets.py
function get_sampling_dataloader (line 8) | def get_sampling_dataloader(data, tokenizer, max_length, batch_size):
class SamplingDataCollator (line 14) | class SamplingDataCollator(RankingDataCollator):
method __call__ (line 15) | def __call__(self, examples):
class SamplingDataset (line 39) | class SamplingDataset(Dataset):
method __init__ (line 45) | def __init__(self, dataset):
method get_label2id (line 61) | def get_label2id(self, sampling_list):
method __len__ (line 64) | def __len__(self):
method __getitem__ (line 67) | def __getitem__(self, idx):
class RejectionSamplingDataset (line 74) | class RejectionSamplingDataset(Dataset):
method __init__ (line 75) | def __init__(self, dataset):
method __len__ (line 87) | def __len__(self):
method __getitem__ (line 90) | def __getitem__(self, index):
FILE: model/model_eval/eval_rm.py
function get_ranking_dataset (line 18) | def get_ranking_dataset(dataset, split):
function batch_inference (line 30) | def batch_inference(inputs, model):
FILE: model/model_eval/manual/create_synth_import.py
function filter_text (line 13) | def filter_text(s: str) -> str:
function format_params (line 23) | def format_params(p: dict) -> str:
function parse_args (line 28) | def parse_args():
function main (line 39) | def main():
FILE: model/model_eval/manual/sampling_report.py
class SamplingConfig (line 27) | class SamplingConfig(pydantic.BaseModel):
class Configuration (line 39) | class Configuration(pydantic.BaseModel):
class SamplingResult (line 44) | class SamplingResult(pydantic.BaseModel):
class PromptResults (line 50) | class PromptResults(pydantic.BaseModel):
class SamplingReport (line 55) | class SamplingReport(pydantic.BaseModel):
function load_jsonl (line 62) | def load_jsonl(input_file_path: str | Path) -> list[dict | str]:
function sample (line 82) | def sample(
function merge_configs (line 147) | def merge_configs(*configs: tuple[Optional[SamplingConfig]]) -> Optional...
function sample_prompt_continuations (line 174) | def sample_prompt_continuations(
function load_configs (line 225) | def load_configs(path: Path) -> Configuration:
function parse_args (line 232) | def parse_args():
function main (line 264) | def main():
FILE: model/model_eval/manual/subsample_dataset.py
function load_message_trees (line 12) | def load_message_trees(
function write_file (line 50) | def write_file(output_file_path: str | Path, items: list) -> None:
function parse_args (line 68) | def parse_args():
function main (line 101) | def main():
FILE: model/model_eval/sampling_score.py
function batch_inference (line 13) | def batch_inference(model, dataloader):
FILE: model/model_eval/utils.py
function load_sampling_data (line 7) | def load_sampling_data(path):
function write_to_json (line 29) | def write_to_json(filename, data):
function describe_samples (line 37) | def describe_samples(samples):
FILE: model/model_training/check_dataset_appearances.py
function argument_parsing (line 31) | def argument_parsing():
function check_in_dataset_row (line 56) | def check_in_dataset_row(row: str | list[str] | tuple[str], matched=dict...
function iterate_over_dataset (line 89) | def iterate_over_dataset(ds):
FILE: model/model_training/check_dataset_counts.py
class Mode (line 14) | class Mode(str, Enum):
method config_name (line 19) | def config_name(self) -> str:
method default_config (line 28) | def default_config(self) -> str:
function read_yaml (line 38) | def read_yaml(dir: str | Path, config_file: str) -> dict[str, Any]:
function argument_parsing (line 43) | def argument_parsing(notebook=False, notebook_args=None):
FILE: model/model_training/custom_datasets/__init__.py
function train_val_dataset (line 81) | def train_val_dataset(dataset, val_split=0.2) -> tuple[Dataset, Dataset ...
function get_one_dataset (line 91) | def get_one_dataset(
FILE: model/model_training/custom_datasets/dialogue_collator.py
class DialogueDataCollator (line 19) | class DialogueDataCollator:
method __post_init__ (line 39) | def __post_init__(self):
method process_one (line 51) | def process_one(self, messages, return_length=False):
method __call__ (line 139) | def __call__(self, features):
FILE: model/model_training/custom_datasets/extra_rm_datasets.py
function load_anthropic_rlhf (line 5) | def load_anthropic_rlhf() -> tuple[Dataset, Dataset]:
function load_shp (line 11) | def load_shp() -> tuple[Dataset, Dataset]:
function load_hellaswag (line 17) | def load_hellaswag() -> tuple[Dataset, Dataset]:
FILE: model/model_training/custom_datasets/formatting.py
function format_system_prefix (line 19) | def format_system_prefix(prefix, eos_token):
function compute_length (line 27) | def compute_length(s: str) -> int:
class Mode (line 31) | class Mode(str, Enum):
class Role (line 37) | class Role(str, Enum):
class Utterance (line 42) | class Utterance(BaseModel):
method between_0_1 (line 52) | def between_0_1(cls, v, field: ModelField) -> float:
method system_tag (line 57) | def system_tag(
class DatasetEntry (line 101) | class DatasetEntry(BaseModel):
class DatasetEntryLm (line 105) | class DatasetEntryLm(DatasetEntry):
class DatasetEntrySft (line 111) | class DatasetEntrySft(DatasetEntry):
method get_formatted (line 117) | def get_formatted(
class DatasetEntryRm (line 150) | class DatasetEntryRm(DatasetEntry):
method get_formatted (line 156) | def get_formatted(
function create_dataset_entry_qa (line 211) | def create_dataset_entry_qa(
function format_pairs (line 246) | def format_pairs(
function format_rl_text (line 261) | def format_rl_text(pairs: list[str]) -> str:
function format_reply (line 266) | def format_reply(text: str, eos_token: str) -> str:
FILE: model/model_training/custom_datasets/instruction.py
class InstructionDataset (line 75) | class InstructionDataset(Dataset):
method __init__ (line 76) | def __init__(
method __len__ (line 141) | def __len__(self):
method __getitem__ (line 144) | def __getitem__(self, idx) -> DatasetEntry:
class RAGDataset (line 160) | class RAGDataset(Dataset):
method __init__ (line 161) | def __init__(
method __len__ (line 175) | def __len__(self):
method __getitem__ (line 178) | def __getitem__(self, idx):
FILE: model/model_training/custom_datasets/oasst_dataset.py
class ListDataset (line 11) | class ListDataset(Dataset):
method __init__ (line 12) | def __init__(self, data: list):
method __len__ (line 16) | def __len__(self):
method __getitem__ (line 19) | def __getitem__(self, index):
function load_oasst_export (line 23) | def load_oasst_export(
FILE: model/model_training/custom_datasets/pretrain_datasets.py
class PretrainDataset (line 12) | class PretrainDataset(Dataset):
method __init__ (line 13) | def __init__(
method __len__ (line 38) | def __len__(self) -> int:
method __getitem__ (line 41) | def __getitem__(self, index) -> DatasetEntryLm:
class RedPajama (line 47) | class RedPajama(PretrainDataset):
method __init__ (line 50) | def __init__(
class FanFics (line 66) | class FanFics(PretrainDataset):
method __init__ (line 69) | def __init__(
FILE: model/model_training/custom_datasets/prompt_dialogue.py
function load_oig_file (line 16) | def load_oig_file(
class Gpt4All (line 107) | class Gpt4All(Dataset):
method __init__ (line 108) | def __init__(self, mode: str, cache_dir: str = None) -> None:
method process_conversation (line 132) | def process_conversation(conv: list[dict[str, None | str]]) -> list[st...
method __len__ (line 161) | def __len__(self):
method __getitem__ (line 164) | def __getitem__(self, index: int) -> list[str] | tuple[str]:
class OrcaChat (line 172) | class OrcaChat(Dataset):
method __init__ (line 175) | def __init__(self, data_files: Union[List[str], str] = "orca-chat-gpt4...
method __len__ (line 178) | def __len__(self):
method __getitem__ (line 181) | def __getitem__(self, idx):
class DolphinMix (line 198) | class DolphinMix(Dataset):
method __init__ (line 201) | def __init__(
method __len__ (line 237) | def __len__(self) -> int:
method __getitem__ (line 240) | def __getitem__(self, idx) -> DatasetEntrySft:
FILE: model/model_training/custom_datasets/qa_datasets.py
function index_squad_v2 (line 33) | def index_squad_v2(example):
function index_uasquad (line 41) | def index_uasquad(example):
function index_trivia_qa_nocontext (line 49) | def index_trivia_qa_nocontext(example):
function index_trivia_qa_context (line 54) | def index_trivia_qa_context(example):
function index_adversarial_qa (line 67) | def index_adversarial_qa(example):
function index_gsm8k (line 71) | def index_gsm8k(example):
function index_wikihow (line 75) | def index_wikihow(example):
function index_essay_instruction (line 79) | def index_essay_instruction(example):
function index_math_qa (line 83) | def index_math_qa(example):
function index_eli5 (line 91) | def index_eli5(example):
function index_gsm_hard (line 95) | def index_gsm_hard(example):
class QADataset (line 103) | class QADataset(Dataset):
method __init__ (line 156) | def __init__(self, dataset, cache_dir, split):
method __len__ (line 180) | def __len__(self):
method __getitem__ (line 183) | def __getitem__(self, idx):
class WebGPT (line 188) | class WebGPT(Dataset):
method __init__ (line 191) | def __init__(self, mode: str = "sft", max_answers: int = 5) -> None:
method __len__ (line 224) | def __len__(self) -> int:
method __getitem__ (line 227) | def __getitem__(self, index) -> DatasetEntry:
class SODA (line 232) | class SODA(Dataset):
method __init__ (line 235) | def __init__(self, cache_dir, mode="sft", input_max_length=32 * 1024) ...
method process_soda_convo (line 246) | def process_soda_convo(self, data: dict[str, Any], input_max_length: i...
method __len__ (line 275) | def __len__(self) -> int:
method __getitem__ (line 278) | def __getitem__(self, index) -> DatasetEntry:
class SODADialogue (line 283) | class SODADialogue(Dataset):
method __init__ (line 284) | def __init__(self, cache_dir, verbose=True):
method __len__ (line 311) | def __len__(self):
method __getitem__ (line 314) | def __getitem__(self, index):
class JokeExplaination (line 318) | class JokeExplaination(Dataset):
method __init__ (line 322) | def __init__(self, cache_dir) -> None:
method __len__ (line 342) | def __len__(self) -> int:
method __getitem__ (line 345) | def __getitem__(self, index) -> DatasetEntry:
class TranslatedQA (line 349) | class TranslatedQA(Dataset):
method __init__ (line 368) | def __init__(self, cache_dir) -> None:
method __len__ (line 401) | def __len__(self):
method __getitem__ (line 404) | def __getitem__(self, index):
class AlpacaBaseDataset (line 408) | class AlpacaBaseDataset(Dataset):
method __init__ (line 409) | def __init__(self, data: list, mode: str):
method __len__ (line 418) | def __len__(self) -> int:
method __getitem__ (line 421) | def __getitem__(self, index: int) -> DatasetEntry:
function load_alpaca_dataset (line 426) | def load_alpaca_dataset(
class Vicuna (line 466) | class Vicuna(Dataset):
method process_vicuna_conversations (line 470) | def process_vicuna_conversations(
method __init__ (line 514) | def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max...
method __len__ (line 533) | def __len__(self) -> int:
method __getitem__ (line 536) | def __getitem__(self, index: int) -> DatasetEntry:
class WizardEvolInstructV2 (line 540) | class WizardEvolInstructV2(Dataset):
method __init__ (line 541) | def __init__(self, cache_dir: str | Path, mode: str = "sft", input_max...
method __len__ (line 561) | def __len__(self) -> int:
method __getitem__ (line 564) | def __getitem__(self, index: int) -> DatasetEntry:
class DatabricksDolly15k (line 569) | class DatabricksDolly15k(Dataset):
method __init__ (line 570) | def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None:
method _process_instruction (line 582) | def _process_instruction(self, row: dict[str, str]) -> DatasetEntry | ...
method __len__ (line 595) | def __len__(self) -> int:
method __getitem__ (line 598) | def __getitem__(self, index: int) -> DatasetEntry:
class Dolly15kMultilingual (line 603) | class Dolly15kMultilingual(Dataset):
method __init__ (line 604) | def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None:
method _process_instruction (line 618) | def _process_instruction(self, row: dict[str, str], lang: str) -> Data...
method __len__ (line 632) | def __len__(self) -> int:
method __getitem__ (line 635) | def __getitem__(self, index: int) -> DatasetEntry:
class AlpacaGpt4 (line 640) | class AlpacaGpt4(Dataset):
method __init__ (line 641) | def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None:
method _process_instruction (line 652) | def _process_instruction(self, row: dict[str, str]) -> DatasetEntry | ...
method __len__ (line 674) | def __len__(self) -> int:
method __getitem__ (line 677) | def __getitem__(self, index: int) -> DatasetEntry:
class GPTeacher_Roleplay (line 682) | class GPTeacher_Roleplay(Dataset):
method __init__ (line 683) | def __init__(self, cache_dir: str | Path, mode: str = "sft") -> None:
method _process_qa (line 707) | def _process_qa(self, row: dict[str, str]) -> DatasetEntry | None:
method __len__ (line 715) | def __len__(self) -> int:
method __getitem__ (line 718) | def __getitem__(self, index: int) -> DatasetEntry:
FILE: model/model_training/custom_datasets/rank_datasets.py
class SHPDataset (line 12) | class SHPDataset(Dataset):
method __init__ (line 19) | def __init__(self, split: str | list[str] | None, max_answers: int = 5):
method __len__ (line 47) | def __len__(self):
method __getitem__ (line 50) | def __getitem__(self, index):
class HellaSwagDataset (line 54) | class HellaSwagDataset(Dataset):
method __init__ (line 64) | def __init__(self, split: str | list[str] | None, seed: int = SEED) ->...
method __len__ (line 80) | def __len__(self) -> int:
method __getitem__ (line 83) | def __getitem__(self, idx) -> tuple[str | None, list[list]]:
class HFDataset (line 88) | class HFDataset(Dataset):
method __init__ (line 96) | def __init__(self, split: str | list[str] | None = None, subset: str =...
method _handle_comparisons (line 118) | def _handle_comparisons(self, dataset):
method _handle_axis (line 126) | def _handle_axis(self, dataset):
method __len__ (line 138) | def __len__(self):
method __getitem__ (line 143) | def __getitem__(self, idx):
class AugmentedOA (line 150) | class AugmentedOA(Dataset):
method __init__ (line 151) | def __init__(self, json_filename: str, split: str = "train") -> None:
method __len__ (line 168) | def __len__(self):
method __getitem__ (line 171) | def __getitem__(self, idx):
class AnthropicRLHF (line 182) | class AnthropicRLHF(Dataset):
method _split_dialogue (line 186) | def _split_dialogue(text: str) -> list[tuple[str, str]]:
method __init__ (line 216) | def __init__(self, split: str = "train") -> None:
method __len__ (line 241) | def __len__(self) -> int:
method __getitem__ (line 244) | def __getitem__(self, index: int) -> tuple[str, list[str]]:
FILE: model/model_training/custom_datasets/ranking_collator.py
class RankingDataCollator (line 11) | class RankingDataCollator:
method process_one (line 26) | def process_one(
method __call__ (line 80) | def __call__(
FILE: model/model_training/custom_datasets/summarization.py
function index_summary_default (line 49) | def index_summary_default(text, summary):
function index_summary_merge (line 53) | def index_summary_merge(text, summary):
class SummarizationDataset (line 57) | class SummarizationDataset(Dataset):
method __init__ (line 58) | def __init__(self, dataset, cache_dir, split, max_words=512):
method __len__ (line 67) | def __len__(self):
method __getitem__ (line 70) | def __getitem__(self, idx):
class HFSummaryPairs (line 101) | class HFSummaryPairs(Dataset):
method __init__ (line 108) | def __init__(self, split="train", mode="sft", conf_threshold=-1) -> None:
method __len__ (line 141) | def __len__(self) -> int:
method __getitem__ (line 144) | def __getitem__(self, index: int) -> tuple | list:
class HFSummary (line 166) | class HFSummary(Dataset):
method __init__ (line 176) | def __init__(self, split="train", mode="sft", conf_threshold=-1, max_c...
method get_sorted_ranks (line 222) | def get_sorted_ranks(comparison_pairs):
method __len__ (line 244) | def __len__(self) -> int:
method __getitem__ (line 247) | def __getitem__(self, index) -> tuple | list:
FILE: model/model_training/custom_datasets/toxic_conversation.py
class ProsocialDialogueExplaination (line 11) | class ProsocialDialogueExplaination(Dataset):
method __init__ (line 21) | def __init__(self, split="train", cache_dir=".cache") -> None:
method __len__ (line 40) | def __len__(self):
method __getitem__ (line 43) | def __getitem__(self, idx):
class ProsocialDialogue (line 47) | class ProsocialDialogue(Dataset):
method __init__ (line 60) | def __init__(self, split="train", cache_dir=".cache") -> None:
method __len__ (line 74) | def __len__(self):
method __getitem__ (line 77) | def __getitem__(self, idx):
FILE: model/model_training/custom_datasets/translation.py
class TranslationPair (line 87) | class TranslationPair(Dataset):
method __init__ (line 88) | def __init__(self, mix_prob=0.2) -> None:
method __len__ (line 94) | def __len__(self):
method __getitem__ (line 99) | def __getitem__(self, index):
class WMT2019 (line 112) | class WMT2019(TranslationPair):
method __init__ (line 113) | def __init__(self, pair="zh-en", split="train", mix_prob=0.2, maximum_...
class DiveMT (line 131) | class DiveMT(TranslationPair):
method __init__ (line 134) | def __init__(self, split="train", mix_prob=0.2) -> None:
class TEDTalk (line 154) | class TEDTalk(TranslationPair):
method __init__ (line 157) | def __init__(self, pair="de-ja", split="train", year="2016", mix_prob=...
FILE: model/model_training/custom_datasets/utils.py
function _filter_by_words (line 44) | def _filter_by_words(text: str, filter_words: list[str] | None = None) -...
FILE: model/model_training/efficiency_utils.py
function rsetattr (line 7) | def rsetattr(obj, attr, val):
function rgetattr (line 12) | def rgetattr(obj, attr, *args):
function fuse_gelu (line 19) | def fuse_gelu(model):
FILE: model/model_training/metrics.py
function reward_accuracy (line 7) | def reward_accuracy(eval_pred):
function kendall_tau (line 30) | def kendall_tau(eval_pred):
function spearmanr (line 54) | def spearmanr(eval_pred):
class RewardMetrics (line 72) | class RewardMetrics:
method __init__ (line 77) | def __init__(self, metrics):
method __call__ (line 92) | def __call__(self, eval_pred):
FILE: model/model_training/models/__init__.py
function freeze_top_n_layers (line 4) | def freeze_top_n_layers(model, target_layers):
function get_specific_model (line 23) | def get_specific_model(
FILE: model/model_training/models/gptj.py
class FrozenBNBLinear (line 12) | class FrozenBNBLinear(nn.Module):
method __init__ (line 13) | def __init__(self, weight, absmax, code, bias=None):
method forward (line 23) | def forward(self, input):
method from_linear (line 30) | def from_linear(cls, linear: nn.Linear) -> "FrozenBNBLinear":
method __repr__ (line 34) | def __repr__(self):
class DequantizeAndLinear (line 38) | class DequantizeAndLinear(torch.autograd.Function):
method forward (line 41) | def forward(
method backward (line 56) | def backward(ctx, grad_output: torch.Tensor):
class FrozenBNBEmbedding (line 66) | class FrozenBNBEmbedding(nn.Module):
method __init__ (line 67) | def __init__(self, weight, absmax, code):
method forward (line 75) | def forward(self, input, **kwargs):
method from_embedding (line 85) | def from_embedding(cls, embedding: nn.Embedding) -> "FrozenBNBEmbedding":
method __repr__ (line 89) | def __repr__(self):
function quantize_blockise_lowmemory (line 93) | def quantize_blockise_lowmemory(matrix: torch.Tensor, chunk_size: int = ...
function convert_to_int8 (line 110) | def convert_to_int8(model):
class GPTJBlock (line 138) | class GPTJBlock(transformers.models.gptj.modeling_gptj.GPTJBlock):
method __init__ (line 139) | def __init__(self, config):
class GPTJModel (line 146) | class GPTJModel(transformers.models.gptj.modeling_gptj.GPTJModel):
method __init__ (line 147) | def __init__(self, config):
class GPTJForCausalLM (line 152) | class GPTJForCausalLM(transformers.models.gptj.modeling_gptj.GPTJForCaus...
method __init__ (line 153) | def __init__(self, config):
function add_adapters (line 158) | def add_adapters(model, adapter_dim=16):
function get_model (line 176) | def get_model(model_name, cache_dir, quantization):
FILE: model/model_training/models/patching.py
function _patched_mlp_forward (line 43) | def _patched_mlp_forward(post_module: nn.Module, module: nn.Module, *arg...
function _patched_attn_forward (line 50) | def _patched_attn_forward(post_module: nn.Module, module: nn.Module, *ar...
function add_dropout (line 57) | def add_dropout(module: nn.Module, patched_fwd: Callable, p_dropout: flo...
function add_flash_attn (line 63) | def add_flash_attn(module: nn.Module, causal: bool = True):
function patch_model (line 89) | def patch_model(
class RopePatch (line 204) | class RopePatch:
method __init__ (line 205) | def __init__(self, model_name, **kwargs):
method from_config (line 234) | def from_config(cls, config):
method patch (line 239) | def patch(self, model):
method patch_falcon_model (line 247) | def patch_falcon_model(self, model, **kwargs):
method patch_llama_model (line 251) | def patch_llama_model(self, model, **kwargs):
FILE: model/model_training/models/patching_falcon.py
function falcon_forward_with_flash_attn (line 9) | def falcon_forward_with_flash_attn(
FILE: model/model_training/models/patching_llama.py
function llama_forward_with_flash_attn (line 14) | def llama_forward_with_flash_attn(
FILE: model/model_training/models/patching_neox.py
function neox_forward_with_flash_attn (line 8) | def neox_forward_with_flash_attn(
FILE: model/model_training/models/patching_utils.py
function compute_flash_attention (line 5) | def compute_flash_attention(flash_attn, q, k, v, attention_mask=None, he...
FILE: model/model_training/models/peft_modeling.py
function load_peft_model (line 10) | def load_peft_model(model, peft_model_path, tokenizer):
function prepare_model_for_gradient_checkpointing (line 33) | def prepare_model_for_gradient_checkpointing(model):
function peft_model (line 49) | def peft_model(model, training_config):
class SaveLoraConfig (line 89) | class SaveLoraConfig:
function save_adapter_model_from_ckpt (line 104) | def save_adapter_model_from_ckpt(save_config: SaveLoraConfig):
FILE: model/model_training/models/prefix_llama.py
function _make_causal_mask (line 47) | def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, p...
function _expand_mask (line 62) | def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Option...
class LlamaRMSNorm (line 76) | class LlamaRMSNorm(nn.Module):
method __init__ (line 77) | def __init__(self, hidden_size, eps=1e-6):
method forward (line 85) | def forward(self, hidden_states):
class LlamaRotaryEmbedding (line 96) | class LlamaRotaryEmbedding(torch.nn.Module):
method __init__ (line 97) | def __init__(self, dim, max_position_embeddings=2048, base=10000, devi...
method forward (line 111) | def forward(self, x, seq_len=None):
function rotate_half (line 128) | def rotate_half(x):
function apply_rotary_pos_emb (line 135) | def apply_rotary_pos_emb(q, k, cos, sin, offset: int = 0):
class LlamaMLP (line 143) | class LlamaMLP(nn.Module):
method __init__ (line 144) | def __init__(
method forward (line 156) | def forward(self, x):
class LlamaAttention (line 160) | class LlamaAttention(nn.Module):
method __init__ (line 163) | def __init__(
method _shape (line 200) | def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
method forward (line 203) | def forward(
class LlamaDecoderLayer (line 272) | class LlamaDecoderLayer(nn.Module):
method __init__ (line 273) | def __init__(self, config: LlamaConfig):
method forward (line 288) | def forward(
class LlamaPreTrainedModel (line 362) | class LlamaPreTrainedModel(PreTrainedModel):
method _init_weights (line 369) | def _init_weights(self, module):
method _set_gradient_checkpointing (line 380) | def _set_gradient_checkpointing(self, module, value=False):
class LlamaModel (line 449) | class LlamaModel(LlamaPreTrainedModel):
method __init__ (line 457) | def __init__(self, config: LlamaConfig):
method get_input_embeddings (line 470) | def get_input_embeddings(self):
method set_input_embeddings (line 473) | def set_input_embeddings(self, value):
method _prepare_decoder_attention_mask (line 477) | def _prepare_decoder_attention_mask(self, attention_mask, input_shape,...
method forward (line 498) | def forward(
class LlamaForCausalLM (line 656) | class LlamaForCausalLM(LlamaPreTrainedModel):
method __init__ (line 659) | def __init__(self, config):
method get_input_embeddings (line 668) | def get_input_embeddings(self):
method set_input_embeddings (line 671) | def set_input_embeddings(self, value):
method get_output_embeddings (line 674) | def get_output_embeddings(self):
method set_output_embeddings (line 677) | def set_output_embeddings(self, new_embeddings):
method set_decoder (line 680) | def set_decoder(self, decoder):
method get_decoder (line 683) | def get_decoder(self):
method forward (line 687) | def forward(
method prepare_inputs_for_generation (line 813) | def prepare_inputs_for_generation(
method _reorder_cache (line 835) | def _reorder_cache(past_key_values, beam_idx):
class LlamaForSequenceClassification (line 857) | class LlamaForSequenceClassification(LlamaPreTrainedModel):
method __init__ (line 860) | def __init__(self, config):
method get_input_embeddings (line 869) | def get_input_embeddings(self):
method set_input_embeddings (line 872) | def set_input_embeddings(self, value):
method forward (line 876) | def forward(
FILE: model/model_training/models/reward_model.py
class GPTNeoXRewardModelConfig (line 11) | class GPTNeoXRewardModelConfig(GPTNeoXConfig):
method __init__ (line 16) | def __init__(
class GPTNeoXRewardModelOutput (line 26) | class GPTNeoXRewardModelOutput(ModelOutput):
class GPTNeoXRewardModel (line 38) | class GPTNeoXRewardModel(GPTNeoXPreTrainedModel):
method __init__ (line 41) | def __init__(self, config):
method forward (line 54) | def forward(
FILE: model/model_training/models/rope.py
function rotate_half (line 5) | def rotate_half(x):
class RWNTKScaledRope (line 10) | class RWNTKScaledRope(torch.nn.Module):
method __init__ (line 16) | def __init__(
method cos_sin (line 33) | def cos_sin(
method forward (line 61) | def forward(self, q, k, past_key_values_length=0):
FILE: model/model_training/tests/test_datasets.py
function test_all_datasets (line 11) | def test_all_datasets():
function test_collate_fn (line 29) | def test_collate_fn():
function test_collate_fn_simple (line 56) | def test_collate_fn_simple():
FILE: model/model_training/tests/test_dialogue_data_collator.py
function pythia_tokenizer (line 10) | def pythia_tokenizer():
function test_dataset_entry_no_context (line 29) | def test_dataset_entry_no_context(pythia_tokenizer):
function test_dataset_entry (line 60) | def test_dataset_entry(pythia_tokenizer):
function test_dialogue_data_collator (line 122) | def test_dialogue_data_collator(pythia_tokenizer):
FILE: model/model_training/tests/test_formatting.py
function test_dataset_entry_formatting_missing_lang (line 5) | def test_dataset_entry_formatting_missing_lang():
function test_dataset_entry (line 39) | def test_dataset_entry():
function test_dataset_entry_float_violations (line 74) | def test_dataset_entry_float_violations():
FILE: model/model_training/tests/test_oasst_dataset.py
function test_load_oasst_export_dataset (line 8) | def test_load_oasst_export_dataset():
FILE: model/model_training/tests/test_patched_falcon.py
function test_flash_attention_falcon_patch (line 7) | def test_flash_attention_falcon_patch(device="cuda:0"):
FILE: model/model_training/tests/test_patched_gpt_neox.py
function test_flash_attention_patch (line 6) | def test_flash_attention_patch(dtype=torch.float16, device="cuda"):
function test_resid_dropout_patch (line 41) | def test_resid_dropout_patch():
FILE: model/model_training/tests/test_patched_llama.py
function test_flash_attention_patch (line 6) | def test_flash_attention_patch(dtype=torch.float16, device="cuda:0", lla...
FILE: model/model_training/tests/test_ranking_collator.py
function pythia_tokenizer (line 20) | def pythia_tokenizer():
function test_ranking_collator_system_tag (line 39) | def test_ranking_collator_system_tag(pythia_tokenizer):
function test_ranking_collator_no_messages (line 91) | def test_ranking_collator_no_messages(pythia_tokenizer):
function test_ranking_collator_local (line 119) | def test_ranking_collator_local(pythia_tokenizer):
function test_rm_datasets (line 175) | def test_rm_datasets():
function test_ranking_collator (line 198) | def test_ranking_collator():
FILE: model/model_training/tests/test_rm_loading.py
function test_convert_model (line 9) | def test_convert_model(
function test_load_reward_model (line 26) | def test_load_reward_model(model_name: str = "andreaskoepf/oasst-rm-1-py...
FILE: model/model_training/tests/test_utils.py
function test_tokenizer (line 7) | def test_tokenizer():
function test_tokenizer_successful_match (line 12) | def test_tokenizer_successful_match():
function test_tokenizer_partial_match (line 18) | def test_tokenizer_partial_match():
function test_tokenizer_failed_match (line 24) | def test_tokenizer_failed_match():
FILE: model/model_training/to_triton.py
class SFTLogitsModel (line 12) | class SFTLogitsModel(nn.Module):
method __init__ (line 13) | def __init__(self, model):
method forward (line 17) | def forward(self, *args, **kwargs):
class RMLogitsModel (line 21) | class RMLogitsModel(nn.Module):
method __init__ (line 22) | def __init__(self, model):
method forward (line 26) | def forward(self, *args, **kwargs):
function load_model_and_tokenizer (line 30) | def load_model_and_tokenizer(triton_mode, config, device="cuda:0"):
function write_traced_module (line 56) | def write_traced_module(
function trace_model (line 89) | def trace_model(model, tokenizer, device="cuda:0", trace_example="reward...
function main (line 99) | def main():
FILE: model/model_training/tools/augment_oasst.py
class AggregateResults (line 20) | class AggregateResults:
method __init__ (line 21) | def __init__(self, reward_model) -> None:
method scoring (line 29) | def scoring(self, prefixes, answer):
method aggregate (line 35) | def aggregate(self, jsonl_filenames, dataset, split="val"):
function r2_conversation (line 80) | def r2_conversation(prefixes, tokenizer, model, top_k=10, temperature=0....
function r0_conversation (line 116) | def r0_conversation(prefixes, tokenizer, model, top_k=10, temperature=0....
function rallio_conversation (line 152) | def rallio_conversation(prefixes, tokenizer, model, top_k=2, temperature...
function augment_conversation (line 200) | def augment_conversation(model_name, dataset, split="train"):
FILE: model/model_training/tools/check_oasst_export.py
function parse_args (line 6) | def parse_args():
function main (line 14) | def main():
FILE: model/model_training/tools/export_model.py
function parse_args (line 9) | def parse_args():
function main (line 33) | def main():
FILE: model/model_training/tools/model_chat.py
class ChatRole (line 19) | class ChatRole(str, Enum):
function talk (line 41) | def talk(human_input: str, history: List[Tuple[str, str]], sep_token: st...
function process_output (line 89) | def process_output(output, method, bot_name):
FILE: model/model_training/tools/sample_rm_data.py
function rank_replies (line 35) | def rank_replies(replies):
function expand_nodes (line 39) | def expand_nodes(tree):
function extract_all_pair_rank (line 57) | def extract_all_pair_rank(tree):
function viz_convo (line 78) | def viz_convo(conversation):
function calculate_total_threads (line 83) | def calculate_total_threads(_target_file):
function process_context (line 104) | def process_context(convo):
FILE: model/model_training/trainer_rl.py
function argument_parsing (line 25) | def argument_parsing(notebook: bool = False, notebook_args: Sequence[str...
function create_reward_fn (line 68) | def create_reward_fn(rank_config, sft_config): # noqa: C901
function main (line 111) | def main():
FILE: model/model_training/trainer_rm.py
class RMTrainer (line 31) | class RMTrainer(Trainer):
method __init__ (line 32) | def __init__(
method compute_loss (line 47) | def compute_loss(self, model, inputs, return_logits=False):
method prediction_step (line 59) | def prediction_step(
method get_train_dataloader (line 80) | def get_train_dataloader(self):
function argument_parsing (line 131) | def argument_parsing(notebook: bool = False, notebook_args: Sequence[str...
function main (line 183) | def main():
FILE: model/model_training/trainer_sft.py
function compute_metrics (line 37) | def compute_metrics(eval_pred, preprocess_fns, metrics):
function preprocess_logits_for_metrics (line 46) | def preprocess_logits_for_metrics(logits, labels):
class SFTTrainer (line 51) | class SFTTrainer(Trainer):
method __init__ (line 52) | def __init__(
method compute_loss (line 68) | def compute_loss(self, model, inputs, return_outputs=False):
method _compute_loss (line 82) | def _compute_loss(self, model, inputs):
method prediction_step (line 100) | def prediction_step(
method get_train_dataloader (line 118) | def get_train_dataloader(self):
function argument_parsing (line 169) | def argument_parsing(notebook: bool = False, notebook_args: Sequence[str...
function tokenizer_sanity_check (line 241) | def tokenizer_sanity_check(tokenizer):
function main (line 284) | def main():
FILE: model/model_training/utils/losses.py
class CrossEntropyLoss (line 6) | class CrossEntropyLoss(nn.CrossEntropyLoss):
method __init__ (line 7) | def __init__(self, weight=None, size_average=None, ignore_index=-100, ...
method forward (line 11) | def forward(self, input, target, mask=None):
class PolyLoss (line 29) | class PolyLoss(nn.Module):
method __init__ (line 30) | def __init__(self, weight=None, size_average=None, ignore_index=-100, ...
method forward (line 38) | def forward(self, input, target, mask=None):
class RMLoss (line 57) | class RMLoss(nn.Module):
method __init__ (line 58) | def __init__(self, reduction="mean", beta=0.001):
method forward (line 63) | def forward(self, logits, cu_lengths=None):
class RMCLSLoss (line 86) | class RMCLSLoss(nn.CrossEntropyLoss):
method __init__ (line 87) | def __init__(self, weight=None, size_average=None, ignore_index=-100, ...
method forward (line 91) | def forward(self, logits, cu_lengths=None):
FILE: model/model_training/utils/ppo_utils.py
class CustomCausalLMHydraWithValueHead (line 35) | class CustomCausalLMHydraWithValueHead(AutoModelForCausalLMWithHydraValu...
method __init__ (line 36) | def __init__(self, *args, **kwargs):
method from_pretrained (line 40) | def from_pretrained(cls, config, tokenizer, kwargs=None, revision=None...
class CustomPPOTrainer (line 126) | class CustomPPOTrainer(AcceleratePPOTrainer, AccelerateRLTrainer):
method __init__ (line 127) | def __init__(self, config, *args, **kwargs):
method decode (line 151) | def decode(
method get_arch (line 225) | def get_arch(self, config):
method generate (line 234) | def generate(self, input_ids, *args, **kwargs):
method generate_eval (line 275) | def generate_eval(self, input_ids, *args, **kwargs):
method make_experience (line 303) | def make_experience(self, num_rollouts: int = 1024, iter_count: int = ...
function triton_server_ref_model (line 538) | def triton_server_ref_model(): # noqa: C901
class CustomPromptPipeline (line 575) | class CustomPromptPipeline(BasePipeline):
method __init__ (line 580) | def __init__(self, prompts: List[str], max_prompt_length: int, tokeniz...
method __getitem__ (line 633) | def __getitem__(self, ix: int):
method __len__ (line 636) | def __len__(self) -> int:
method create_loader (line 639) | def create_loader(self, batch_size: int, shuffle=False) -> DataLoader:
FILE: model/model_training/utils/utils.py
function _strtobool (line 27) | def _strtobool(x):
function init_rng (line 31) | def init_rng(conf: argparse.Namespace) -> None:
class PerDatasetSampler (line 38) | class PerDatasetSampler(DistributedSampler):
method __init__ (line 63) | def __init__(
method set_epoch (line 93) | def set_epoch(self, epoch: int) -> None:
method __len__ (line 96) | def __len__(self) -> int:
method __iter__ (line 99) | def __iter__(self):
method build_sampler_from_config (line 130) | def build_sampler_from_config(cls, training_conf, datasets: List[Datas...
function get_dataset_fractions (line 138) | def get_dataset_fractions(conf, dataset_sizes: List[int], verbose: bool ...
class SpecialTokens (line 166) | class SpecialTokens(NamedTuple):
class TokenizerConfig (line 172) | class TokenizerConfig(NamedTuple):
function match_tokenizer_name (line 194) | def match_tokenizer_name(model_name: str) -> TokenizerConfig:
function get_tokenizer (line 208) | def get_tokenizer(conf) -> transformers.AutoTokenizer:
function default_preprocess (line 246) | def default_preprocess(eval_pred, ignote_negative_labels=True):
function preprocess_qa (line 257) | def preprocess_qa(eval_pred):
function get_metrics (line 282) | def get_metrics(conf, tokenizer):
function get_model (line 301) | def get_model(conf, tokenizer, pad_vocab_size_to_multiple_of=16, check_f...
function get_dataset_name_and_kwargs_from_data_config (line 360) | def get_dataset_name_and_kwargs_from_data_config(data_config):
function get_dataset (line 374) | def get_dataset(
function get_loss (line 393) | def get_loss(loss, poly_eps: float = 1.0, score_l2_reg: float = 0.001):
function read_yamls (line 406) | def read_yamls(dir):
function train_val_dataset (line 421) | def train_val_dataset(dataset, val_split=0.2):
function process_output (line 431) | def process_output(output: str, method: str = "v2", bot_name: str = "Joi...
function merge_dicts (line 441) | def merge_dicts(default: dict, config: dict):
function get_all_linear_layers (line 452) | def get_all_linear_layers(model):
FILE: model/model_training/utils/utils_rl.py
function prepare_tensor (line 5) | def prepare_tensor(name: str, input):
FILE: model/pretokenizer/create_hf_tokenizer_config.py
function strtobool (line 9) | def strtobool(s: str) -> bool:
function parse_args (line 13) | def parse_args():
function main (line 38) | def main():
FILE: model/pretokenizer/indexed_dataset.py
function __best_fitting_dtype (line 29) | def __best_fitting_dtype(vocab_size=None):
function get_available_dataset_impl (line 36) | def get_available_dataset_impl():
function infer_dataset_impl (line 40) | def infer_dataset_impl(path):
function make_builder (line 56) | def make_builder(out_file, impl, vocab_size=None):
function make_dataset (line 63) | def make_dataset(path, impl: str, skip_warmup=False):
function dataset_exists (line 81) | def dataset_exists(path, impl):
function read_longs (line 88) | def read_longs(f, n):
function write_longs (line 94) | def write_longs(f, a):
function code (line 101) | def code(dtype):
function index_file_path (line 108) | def index_file_path(prefix_path):
function data_file_path (line 112) | def data_file_path(prefix_path):
function create_doc_idx (line 116) | def create_doc_idx(sizes):
class IndexedDataset (line 124) | class IndexedDataset(torch.utils.data.Dataset):
method __init__ (line 129) | def __init__(self, path):
method read_index (line 135) | def read_index(self, path):
method read_data (line 152) | def read_data(self, path):
method check_index (line 155) | def check_index(self, i):
method __del__ (line 159) | def __del__(self):
method __getitem__ (line 164) | def __getitem__(self, idx):
method __len__ (line 188) | def __len__(self):
method num_tokens (line 191) | def num_tokens(self, index):
method size (line 194) | def size(self, index):
method exists (line 198) | def exists(path):
method supports_prefetch (line 202) | def supports_prefetch(self):
class IndexedCachedDataset (line 206) | class IndexedCachedDataset(IndexedDataset):
method __init__ (line 207) | def __init__(self, path):
method supports_prefetch (line 213) | def supports_prefetch(self):
method prefetch (line 216) | def prefetch(self, indices):
method __getitem__ (line 241) | def __getitem__(self, idx):
class IndexedDatasetBuilder (line 258) | class IndexedDatasetBuilder(object):
method __init__ (line 261) | def __init__(self, out_file, dtype=np.int32):
method add_item (line 270) | def add_item(self, tensor):
method end_document (line 277) | def end_document(self):
method merge_file_ (line 280) | def merge_file_(self, another_file):
method finalize (line 305) | def finalize(self, index_file):
function _warmup_mmap_file (line 320) | def _warmup_mmap_file(path):
class MMapIndexedDataset (line 326) | class MMapIndexedDataset(torch.utils.data.Dataset):
class Index (line 327) | class Index(object):
method writer (line 331) | def writer(cls, path, dtype):
method __init__ (line 376) | def __init__(self, path, skip_warmup=False):
method __del__ (line 413) | def __del__(self):
method dtype (line 418) | def dtype(self):
method sizes (line 422) | def sizes(self):
method doc_idx (line 426) | def doc_idx(self):
method __getitem__ (line 430) | def __getitem__(self, i):
method __len__ (line 433) | def __len__(self):
method __init__ (line 436) | def __init__(self, path, skip_warmup=False):
method __getstate__ (line 445) | def __getstate__(self):
method __setstate__ (line 448) | def __setstate__(self, state):
method _do_init (line 451) | def _do_init(self, path, skip_warmup):
method __del__ (line 463) | def __del__(self):
method __len__ (line 468) | def __len__(self):
method __getitem__ (line 472) | def __getitem__(self, idx):
method get (line 491) | def get(self, idx, offset=0, length=None):
method sizes (line 505) | def sizes(self):
method doc_idx (line 509) | def doc_idx(self):
method get_doc_idx (line 512) | def get_doc_idx(self):
method set_doc_idx (line 515) | def set_doc_idx(self, doc_idx_):
method supports_prefetch (line 519) | def supports_prefetch(self):
method exists (line 523) | def exists(path):
class MMapIndexedDatasetBuilder (line 527) | class MMapIndexedDatasetBuilder(object):
method __init__ (line 528) | def __init__(self, out_file, dtype=np.int64):
method add_item (line 534) | def add_item(self, tensor):
method add_doc (line 539) | def add_doc(self, tensor, sizes):
method end_document (line 545) | def end_document(self):
method merge_file_ (line 548) | def merge_file_(self, another_file):
method finalize (line 561) | def finalize(self, index_file):
FILE: model/pretokenizer/pretokenize.py
class IntRole (line 18) | class IntRole(IntEnum):
class Encoder (line 25) | class Encoder(object):
method __init__ (line 26) | def __init__(self, args):
method encode_text (line 30) | def encode_text(self, text: str) -> list[int]:
method decode (line 33) | def decode(self, tokens: list[int]) -> str:
method special_tokens (line 37) | def special_tokens(self) -> dict:
class DatasetWriter (line 41) | class DatasetWriter:
method __init__ (line 42) | def __init__(
method add_item (line 53) | def add_item(self, tokenized_item):
method finalize (line 56) | def finalize(self):
function format_pairs (line 60) | def format_pairs(pairs: list[str] | tuple[str]) -> tuple[list[str], list...
function format_sft_entry (line 69) | def format_sft_entry(entry: DatasetEntrySft) -> tuple[list[str], list[in...
function format_conversation (line 88) | def format_conversation(messages) -> str:
function get_dataset_name (line 97) | def get_dataset_name(d: Dataset):
class TokenStats (line 112) | class TokenStats:
method __init__ (line 113) | def __init__(self, name: str, total_samples: int, fraction: float = 1):
method processed_samples (line 125) | def processed_samples(self) -> int:
method skip (line 128) | def skip(self, tokens: list[int]) -> None:
method add (line 132) | def add(self, tokens: list[int]) -> None:
function tokenize_dataset (line 142) | def tokenize_dataset(
function parse_args (line 293) | def parse_args():
function main (line 368) | def main():
FILE: model/pretokenizer/tokenizer.py
function build_tokenizer (line 11) | def build_tokenizer(args):
function _vocab_size_with_padding (line 39) | def _vocab_size_with_padding(orig_vocab_size, args):
class AbstractTokenizer (line 56) | class AbstractTokenizer(ABC):
method __init__ (line 59) | def __init__(self, name):
method vocab_size (line 65) | def vocab_size(self):
method vocab (line 70) | def vocab(self):
method inv_vocab (line 76) | def inv_vocab(self):
method tokenize (line 81) | def tokenize(self, text):
method detokenize (line 84) | def detokenize(self, token_ids):
method cls (line 88) | def cls(self):
method sep (line 92) | def sep(self):
method pad (line 96) | def pad(self):
method eod (line 100) | def eod(self):
method mask (line 104) | def mask(self):
class _FalconTokenizer (line 108) | class _FalconTokenizer(AbstractTokenizer):
method __init__ (line 111) | def __init__(self, vocab_extra_ids_list=None, new_tokens=True):
method vocab_size (line 128) | def vocab_size(self):
method vocab (line 132) | def vocab(self):
method tokenize (line 135) | def tokenize(self, text):
method detokenize (line 138) | def detokenize(self, token_ids):
method inv_vocab (line 142) | def inv_vocab(self):
method eod (line 146) | def eod(self):
method pad (line 150) | def pad(self):
method eos_token_id (line 154) | def eos_token_id(self):
class _SentencePieceTokenizer (line 158) | class _SentencePieceTokenizer(AbstractTokenizer):
method __init__ (line 161) | def __init__(self, model_file, vocab_extra_ids=0, vocab_extra_ids_list...
method _initalize (line 171) | def _initalize(self, vocab_extra_ids, vocab_extra_ids_list, new_tokens):
method vocab_size (line 238) | def vocab_size(self):
method vocab (line 242) | def vocab(self):
method inv_vocab (line 246) | def inv_vocab(self):
method tokenize (line 251) | def tokenize(self, text):
method detokenize (line 277) | def detokenize(self, ids):
method cls (line 290) | def cls(self):
method sep (line 294) | def sep(self):
method pad (line 298) | def pad(self):
method bos_token_id (line 302) | def bos_token_id(self):
method bos (line 306) | def bos(self):
method eod (line 310) | def eod(self):
method eos_token_id (line 316) | def eos_token_id(self):
method eos (line 322) | def eos(self):
method mask (line 326) | def mask(self):
method additional_special_tokens_ids (line 330) | def additional_special_tokens_ids(self):
FILE: notebooks/data-augmentation/anthropic/trainer.py
class ProSocialDataset (line 19) | class ProSocialDataset(Dataset):
method __init__ (line 20) | def __init__(self, split):
method __len__ (line 28) | def __len__(self):
method __getitem__ (line 31) | def __getitem__(self, idx):
function compute_metrics (line 55) | def compute_metrics(eval_pred):
FILE: oasst-data/examples/clean_dataset.py
function parse_args (line 11) | def parse_args():
function main (line 29) | def main():
FILE: oasst-data/examples/filter_messages.py
function parse_args (line 9) | def parse_args():
function main (line 87) | def main():
FILE: oasst-data/examples/filter_trees.py
function parse_args (line 8) | def parse_args():
function main (line 32) | def main():
FILE: oasst-data/examples/split_dataset.py
function parse_args (line 7) | def parse_args():
function main (line 36) | def main():
FILE: oasst-data/examples/tree_to_messages.py
function parse_args (line 6) | def parse_args():
function main (line 23) | def main():
FILE: oasst-data/oasst_data/reader.py
function open_jsonl_read (line 12) | def open_jsonl_read(input_file_path: str | Path) -> TextIO:
function read_oasst_obj (line 21) | def read_oasst_obj(obj_dict: dict) -> ExportMessageTree | ExportMessageN...
function read_oasst_jsonl (line 31) | def read_oasst_jsonl(
function read_message_trees (line 41) | def read_message_trees(input_file_path: str | Path) -> Iterable[ExportMe...
function read_message_tree_list (line 47) | def read_message_tree_list(
function convert_hf_message (line 54) | def convert_hf_message(row: dict) -> None:
function read_messages (line 66) | def read_messages(input_file_path: str | Path) -> Iterable[ExportMessage...
function read_message_list (line 72) | def read_message_list(
function read_dataset_message_trees (line 79) | def read_dataset_message_trees(
function read_dataset_messages (line 119) | def read_dataset_messages(
FILE: oasst-data/oasst_data/schemas.py
class LabelAvgValue (line 9) | class LabelAvgValue(BaseModel):
class ExportMessageEvent (line 17) | class ExportMessageEvent(BaseModel):
class ExportMessageEventEmoji (line 22) | class ExportMessageEventEmoji(ExportMessageEvent):
class ExportMessageEventRating (line 27) | class ExportMessageEventRating(ExportMessageEvent):
class ExportMessageEventRanking (line 32) | class ExportMessageEventRanking(ExportMessageEvent):
class ExportMessageEventReport (line 41) | class ExportMessageEventReport(ExportMessageEvent):
class ExportMessageEventScore (line 47) | class ExportMessageEventScore(ExportMessageEvent):
class DetoxifyRating (line 52) | class DetoxifyRating(BaseModel):
class ExportMessageNode (line 62) | class ExportMessageNode(BaseModel):
method get_label_value (line 85) | def get_label_value(self, name: str) -> float | None:
class ExportMessageTree (line 91) | class ExportMessageTree(BaseModel):
FILE: oasst-data/oasst_data/traversal.py
function visit_threads_depth_first (line 6) | def visit_threads_depth_first(
function visit_messages_depth_first (line 24) | def visit_messages_depth_first(
FILE: oasst-data/oasst_data/writer.py
function default_serializer (line 10) | def default_serializer(obj):
function open_jsonl_write (line 17) | def open_jsonl_write(file_name: str | Path) -> TextIO:
function write_tree (line 25) | def write_tree(
function write_message_trees (line 34) | def write_message_trees(
function write_message (line 45) | def write_message(
function write_messages (line 59) | def write_messages(
FILE: oasst-shared/oasst_shared/api_client.py
class TaskType (line 16) | class TaskType(str, enum.Enum):
class OasstApiClient (line 33) | class OasstApiClient:
method __init__ (line 36) | def __init__(self, backend_url: str, api_key: str, session: Optional[a...
method post (line 68) | async def post(self, path: str, data: dict[str, t.Any]) -> Optional[di...
method _parse_task (line 104) | def _parse_task(self, data: Optional[dict[str, t.Any]]) -> protocol_sc...
method fetch_task (line 115) | async def fetch_task(
method fetch_random_task (line 129) | async def fetch_random_task(
method ack_task (line 136) | async def ack_task(self, task_id: str | UUID, message_id: str) -> None:
method nack_task (line 142) | async def nack_task(self, task_id: str | UUID, reason: str) -> None:
method post_interaction (line 148) | async def post_interaction(self, interaction: protocol_schema.Interact...
method close (line 154) | async def close(self):
FILE: oasst-shared/oasst_shared/exceptions/oasst_api_error.py
class OasstErrorCode (line 5) | class OasstErrorCode(IntEnum):
class OasstError (line 93) | class OasstError(Exception):
method __init__ (line 100) | def __init__(self, message: str, error_code: OasstErrorCode, http_stat...
method __repr__ (line 106) | def __repr__(self) -> str:
FILE: oasst-shared/oasst_shared/model_configs.py
class ModelConfig (line 4) | class ModelConfig(pydantic.BaseModel):
method is_llama (line 11) | def is_llama(self) -> bool:
method is_lorem (line 15) | def is_lorem(self) -> bool:
method compat_hash (line 19) | def compat_hash(self) -> str:
FILE: oasst-shared/oasst_shared/schemas/inference.py
class WorkerGpuInfo (line 16) | class WorkerGpuInfo(pydantic.BaseModel):
class WorkerHardwareInfo (line 21) | class WorkerHardwareInfo(pydantic.BaseModel):
method __init__ (line 36) | def __init__(self, **data):
class WorkerConfig (line 69) | class WorkerConfig(pydantic.BaseModel):
method compat_hash (line 74) | def compat_hash(self) -> str:
class WorkerInfo (line 78) | class WorkerInfo(pydantic.BaseModel):
class GpuMetricsInfo (line 83) | class GpuMetricsInfo(pydantic.BaseModel):
class WorkerMetricsInfo (line 88) | class WorkerMetricsInfo(pydantic.BaseModel):
method __init__ (line 95) | def __init__(self, **data):
class SamplingParameters (line 118) | class SamplingParameters(pydantic.BaseModel):
class PluginApiType (line 127) | class PluginApiType(pydantic.BaseModel):
class PluginAuthType (line 136) | class PluginAuthType(pydantic.BaseModel):
class PluginOpenAPIParameter (line 140) | class PluginOpenAPIParameter(pydantic.BaseModel):
class PluginOpenAPIEndpoint (line 148) | class PluginOpenAPIEndpoint(pydantic.BaseModel):
class PluginConfig (line 158) | class PluginConfig(pydantic.BaseModel):
class PluginEntry (line 172) | class PluginEntry(pydantic.BaseModel):
class PluginExecutionDetails (line 180) | class PluginExecutionDetails(pydantic.BaseModel):
class PluginUsed (line 190) | class PluginUsed(pydantic.BaseModel):
function make_seed (line 197) | def make_seed() -> int:
class WorkParameters (line 201) | class WorkParameters(pydantic.BaseModel):
class ReportType (line 217) | class ReportType(str, enum.Enum):
class Vote (line 223) | class Vote(pydantic.BaseModel):
class Report (line 228) | class Report(pydantic.BaseModel):
class MessageState (line 234) | class MessageState(str, enum.Enum):
class MessageRead (line 244) | class MessageRead(pydantic.BaseModel):
method is_assistant (line 263) | def is_assistant(self) -> bool:
class Thread (line 267) | class Thread(pydantic.BaseModel):
class SafetyParameters (line 271) | class SafetyParameters(pydantic.BaseModel):
method level_must_be_in_range (line 275) | def level_must_be_in_range(cls, v):
class SafetyRequest (line 281) | class SafetyRequest(pydantic.BaseModel):
class SafetyResponse (line 286) | class SafetyResponse(pydantic.BaseModel):
class WorkerRequestBase (line 290) | class WorkerRequestBase(pydantic.BaseModel):
class WorkRequest (line 294) | class WorkRequest(WorkerRequestBase):
class PingRequest (line 304) | class PingRequest(WorkerRequestBase):
class ErrorRequest (line 308) | class ErrorRequest(WorkerRequestBase):
class UpgradeProtocolRequest (line 313) | class UpgradeProtocolRequest(WorkerRequestBase):
class WrongApiKeyRequest (line 317) | class WrongApiKeyRequest(WorkerRequestBase):
class TerminateRequest (line 321) | class TerminateRequest(WorkerRequestBase):
class WorkerResponseBase (line 325) | class WorkerResponseBase(pydantic.BaseModel):
class PongResponse (line 329) | class PongResponse(WorkerResponseBase):
class SafePromptResponse (line 334) | class SafePromptResponse(WorkerResponseBase):
class PluginIntermediateResponse (line 342) | class PluginIntermediateResponse(WorkerResponseBase):
class TokenResponse (line 351) | class TokenResponse(WorkerResponseBase):
class GeneratedTextResponse (line 358) | class GeneratedTextResponse(WorkerResponseBase):
class InternalFinishedMessageResponse (line 366) | class InternalFinishedMessageResponse(WorkerResponseBase):
class InternalErrorResponse (line 371) | class InternalErrorResponse(WorkerResponseBase):
class ErrorResponse (line 377) | class ErrorResponse(WorkerResponseBase):
class GeneralErrorResponse (line 383) | class GeneralErrorResponse(WorkerResponseBase):
FILE: oasst-shared/oasst_shared/schemas/protocol.py
class TaskRequestType (line 11) | class TaskRequestType(str, enum.Enum):
class User (line 26) | class User(BaseModel):
class Account (line 32) | class Account(BaseModel):
class Token (line 38) | class Token(BaseModel):
class TokenPair (line 43) | class TokenPair(BaseModel):
class FrontEndUser (line 48) | class FrontEndUser(User):
class PageResult (line 61) | class PageResult(BaseModel):
class FrontEndUserPage (line 69) | class FrontEndUserPage(PageResult):
class ConversationMessage (line 73) | class ConversationMessage(BaseModel):
class Conversation (line 88) | class Conversation(BaseModel):
method __len__ (line 93) | def __len__(self):
method is_prompter_turn (line 97) | def is_prompter_turn(self) -> bool:
class Message (line 106) | class Message(ConversationMessage):
class MessageRevision (line 120) | class MessageRevision(BaseModel):
class MessagePage (line 129) | class MessagePage(PageResult):
class MessageTree (line 133) | class MessageTree(BaseModel):
class TaskRequest (line 140) | class TaskRequest(BaseModel):
class TaskAck (line 151) | class TaskAck(BaseModel):
class TaskNAck (line 157) | class TaskNAck(BaseModel):
class TaskClose (line 163) | class TaskClose(BaseModel):
class Task (line 169) | class Task(BaseModel):
class SummarizeStoryTask (line 176) | class SummarizeStoryTask(Task):
class RatingScale (line 183) | class RatingScale(BaseModel):
class AbstractRatingTask (line 188) | class AbstractRatingTask(Task):
class RateSummaryTask (line 194) | class RateSummaryTask(AbstractRatingTask):
class WithHintMixin (line 202) | class WithHintMixin(BaseModel):
class InitialPromptTask (line 206) | class InitialPromptTask(Task, WithHintMixin):
class ReplyToConversationTask (line 212) | class ReplyToConversationTask(Task):
class PrompterReplyTask (line 219) | class PrompterReplyTask(ReplyToConversationTask, WithHintMixin):
class AssistantReplyTask (line 225) | class AssistantReplyTask(ReplyToConversationTask):
class RankInitialPromptsTask (line 231) | class RankInitialPromptsTask(Task):
class RankConversationRepliesTask (line 239) | class RankConversationRepliesTask(Task):
class RankPrompterRepliesTask (line 251) | class RankPrompterRepliesTask(RankConversationRepliesTask):
class RankAssistantRepliesTask (line 257) | class RankAssistantRepliesTask(RankConversationRepliesTask):
class LabelTaskMode (line 263) | class LabelTaskMode(str, enum.Enum):
class LabelTaskDisposition (line 270) | class LabelTaskDisposition(str, enum.Enum):
class LabelDescription (line 277) | class LabelDescription(BaseModel):
class AbstractLabelTask (line 284) | class AbstractLabelTask(Task):
class LabelInitialPromptTask (line 294) | class LabelInitialPromptTask(AbstractLabelTask):
class LabelConversationReplyTask (line 301) | class LabelConversationReplyTask(AbstractLabelTask):
class LabelPrompterReplyTask (line 308) | class LabelPrompterReplyTask(LabelConversationReplyTask):
class LabelAssistantReplyTask (line 314) | class LabelAssistantReplyTask(LabelConversationReplyTask):
class TaskDone (line 320) | class TaskDone(Task):
class Interaction (line 345) | class Interaction(BaseModel):
class TextReplyToMessage (line 352) | class TextReplyToMessage(Interaction):
class MessageRating (line 362) | class MessageRating(Interaction):
class MessageRanking (line 370) | class MessageRanking(Interaction):
class LabelWidget (line 379) | class LabelWidget(str, enum.Enum):
class TextLabel (line 385) | class TextLabel(str, enum.Enum):
method __new__ (line 388) | def __new__(cls, label: str, widget: LabelWidget, display_text: str = ...
class TextLabels (line 429) | class TextLabels(Interaction):
method has_message_id (line 440) | def has_message_id(self) -> bool:
method check_label_values (line 446) | def check_label_values(cls, v):
class SystemStats (line 461) | class SystemStats(BaseModel):
class UserScore (line 469) | class UserScore(BaseModel):
class LeaderboardStats (line 504) | class LeaderboardStats(BaseModel):
class TrollScore (line 510) | class TrollScore(BaseModel):
class TrollboardStats (line 548) | class TrollboardStats(BaseModel):
class OasstErrorResponse (line 554) | class OasstErrorResponse(BaseModel):
class EmojiCode (line 561) | class EmojiCode(str, enum.Enum):
class EmojiOp (line 580) | class EmojiOp(str, enum.Enum):
class MessageEmojiRequest (line 586) | class MessageEmojiRequest(BaseModel):
class MessageEditRequest (line 592) | class MessageEditRequest(BaseModel):
class CreateFrontendUserRequest (line 597) | class CreateFrontendUserRequest(User):
class CachedStatsName (line 604) | class CachedStatsName(str, enum.Enum):
class CachedStatsResponse (line 612) | class CachedStatsResponse(BaseModel):
class AllCachedStatsResponse (line 618) | class AllCachedStatsResponse(BaseModel):
FILE: oasst-shared/oasst_shared/utils.py
function utcnow (line 12) | def utcnow() -> datetime:
function unaware_to_utc (line 17) | def unaware_to_utc(d: datetime | None) -> datetime:
class TimerError (line 24) | class TimerError(Exception):
class ScopeTimer (line 28) | class ScopeTimer:
method __init__ (line 29) | def __init__(self):
method start (line 32) | def start(self) -> None:
method stop (line 36) | def stop(self) -> float:
method __enter__ (line 41) | def __enter__(self):
method __exit__ (line 46) | def __exit__(self, *exc_info):
function log_timing (line 51) | def log_timing(func=None, *, log_kwargs: bool = False, level: int | str ...
function sha256_hash (line 72) | def sha256_hash(key: str, seed: int) -> str:
class Anonymizer (line 76) | class Anonymizer:
method __init__ (line 77) | def __init__(self, seed, value_generator=lambda key, seed: sha256_hash...
method __getitem__ (line 83) | def __getitem__(self, key):
method anonymize (line 92) | def anonymize(self, collection: str, key: str | None) -> str | None:
FILE: oasst-shared/tests/test_oasst_api_client.py
function oasst_api_client_mocked (line 13) | def oasst_api_client_mocked():
class MockClientSession (line 27) | class MockClientSession(aiohttp.ClientSession):
method set_response (line 30) | def set_response(self, response: Any):
method post (line 33) | async def post(self, *args, **kwargs):
function mock_http_session (line 38) | def mock_http_session():
function oasst_api_client_fake_http (line 43) | def oasst_api_client_fake_http(mock_http_session):
function test_can_fetch_task (line 53) | async def test_can_fetch_task(task_type: protocol_schema.TaskRequestType...
function test_can_ack_task (line 58) | async def test_can_ack_task(oasst_api_client_mocked: OasstApiClient):
function test_can_nack_task (line 63) | async def test_can_nack_task(oasst_api_client_mocked: OasstApiClient):
function test_can_post_interaction (line 68) | async def test_can_post_interaction(oasst_api_client_mocked: OasstApiCli...
function test_can_handle_oasst_error_from_api (line 89) | async def test_can_handle_oasst_error_from_api(
function test_can_handle_unknown_error_from_api (line 113) | async def test_can_handle_unknown_error_from_api(
FILE: scripts/data-collection/twitter/twitter_create_convs.py
function role_decide (line 81) | def role_decide(user_id, prompt_user):
class ConversationTreeNode (line 88) | class ConversationTreeNode:
method __init__ (line 89) | def __init__(self, tweet_id, prompt_user, from_df, children_df, metada...
method get_children (line 102) | def get_children(self, tweet_id, children_df):
class ConversationTree (line 118) | class ConversationTree:
method __init__ (line 119) | def __init__(self, tweet_id, prompt_user, from_df, children_df, r_meta...
FILE: scripts/data-collection/twitter/twitter_process_json.py
function main (line 71) | def main(file_list_pkl, folder_path, processed_max_buffer):
function get_file_paths (line 85) | def get_file_paths(file_list_pkl, folder_path):
function get_processed_list (line 100) | def get_processed_list(processed_file_list_pkl):
function modify_dict_cols (line 111) | def modify_dict_cols(j_dict):
function process_single_file (line 145) | def process_single_file(f, processed_list):
function process_json (line 176) | def process_json(file_list, processed_max_buffer):
FILE: scripts/data_augment/data_augment.py
class DataAugmenter (line 31) | class DataAugmenter:
method __init__ (line 32) | def __init__(self):
method parse (line 35) | def parse(self, essays):
method parse_single (line 47) | def parse_single(self, essay):
class EssayInstructor (line 51) | class EssayInstructor(DataAugmenter):
method __init__ (line 52) | def __init__(self, model_name=None):
method parse_single (line 58) | def parse_single(self, essay):
class EssayReviser (line 89) | class EssayReviser(DataAugmenter):
method __init__ (line 90) | def __init__(self):
method parse_single (line 94) | def parse_single(self, essay):
class StackExchangeBuilder (line 135) | class StackExchangeBuilder(DataAugmenter):
method __init__ (line 136) | def __init__(self, base_url=None, filter_opts=None):
method get_all_filenames (line 146) | def get_all_filenames(self):
method xml_to_df (line 160) | def xml_to_df(self, response: str):
method filter (line 200) | def filter(self, df):
method parse (line 242) | def parse(self, _):
class HierachicalSummarizer (line 274) | class HierachicalSummarizer(DataAugmenter):
method __init__ (line 275) | def __init__(self):
method cleanup_summary (line 295) | def cleanup_summary(self, out):
method parse_single (line 307) | def parse_single(self, essay):
class EntityRecognizedSummarizer (line 345) | class EntityRecognizedSummarizer(DataAugmenter):
method __init__ (line 346) | def __init__(self):
method parse_single (line 349) | def parse_single(self, essay):
class CodeBugger (line 360) | class CodeBugger(DataAugmenter):
method __init__ (line 375) | def __init__(self):
method parse_single (line 379) | def parse_single(self, code):
class CodeInstructor (line 394) | class CodeInstructor(DataAugmenter):
method __init__ (line 395) | def __init__(self):
method parse (line 399) | def parse(self, codes):
function recognize_entities (line 426) | def recognize_entities(text, model, n=4, person="ignore"):
function parse_arguments (line 445) | def parse_arguments():
function read_data (line 460) | def read_data(args):
function get_augmenter (line 472) | def get_augmenter(args):
function main (line 502) | def main(args):
FILE: scripts/discord/stats.py
class Settings (line 18) | class Settings(pydantic.BaseSettings):
function on_ready (line 31) | async def on_ready():
FILE: scripts/discord/verify-lobby.py
class Settings (line 18) | class Settings(pydantic.BaseSettings):
function on_ready (line 31) | async def on_ready():
FILE: scripts/frontend-development/find-missing-locales.py
function get_not_translated (line 12) | def get_not_translated(en_json, translation_json, parent_key=None):
function get_missing (line 26) | def get_missing(en_json, translation_json):
function print_result (line 30) | def print_result(missing, not_translated, file):
function audit (line 39) | def audit(file, en_file):
function main (line 45) | def main():
FILE: scripts/postprocessing/importance_selection.py
function least_squares_fit (line 14) | def least_squares_fit(features, target, scaling=1):
function get_df (line 41) | def get_df(study_label):
function reweight_features (line 107) | def reweight_features(features, weights, noise_scale=0.0):
function get_subframe (line 119) | def get_subframe(arr, columns_to_filter):
function sample_importance_weights (line 132) | def sample_importance_weights(importance_weights, temperature=1.0):
function make_random_testframe (line 140) | def make_random_testframe(num_rows, num_cols, frac_missing):
function combine_underrepresented_columns (line 147) | def combine_underrepresented_columns(arr, num_instances):
function importance_votes (line 171) | def importance_votes(arr, to_fit=10, init_weight=None):
function select_ids (line 218) | def select_ids(arr, pick_frac, minima=(50, 500), folds=50, to_fit=200, f...
FILE: scripts/postprocessing/infogain_selector.py
function analytic_solution (line 61) | def analytic_solution(a_post, a_prior):
function uniform_expected_infogain (line 79) | def uniform_expected_infogain(a_prior):
FILE: scripts/postprocessing/ranking_disagreement.py
function normalised_kendall_tau_distance (line 12) | def normalised_kendall_tau_distance(values1, values2):
function get_df (line 25) | def get_df():
FILE: scripts/postprocessing/rankings.py
function head_to_head_votes (line 6) | def head_to_head_votes(ranks: List[List[int]]):
function cycle_detect (line 24) | def cycle_detect(pairs):
function get_winner (line 54) | def get_winner(pairs):
function get_ranking (line 67) | def get_ranking(pairs):
function ranked_pairs (line 82) | def ranked_pairs(ranks: List[List[int]]):
FILE: scripts/postprocessing/regex_pii_detector.py
class PIIDetector (line 78) | class PIIDetector:
method __init__ (line 80) | def __init__(self):
method get_pii (line 87) | def get_pii(self, input_text: str):
method formatted_output (line 96) | def formatted_output(self, match_list: list):
FILE: scripts/postprocessing/scoring.py
class Voter (line 10) | class Voter:
method voter_quality (line 31) | def voter_quality(self):
method rank_quality (line 34) | def rank_quality(self):
method prompt_quality (line 37) | def prompt_quality(self):
method is_well_behaved (line 40) | def is_well_behaved(self, threshhold_vote, threshhold_prompt, threshho...
method total_points (line 47) | def total_points(self, voting_weight, prompt_weight, ranking_weight):
function score_update_votes (line 55) | def score_update_votes(new_vote: int, consensus: npt.ArrayLike, voter_da...
function score_update_prompts (line 86) | def score_update_prompts(consensus: npt.ArrayLike, voter_data: Voter) ->...
function score_update_ranking (line 120) | def score_update_ranking(user_ranking: npt.ArrayLike, consensus_ranking:...
FILE: scripts/postprocessing/task_schedule.py
class Task (line 7) | class Task(Enum):
function task_selection (line 14) | def task_selection(
function next_answer_task (line 49) | def next_answer_task(possible_prompts, answers_per_prompt):
FILE: scripts/xor-codec/xor_codec.py
function xor_uncompressed (line 10) | def xor_uncompressed(dst, src_payload, src_base, block_size=4096):
function xor_encode (line 30) | def xor_encode(dst, src_payload, src_base, block_size=4096):
function xor_decode (line 50) | def xor_decode(dst, src_payload, src_base, block_size=4096):
function xor_dir (line 70) | def xor_dir(dst, src_payload, src_base, decode=True, compress=True):
FILE: text-frontend/__main__.py
function _random_message_id (line 16) | def _random_message_id():
function _render_message (line 20) | def _render_message(message: dict) -> str:
function main (line 28) | def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "123...
FILE: text-frontend/auto_main.py
function _random_message_id (line 15) | def _random_message_id():
function _render_message (line 19) | def _render_message(message: dict) -> str:
function main (line 27) | def main(
FILE: website/.storybook/preview.js
constant MOCK_VALID_LABELS (line 12) | const MOCK_VALID_LABELS= [
FILE: website/cypress.config.js
method setupNodeEvents (line 7) | setupNodeEvents(on, config) {
method setupNodeEvents (line 20) | setupNodeEvents(on, config) {
FILE: website/cypress/support/component.ts
type Chainable (line 32) | interface Chainable {
FILE: website/cypress/support/index.ts
type Chainable (line 6) | interface Chainable {
FILE: website/jest.setup.js
constant CONSOLE_FAIL_TYPES (line 14) | const CONSOLE_FAIL_TYPES = ["error", "warn"];
FILE: website/next-lint.js
function npmLint (line 3) | async function npmLint() {
FILE: website/next.config.js
method redirects (line 33) | async redirects() {
FILE: website/prisma/migrations/20230326131923_initial_migration/migration.sql
type "Account" (line 2) | CREATE TABLE "Account" (
type "Session" (line 20) | CREATE TABLE "Session" (
type "User" (line 30) | CREATE TABLE "User" (
type "VerificationToken" (line 43) | CREATE TABLE "VerificationToken" (
type "RegisteredTask" (line 50) | CREATE TABLE "RegisteredTask" (
type "TaskInteraction" (line 59) | CREATE TABLE "TaskInteraction" (
type "Account" (line 68) | CREATE UNIQUE INDEX "Account_provider_providerAccountId_key" ON "Account...
type "Session" (line 71) | CREATE UNIQUE INDEX "Session_sessionToken_key" ON "Session"("sessionToken")
type "User" (line 74) | CREATE UNIQUE INDEX "User_email_key" ON "User"("email")
type "VerificationToken" (line 77) | CREATE UNIQUE INDEX "VerificationToken_token_key" ON "VerificationToken"...
type "VerificationToken" (line 80) | CREATE UNIQUE INDEX "VerificationToken_identifier_token_key" ON "Verific...
FILE: website/prisma/seed.ts
function main (line 11) | async function main() {
FILE: website/public/mockServiceWorker.js
constant INTEGRITY_CHECKSUM (line 11) | const INTEGRITY_CHECKSUM = '3d6b9f06410d179a7f7404d4bf4c3c70'
function handleRequest (line 140) | async function handleRequest(event, requestId) {
function resolveMainClient (line 174) | async function resolveMainClient(event) {
function getResponse (line 197) | async function getResponse(event, client, requestId) {
function sendToClient (line 278) | function sendToClient(client, message) {
function sleep (line 294) | function sleep(timeMs) {
function respondWithMock (line 300) | async function respondWithMock(response) {
FILE: website/src/components/AuthLayout.tsx
function AuthLayout (line 4) | function AuthLayout({ children }: { children: ReactNode }) {
FILE: website/src/components/Buttons/Skip.tsx
type SkipButtonProps (line 15) | interface SkipButtonProps extends ButtonProps {
FILE: website/src/components/CallToAction.tsx
constant CIRCLE_HEIGHT (line 9) | const CIRCLE_HEIGHT = 558;
constant CIRCLE_WIDTH (line 10) | const CIRCLE_WIDTH = 558;
function CircleBackground (line 12) | function CircleBackground() {
function CallToAction (line 44) | function CallToAction() {
FILE: website/src/components/Chat/ChatAssistantDraftPager.tsx
type DraftPickedParams (line 10) | type DraftPickedParams = { chatId: string; regenIndex: number; messageIn...
type OnDraftPickedFn (line 12) | type OnDraftPickedFn = (params: DraftPickedParams) => void;
type OnRetryFn (line 13) | type OnRetryFn = (params: { parentId: string; chatId: string }) => void;
type ChatAssistantDraftPagerProps (line 15) | type ChatAssistantDraftPagerProps = {
FILE: website/src/components/Chat/ChatAssistantDraftViewer.tsx
type OnDraftPickedFn (line 10) | type OnDraftPickedFn = (message_index: number) => void;
type ChatAssistantDraftViewerProps (line 12) | type ChatAssistantDraftViewerProps = {
FILE: website/src/components/Chat/ChatConfigForm.tsx
type NumberInputSliderProps (line 381) | type NumberInputSliderProps = {
FILE: website/src/components/Chat/ChatConfigSummary.tsx
function ChatConfigSummary (line 7) | function ChatConfigSummary() {
FILE: website/src/components/Chat/ChatContext.tsx
type ChatStateContext (line 4) | type ChatStateContext = {
type ChatActionContext (line 8) | type ChatActionContext = {
FILE: website/src/components/Chat/ChatConversation.tsx
type ChatConversationProps (line 30) | interface ChatConversationProps {
method onSuccess (line 59) | onSuccess(data) {
FILE: website/src/components/Chat/ChatConversationTree.tsx
type ChatConversationTreeProps (line 15) | type ChatConversationTreeProps = {
constant LAST_ASSISTANT_MESSAGE_ID (line 21) | const LAST_ASSISTANT_MESSAGE_ID = "last_assistant_message";
FILE: website/src/components/Chat/ChatForm.tsx
type ChatFormProps (line 11) | type ChatFormProps = {
FILE: website/src/components/Chat/ChatInitialDataContext.tsx
type ChatInitialDataContext (line 4) | type ChatInitialDataContext = {
FILE: website/src/components/Chat/ChatListItem.tsx
type ChatListItemIconButtonProps (line 300) | type ChatListItemIconButtonProps = {
FILE: website/src/components/Chat/ChatMessageEntry.tsx
type EditPromptParams (line 29) | type EditPromptParams = { parentId: string; chatId: string; content: str...
type ChatMessageEntryProps (line 31) | type ChatMessageEntryProps = {
type PendingMessageEntryProps (line 219) | type PendingMessageEntryProps = {
FILE: website/src/components/Chat/CreateChatButton.tsx
type CreateChatButtonProps (line 19) | interface CreateChatButtonProps extends ButtonProps {
FILE: website/src/components/Chat/CustomInstructions.tsx
constant CHAR_LIMIT (line 20) | const CHAR_LIMIT = 256;
function createChangeHandler (line 40) | function createChangeHandler(key: string) {
FILE: website/src/components/Chat/PluginsChooser.tsx
type PluginsChooserProps (line 40) | type PluginsChooserProps = {
FILE: website/src/components/Chat/useListChatPagination.ts
type ChatListViewSelection (line 7) | type ChatListViewSelection = "visible" | "visible_hidden";
function useListChatPagination (line 9) | function useListChatPagination(view: ChatListViewSelection) {
FILE: website/src/components/Container.tsx
function Container (line 3) | function Container({ className, ...props }) {
FILE: website/src/components/Dashboard/LeaderboardWidget.tsx
function LeaderboardWidget (line 7) | function LeaderboardWidget() {
FILE: website/src/components/Dashboard/SlimFooter.tsx
function SlimFooter (line 7) | function SlimFooter() {
FILE: website/src/components/Dashboard/TaskOption.tsx
type TaskCategoryItem (line 23) | type TaskCategoryItem = { taskType: TaskType; count: number };
type TasksOptionProps (line 25) | interface TasksOptionProps {
FILE: website/src/components/Dashboard/WelcomeCard.tsx
function WelcomeCard (line 5) | function WelcomeCard() {
FILE: website/src/components/DataTable/DataTable.tsx
type ColumnMeta (line 48) | interface ColumnMeta<TData extends RowData, TValue> {
type DataTableColumnDef (line 55) | type DataTableColumnDef<T> = ColumnDef<T, any> & {
type FilterItem (line 60) | type FilterItem = {
type DataTableRowPropsCallback (line 65) | type DataTableRowPropsCallback<T> = (row: Row<T>) => TableRowProps;
type DataTableCellPropsCallback (line 67) | type DataTableCellPropsCallback<T> = (cell: Cell<T, unknown>) => TableCe...
type DataTableProps (line 69) | type DataTableProps<T> = {
type WithSpanCell (line 181) | type WithSpanCell<T> = Cell<T, unknown> & { span?: number };
FILE: website/src/components/DataTable/DataTableAction.tsx
type DataTableActionProps (line 4) | type DataTableActionProps = Omit<IconButtonProps, "icon" | "size"> & { i...
FILE: website/src/components/DataTable/jsonExpandRowModel.tsx
type ExpandableRow (line 5) | type ExpandableRow<T> = Omit<T, "shouldExpand"> & {
FILE: website/src/components/DataTable/useCursorPagination.ts
type CursorPaginationState (line 3) | type CursorPaginationState = {
FILE: website/src/components/EmptyState.tsx
type EmptyStateProps (line 6) | type EmptyStateProps = {
FILE: website/src/components/Explain.tsx
type ExplainProps (line 14) | interface ExplainProps {
FILE: website/src/components/Faq.tsx
constant FAQS (line 6) | const FAQS = Array.from({ length: 6 });
function Faq (line 8) | function Faq() {
FILE: website/src/components/Footer.tsx
function Footer (line 7) | function Footer() {
FILE: website/src/components/Header/Header.tsx
function AccountButton (line 17) | function AccountButton() {
constant HEADER_HEIGHT (line 34) | const HEADER_HEIGHT = "82px";
type HeaderProps (line 36) | type HeaderProps = { preLogoSlot?: ReactNode; fixed?: boolean };
constant ANNOUNCEMENT_CACHE_KEY (line 38) | const ANNOUNCEMENT_CACHE_KEY = "announcement";
function Header (line 40) | function Header({ preLogoSlot, fixed = true }: HeaderProps) {
FILE: website/src/components/Header/UserMenu.tsx
type MenuOption (line 23) | interface MenuOption {
function UserMenu (line 30) | function UserMenu() {
FILE: website/src/components/Hero.tsx
function Hero (line 10) | function Hero() {
FILE: website/src/components/JsonCard.tsx
type JsonCardProps (line 5) | type JsonCardProps = StrictOmit<CardProps, "children"> & {
FILE: website/src/components/Layout.tsx
type NextPageWithLayout (line 13) | type NextPageWithLayout<P = unknown, IP = P> = NextPage<P, IP> & {
FILE: website/src/components/
Copy disabled (too large)
Download .json
Condensed preview — 1538 files, each showing path, character count, and a content snippet. Download the .json file for the full structured content (12,851K chars).
[
{
"path": ".devcontainer/README.md",
"chars": 4717,
"preview": "# devcontainer\n\n## example usage\n\nBelow are some example use cases you might want to run from within the\ndevcontainer (e"
},
{
"path": ".devcontainer/backend-dev/devcontainer.json",
"chars": 553,
"preview": "{\n \"name\": \"Backend Development\",\n \"image\": \"mcr.microsoft.com/vscode/devcontainers/universal\",\n \"features\": {\n \"g"
},
{
"path": ".devcontainer/backend-dev/post_create_command.sh",
"chars": 728,
"preview": "#!/bin/bash\n\n\n# allow git usage\ngit config --global --add safe.directory \"*\"\n\n# ensure pre-commit is installed\npre-commi"
},
{
"path": ".devcontainer/devcontainer.json",
"chars": 430,
"preview": "{\n \"name\": \"Generic\",\n \"image\": \"mcr.microsoft.com/vscode/devcontainers/universal\",\n \"features\": {\n \"ghcr.io/devco"
},
{
"path": ".devcontainer/frontend-dev/devcontainer.json",
"chars": 405,
"preview": "{\n \"name\": \"Front-end Development\",\n \"image\": \"mcr.microsoft.com/vscode/devcontainers/universal\",\n \"features\": {\n "
},
{
"path": ".devcontainer/frontend-dev/post_create_command.sh",
"chars": 121,
"preview": "#!/bin/bash\n\n# ensure pre-commit is installed\npre-commit install\n\n# npm install in /website\ncd website\nnpm install\ncd .."
},
{
"path": ".devcontainer/post_create_command.sh",
"chars": 191,
"preview": "#!/bin/bash\n\n# do as little as possible in this script to keep the container creation fast.\n# for more specific devconta"
},
{
"path": ".dockerignore",
"chars": 30,
"preview": "**/node_modules\nwebsite/.next\n"
},
{
"path": ".gitattributes",
"chars": 50,
"preview": "* text=auto eol=lf\n*.ipynb linguist-documentation\n"
},
{
"path": ".github/dependabot.yml",
"chars": 114,
"preview": "version: 2\nupdates:\n - package-ecosystem: \"npm\"\n directory: \"/website\"\n schedule:\n interval: \"weekly\"\n"
},
{
"path": ".github/workflows/build-frontend.yaml",
"chars": 554,
"preview": "name: Build Frontend\n\non:\n push:\n branches:\n - main\n paths:\n - website/**\n pull_request:\n paths:\n "
},
{
"path": ".github/workflows/build-postgres.yaml",
"chars": 343,
"preview": "name: Build OASST Postgres image\n\non:\n push:\n branches:\n - main\n paths:\n - docker/oasst-postgres/**\n\njo"
},
{
"path": ".github/workflows/deploy-docs-site.yaml",
"chars": 975,
"preview": "name: Deploy docs to gitHub pages\n\non:\n push:\n branches:\n - main\n paths:\n - \".github/workflows/deploy-d"
},
{
"path": ".github/workflows/deploy-to-node.yaml",
"chars": 7451,
"preview": "name: Deploy to node\n\non:\n workflow_call:\n inputs:\n stack-name:\n required: false\n type: string\n "
},
{
"path": ".github/workflows/docker-build.yaml",
"chars": 1907,
"preview": "name: Build\n\non:\n workflow_call:\n inputs:\n dockerfile:\n required: true\n type: string\n contex"
},
{
"path": ".github/workflows/pre-commit.yaml",
"chars": 1132,
"preview": "name: pre-commit\n\non:\n workflow_call:\n pull_request_target:\n\njobs:\n pre-commit:\n runs-on: ubuntu-latest\n steps:"
},
{
"path": ".github/workflows/production-deploy.yaml",
"chars": 330,
"preview": "name: Deploy to prod\n\non:\n push:\n branches:\n - production\n\njobs:\n deploy-to-prod:\n uses: ./.github/workflow"
},
{
"path": ".github/workflows/production2-deploy.yaml",
"chars": 333,
"preview": "name: Deploy to prod2\n\non:\n push:\n branches:\n - production2\n\njobs:\n deploy-to-prod:\n uses: ./.github/workfl"
},
{
"path": ".github/workflows/release.yaml",
"chars": 2941,
"preview": "name: Release\n\non:\n push:\n branches:\n - main\n release:\n types:\n - released\n\njobs:\n pre-commit:\n us"
},
{
"path": ".github/workflows/test-api-contract.yaml",
"chars": 1370,
"preview": "name: Test API Contract\n\non:\n push:\n branches:\n - main\n paths:\n - \"oasst-shared/**\"\n - \"backend/**"
},
{
"path": ".github/workflows/test-e2e.yaml",
"chars": 959,
"preview": "name: E2E Tests (Website)\n\non:\n push:\n branches:\n - main\n paths:\n - oasst-shared/**\n - backend/**\n"
},
{
"path": ".gitignore",
"chars": 358,
"preview": ".venv*\nvenv\n.env\n*.pyc\n*.swp\n*.egg-info\n__pycache__\n.DS_Store\n\n# Generated files\nbackend/oasst-openapi.json\nbackend/open"
},
{
"path": ".pre-commit-config.yaml",
"chars": 3516,
"preview": "# WARNING!\n#\n# When making changes to auto-formatters used in pre-commit hooks, you are\n# likely to cause merge conflict"
},
{
"path": ".python-version",
"chars": 7,
"preview": "3.10.8\n"
},
{
"path": ".vscode/extensions.json",
"chars": 186,
"preview": "{\n \"recommendations\": [\n \"ms-vscode-remote.remote-containers\",\n \"ms-vscode-remote.vscode-remote-extensionpack\",\n "
},
{
"path": ".vscode/launch.json",
"chars": 2908,
"preview": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Python: Current File\",\n \"type\": \"python\",\n "
},
{
"path": ".vscode/settings.json",
"chars": 316,
"preview": "{\n \"python.analysis.extraPaths\": [\"${workspaceFolder}/oasst-shared\"],\n \"prettier.singleQuote\": false,\n \"eslint.workin"
},
{
"path": "CODEOWNERS",
"chars": 865,
"preview": "* @yk @andreaskoepf\n/website/ @AbdBarho @notmd @yk @andreaskoepf\n/website/src/data/team.json @yk @andreaskoepf @fozzieth"
},
{
"path": "CONTRIBUTING.md",
"chars": 7891,
"preview": "<a href=\"https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/CONTRIBUTING.md?_x_tr_sl=auto&_x_tr_tl=en&_"
},
{
"path": "LICENSE",
"chars": 11357,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 9283,
"preview": "<h1 align=\"center\">\n <span>Open-Assistant</span>\n <img width=\"auto\" height=\"50px\" src=\"https://github.com/LAION-AI/O"
},
{
"path": "ansible/.gitignore",
"chars": 33,
"preview": "dev/\ndev-inference/\n*.local.yaml\n"
},
{
"path": "ansible/README.md",
"chars": 455,
"preview": "To test the ansible playbook on localhost run\n`ansible-playbook -i test.inventory.ini dev.yaml`.\\\nIn case you're missing"
},
{
"path": "ansible/deploy-to-node.yaml",
"chars": 12629,
"preview": "# ansible playbook to set up some docker containers\n\n- name: Deploy to node\n hosts: dev\n gather_facts: true\n vars:\n "
},
{
"path": "ansible/inference/deploy-server.yaml",
"chars": 6709,
"preview": "# Ansible playbook to setup inference server\n\n- name: Deploy inference\n hosts: dev\n gather_facts: true\n vars:\n sta"
},
{
"path": "ansible/inference/deploy-worker.yaml",
"chars": 1079,
"preview": "# Ansible playbook to setup inference worker\n\n- name: Deploy inference\n hosts: dev\n gather_facts: true\n vars:\n sta"
},
{
"path": "ansible/inference/redis.conf",
"chars": 46,
"preview": "maxmemory 1024mb\nmaxmemory-policy allkeys-lru\n"
},
{
"path": "ansible/inference/test.inventory.ini",
"chars": 36,
"preview": "[test]\ndev ansible_connection=local\n"
},
{
"path": "ansible/pgbackrest.conf",
"chars": 477,
"preview": "[oasst]\npg1-path=/var/lib/postgresql/data\n\n[global]\nrepo1-retention-full=3\nrepo1-type=s3\nrepo1-path=/oasst-prod\nrepo1-s3"
},
{
"path": "ansible/redis.conf",
"chars": 45,
"preview": "maxmemory 100mb\nmaxmemory-policy allkeys-lru\n"
},
{
"path": "ansible/test.inventory.ini",
"chars": 36,
"preview": "[test]\ndev ansible_connection=local\n"
},
{
"path": "backend/.gitignore",
"chars": 54,
"preview": "__pycache__\n.env\nnotes.txt\nvenv\ncelerybeat-schedule.*\n"
},
{
"path": "backend/README.md",
"chars": 4674,
"preview": "<a href=\"https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/backend/README.md?_x_tr_sl=auto&_x_tr_tl=en"
},
{
"path": "backend/alembic/README",
"chars": 39,
"preview": "Generic single-database configuration.\n"
},
{
"path": "backend/alembic/env.py",
"chars": 2283,
"preview": "from logging.config import fileConfig\n\nimport sqlmodel\nfrom alembic import context\nfrom oasst_backend import models # n"
},
{
"path": "backend/alembic/script.py.mako",
"chars": 526,
"preview": "\"\"\"${message}\n\nRevision ID: ${up_revision}\nRevises: ${down_revision | comma,n}\nCreate Date: ${create_date}\n\n\"\"\"\nfrom ale"
},
{
"path": "backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py",
"chars": 2699,
"preview": "\"\"\"first revision\n\nRevision ID: 23e5fea252dd\nRevises:\nCreate Date: 2022-12-12 12:47:28.801354\n\n\"\"\"\nimport sqlalchemy as "
},
{
"path": "backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py",
"chars": 9301,
"preview": "\"\"\"v1 db structure\n\nRevision ID: cd7de470586e\nRevises: 23e5fea252dd\nCreate Date: 2022-12-15 11:15:32.830225\n\n\"\"\"\nimport "
},
{
"path": "backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py",
"chars": 817,
"preview": "\"\"\"add auth_method to person\n\nRevision ID: 6368515778c5\nRevises: cd7de470586e\nCreate Date: 2022-12-17 17:57:33.022549\n\n\""
},
{
"path": "backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py",
"chars": 918,
"preview": "\"\"\"add_auth_method_to_ix_person_username\n\nRevision ID: 0daec5f8135f\nRevises: 6368515778c5\nCreate Date: 2022-12-22 18:35:"
},
{
"path": "backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py",
"chars": 1503,
"preview": "\"\"\"Adds text labels table.\n\nRevision ID: 067c4002f2d9\nRevises: 0daec5f8135f\nCreate Date: 2022-12-25 17:05:21.208843\n\n\"\"\""
},
{
"path": "backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py",
"chars": 2632,
"preview": "\"\"\"add_journal_table\n\nRevision ID: 3358eb6834e6\nRevises: 067c4002f2d9\nCreate Date: 2022-12-27 14:44:59.483868\n\n\"\"\"\nimpor"
},
{
"path": "backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py",
"chars": 2373,
"preview": "\"\"\"post ref for work_package\n\nRevision ID: d24b37426857\nRevises: 3358eb6834e6\nCreate Date: 2022-12-28 11:42:26.773704\n\n\""
},
{
"path": "backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py",
"chars": 763,
"preview": "\"\"\"Added lang column for ISO-639-1 codes\n\nRevision ID: ef0b52902560\nRevises: d24b37426857\nCreate Date: 2022-12-28 18:24:"
},
{
"path": "backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py",
"chars": 749,
"preview": "\"\"\"add collective flag to task\n\nRevision ID: 464ec4667aae\nRevises: ef0b52902560\nCreate Date: 2022-12-29 21:03:06.841962\n"
},
{
"path": "backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py",
"chars": 726,
"preview": "\"\"\"add field trusted api client\n\nRevision ID: 73ce3675c1f5\nRevises: 464ec4667aae\nCreate Date: 2022-12-30 01:09:06.446020"
},
{
"path": "backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py",
"chars": 18028,
"preview": "\"\"\"name changes: person->user, post->message, work_package->task\n\nRevision ID: abb47e9d145a\nRevises: 73ce3675c1f5\nCreate"
},
{
"path": "backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py",
"chars": 717,
"preview": "\"\"\"add deleted field to post\n\nRevision ID: 8d269bc4fdbd\nRevises: abb47e9d145a\nCreate Date: 2022-12-31 04:38:41.799206\n\n\""
},
{
"path": "backend/alembic/versions/2023_01_05_1144-d4161e384f83_added_messagetreestate_table.py",
"chars": 1899,
"preview": "\"\"\"added MessageTreeState table\n\nRevision ID: d4161e384f83\nRevises: 8d269bc4fdbd\nCreate Date: 2023-01-05 11:44:02.630633"
},
{
"path": "backend/alembic/versions/2023_01_05_1346-3b0adfadbef9_removed_date_created_and_deleted_flag_.py",
"chars": 1186,
"preview": "\"\"\"removed date_created and deleted flag from message_tree_state\n\nRevision ID: 3b0adfadbef9\nRevises: d4161e384f83\nCreate"
},
{
"path": "backend/alembic/versions/2023_01_05_1745-20cd871f4ec7_added_user_to_textlabels.py",
"chars": 891,
"preview": "\"\"\"Added user to TextLabels\n\nRevision ID: 20cd871f4ec7\nRevises: d4161e384f83\nCreate Date: 2023-01-05 17:45:15.696468\n\n\"\""
},
{
"path": "backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py",
"chars": 509,
"preview": "\"\"\"added frontend_type to api_client\n\nRevision ID: ba61fe17fb6e\nRevises: 20cd871f4ec7\nCreate Date: 2023-01-07 12:50:32.1"
},
{
"path": "backend/alembic/versions/2023_01_08_1106-3d96bb92e33a_added_minilm_embedding_column_to_message.py",
"chars": 724,
"preview": "\"\"\"added miniLM_embedding column to message\n\nRevision ID: 023548d474f7\nRevises: ba61fe17fb6e\nCreate Date: 2023-01-08 11:"
},
{
"path": "backend/alembic/versions/2023_01_08_1603-35bdc1a08bb8_embedding_for_message_now_in_its_own_.py",
"chars": 1417,
"preview": "\"\"\"embedding for message now in its own table\n\nRevision ID: 35bdc1a08bb8\nRevises: 023548d474f7\nCreate Date: 2023-01-08 1"
},
{
"path": "backend/alembic/versions/2023_01_08_2128-aac6b2f66006_created_date.py",
"chars": 770,
"preview": "\"\"\"Created date\n\nRevision ID: aac6b2f66006\nRevises: 35bdc1a08bb8\nCreate Date: 2023-01-08 21:28:27.342729\n\n\"\"\"\nimport sql"
},
{
"path": "backend/alembic/versions/2023_01_08_2200-bcc2fe18d214_messagetoxicity.py",
"chars": 1199,
"preview": "\"\"\"MessageToxicity\n\nRevision ID: bcc2fe18d214\nRevises: 20cd871f4ec7\nCreate Date: 2023-01-08 22:00:43.297719\n\n\"\"\"\nimport "
},
{
"path": "backend/alembic/versions/2023_01_08_2208-92a367bb9f40_restructure_message_tree_state_table.py",
"chars": 2667,
"preview": "\"\"\"restructure message_tree_state table\n\nRevision ID: 92a367bb9f40\nRevises: ba61fe17fb6e\nCreate Date: 2023-01-08 22:08:4"
},
{
"path": "backend/alembic/versions/2023_01_09_0047-05975b274a81_add_review_count_ranking_count_to_.py",
"chars": 1071,
"preview": "\"\"\"add review_count & ranking_count to message\n\nRevision ID: 05975b274a81\nRevises: 92a367bb9f40\nCreate Date: 2023-01-09 "
},
{
"path": "backend/alembic/versions/2023_01_10_1733-846cc08ac79f_add_enabled_deleted_notes_fields_to_user.py",
"chars": 1079,
"preview": "\"\"\"Add enabled, deleted, notes fields to User\n\nRevision ID: 846cc08ac79f\nRevises: aac6b2f66006\nCreate Date: 2023-01-10 1"
},
{
"path": "backend/alembic/versions/2023_01_12_0119-befa42582ea4_remove_accepted_messages_from_message_.py",
"chars": 784,
"preview": "\"\"\"remove accepted_messages from message_tree_state\n\nRevision ID: befa42582ea4\nRevises: 05975b274a81\nCreate Date: 2023-0"
},
{
"path": "backend/alembic/versions/2023_01_14_1509-619255ae9076_add_rank_to_message_table.py",
"chars": 5203,
"preview": "\"\"\"add rank to message table\n\nRevision ID: 619255ae9076\nRevises: bcc2fe18d214\nCreate Date: 2023-01-14 15:09:03.462482\n\n\""
},
{
"path": "backend/alembic/versions/2023_01_15_0002-7c98102efbca_change_user_stats_ranking_counts.py",
"chars": 3421,
"preview": "\"\"\"change user_stats ranking counts\n\nRevision ID: 7c98102efbca\nRevises: 619255ae9076\nCreate Date: 2023-01-15 00:02:45.62"
},
{
"path": "backend/alembic/versions/2023_01_15_1139-423557e869e4_add_indices_for_created_date.py",
"chars": 1100,
"preview": "\"\"\"add indices for created_date\n\nRevision ID: 423557e869e4\nRevises: 7c98102efbca\nCreate Date: 2023-01-15 11:39:10.407859"
},
{
"path": "backend/alembic/versions/2023_01_15_1654-0964ac95170d_add_rank_and_indices_to_user_stats.py",
"chars": 1108,
"preview": "\"\"\"add rank and indices to user_stats\n\nRevision ID: 0964ac95170d\nRevises: 423557e869e4\nCreate Date: 2023-01-15 16:54:09."
},
{
"path": "backend/alembic/versions/2023_01_19_2153-7f0a28a156f4_switch_to_timestamp_with_tz.py",
"chars": 2931,
"preview": "\"\"\"switch to timestamp with tz\n\nRevision ID: 7f0a28a156f4\nRevises: 0964ac95170d\nCreate Date: 2023-01-19 21:53:01.107137\n"
},
{
"path": "backend/alembic/versions/2023_01_19_2200-4f26fec4d204_add_ix_user_display_name_id.py",
"chars": 689,
"preview": "\"\"\"add ix_user_display_name_id\n\nRevision ID: 4f26fec4d204\nRevises: 0964ac95170d\nCreate Date: 2023-01-19 22:00:00\n\n\"\"\"\nfr"
},
{
"path": "backend/alembic/versions/2023_01_20_1650-160ac010efcc_use_en_instead_en_us_as_default_lang.py",
"chars": 862,
"preview": "\"\"\"use 'en' instead 'en-US' as default lang\n\nRevision ID: 160ac010efcc\nRevises: 4f26fec4d204\nCreate Date: 2023-01-20 16:"
},
{
"path": "backend/alembic/versions/2023_01_24_1134-8ba17b5f467a_add_message_id_to_message_reaction.py",
"chars": 1262,
"preview": "\"\"\"add message_id to message_reaction\n\nRevision ID: 8ba17b5f467a\nRevises: 160ac010efcc\nCreate Date: 2023-01-24 11:34:42."
},
{
"path": "backend/alembic/versions/2023_01_24_2256-40ed93df0ed5_add_message_emoji.py",
"chars": 1676,
"preview": "\"\"\"add message_emoji\n\nRevision ID: 40ed93df0ed5\nRevises: 8ba17b5f467a\nCreate Date: 2023-01-24 22:56:28.229408\n\n\"\"\"\nimpor"
},
{
"path": "backend/alembic/versions/2023_01_26_1835-c84fcd6900dc_add_task_created_date_index.py",
"chars": 697,
"preview": "\"\"\"add task created date index\n\nRevision ID: c84fcd6900dc\nRevises: 40ed93df0ed5\nCreate Date: 2023-01-26 18:35:43.061589\n"
},
{
"path": "backend/alembic/versions/2023_01_27_2013-f856bf19d32b_add_user_show_on_leaderboard.py",
"chars": 751,
"preview": "\"\"\"add user.show_on_leaderboard\n\nRevision ID: f856bf19d32b\nRevises: c84fcd6900dc\nCreate Date: 2023-01-27 20:13:56.533374"
},
{
"path": "backend/alembic/versions/2023_01_28_1157-49d8445b4c90_add_origin_column_to_message_tree_state.py",
"chars": 1023,
"preview": "\"\"\"add origin column to message_tree_state\n\nRevision ID: 49d8445b4c90\nRevises: f856bf19d32b\nCreate Date: 2023-01-28 11:5"
},
{
"path": "backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py",
"chars": 1228,
"preview": "\"\"\"move user_streak from user_stats to user table\n\nRevision ID: 7b8f0011e0b0\nRevises: 8a5feed819ee\nCreate Date: 2023-01-"
},
{
"path": "backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py",
"chars": 1118,
"preview": "\"\"\"add tos_acceptance_date to user\n\nRevision ID: 55361f323d12\nRevises: 7b8f0011e0b0\nCreate Date: 2023-02-01 00:22:08.280"
},
{
"path": "backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py",
"chars": 760,
"preview": "\"\"\"add won_prompt_lottery_date to mts\n\nRevision ID: f60958968ff8\nRevises: 7b8f0011e0b0\nCreate Date: 2023-02-01 10:10:38."
},
{
"path": "backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py",
"chars": 892,
"preview": "\"\"\"add skip bool & skip_reason to task\n\nRevision ID: 9e7ec4a9e3f2\nRevises: 7b8f0011e0b0\nCreate Date: 2023-02-01 21:46:49"
},
{
"path": "backend/alembic/versions/2023_02_02_1544-4d7e0b0ebe84_add_troll_stats.py",
"chars": 2515,
"preview": "\"\"\"add troll_stats\n\nRevision ID: 4d7e0b0ebe84\nRevises: 9e7ec4a9e3f2\nCreate Date: 2023-02-02 15:44:12.647260\n\n\"\"\"\nimport "
},
{
"path": "backend/alembic/versions/2023_02_02_1817-8c8241d1f973_add_account_table.py",
"chars": 1309,
"preview": "\"\"\"Add Account table\n\nRevision ID: 8c8241d1f973\nRevises: 4d7e0b0ebe84\nCreate Date: 2023-01-30 15:10:58.776315\n\n\"\"\"\nimpor"
},
{
"path": "backend/alembic/versions/2023_02_07_1922-caee1e8ee0bc_added_new_table_for_flagged_messages.py",
"chars": 1520,
"preview": "\"\"\"Added new table for flagged messages\n\nRevision ID: caee1e8ee0bc\nRevises: 8c8241d1f973\nCreate Date: 2023-02-07 19:22:1"
},
{
"path": "backend/alembic/versions/2023_02_11_1030-ba40d055714a_add_cached_stats.py",
"chars": 1059,
"preview": "\"\"\"add cached_stats\n\nRevision ID: ba40d055714a\nRevises: caee1e8ee0bc\nCreate Date: 2023-02-11 10:30:21.996198\n\n\"\"\"\nimport"
},
{
"path": "backend/alembic/versions/2023_02_14_1756-165b55de5a94_add_text_labels_message_id_index.py",
"chars": 724,
"preview": "\"\"\"add text_labels message_id index\n\nRevision ID: 165b55de5a94\nRevises: ba40d055714a\nCreate Date: 2023-02-14 17:56:48.26"
},
{
"path": "backend/alembic/versions/2023_02_15_1754-8cd0c34d0c3c_message_review_result_nullable.py",
"chars": 915,
"preview": "\"\"\"message review_result nullable\n\nRevision ID: 8cd0c34d0c3c\nRevises: 165b55de5a94\nCreate Date: 2023-02-15 17:54:58.0292"
},
{
"path": "backend/alembic/versions/2023_02_26_0052-9db92d504f64_add_lang_to_message_tree_state.py",
"chars": 1353,
"preview": "\"\"\"add lang to message_tree_state\n\nRevision ID: 9db92d504f64\nRevises: 8cd0c34d0c3c\nCreate Date: 2023-02-26 00:52:40.6248"
},
{
"path": "backend/alembic/versions/2023_05_07_2129-1b6e3ae16e9d_add_text_search.py",
"chars": 679,
"preview": "\"\"\"add text search\n\nRevision ID: 1b6e3ae16e9d\nRevises: 9db92d504f64\nCreate Date: 2023-05-07 21:29:35.545612\n\"\"\"\nimport s"
},
{
"path": "backend/alembic/versions/2023_06_06_1505-c181661eba3a_add_message_revisions.py",
"chars": 1738,
"preview": "\"\"\"add_message_revisions\n\nRevision ID: c181661eba3a\nRevises: 1b6e3ae16e9d\nCreate Date: 2023-06-06 15:05:58.079120\n\n\"\"\"\ni"
},
{
"path": "backend/alembic.ini",
"chars": 3309,
"preview": "# A generic, single database configuration.\n\n[alembic]\n# path to migration scripts\nscript_location = %(here)s/alembic\n\n#"
},
{
"path": "backend/export.py",
"chars": 15913,
"preview": "import argparse\nfrom pathlib import Path\nfrom typing import List, Optional\nfrom uuid import UUID\n\nimport sqlalchemy as s"
},
{
"path": "backend/import.py",
"chars": 8077,
"preview": "import argparse\nimport json\nimport sys\nfrom pathlib import Path\nfrom typing import Optional\nfrom uuid import UUID\n\nimpor"
},
{
"path": "backend/main.py",
"chars": 13969,
"preview": "import json\nfrom datetime import datetime\nfrom http import HTTPStatus\nfrom math import ceil\nfrom pathlib import Path\nfro"
},
{
"path": "backend/oasst_backend/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/oasst_backend/api/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/oasst_backend/api/deps.py",
"chars": 7659,
"preview": "from http import HTTPStatus\nfrom secrets import token_hex\nfrom typing import Generator, NamedTuple, Optional\nfrom uuid i"
},
{
"path": "backend/oasst_backend/api/v1/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/oasst_backend/api/v1/admin.py",
"chars": 10308,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport pydantic\nfrom fastapi import API"
},
{
"path": "backend/oasst_backend/api/v1/api.py",
"chars": 1287,
"preview": "from fastapi import APIRouter\nfrom oasst_backend.api.v1 import (\n admin,\n auth,\n frontend_messages,\n fronten"
},
{
"path": "backend/oasst_backend/api/v1/auth.py",
"chars": 1404,
"preview": "from typing import Union\n\nfrom cryptography.hazmat.primitives import hashes\nfrom cryptography.hazmat.primitives.kdf.hkdf"
},
{
"path": "backend/oasst_backend/api/v1/frontend_messages.py",
"chars": 4418,
"preview": "from typing import Optional\n\nfrom fastapi import APIRouter, Depends\nfrom oasst_backend.api import deps\nfrom oasst_backen"
},
{
"path": "backend/oasst_backend/api/v1/frontend_users.py",
"chars": 5672,
"preview": "import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends, Query\nfrom oa"
},
{
"path": "backend/oasst_backend/api/v1/hugging_face.py",
"chars": 1108,
"preview": "from typing import List\n\nfrom fastapi import APIRouter, Depends\nfrom oasst_backend.api import deps\nfrom oasst_backend.mo"
},
{
"path": "backend/oasst_backend/api/v1/leaderboards.py",
"chars": 1953,
"preview": "from typing import Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends, Query\nfrom oasst_backend.api "
},
{
"path": "backend/oasst_backend/api/v1/login.py",
"chars": 2937,
"preview": "import aiohttp\nfrom fastapi import APIRouter, Depends, HTTPException, Request\nfrom oasst_backend import auth\nfrom oasst_"
},
{
"path": "backend/oasst_backend/api/v1/messages.py",
"chars": 13953,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends,"
},
{
"path": "backend/oasst_backend/api/v1/stats.py",
"chars": 2840,
"preview": "from fastapi import APIRouter, Depends\nfrom oasst_backend.api import deps\nfrom oasst_backend.cached_stats_repository imp"
},
{
"path": "backend/oasst_backend/api/v1/tasks.py",
"chars": 6895,
"preview": "from typing import Any, Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends\nfrom fastapi.security.api"
},
{
"path": "backend/oasst_backend/api/v1/text_labels.py",
"chars": 3192,
"preview": "from typing import Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom fastapi.se"
},
{
"path": "backend/oasst_backend/api/v1/trollboards.py",
"chars": 802,
"preview": "from typing import Optional\n\nfrom fastapi import APIRouter, Depends, Query\nfrom oasst_backend.api import deps\nfrom oasst"
},
{
"path": "backend/oasst_backend/api/v1/users.py",
"chars": 11631,
"preview": "import datetime\nfrom typing import Callable, Optional\nfrom uuid import UUID\n\nfrom fastapi import APIRouter, Depends, Que"
},
{
"path": "backend/oasst_backend/api/v1/utils.py",
"chars": 2870,
"preview": "import re\nfrom uuid import UUID\n\nfrom oasst_backend.models import Message, MessageRevision\nfrom oasst_shared.schemas imp"
},
{
"path": "backend/oasst_backend/auth.py",
"chars": 1040,
"preview": "from datetime import datetime, timedelta\nfrom typing import Optional\n\nfrom jose import jwt\nfrom oasst_backend.config imp"
},
{
"path": "backend/oasst_backend/cached_stats_repository.py",
"chars": 4428,
"preview": "from oasst_backend.models import CachedStats, Message, MessageTreeState, User\nfrom oasst_shared.exceptions.oasst_api_err"
},
{
"path": "backend/oasst_backend/celery_worker.py",
"chars": 1019,
"preview": "import os\n\nfrom celery import Celery\nfrom loguru import logger\n\n\"\"\"\nTo run the worker run `celery run -A oasst_backend.c"
},
{
"path": "backend/oasst_backend/config.py",
"chars": 10456,
"preview": "from pathlib import Path\nfrom typing import Any, Dict, List, Optional\n\nfrom oasst_shared.schemas.protocol import TextLab"
},
{
"path": "backend/oasst_backend/database.py",
"chars": 495,
"preview": "from oasst_backend.config import settings\nfrom oasst_shared.exceptions import OasstError, OasstErrorCode\nfrom sqlmodel i"
},
{
"path": "backend/oasst_backend/journal_writer.py",
"chars": 3713,
"preview": "import enum\nfrom typing import Literal, Optional\nfrom uuid import UUID\n\nfrom oasst_backend.models import ApiClient, Jour"
},
{
"path": "backend/oasst_backend/models/__init__.py",
"chars": 1010,
"preview": "from .api_client import ApiClient\nfrom .cached_stats import CachedStats\nfrom .flagged_message import FlaggedMessage\nfrom"
},
{
"path": "backend/oasst_backend/models/api_client.py",
"chars": 839,
"preview": "from typing import Optional\nfrom uuid import UUID, uuid4\n\nimport sqlalchemy as sa\nimport sqlalchemy.dialects.postgresql "
},
{
"path": "backend/oasst_backend/models/cached_stats.py",
"chars": 571,
"preview": "from datetime import datetime\n\nimport sqlalchemy as sa\nimport sqlalchemy.dialects.postgresql as pg\nfrom sqlmodel import "
},
{
"path": "backend/oasst_backend/models/db_payload.py",
"chars": 3927,
"preview": "from typing import Literal, Optional\nfrom uuid import UUID\n\nfrom oasst_backend.models.payload_column_type import payload"
},
{
"path": "backend/oasst_backend/models/flagged_message.py",
"chars": 731,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/journal.py",
"chars": 2538,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID, uuid1, uuid4\n\nimport sqlalchemy as sa\ni"
},
{
"path": "backend/oasst_backend/models/message.py",
"chars": 4623,
"preview": "from datetime import datetime\nfrom http import HTTPStatus\nfrom typing import Any, Optional\nfrom uuid import UUID, uuid4\n"
},
{
"path": "backend/oasst_backend/models/message_embedding.py",
"chars": 860,
"preview": "from datetime import datetime\nfrom typing import List, Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sq"
},
{
"path": "backend/oasst_backend/models/message_emoji.py",
"chars": 1013,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/message_reaction.py",
"chars": 1164,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/message_revision.py",
"chars": 1085,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/message_toxicity.py",
"chars": 938,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/message_tree_state.py",
"chars": 3551,
"preview": "from datetime import datetime\nfrom enum import Enum\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy"
},
{
"path": "backend/oasst_backend/models/payload_column_type.py",
"chars": 3116,
"preview": "import json\nfrom typing import Any, Generic, Type, TypeVar\n\nimport sqlalchemy.dialects.postgresql as pg\nfrom fastapi.enc"
},
{
"path": "backend/oasst_backend/models/task.py",
"chars": 1890,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID, uuid4\n\nimport sqlalchemy as sa\nimport s"
},
{
"path": "backend/oasst_backend/models/text_labels.py",
"chars": 1220,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID, uuid4\n\nimport sqlalchemy as sa\nimport s"
},
{
"path": "backend/oasst_backend/models/troll_stats.py",
"chars": 1912,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy as sa\nimport sqlalche"
},
{
"path": "backend/oasst_backend/models/user.py",
"chars": 3352,
"preview": "from datetime import datetime\nfrom typing import Optional\nfrom uuid import UUID, uuid4\n\nimport sqlalchemy as sa\nimport s"
},
{
"path": "backend/oasst_backend/models/user_stats.py",
"chars": 2132,
"preview": "from datetime import datetime\nfrom enum import Enum\nfrom typing import Optional\nfrom uuid import UUID\n\nimport sqlalchemy"
},
{
"path": "backend/oasst_backend/prompt_repository.py",
"chars": 58799,
"preview": "import random\nimport re\nfrom collections import defaultdict\nfrom datetime import datetime, timedelta\nfrom http import HT"
},
{
"path": "backend/oasst_backend/scheduled_tasks.py",
"chars": 4558,
"preview": "from __future__ import absolute_import, unicode_literals\n\nfrom datetime import timedelta\nfrom typing import Any, Dict, L"
},
{
"path": "backend/oasst_backend/schemas/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/oasst_backend/schemas/hugging_face.py",
"chars": 106,
"preview": "from pydantic import BaseModel\n\n\nclass ToxicityClassification(BaseModel):\n label: str\n score: float\n"
},
{
"path": "backend/oasst_backend/schemas/message_tree.py",
"chars": 328,
"preview": "from uuid import UUID\n\nfrom oasst_backend.models.message_tree_state import State as TreeState\nfrom pydantic import BaseM"
},
{
"path": "backend/oasst_backend/schemas/text_labels.py",
"chars": 171,
"preview": "from oasst_shared.schemas.protocol import LabelDescription\nfrom pydantic import BaseModel\n\n\nclass ValidLabelsResponse(Ba"
},
{
"path": "backend/oasst_backend/task_repository.py",
"chars": 10620,
"preview": "from datetime import datetime, timedelta\nfrom typing import Optional\nfrom uuid import UUID\n\nimport oasst_backend.models."
},
{
"path": "backend/oasst_backend/tree_manager.py",
"chars": 83886,
"preview": "import random\nfrom datetime import datetime, timedelta\nfrom enum import Enum\nfrom http import HTTPStatus\nfrom typing imp"
},
{
"path": "backend/oasst_backend/user_repository.py",
"chars": 14593,
"preview": "from typing import Optional\nfrom uuid import UUID\n\nimport oasst_backend.models as models\nfrom oasst_backend.config impor"
},
{
"path": "backend/oasst_backend/user_stats_repository.py",
"chars": 24154,
"preview": "from datetime import datetime, timedelta\nfrom typing import Optional\nfrom uuid import UUID\n\nimport numpy as np\nimport sq"
},
{
"path": "backend/oasst_backend/utils/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/oasst_backend/utils/database_utils.py",
"chars": 12054,
"preview": "from enum import IntEnum\nfrom functools import wraps\nfrom http import HTTPStatus\nfrom typing import Callable\n\nfrom logur"
},
{
"path": "backend/oasst_backend/utils/discord.py",
"chars": 2482,
"preview": "from uuid import UUID\n\nimport requests\nfrom loguru import logger\nfrom oasst_backend.celery_worker import app as celery_a"
},
{
"path": "backend/oasst_backend/utils/exported_tree_loading.py",
"chars": 2433,
"preview": "import json\nfrom collections import defaultdict\nfrom typing import List\n\nimport pandas as pd\n\n\ndef load_jsonl(filepaths)"
},
{
"path": "backend/oasst_backend/utils/hugging_face.py",
"chars": 2262,
"preview": "from enum import Enum\nfrom typing import Any, Dict\n\nimport aiohttp\nfrom loguru import logger\nfrom oasst_backend.config i"
},
{
"path": "backend/oasst_backend/utils/language_classification.py",
"chars": 3679,
"preview": "import os\nimport pickle\nfrom collections import Counter\n\nfrom sklearn import metrics\nfrom sklearn.feature_extraction.tex"
},
{
"path": "backend/oasst_backend/utils/message_tree_topic_modeling.py",
"chars": 4291,
"preview": "import argparse\n\nfrom bertopic import BERTopic\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom bertopi"
},
{
"path": "backend/oasst_backend/utils/ranking.py",
"chars": 6399,
"preview": "from typing import List\n\nimport numpy as np\n\n\ndef head_to_head_votes(ranks: List[List[int]]):\n tallies = np.zeros((le"
},
{
"path": "backend/oasst_backend/utils/similarity_functions.py",
"chars": 6520,
"preview": "import math\n\nimport numpy as np\nimport scipy.sparse as sp\nimport torch\nimport torch.nn.functional as F\nfrom pandas impor"
},
{
"path": "backend/oasst_backend/utils/topic_model_requirments.txt",
"chars": 44,
"preview": "pandas\nsentence-transformers\nbertopic\nscipy\n"
},
{
"path": "backend/oasst_backend/utils/tree_export.py",
"chars": 6540,
"preview": "from __future__ import annotations\n\nimport contextlib\nimport gzip\nimport json\nimport sys\nimport uuid\nfrom collections im"
},
{
"path": "backend/requirements.txt",
"chars": 448,
"preview": "aiohttp==3.8.3\nalembic==1.8.1\nasgiref==3.6.0\nCelery==5.2.0\ncryptography==41.0.0\nfastapi==0.88.0\nfastapi-limiter==0.1.5\nf"
},
{
"path": "backend/requirements_worker.txt",
"chars": 403,
"preview": "aiohttp==3.8.3\nalembic==1.8.1\nasgiref==3.6.0\nCelery==5.2.0\ncryptography==39.0.0\nfastapi==0.88.0\nfastapi-limiter==0.1.5\nf"
},
{
"path": "backend/rerank.py",
"chars": 2995,
"preview": "import argparse\nfrom uuid import UUID\n\nimport oasst_backend.utils.database_utils as db_utils\nfrom export import fetch_tr"
},
{
"path": "backend/sql_snippets.md",
"chars": 5420,
"preview": "# Collection of SQL Snippets\n\nHere are find some SQL queries to inspect the current OA postgres DB.\n\n# Basic Stats\n\n```s"
},
{
"path": "backend/test_data/generic/test_generic_data.json",
"chars": 2089,
"preview": "[\n {\n \"task_message_id\": \"de111fa8\",\n \"user_message_id\": \"6f1d0711\",\n \"parent_message_id\": null,\n \"text\": \""
},
{
"path": "backend/test_data/realistic/realistic_seed_data.json",
"chars": 137110,
"preview": "[\n {\n \"task_message_id\": \"qv8uup5z\",\n \"user_message_id\": \"b9d4wm7n\",\n \"parent_message_id\": null,\n \"text\": \""
},
{
"path": "backend/tests/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "backend/tests/test_settings.py",
"chars": 1058,
"preview": "from oasst_backend.config import Settings\n\n\ndef test_create_default_settings():\n \"\"\"\n Make sure we can create one "
},
{
"path": "backend/tests/test_tree_manager_config.py",
"chars": 181,
"preview": "from oasst_backend.config import TreeManagerConfiguration\n\n\ndef test_tree_manager_config():\n \"\"\"\n Just test that w"
},
{
"path": "backend/update_message_attributes.py",
"chars": 3288,
"preview": "import time\n\nfrom loguru import logger\nfrom oasst_backend.models import ApiClient, Message\nfrom oasst_backend.scheduled_"
},
{
"path": "copilot/.workspace",
"chars": 28,
"preview": "application: open-assistant\n"
},
{
"path": "copilot/README.md",
"chars": 2244,
"preview": "<a href=\"https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/copilot/README.md?_x_tr_sl=auto&_x_tr_tl=en"
},
{
"path": "copilot/api/manifest.yml",
"chars": 911,
"preview": "# The manifest for the \"api\" service.\n# Read the full specification for the \"Load Balanced Web Service\" type at:\n# http"
},
{
"path": "copilot/environments/staging/manifest.yml",
"chars": 306,
"preview": "# The manifest for the \"staging\" environment.\n# Read the full specification for the \"Environment\" type at:\n# https://aw"
},
{
"path": "copilot/web/manifest.yml",
"chars": 1707,
"preview": "# The manifest for the \"web\" service.\n# Read the full specification for the \"Load Balanced Web Service\" type at:\n# http"
},
{
"path": "data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "data/datasets/README.md",
"chars": 8163,
"preview": "<a href=\"https://github-com.translate.goog/LAION-AI/Open-Assistant/blob/main/data/datasets/README.md?_x_tr_sl=auto&_x_tr"
},
{
"path": "data/datasets/TSSB-3M/README.md",
"chars": 619,
"preview": "# Dataset summary\n\nThis dataset contains over 3 million isolated single statement bug fixes. Each\nbug fix is related to "
},
{
"path": "data/datasets/TSSB-3M/generate_dataset.py",
"chars": 7347,
"preview": "\"\"\"Convert the source TSSB-3M dataset to instruction data\n\"\"\"\n\nimport json\nimport random\nimport re\nfrom os.path import "
},
{
"path": "data/datasets/TSSB-3M/invalid_commit_messages.tsv",
"chars": 75483,
"preview": "fix\t19758\nminor fix\t17527\nbugfix\t14972\nbug fix\t13735\nMinor fix\t7833\nfix test\t7433\nfix bug\t6962\ndebug\t6679\nsmall fix\t6115"
},
{
"path": "data/datasets/TSSB-3M/load_script.py",
"chars": 129,
"preview": "from datasets import load_dataset\n\nif __name__ == \"__main__\":\n ds = load_dataset(\"zirui3/TSSB-3M-instructions\")\n p"
},
{
"path": "data/datasets/__init__.py",
"chars": 2291,
"preview": "TEXT_DATASETS = {\n \"gutenberg_english\": \"sedthh/gutenberg_english\", # Gutenberg eBooks in English\n \"gutenberg_mul"
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/1_clean_wikitext.py",
"chars": 2676,
"preview": "import os\nimport re\nimport time\nimport timeit\n\nimport pandas as pd\nimport psutil\nfrom datasets import load_dataset\nfrom "
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/2_wikitext_doc2query.ipynb",
"chars": 121362,
"preview": "{\n \"nbformat\": 4,\n \"nbformat_minor\": 0,\n \"metadata\": {\n \"colab\": {\n \"provenance\": [],\n \"machine_shape\": \"hm\"\n },\n "
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/3_10k_bart_trial.ipynb",
"chars": 166979,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n "
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/3_10k_bart_trial.py",
"chars": 1400,
"preview": "import pandas as pd\nimport tiktoken\nimport torch\nimport tqdm\nfrom transformers import pipeline\n\n\ndef num_tokens_from_str"
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/4_convert_to_oa_format.py",
"chars": 878,
"preview": "import json\n\nimport pandas as pd\n\nif __name__ == \"__main__\":\n raw_df = pd.read_csv(r\"...\\wiki_qa_bart_10000row.csv\")\n"
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/5_test_downloading_my_dataset.py",
"chars": 171,
"preview": "if __name__ == \"__main__\":\n from datasets import load_dataset\n\n dataset = load_dataset(\"michaelthwan/oa_wiki_qa_ba"
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/README.md",
"chars": 1110,
"preview": "# Dataset: Retrieval-based grounded model generated Q-A pairs #2004\n\nRelated to [Issue #2004](https://github.com/LAION-A"
},
{
"path": "data/datasets/bart_searchgpt_wiki_nlp_augment/requirement.txt",
"chars": 151,
"preview": "## python 3.10.8\n# basic\npandas==1.5.3\n# PyYAML==6.0\n\n# embedding\nmatplotlib==3.7.1\nseaborn\n\n# misc\npsutil==5.9.4\n\n# hug"
},
{
"path": "data/datasets/biostars_qa/README.md",
"chars": 1191,
"preview": "### Dataset Summary\n\nThis dataset contains 4803 question/answer pairs extracted from the\n[BioStars](https://www.biostars"
},
{
"path": "data/datasets/biostars_qa/get_biostars_dataset.py",
"chars": 5417,
"preview": "import json\nimport os\nimport re\nimport time\n\nimport pandas as pd\nimport requests\nfrom tqdm import tqdm\n\n\ndef get_biostar"
},
{
"path": "data/datasets/biostars_qa/requirements.txt",
"chars": 29,
"preview": "pandas\npyarrow\nrequests\ntdqm\n"
},
{
"path": "data/datasets/cmu_wiki_qa/README.md",
"chars": 1380,
"preview": "---\ndataset_info:\n features:\n - name: INSTRUCTION\n dtype: string\n - name: RESPONSE\n dtype: string\n -"
},
{
"path": "data/datasets/cmu_wiki_qa/cmu_parser.ipynb",
"chars": 15314,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Parser for Question-Answer Datase"
},
{
"path": "data/datasets/cmu_wiki_qa/requirements.txt",
"chars": 35,
"preview": "numpy\npandas\npyarrow\nrequests\ntqdm\n"
}
]
// ... and 1338 more files (download for full content)
About this extraction
This page contains the full source code of the LAION-AI/Open-Assistant GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 1538 files (11.6 MB), approximately 3.1M tokens, and a symbol index with 2537 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.