[ { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "

\n \"qa-pilot\"\n

\n\nQA-Pilot is an interactive chat project that leverages online/local LLM for rapid understanding and navigation of GitHub code repository.\n\n### Features\n\n* Chat with github public repository with git clone way\n* Store the chat history \n* Easy to set the configuration\n* Multiple chat sessions\n* Locate the session quicly with search function\n* Integrate with `codegraph` to view the python file\n* Support the different LLM models\n * ollama(deepseek, llama3.1, phi3, llama3, gemma2)\n * openai(gpt-4o, gpt-4-turbo, gpt-4, and gpt-3.5-turbo)\n * mistralai(mistral-tiny, mistral-tiny, mistral-small-latest, mistral-medium-latest, mistral-large-latest, codestral-lates)\n * localai(gpt-4, more)\n * zhipuai(glm-4-0520, glm-4, glm-4-air, glm-4-airx, glm-4-flash)\n * anthropic(claude-3-opus-20240229, claude-3-sonnet-20240229, claude-3-haiku-20240307, claude-3-5-sonnet-20240620)\n * llamacpp\n * nvidia(meta/llama3-70b-instruct, more)\n * tongyi(qwen-turbo, qwen-plus, qwen-max, more)\n * moonshot(moonshot-v1-8k, moonshot-v1-32k, moonshot-v1-128k)\n\n\n### Release\n\n* 2024-07-03 update langchain to `0.2.6` version and add `moonshot` API support\n\n* 2024-06-30 add `Go Codegraph`\n\n* 2024-06-27 add `nvidia/tongyi` API support\n\n* 2024-06-19 add `llamacpp` API support, improve the `settings` list in the sidebar and add upload model function for `llamacpp`, add `prompt templates` setting\n\n* 2024-06-15 add `anthropic` API support, refactor some functions, and fix chat show messages\n\n* 2024-06-12 add `zhipuai` API support\n\n* 2024-06-10 Convert `flask` to `fastapi` and add `localai` API support\n\n* 2024-06-07 Add `rr:` option and use `FlashRank` for the search \n\n* 2024-06-05 Upgrade `langchain` to `v0.2` and add `ollama embeddings`\n\n* 2024-05-26 Release v2.0.1: Refactoring to replace `Streamlit` fontend with `Svelte` to improve the performance.\n\n### Disclaimer\n\n* This is a test project to validate the feasibility of a fully local solution for question answering using LLMs and Vector embeddings. It is not production ready, and it is not meant to be used in production. \n* `Do not use models for analyzing your critical or production data!!`\n* `Do not use models for analyzing customer data to ensure data privacy and security!!`\n* `Do not use models for analyzing you private/sensitivity code respository!!`\n\n#### QA-Pilot\n![qa-demo-new](https://github.com/reid41/QA-Pilot/assets/25558653/8198730f-32ec-4664-a10c-43b3f40c99ad)\n\n\n#### CodeGraph\n![code-graph-new](https://github.com/reid41/QA-Pilot/assets/25558653/8c47ea00-d703-42b5-b43b-d40796e7de1d)\n\nTo deploy QA-Pilot, you can follow the below steps:\n\n1. Clone the QA-Pilot repository:\n\n```shell\ngit clone https://github.com/reid41/QA-Pilot.git\ncd QA-Pilot\n```\n\n2. Install [conda](https://www.anaconda.com/download) for virtual environment management. Create and activate a new virtual environment.\n\n```shell\nconda create -n QA-Pilot python=3.10.14\nconda activate QA-Pilot\n```\n\n3. Install the required dependencies:\n\n```shell\npip install -r requirements.txt\n```\n\n4. Install the pytorch with cuda [pytorch](https://pytorch.org/get-started/locally/)\n\n5. Setup providers\n\n* For setup [ollama website](https://ollama.com/) and [ollama github](https://github.com/ollama/ollama) to manage the local LLM. \ne.g.\n\n```shell\nollama pull \n\nollama list\n```\n\n* For setup [localAI](https://localai.io/) and [LocalAI github](https://github.com/mudler/LocalAI) to manage the local LLM, set the localAI `base_url` in config/config.ini.\ne.g.\n```shell\ndocker run -p 8080:8080 --name local-ai -ti localai/localai:latest-aio-cpu\n# Do you have a Nvidia GPUs? Use this instead\n# CUDA 11\n# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-nvidia-cuda-11\n# CUDA 12\n# docker run -p 8080:8080 --gpus all --name local-ai -ti localai/localai:latest-aio-gpu-nvidia-cuda-12\n\n# quick check the service with http://:8080/\n# quick check the models with http://:8080/models/\n```\n\n* For setup llamacpp with [llama-cpp-python](https://github.com/abetlen/llama-cpp-python#windows-remarks)\n - upload the model to `llamacpp_models` dir or upload from the `llamacpp models` under the `Settings`\n - set the model in `llamacpp_llm_models` section in `config/config.ini`\n\n* For setup API key in `.env`\n - [OpenAI](https://platform.openai.com/docs/overview): OPENAI_API_KEY=''\n - [MistralAI](https://docs.mistral.ai/): MISTRAL_API_KEY=''\n - [ZhipuAI](https://open.bigmodel.cn/): ZHIPUAI_API_KEY=''\n - [Anthropic](https://console.anthropic.com/settings/keys): ANTHROPIC_API_KEY=''\n - [Nvidia](https://build.nvidia.com/explore/discover): NVIDIA_API_KEY=''\n - [TongYi](https://help.aliyun.com/document_detail/611472.html?spm=a2c4g.2399481.0.0): DASHSCOPE_API_KEY=''\n - [Moonshot](https://platform.moonshot.cn/): MOONSHOT_API_KEY=''\n\n* For `Go codegraph`, make sure setup [GO](https://go.dev/doc/install) env, compile go file and test\n```shell\ngo build -o parser parser.go\n\n# test\n./parser /path/test.go\n```\n\n6. Set the related parameters in `config/config.ini`, e.g. `model provider`, `model`, `variable`, `Ollama API url` and setup the [Postgresql](https://www.postgresql.org/download/) env\n```shell\n# create the db, e.g.\nCREATE DATABASE qa_pilot_chatsession_db;\nCREATE USER qa_pilot_user WITH ENCRYPTED PASSWORD 'qa_pilot_p';\nGRANT ALL PRIVILEGES ON DATABASE qa_pilot_chatsession_db TO qa_pilot_user;\n\n# set the connection\ncat config/config.ini\n[database]\ndb_name = qa_pilot_chatsession_db\ndb_user = qa_pilot_user\ndb_password = qa_pilot_p\ndb_host = localhost\ndb_port = 5432\n\n\n# set the arg in script and test connection\npython check_postgresql_connection.py\n```\n\n7. Download and install [node.js](https://nodejs.org/en/download/package-manager) and Set up the fontend env in one terminal\n```shell\n# make sure the backend server host ip is correct, localhost is by default\ncat svelte-app/src/config.js\nexport const API_BASE_URL = 'http://localhost:5000';\n\n# install deps\ncd svelte-app\nnpm install\n\nnpm run dev\n```\n\n8. Run the backend QA-Pilot in another terminal:\n\n```shell\npython qa_pilot_run.py\n```\n\n### Tips\n* Do not use url and upload at the same time.\n* The remove button cannot really remove the local chromadb, need to remove it manually when stop it.\n* Switch to `New Source Button` to add a new project\n* Use `rsd:` to start the input and get the source document\n* Use `rr:` to start the input and use the `FlashrankRerank` for the search\n* Click `Open Code Graph` in `QA-Pilot` to view the code(make sure the the already in the project session and loaded before click), curretly support `python` and `go`\n\n" }, { "path": "app.py", "content": "from fastapi import FastAPI, Request, HTTPException, UploadFile, File, Form\nfrom fastapi.responses import JSONResponse\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom fastapi.templating import Jinja2Templates\nimport configparser\nimport os\nfrom dotenv import load_dotenv, set_key\nfrom utils.helper import (\n DataHandler,\n remove_directory,\n encode_kwargs,\n model_kwargs,\n)\nimport psycopg2\nfrom psycopg2 import sql\nimport ast \nfrom qa_model_apis import (\n get_chat_model,\n get_embedding_model,\n)\nfrom utils.codegraph import (\n parse_python_code,\n read_current_repo_path,\n build_file_tree,\n)\n\nfrom utils.go_codegraph import(\n parse_go_code,\n go_build_file_tree,\n)\n\napp = FastAPI()\n\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"],\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\nconfig_path = os.path.join('config', 'config.ini')\nprompt_templates_path = os.path.join('config', 'prompt_templates.ini')\nconfig = configparser.ConfigParser()\nconfig.read(config_path)\n\ntemplates = Jinja2Templates(directory=\"templates\")\n\nDB_NAME = config['database']['db_name']\nDB_USER = config['database']['db_user']\nDB_PASSWORD = config['database']['db_password']\nDB_HOST = config['database']['db_host']\nDB_PORT = config['database']['db_port']\n\n# for analyse code\ncurrent_session = None\n\ncurrent_model_info = {\n \"provider\": None,\n \"model\": None,\n \"eb_provider\": None,\n \"eb_model\": None,\n \"chat_model\": None,\n \"embedding_model\": None\n}\n\ndef init_db():\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n cursor.execute('''\n CREATE TABLE IF NOT EXISTS sessions (\n id BIGINT PRIMARY KEY,\n name TEXT NOT NULL,\n url TEXT NOT NULL\n )\n ''')\n conn.commit()\n # fix the first time to set the session\n cursor.execute('SELECT id, name, url FROM sessions LIMIT 1')\n session = cursor.fetchone()\n if session:\n global current_session\n current_session = {'id': session[0], 'name': session[1], 'url': session[2]}\n print(\"Default session set to:\", current_session)\n conn.close()\n\ndef load_models_if_needed():\n selected_provider = config.get('model_providers', 'selected_provider')\n selected_model = config.get(f\"{selected_provider}_llm_models\", 'selected_model')\n eb_selected_provider = config.get('embedding_model_providers', 'selected_provider')\n eb_selected_model = config.get(f\"{eb_selected_provider}_embedding_models\", 'selected_model')\n \n if (current_model_info[\"provider\"] != selected_provider or \n current_model_info[\"model\"] != selected_model or \n current_model_info[\"eb_provider\"] != eb_selected_provider or \n current_model_info[\"eb_model\"] != eb_selected_model):\n\n current_model_info[\"provider\"] = selected_provider\n current_model_info[\"model\"] = selected_model\n current_model_info[\"eb_provider\"] = eb_selected_provider\n current_model_info[\"eb_model\"] = eb_selected_model\n current_model_info[\"chat_model\"] = get_chat_model(selected_provider, selected_model)\n current_model_info[\"embedding_model\"] = get_embedding_model(eb_selected_provider, eb_selected_model, model_kwargs, encode_kwargs)\n print(f\"Loaded new models: provider={selected_provider}, model={selected_model}\")\n \n print(f\"Loaded models: provider={selected_provider}, model={selected_model}\")\n\ndef create_message_table(session_id):\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n table_name = sql.Identifier(f'session_{session_id}')\n cursor.execute(sql.SQL('''\n CREATE TABLE IF NOT EXISTS {} (\n id BIGSERIAL PRIMARY KEY,\n sender TEXT NOT NULL,\n text TEXT NOT NULL\n )\n ''').format(table_name))\n conn.commit()\n conn.close()\n\ninit_db()\n\ndef load_config():\n config.read(config_path)\n return config\n\n@app.get('/get_config')\nasync def get_config():\n config = load_config()\n config_dict = {section: dict(config.items(section)) for section in config.sections()}\n return JSONResponse(content=config_dict)\n\n@app.post('/save_config')\nasync def save_config(request: Request):\n new_config = await request.json()\n config = load_config()\n for section, section_values in new_config.items():\n if not config.has_section(section):\n config.add_section(section)\n for key, value in section_values.items():\n config.set(section, key, value)\n with open(config_path, 'w') as configfile:\n config.write(configfile)\n return JSONResponse(content={\"message\": \"Configuration saved successfully!\"})\n\n@app.post('/update_provider')\nasync def update_provider(request: Request):\n data = await request.json()\n selected_provider = data.get('selected_provider')\n config.set('model_providers', 'selected_provider', selected_provider)\n with open(config_path, 'w') as configfile:\n config.write(configfile)\n return JSONResponse(content={\"message\": \"Provider updated successfully!\"})\n\n@app.post('/update_model')\nasync def update_model(request: Request):\n data = await request.json()\n selected_provider = data.get('selected_provider')\n selected_model = data.get('selected_model')\n config.set(f'{selected_provider}_llm_models', 'selected_model', selected_model)\n with open(config_path, 'w') as configfile:\n config.write(configfile)\n return JSONResponse(content={\"message\": \"Model updated successfully!\"})\n\n@app.post('/load_repo')\nasync def load_repo(request: Request):\n data = await request.json()\n git_url = data.get('git_url')\n if not git_url:\n raise HTTPException(status_code=400, detail=\"Git URL is required\")\n\n load_models_if_needed()\n chat_model = current_model_info[\"chat_model\"]\n embedding_model = current_model_info[\"embedding_model\"]\n data_handler = DataHandler(git_url, chat_model, embedding_model)\n try:\n data_handler.git_clone_repo()\n data_handler.load_into_db()\n return JSONResponse(content={\"message\": f\"Repository {git_url} loaded successfully!\"})\n except Exception as e:\n raise HTTPException(status_code=500, detail=str(e))\n\n@app.post('/chat')\nasync def chat(request: Request):\n data = await request.json()\n load_models_if_needed()\n chat_model = current_model_info[\"chat_model\"]\n embedding_model = current_model_info[\"embedding_model\"]\n\n user_message = data.get('message')\n current_repo = data.get('current_repo')\n session_id = data.get('session_id')\n if not user_message or not current_repo or not session_id:\n raise HTTPException(status_code=400, detail=\"Message, current_repo and session_id are required\")\n\n try:\n data_handler = DataHandler(current_repo, chat_model, embedding_model)\n data_handler.load_into_db()\n rsd = False\n rr = False\n\n # return source documents\n if user_message.startswith('rsd:'):\n user_message = user_message[4:].strip()\n rsd = True\n # use reranker\n elif user_message.startswith('rr:'):\n user_message = user_message[3:].strip()\n rr = True\n bot_response = data_handler.retrieval_qa(user_message, rsd=rsd, rr=rr)\n\n # Save user message and bot response to the session table\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n table_name = sql.Identifier(f'session_{session_id}')\n cursor.execute(sql.SQL('INSERT INTO {} (sender, text) VALUES (%s, %s)').format(table_name), ('You', user_message))\n cursor.execute(sql.SQL('INSERT INTO {} (sender, text) VALUES (%s, %s)').format(table_name), ('QA-Pilot', bot_response))\n conn.commit()\n conn.close()\n\n return JSONResponse(content={\"response\": bot_response})\n except Exception as e:\n raise HTTPException(status_code=500, detail=str(e))\n\n@app.get('/sessions')\nasync def get_sessions():\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n cursor.execute('SELECT id, name, url FROM sessions')\n sessions = [{'id': row[0], 'name': row[1], 'url': row[2]} for row in cursor.fetchall()]\n conn.close()\n print(f\"Fetched sessions from DB: {sessions}\")\n return JSONResponse(content=sessions)\n\n@app.post('/sessions')\nasync def save_sessions(request: Request):\n sessions = await request.json()\n print(f\"Received sessions to save: {sessions}\")\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n for session in sessions:\n cursor.execute('INSERT INTO sessions (id, name, url) VALUES (%s, %s, %s) ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, url = EXCLUDED.url',\n (session['id'], session['name'], session['url']))\n create_message_table(session['id'])\n conn.commit()\n conn.close()\n print(\"Saved sessions to DB\")\n return JSONResponse(content={\"message\": \"Sessions saved successfully!\"})\n\n@app.get('/messages/{session_id}')\nasync def get_messages(session_id: int):\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n table_name = sql.Identifier(f'session_{session_id}')\n cursor.execute(sql.SQL('SELECT sender, text FROM {}').format(table_name))\n messages = [{'sender': row[0], 'text': row[1]} for row in cursor.fetchall()]\n conn.close()\n print(f\"Fetched messages from session {session_id}\")\n return JSONResponse(content=messages)\n\n@app.post('/update_current_session')\nasync def update_current_session(request: Request):\n global current_session\n current_session = await request.json()\n return JSONResponse(content={\"message\": \"Current session updated successfully!\"})\n\n@app.delete('/sessions/{session_id}')\nasync def delete_session(session_id: int):\n print(f\"Deleting session with ID: {session_id}\")\n\n conn = psycopg2.connect(\n dbname=DB_NAME,\n user=DB_USER,\n password=DB_PASSWORD,\n host=DB_HOST,\n port=DB_PORT\n )\n cursor = conn.cursor()\n\n try:\n cursor.execute('SELECT name FROM sessions WHERE id = %s', (session_id,))\n session = cursor.fetchone()\n if session:\n session_name = session[0]\n print(\"anem\", session_name)\n cursor.execute('DELETE FROM sessions WHERE id = %s', (session_id,))\n conn.commit()\n cursor.execute(sql.SQL('DROP TABLE IF EXISTS {}').format(sql.Identifier(f'session_{session_id}')))\n conn.commit()\n # remove the git clone project\n remove_project_path = os.path.join(\"projects\", session_name)\n remove_directory(remove_project_path)\n print(\"Session deleted successfully\")\n return JSONResponse(content={\"message\": \"Session deleted successfully!\"})\n except Exception as e:\n print(f\"Error deleting session: {e}\")\n raise HTTPException(status_code=500, detail=str(e))\n finally:\n conn.close()\n\n# api key handling functions\n@app.post('/check_api_key')\nasync def check_api_key(request: Request):\n data = await request.json()\n provider = data.get('provider')\n key_var = f\"{provider.upper()}_API_KEY\"\n \n load_dotenv()\n api_key = os.getenv(key_var)\n \n return JSONResponse(content={'exists': bool(api_key)})\n\n@app.post('/save_api_key')\nasync def save_api_key(request: Request):\n data = await request.json()\n provider = data.get('provider')\n api_key = data.get('api_key')\n key_var = f\"{provider.upper()}_API_KEY\"\n \n dotenv_path = '.env'\n set_key(dotenv_path, key_var, api_key)\n load_dotenv()\n \n return JSONResponse(content={'message': 'API Key saved successfully!'})\n\n\n# hanlde llamacpp models(list/upload/delete)\n@app.get('/llamacpp_models')\nasync def list_llamacpp_models():\n upload_dir = \"llamacpp_models\"\n if not os.path.exists(upload_dir):\n return JSONResponse(content=[])\n models = os.listdir(upload_dir)\n return JSONResponse(content=models)\n\n\n@app.post('/llamacpp_models', status_code=201)\nasync def upload_llamacpp_model(file: UploadFile = File(...), chunk: int = Form(...), totalChunks: int = Form(...)):\n try:\n upload_dir = \"llamacpp_models\"\n os.makedirs(upload_dir, exist_ok=True)\n chunk_dir = os.path.join(upload_dir, \"chunks\")\n os.makedirs(chunk_dir, exist_ok=True)\n chunk_file_path = os.path.join(chunk_dir, f\"{file.filename}.part{chunk}\")\n\n with open(chunk_file_path, \"wb\") as f:\n f.write(await file.read())\n\n # Check if all chunks are uploaded\n if len(os.listdir(chunk_dir)) == totalChunks:\n final_file_path = os.path.join(upload_dir, file.filename)\n with open(final_file_path, \"wb\") as final_file:\n for i in range(totalChunks):\n chunk_file_path = os.path.join(chunk_dir, f\"{file.filename}.part{i}\")\n with open(chunk_file_path, \"rb\") as chunk_file:\n final_file.write(chunk_file.read())\n os.remove(chunk_file_path)\n os.rmdir(chunk_dir) # Remove the chunks directory\n\n return JSONResponse(content={\"message\": \"Chunk uploaded successfully!\"})\n except Exception as e:\n print(f\"Error uploading model: {e}\")\n raise HTTPException(status_code=500, detail=\"Failed to upload chunk\")\n \n\n@app.delete('/llamacpp_models/{model_name}')\nasync def delete_llamacpp_model(model_name: str):\n file_path = os.path.join(\"llamacpp_models\", model_name)\n if os.path.exists(file_path):\n os.remove(file_path)\n return JSONResponse(content={\"message\": \"Model deleted successfully!\"})\n else:\n raise HTTPException(status_code=404, detail=\"Model not found\")\n\n# handle prompt templates\n@app.get('/get_prompt_templates')\nasync def get_prompt_templates():\n prompt_templates_path = os.path.join('config', 'prompt_templates.ini')\n templates_config = configparser.ConfigParser()\n\n if not os.path.exists(prompt_templates_path):\n return JSONResponse(content={})\n\n with open(prompt_templates_path, 'r', encoding='utf-8') as file:\n file_content = file.read()\n # Ensure the content has a section header\n if '[qa_prompt_templates]' not in file_content:\n file_content = '[qa_prompt_templates]\\n' + file_content\n\n try:\n templates_config.read_string(file_content)\n except configparser.ParsingError as e:\n print(f\"Error parsing config: {e}\")\n raise HTTPException(status_code=500, detail=\"Error parsing prompt templates\")\n\n templates = {k: v.replace('\\\\n', '\\n') for k, v in templates_config.items('qa_prompt_templates')}\n return JSONResponse(content=templates)\n\n\n@app.post('/delete_prompt_template')\nasync def delete_prompt_template(request: Request):\n data = await request.json()\n template_name = data.get('template_name')\n\n prompt_templates_path = os.path.join('config', 'prompt_templates.ini')\n templates_config = configparser.ConfigParser()\n templates_config.read(prompt_templates_path)\n\n if template_name in templates_config['qa_prompt_templates']:\n templates_config.remove_option('qa_prompt_templates', template_name)\n with open(prompt_templates_path, 'w', encoding='utf-8') as configfile:\n templates_config.write(configfile)\n return JSONResponse(content={\"message\": \"Template deleted successfully!\"})\n else:\n raise HTTPException(status_code=404, detail=\"Template not found\")\n\n\n@app.post('/save_prompt_templates')\nasync def save_prompt_templates(request: Request):\n new_templates = await request.json()\n prompt_templates_path = os.path.join('config', 'prompt_templates.ini')\n templates_config = configparser.ConfigParser()\n templates_config['qa_prompt_templates'] = {k: v.replace('\\n', '\\\\n') for k, v in new_templates.items()}\n\n with open(prompt_templates_path, 'w', encoding='utf-8') as configfile:\n templates_config.write(configfile)\n return JSONResponse(content={\"message\": \"Templates saved successfully!\"})\n\n#############################python codegraph############################\n@app.get('/codegraph')\nasync def codegraph_home(request: Request):\n return templates.TemplateResponse('index.html', {'request': request})\n\n\n@app.get('/data')\nasync def data(filepath: str):\n code_data = parse_python_code(filepath) # Ensure the path points to your Python code file\n return JSONResponse(content=code_data)\n\n@app.get('/directory')\nasync def directory():\n current_repo_path = read_current_repo_path(current_session)\n if current_repo_path is None:\n raise HTTPException(status_code=404, detail=\"Repository path not set or not found\")\n dir_tree = build_file_tree(current_repo_path) # Ensure the path points to your code directory\n return JSONResponse(content=dir_tree)\n\n@app.post('/analyze')\nasync def analyze(request: Request):\n data = await request.json()\n load_models_if_needed()\n chat_model = current_model_info[\"chat_model\"]\n embedding_model = current_model_info[\"embedding_model\"]\n code = data.get('code', '')\n # send the code to LLM\n data_handler = DataHandler(git_url='', chat_model=chat_model, embedding_model=embedding_model)\n code_analysis = data_handler.restrieval_qa_for_code(code)\n return JSONResponse(content={'analysis': code_analysis})\n\n#####go codegraph#####\n@app.get('/go_codegraph')\nasync def go_codegraph_home(request: Request):\n return templates.TemplateResponse('go_index.html', {'request': request})\n\n@app.get('/go_data')\nasync def go_data(filepath: str):\n if os.path.isdir(filepath):\n raise HTTPException(status_code=400, detail=\"The specified path is a directory, not a file.\")\n code_data = parse_go_code(filepath)\n return JSONResponse(content=code_data)\n\n@app.get('/go_directory')\nasync def directory():\n current_repo_path = read_current_repo_path(current_session)\n if current_repo_path is None:\n raise HTTPException(status_code=404, detail=\"Repository path not set or not found\")\n dir_tree = go_build_file_tree(current_repo_path) # Ensure the path points to your code directory\n return JSONResponse(content=dir_tree)\n\nif __name__ == \"__main__\":\n import uvicorn\n uvicorn.run(app, host=\"0.0.0.0\", port=5003, log_level=\"debug\")\n" }, { "path": "check_postgresql_connection.py", "content": "import psycopg2\nimport configparser\n\nconfig = configparser.ConfigParser()\nconfig.read('config/config.ini')\ndb_config = config['database']\n\ntry:\n conn = psycopg2.connect(\n dbname=db_config['db_name'],\n user=db_config['db_user'],\n password=db_config['db_password'],\n host=db_config['db_host'],\n port=db_config['db_port']\n )\n print(\"Connection successful\")\n cur = conn.cursor()\n cur.execute(\"SELECT version();\")\n db_version = cur.fetchone()\n print(f\"Database version: {db_version}\")\n cur.close()\n conn.close()\nexcept psycopg2.Error as e:\n print(f\"Error: {e}\")" }, { "path": "config/config.ini", "content": "[app_setting]\nversion = v2.15.6\n\n[model_providers]\nprovider_list = openai, ollama, mistralai, localai, zhipuai, anthropic, llamacpp, nvidia, tongyi, moonshot\nselected_provider = ollama\n\n[openai_llm_models]\nmodel_list = gpt-3.5-turbo, gpt-4o\nselected_model = gpt-4o\n\n[mistralai_llm_models]\nmodel_list = mistral-tiny, mistral-small\nselected_model = mistral-small\n\n[ollama_llm_models]\nmodel_list = qwen2.5-coder:14b, qwen2.5:14b, deepseek-r1:14b\nselected_model = qwen2.5:14b\nbase_url = http://localhost:11434\n\n[localai_llm_models]\nmodel_list = gpt-4\nselected_model = gpt-4\nbase_url = http://localhost:8080/v1\n\n[zhipuai_llm_models]\nmodel_list = glm-4\nselected_model = glm-4\n\n[anthropic_llm_models]\nmodel_list = claude-3-opus-20240229\nselected_model = claude-3-opus-20240229\n\n[llamacpp_llm_models]\nmodel_list = \nselected_model = \n\n[nvidia_llm_models]\nmodel_list = meta/llama3-70b-instruct\nselected_model = meta/llama3-70b-instruct\n\n[tongyi_llm_models]\nmodel_list = qwen-turbo\nselected_model = qwen-turbo\n\n[moonshot_llm_models]\nmodel_list = moonshot-v1-8k, moonshot-v1-32k, moonshot-v1-128k\nselected_model = moonshot-v1-8k\n\n[embedding_model_providers]\nprovider_list = huggingface, ollama\nselected_provider = huggingface\n\n[huggingface_embedding_models]\nmodel_list = sentence-transformers/all-MiniLM-L6-v2\nselected_model = sentence-transformers/all-MiniLM-L6-v2\n\n[ollama_embedding_models]\nmodel_list = nomic-embed-text, mxbai-embed-large\nselected_model = nomic-embed-text\n\n[prompt_templates]\nqa_selected_prompt = qa_template\ncode_selected_prompt = code_template\nlocalai_selected_prompt = code_template_localai\n\n[the_project_dirs]\nvectorstore_dir = VectorStore\nsessions_dir = sessions\nproject_dir = projects\n\n[for_loop_dirs_depth]\nmax_dir_depth = 5\n\n[chunk_setting]\nchunk_size = 3000\nchunk_overlap = 200\n\n[database]\ndb_name = qa_pilot_chatsession_db\ndb_user = qa_pilot_user\ndb_password = qa_pilot_p\ndb_host = localhost\ndb_port = 5432\n\n" }, { "path": "config/prompt_templates.ini", "content": "[qa_prompt_templates]\nqa_template = \"\"\"I want you to act as a very senior code developer who is familar with github/gitlab community. I will provide you the code project, you need to provide answers which are based on the project. {context}\"\"\"\ncode_template = \"\"\"I want you to act as a Senior Python developer. I will provide you the code project, you provide detailed exaplanation. Human: {input}History: {history} AI:\"\"\"\ncode_template_localai = \"\"\"I want you to act as a Senior Python developer. I will provide you the code project, you provide detailed exaplanation. Human: {input} AI:\"\"\"\n\n" }, { "path": "llamacpp_models/model_dir", "content": "" }, { "path": "parser.go", "content": "package main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"go/ast\"\n\t\"go/parser\"\n\t\"go/token\"\n\t\"os\"\n\t\"strings\"\n)\n\ntype Node struct {\n\tName string\n\tType string // \"func\", \"method\", \"type\", \"interface\", \"int\", or \"import\"\n\tCalls []string // list of called functions/methods\n\tCode string // source code of the node\n\tPosition string // file position of the node\n}\n\nfunc main() {\n\tif len(os.Args) < 2 {\n\t\tfmt.Println(\"Usage: ./parser \")\n\t\tos.Exit(1)\n\t}\n\n\tfilePath := os.Args[1]\n\tfset := token.NewFileSet()\n\tnode, err := parser.ParseFile(fset, filePath, nil, parser.ParseComments)\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\n\tnodes := make(map[string]*Node)\n\tast.Inspect(node, func(n ast.Node) bool {\n\t\tswitch x := n.(type) {\n\t\tcase *ast.FuncDecl:\n\t\t\tfuncName := x.Name.Name\n\t\t\tfuncType := \"func\"\n\t\t\tif x.Recv != nil {\n\t\t\t\trecvType := fmt.Sprintf(\"%s\", x.Recv.List[0].Type)\n\t\t\t\tif len(recvType) > 1 && recvType[0] == '*' {\n\t\t\t\t\trecvType = recvType[1:]\n\t\t\t\t}\n\t\t\t\tfuncType = \"method\"\n\t\t\t\tfuncName = fmt.Sprintf(\"%s.%s\", cleanType(recvType), funcName)\n\t\t\t}\n\t\t\tpos := fset.Position(x.Pos())\n\t\t\tcode := getNodeCode(fset, x.Pos(), x.End(), filePath)\n\t\t\tnodes[funcName] = &Node{\n\t\t\t\tName: funcName,\n\t\t\t\tType: funcType,\n\t\t\t\tCalls: []string{},\n\t\t\t\tCode: code,\n\t\t\t\tPosition: pos.String(),\n\t\t\t}\n\t\tcase *ast.GenDecl:\n\t\t\tif x.Tok == token.TYPE {\n\t\t\t\tfor _, spec := range x.Specs {\n\t\t\t\t\ttypeSpec := spec.(*ast.TypeSpec)\n\t\t\t\t\ttypeName := typeSpec.Name.Name\n\t\t\t\t\tpos := fset.Position(typeSpec.Pos())\n\t\t\t\t\tcode := getNodeCode(fset, typeSpec.Pos(), typeSpec.End(), filePath)\n\t\t\t\t\tnodes[typeName] = &Node{\n\t\t\t\t\t\tName: typeName,\n\t\t\t\t\t\tType: \"type\",\n\t\t\t\t\t\tCalls: []string{},\n\t\t\t\t\t\tCode: code,\n\t\t\t\t\t\tPosition: pos.String(),\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else if x.Tok == token.IMPORT {\n\t\t\t\tfor _, spec := range x.Specs {\n\t\t\t\t\timportSpec := spec.(*ast.ImportSpec)\n\t\t\t\t\timportPath := importSpec.Path.Value\n\t\t\t\t\tpos := fset.Position(importSpec.Pos())\n\t\t\t\t\tcode := importSpec.Path.Value\n\t\t\t\t\tnodes[importPath] = &Node{\n\t\t\t\t\t\tName: importPath,\n\t\t\t\t\t\tType: \"import\",\n\t\t\t\t\t\tCalls: []string{},\n\t\t\t\t\t\tCode: code,\n\t\t\t\t\t\tPosition: pos.String(),\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\t// Collect function and method calls\n\tast.Inspect(node, func(n ast.Node) bool {\n\t\tswitch x := n.(type) {\n\t\tcase *ast.CallExpr:\n\t\t\tcaller := \"\"\n\t\t\tif sel, ok := x.Fun.(*ast.SelectorExpr); ok {\n\t\t\t\tif ident, ok := sel.X.(*ast.Ident); ok {\n\t\t\t\t\tcaller = fmt.Sprintf(\"%s.%s\", ident.Name, sel.Sel.Name)\n\t\t\t\t}\n\t\t\t} else if ident, ok := x.Fun.(*ast.Ident); ok {\n\t\t\t\tcaller = ident.Name\n\t\t\t}\n\t\t\tif caller != \"\" {\n\t\t\t\tif parentFunc := getParentFunc(node, x.Pos(), fset); parentFunc != \"\" {\n\t\t\t\t\tif _, ok := nodes[parentFunc]; ok {\n\t\t\t\t\t\tnodes[parentFunc].Calls = append(nodes[parentFunc].Calls, caller)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\n\tjsonOutput, err := json.Marshal(nodes)\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\tos.Exit(1)\n\t}\n\tfmt.Println(string(jsonOutput))\n}\n\nfunc getNodeCode(fset *token.FileSet, start, end token.Pos, filePath string) string {\n\tstartOffset := fset.Position(start).Offset\n\tendOffset := fset.Position(end).Offset\n\tcode, _ := os.ReadFile(filePath)\n\treturn string(code[startOffset:endOffset])\n}\n\nfunc getParentFunc(node *ast.File, pos token.Pos, fset *token.FileSet) string {\n\tvar parentFunc string\n\tast.Inspect(node, func(n ast.Node) bool {\n\t\tif fd, ok := n.(*ast.FuncDecl); ok {\n\t\t\tif fd.Pos() < pos && fd.End() > pos {\n\t\t\t\tif fd.Recv != nil {\n\t\t\t\t\trecvType := fmt.Sprintf(\"%s\", fd.Recv.List[0].Type)\n\t\t\t\t\tif len(recvType) > 1 && recvType[0] == '*' {\n\t\t\t\t\t\trecvType = recvType[1:]\n\t\t\t\t\t}\n\t\t\t\t\tparentFunc = fmt.Sprintf(\"%s.%s\", cleanType(recvType), fd.Name.Name)\n\t\t\t\t} else {\n\t\t\t\t\tparentFunc = fd.Name.Name\n\t\t\t\t}\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\t})\n\treturn parentFunc\n}\n\nfunc cleanType(typeName string) string {\n\t// Remove generic type parameters if present\n\treturn strings.TrimSpace(strings.Split(typeName, \"[\")[0])\n}\n" }, { "path": "qa_model_apis.py", "content": "from langchain_community.chat_models import ChatOllama\nfrom langchain_mistralai.chat_models import ChatMistralAI\nfrom langchain_openai import ChatOpenAI\nfrom langchain_huggingface.embeddings import HuggingFaceEmbeddings\nfrom langchain_community.embeddings import OllamaEmbeddings\nimport os\nimport configparser\nfrom dotenv import load_dotenv\nfrom langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\nfrom llama_index.llms.openai_like import OpenAILike\nfrom langchain_community.chat_models import ChatZhipuAI\nfrom langchain_anthropic import ChatAnthropic\nimport multiprocessing\nfrom langchain_community.chat_models import ChatLlamaCpp\nfrom langchain_nvidia_ai_endpoints import ChatNVIDIA\nfrom langchain_community.chat_models.tongyi import ChatTongyi\nfrom langchain_community.chat_models.moonshot import MoonshotChat\n\n# read from the config.ini\nconfig_path = os.path.join('config', 'config.ini')\nconfig = configparser.ConfigParser()\nconfig.read(config_path)\nollama_base_url = config.get('ollama_llm_models', 'base_url')\nlocalai_base_url = config.get('localai_llm_models', 'base_url')\n\n\n# get the chat model from config\ndef get_chat_model(provider, model_name=''): \n if provider == 'ollama':\n return ChatOllama(\n base_url=ollama_base_url,\n model=model_name,\n streaming=True,\n callbacks=[StreamingStdOutCallbackHandler()]\n )\n elif provider == 'openai':\n load_dotenv()\n return ChatOpenAI(model_name=model_name)\n elif provider == 'mistralai':\n load_dotenv()\n return ChatMistralAI(model_name=model_name)\n elif provider == 'localai':\n return OpenAILike( \n api_base=localai_base_url, \n api_key=\"qa_pilot\", \n is_chat_model=True, \n context_window=32768, \n model=model_name\n )\n elif provider == 'zhipuai':\n load_dotenv()\n return ChatZhipuAI(\n model=model_name,\n temperature=0.5,\n )\n elif provider == 'anthropic':\n load_dotenv()\n return ChatAnthropic(\n model=model_name\n )\n elif provider == 'llamacpp':\n local_model = os.path.join(\"llamacpp_models\", model_name)\n return ChatLlamaCpp(\n temperature=0.5,\n model_path=local_model,\n n_ctx=10000,\n n_gpu_layers=8,\n n_batch=300, # Should be between 1 and n_ctx, consider the amount of VRAM in your GPU.\n max_tokens=512,\n n_threads=multiprocessing.cpu_count() - 1,\n repeat_penalty=1.5,\n top_p=0.5,\n verbose=True,\n )\n elif provider == 'nvidia':\n load_dotenv()\n return ChatNVIDIA(\n model=model_name\n )\n elif provider == 'tongyi':\n load_dotenv()\n return ChatTongyi(\n model=model_name\n ) \n elif provider == 'moonshot':\n load_dotenv()\n return MoonshotChat(\n model=model_name\n ) \n else:\n raise ValueError(f\"Unsupported model provider: {provider}\")\n \n\ndef get_embedding_model(eb_provider, model_name='', model_kwargs='', encode_kwargs=''):\n if eb_provider == 'huggingface': \n return HuggingFaceEmbeddings(\n model_name=model_name,\n model_kwargs=model_kwargs,\n encode_kwargs=encode_kwargs,\n cache_folder=\"./cache_embeddings/\"\n )\n elif eb_provider == 'ollama':\n return OllamaEmbeddings(\n model=model_name,\n model_kwargs=model_kwargs\n )\n else:\n raise ValueError(f\"Unsupported embedding model provider: {eb_provider}\")" }, { "path": "qa_pilot_run.py", "content": "import uvicorn\nfrom app import app\n\nif __name__ == \"__main__\":\n print(\"Starting server on http://0.0.0.0:5000\")\n uvicorn.run(app, host=\"0.0.0.0\", port=5000)" }, { "path": "requirements.txt", "content": "aiohttp==3.9.3\naiosignal==1.3.1\naltair==5.2.0\nannotated-types==0.6.0\nanthropic==0.28.1\nanyio==4.3.0\nasgiref==3.8.1\nasync-timeout==4.0.3\nattrs==23.2.0\nbackoff==2.2.1\nbcrypt==4.1.2\nblinker==1.7.0\nboto3==1.34.71\nbotocore==1.34.71\nbuild==1.1.1\ncachetools==5.3.3\ncertifi==2024.2.2\ncharset-normalizer==3.3.2\nchroma-hnswlib==0.7.3\nchromadb==0.4.24\nclick==8.1.7\ncolorama==0.4.6\ncoloredlogs==15.0.1\ndashscope==1.20.0\ndataclasses-json==0.6.4\ndefusedxml==0.7.1\nDeprecated==1.2.14\ndill==0.3.8\ndirtyjson==1.0.8\ndiskcache==5.6.3\ndistro==1.9.0\nexceptiongroup==1.2.0\nfastapi==0.110.0\nfilelock==3.13.3\nFlashRank==0.2.5\nFlask==3.0.3\nFlask-Cors==4.0.1\nflatbuffers==24.3.25\nfrozenlist==1.4.1\nfsspec==2024.3.1\ngitdb==4.0.11\nGitPython==3.1.42\ngoogle-auth==2.29.0\ngoogleapis-common-protos==1.63.0\ngreenlet==3.0.3\ngrpcio==1.62.1\nh11==0.14.0\nhttpcore==1.0.4\nhttptools==0.6.1\nhttpx==0.27.0\nhttpx-sse==0.4.0\nhuggingface-hub==0.23.2\nhumanfriendly==10.0\nhumbug==0.3.2\nidna==3.6\nimportlib-metadata==6.11.0\nimportlib_resources==6.4.0\nintel-openmp==2021.4.0\nitsdangerous==2.2.0\nJinja2==3.1.3\njiter==0.4.2\njmespath==1.0.1\njoblib==1.3.2\njsonpatch==1.33\njsonpointer==2.4\njsonschema==4.21.1\njsonschema-specifications==2023.12.1\nkubernetes==29.0.0\nlangchain==0.2.6\nlangchain-anthropic==0.1.15\nlangchain-chroma==0.1.1\nlangchain-community==0.2.6\nlangchain-core==0.2.11\nlangchain-huggingface==0.0.2\nlangchain-mistralai==0.1.8\nlangchain-nvidia-ai-endpoints==0.1.2\nlangchain-openai==0.1.8\nlangchain-text-splitters==0.2.1\nlangsmith==0.1.80\nllama-index-core==0.10.43.post1\nllama-index-llms-openai==0.1.22\nllama-index-llms-openai-like==0.1.3\nllama_cpp_python==0.2.67\nllamaindex-py-client==0.1.19\nlz4==4.3.3\nmarkdown-it-py==3.0.0\nMarkupSafe==2.1.5\nmarshmallow==3.21.1\nmdurl==0.1.2\nmkl==2021.4.0\nmmh3==4.1.0\nmonotonic==1.6\nmpmath==1.3.0\nmultidict==6.0.5\nmultiprocess==0.70.16\nmypy-extensions==1.0.0\nnest-asyncio==1.6.0\nnetworkx==3.2.1\nnltk==3.8.1\nnumpy==1.26.4\noauthlib==3.2.2\nonnxruntime==1.17.1\nopenai==1.28.0\nopentelemetry-api==1.23.0\nopentelemetry-exporter-otlp-proto-common==1.23.0\nopentelemetry-exporter-otlp-proto-grpc==1.23.0\nopentelemetry-instrumentation==0.44b0\nopentelemetry-instrumentation-asgi==0.44b0\nopentelemetry-instrumentation-fastapi==0.44b0\nopentelemetry-proto==1.23.0\nopentelemetry-sdk==1.23.0\nopentelemetry-semantic-conventions==0.44b0\nopentelemetry-util-http==0.44b0\norjson==3.9.15\noverrides==7.7.0\npackaging==23.2\npandas==2.2.1\npathos==0.3.2\npillow==10.2.0\nposthog==3.5.0\npox==0.3.4\nppft==1.7.6.8\nprotobuf==4.25.3\npsycopg2-binary==2.9.9\npulsar-client==3.4.0\npyarrow==15.0.2\npyasn1==0.6.0\npyasn1_modules==0.4.0\npydantic==2.6.4\npydantic_core==2.16.3\npydeck==0.8.1b0\nPygments==2.17.2\nPyJWT==2.8.0\nPyPika==0.48.9\npyproject_hooks==1.0.0\npyreadline3==3.4.1\npython-dateutil==2.9.0.post0\npython-dotenv==1.0.1\npython-multipart==0.0.9\npytz==2024.1\nPyYAML==6.0.1\nreferencing==0.34.0\nregex==2023.12.25\nrequests==2.31.0\nrequests-oauthlib==2.0.0\nrich==13.7.1\nrpds-py==0.18.0\nrsa==4.9\ns3transfer==0.10.1\nsafetensors==0.4.2\nscikit-learn==1.4.1.post1\nscipy==1.12.0\nsentence-transformers==2.6.1\nsix==1.16.0\nsmmap==5.0.1\nsniffio==1.3.1\nSQLAlchemy==2.0.29\nstarlette==0.36.3\nstreamlit==1.32.2\nsympy==1.12\ntbb==2021.12.0\ntenacity==8.2.3\ntext-generation==0.7.0\nthreadpoolctl==3.4.0\ntiktoken==0.7.0\ntokenizers==0.19.1\ntoml==0.10.2\ntomli==2.0.1\ntoolz==0.12.1\ntornado==6.4\ntqdm==4.66.2\ntransformers==4.41.2\ntyper==0.11.0\ntyping-inspect==0.9.0\ntyping_extensions==4.10.0\ntzdata==2024.1\nurllib3==2.2.1\nuvicorn==0.29.0\nwaitress==3.0.0\nwatchdog==4.0.0\nwatchfiles==0.21.0\nwebsocket-client==1.7.0\nwebsockets==12.0\nWerkzeug==3.0.3\nwrapt==1.16.0\nyarl==1.9.4\nzipp==3.18.1\n" }, { "path": "svelte-app/.gitignore", "content": "/node_modules/\n/public/build/\n\n.DS_Store\n" }, { "path": "svelte-app/README.md", "content": "This repo is no longer maintained. Consider using `npm init vite` and selecting the `svelte` option or — if you want a full-fledged app framework — use [SvelteKit](https://kit.svelte.dev), the official application framework for Svelte.\n\n---\n\n# svelte app\n\nThis is a project template for [Svelte](https://svelte.dev) apps. It lives at https://github.com/sveltejs/template.\n\nTo create a new project based on this template using [degit](https://github.com/Rich-Harris/degit):\n\n```bash\nnpx degit sveltejs/template svelte-app\ncd svelte-app\n```\n\n*Note that you will need to have [Node.js](https://nodejs.org) installed.*\n\n\n## Get started\n\nInstall the dependencies...\n\n```bash\ncd svelte-app\nnpm install\n```\n\n...then start [Rollup](https://rollupjs.org):\n\n```bash\nnpm run dev\n```\n\nNavigate to [localhost:8080](http://localhost:8080). You should see your app running. Edit a component file in `src`, save it, and reload the page to see your changes.\n\nBy default, the server will only respond to requests from localhost. To allow connections from other computers, edit the `sirv` commands in package.json to include the option `--host 0.0.0.0`.\n\nIf you're using [Visual Studio Code](https://code.visualstudio.com/) we recommend installing the official extension [Svelte for VS Code](https://marketplace.visualstudio.com/items?itemName=svelte.svelte-vscode). If you are using other editors you may need to install a plugin in order to get syntax highlighting and intellisense.\n\n## Building and running in production mode\n\nTo create an optimised version of the app:\n\n```bash\nnpm run build\n```\n\nYou can run the newly built app with `npm run start`. This uses [sirv](https://github.com/lukeed/sirv), which is included in your package.json's `dependencies` so that the app will work when you deploy to platforms like [Heroku](https://heroku.com).\n\n\n## Single-page app mode\n\nBy default, sirv will only respond to requests that match files in `public`. This is to maximise compatibility with static fileservers, allowing you to deploy your app anywhere.\n\nIf you're building a single-page app (SPA) with multiple routes, sirv needs to be able to respond to requests for *any* path. You can make it so by editing the `\"start\"` command in package.json:\n\n```js\n\"start\": \"sirv public --single\"\n```\n\n## Using TypeScript\n\nThis template comes with a script to set up a TypeScript development environment, you can run it immediately after cloning the template with:\n\n```bash\nnode scripts/setupTypeScript.js\n```\n\nOr remove the script via:\n\n```bash\nrm scripts/setupTypeScript.js\n```\n\nIf you want to use `baseUrl` or `path` aliases within your `tsconfig`, you need to set up `@rollup/plugin-alias` to tell Rollup to resolve the aliases. For more info, see [this StackOverflow question](https://stackoverflow.com/questions/63427935/setup-tsconfig-path-in-svelte).\n\n## Deploying to the web\n\n### With [Vercel](https://vercel.com)\n\nInstall `vercel` if you haven't already:\n\n```bash\nnpm install -g vercel\n```\n\nThen, from within your project folder:\n\n```bash\ncd public\nvercel deploy --name my-project\n```\n\n### With [surge](https://surge.sh/)\n\nInstall `surge` if you haven't already:\n\n```bash\nnpm install -g surge\n```\n\nThen, from within your project folder:\n\n```bash\nnpm run build\nsurge public my-project.surge.sh\n```\n" }, { "path": "svelte-app/package.json", "content": "{\n \"name\": \"svelte-app\",\n \"version\": \"1.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build\": \"rollup -c\",\n \"dev\": \"rollup -c -w\",\n \"start\": \"sirv public --no-clear --port 5001 --host 0.0.0.0\"\n },\n \"devDependencies\": {\n \"@rollup/plugin-commonjs\": \"^24.0.0\",\n \"@rollup/plugin-node-resolve\": \"^15.0.0\",\n \"@rollup/plugin-terser\": \"^0.4.4\",\n \"rollup\": \"^3.15.0\",\n \"rollup-plugin-css-only\": \"^4.3.0\",\n \"rollup-plugin-livereload\": \"^2.0.0\",\n \"rollup-plugin-svelte\": \"^7.1.2\",\n \"svelte\": \"^3.55.0\"\n },\n \"dependencies\": {\n \"cytoscape\": \"^3.29.2\",\n \"cytoscape-dagre\": \"^2.5.0\",\n \"highlight.js\": \"^11.9.0\",\n \"marked\": \"^12.0.2\",\n \"sirv-cli\": \"^2.0.0\",\n \"svelte-spa-router\": \"^4.0.1\"\n }\n}\n" }, { "path": "svelte-app/public/global.css", "content": "html, body {\n\tposition: relative;\n\twidth: 100%;\n\theight: 100%;\n}\n\nbody {\n\tcolor: #333;\n\tmargin: 0;\n\tpadding: 8px;\n\tbox-sizing: border-box;\n\tfont-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, Oxygen-Sans, Ubuntu, Cantarell, \"Helvetica Neue\", sans-serif;\n}\n\na {\n\tcolor: rgb(0,100,200);\n\ttext-decoration: none;\n}\n\na:hover {\n\ttext-decoration: underline;\n}\n\na:visited {\n\tcolor: rgb(0,80,160);\n}\n\nlabel {\n\tdisplay: block;\n}\n\ninput, button, select, textarea {\n\tfont-family: inherit;\n\tfont-size: inherit;\n\t-webkit-padding: 0.4em 0;\n\tpadding: 0.4em;\n\tmargin: 0 0 0.5em 0;\n\tbox-sizing: border-box;\n\tborder: 1px solid #ccc;\n\tborder-radius: 2px;\n}\n\ninput:disabled {\n\tcolor: #ccc;\n}\n\nbutton {\n\tcolor: #333;\n\tbackground-color: #f4f4f4;\n\toutline: none;\n}\n\nbutton:disabled {\n\tcolor: #999;\n}\n\nbutton:not(:disabled):active {\n\tbackground-color: #ddd;\n}\n\nbutton:focus {\n\tborder-color: #666;\n}\n" }, { "path": "svelte-app/public/index.html", "content": "\n\n\n\t\n\t\n\n\tSvelte app\n\n\t\n\t\n\t\n\n\t\n\n\n\n\n\n" }, { "path": "svelte-app/rollup.config.js", "content": "import { spawn } from 'child_process';\nimport svelte from 'rollup-plugin-svelte';\nimport commonjs from '@rollup/plugin-commonjs';\nimport terser from '@rollup/plugin-terser';\nimport resolve from '@rollup/plugin-node-resolve';\nimport livereload from 'rollup-plugin-livereload';\nimport css from 'rollup-plugin-css-only';\n\nconst production = !process.env.ROLLUP_WATCH;\n\nfunction serve() {\n\tlet server;\n\n\tfunction toExit() {\n\t\tif (server) server.kill(0);\n\t}\n\n\treturn {\n\t\twriteBundle() {\n\t\t\tif (server) return;\n\t\t\tserver = spawn('npm', ['run', 'start', '--', '--dev'], {\n\t\t\t\tstdio: ['ignore', 'inherit', 'inherit'],\n\t\t\t\tshell: true\n\t\t\t});\n\n\t\t\tprocess.on('SIGTERM', toExit);\n\t\t\tprocess.on('exit', toExit);\n\t\t}\n\t};\n}\n\nexport default {\n\tinput: 'src/main.js',\n\toutput: {\n\t\tsourcemap: true,\n\t\tformat: 'iife',\n\t\tname: 'app',\n\t\tfile: 'public/build/bundle.js'\n\t},\n\tplugins: [\n\t\tsvelte({\n\t\t\tcompilerOptions: {\n\t\t\t\t// enable run-time checks when not in production\n\t\t\t\tdev: !production\n\t\t\t}\n\t\t}),\n\t\t// we'll extract any component CSS out into\n\t\t// a separate file - better for performance\n\t\tcss({ output: 'bundle.css' }),\n\n\t\t// If you have external dependencies installed from\n\t\t// npm, you'll most likely need these plugins. In\n\t\t// some cases you'll need additional configuration -\n\t\t// consult the documentation for details:\n\t\t// https://github.com/rollup/plugins/tree/master/packages/commonjs\n\t\tresolve({\n\t\t\tbrowser: true,\n\t\t\tdedupe: ['svelte'],\n\t\t\texportConditions: ['svelte']\n\t\t}),\n\t\tcommonjs(),\n\n\t\t// In dev mode, call `npm run start` once\n\t\t// the bundle has been generated\n\t\t!production && serve(),\n\n\t\t// Watch the `public` directory and refresh the\n\t\t// browser on changes when not in production\n\t\t!production && livereload('public'),\n\n\t\t// If we're building for production (npm run build\n\t\t// instead of npm run dev), minify\n\t\tproduction && terser()\n\t],\n\twatch: {\n\t\tclearScreen: false\n\t}\n};\n" }, { "path": "svelte-app/scripts/setupTypeScript.js", "content": "// @ts-check\n\n/** This script modifies the project to support TS code in .svelte files like:\n\n \n \n As well as validating the code for CI.\n */\n\n/** To work on this script:\n rm -rf test-template template && git clone sveltejs/template test-template && node scripts/setupTypeScript.js test-template\n*/\n\nimport fs from \"fs\"\nimport path from \"path\"\nimport { argv } from \"process\"\nimport url from 'url';\n\nconst __filename = url.fileURLToPath(import.meta.url);\nconst __dirname = url.fileURLToPath(new URL('.', import.meta.url));\nconst projectRoot = argv[2] || path.join(__dirname, \"..\")\n\n// Add deps to pkg.json\nconst packageJSON = JSON.parse(fs.readFileSync(path.join(projectRoot, \"package.json\"), \"utf8\"))\npackageJSON.devDependencies = Object.assign(packageJSON.devDependencies, {\n \"svelte-check\": \"^3.0.0\",\n \"svelte-preprocess\": \"^5.0.0\",\n \"@rollup/plugin-typescript\": \"^11.0.0\",\n \"typescript\": \"^4.9.0\",\n \"tslib\": \"^2.5.0\",\n \"@tsconfig/svelte\": \"^3.0.0\"\n})\n\n// Add script for checking\npackageJSON.scripts = Object.assign(packageJSON.scripts, {\n \"check\": \"svelte-check\"\n})\n\n// Write the package JSON\nfs.writeFileSync(path.join(projectRoot, \"package.json\"), JSON.stringify(packageJSON, null, \" \"))\n\n// mv src/main.js to main.ts - note, we need to edit rollup.config.js for this too\nconst beforeMainJSPath = path.join(projectRoot, \"src\", \"main.js\")\nconst afterMainTSPath = path.join(projectRoot, \"src\", \"main.ts\")\nfs.renameSync(beforeMainJSPath, afterMainTSPath)\n\n// Switch the app.svelte file to use TS\nconst appSveltePath = path.join(projectRoot, \"src\", \"App.svelte\")\nlet appFile = fs.readFileSync(appSveltePath, \"utf8\")\nappFile = appFile.replace(\"\n\n\n\n{#if isOpen}\n
\n
\n

API Key Settings

\n \n \n {#if successMessage}\n

{successMessage}

\n {/if}\n {#if errorMessage}\n

{errorMessage}

\n {/if}\n
\n \n \n
\n
\n{/if}\n\n\n{#if apiMessage}\n
\n {apiMessage}\n
\n{/if}\n" }, { "path": "svelte-app/src/App.svelte", "content": "\n\n\n\n
\n
\n \n \n\n \n \n\n \n
\n \n \n
\n\n \n
\n \n \n \n \n
\n\n \n\n \n\n {#each filteredSessions as session}\n
\n \n \n
\n {/each}\n
\n
\n
{currentSessionIndex !== -1 ? sessions[currentSessionIndex].name : 'QA-Pilot'}
\n {#if currentSessionIndex !== -1}\n \n {:else}\n
\n \"Placeholder\n

1. Please check your config with \"Edit QA-Pilot Settings\" button.

\n

2. Click \"New Source Button\" to input the github URL.

\n
\n {/if}\n
\n
\n\n{#if showConfigEditor}\n \n{/if}\n\n{#if showNewSourceModal}\n \n{/if}\n\n{#if showApiKeyModal}\n \n{/if}\n\n{#if showLlamaCppModelsModal}\n \n{/if}\n\n{#if showPromptTemplatesModal}\n \n{/if}\n\n{#if showDeleteModal}\n \n{/if}\n" }, { "path": "svelte-app/src/Chat.svelte", "content": "\n\n\n\n
\n
\n {#each messages as message}\n
\n {#if message.sender === 'loader'}\n
\n
\n
Thinking...
\n {:else}\n
{message.sender}
\n {#if formatMessage(message.text).think}\n
{@html marked(formatMessage(message.text).think)}
\n {/if}\n\n {#if formatMessage(message.text).normal}\n
{@html marked(formatMessage(message.text).normal)}
\n {/if}\n {#if message.sender !== 'You'}\n
\n \n
\n {/if}\n {/if}\n
\n {/each}\n
\n
\n \n \n \n
\n
\n" }, { "path": "svelte-app/src/ConfigEditor.svelte", "content": "\n\n \n \n \n
\n

Edit QA-Pilot Settings

\n {#if Object.keys(configData).length === 0}\n

Loading configuration...

\n {:else}\n {#each configOrder as section}\n
\n

{section}

\n {#each Object.keys(configData[section]) as key}\n \n \n {/each}\n
\n {/each}\n
\n \n \n
\n {/if}\n
\n " }, { "path": "svelte-app/src/DeleteConfirmationModal.svelte", "content": "\n\n\n\n{#if isOpen}\n
\n
\n

Are you sure you want to delete the session: {sessionName}?

\n
\n \n \n
\n
\n{/if}\n" }, { "path": "svelte-app/src/DeleteTemplateModal.svelte", "content": "\n\n\n\n{#if isOpen}\n
\n
\n

Are you sure you want to delete the template: {templateName}?

\n {#if isDefaultTemplate}\n

Warning: You are about to delete a default template. Ensure there is a replacement before proceeding.

\n {/if}\n
\n \n \n
\n
\n{/if}\n" }, { "path": "svelte-app/src/LlamaCppModelsModal.svelte", "content": "\n\n\n\n{#if isOpen}\n
\n
\n

LlamaCpp Models

\n {#each models as model}\n
\n {model}\n \n
\n {/each}\n newModelFile = e.target.files[0]} />\n {#if showProgressBar}\n
\n
\n
\n {/if}\n {#if showMessage}\n
{messageText}
\n {/if}\n
\n \n \n
\n
\n{/if}\n" }, { "path": "svelte-app/src/NewSourceModal.svelte", "content": "\n\n\n\n{#if isOpen}\n
\n
\n

Enter GitHub URL

\n \n
\n \n \n
\n
\n{/if}\n" }, { "path": "svelte-app/src/PromptTemplatesModal.svelte", "content": "\n\n\n\n{#if isOpen}\n
\n
\n

Prompt Templates

\n \n \n
\n \n \n \n
\n \n \n \n {#if showMessage}\n
{messageText}
\n {/if}\n
\n{/if}\n\n showDeleteModal = false} \n/>\n" }, { "path": "svelte-app/src/config.js", "content": "export const API_BASE_URL = 'http://localhost:5000';\n" }, { "path": "svelte-app/src/global.css", "content": "a {\n color: #00ffff !important;\n text-decoration: none !important; \n}\n\na:hover {\n text-decoration: underline !important; \n}\n" }, { "path": "svelte-app/src/main.js", "content": "import './global.css'\nimport App from './App.svelte';\n\nconst app = new App({\n target: document.body\n});\n\nexport default app;\n" }, { "path": "templates/go_index.html", "content": "\n\n\n \n Go CodeGPS\n \n \n \n \n \n \n \n\n\n
\n
\n
\n
\n
    \n
    \n
    \n \n \n
    \n
    \n \n\n\n" }, { "path": "templates/index.html", "content": "\n\n\n \n CodeGPS\n \n \n \n \n \n \n \n\n\n
    \n
    \n
    \n
    \n
      \n
      \n
      \n \n \n
      \n
      \n \n\n\n" }, { "path": "utils/codegraph.py", "content": "import ast \nimport os\n\ndef parse_python_code(filepath):\n \"\"\"Parse Python code file, extract classes, methods, global functions, imported modules and their source code, and the call relationships.\"\"\"\n with open(filepath, 'r', encoding='utf-8') as file:\n source = file.read()\n node = ast.parse(source)\n\n classes = []\n methods = []\n functions = []\n imports = []\n links = []\n method_calls = {}\n function_calls = {}\n import_calls = {}\n class_inheritance = {}\n\n # Extract classes and methods\n for item in ast.walk(node):\n if isinstance(item, ast.ClassDef):\n class_start_line = item.lineno - 1\n class_end_line = item.end_lineno\n class_source = \"\\n\".join(source.splitlines()[class_start_line:class_end_line])\n classes.append({'key': item.name, 'name': item.name, 'class': 'class', 'color': 'lightblue', 'source': class_source})\n\n for base in item.bases:\n if isinstance(base, ast.Name):\n class_inheritance[item.name] = base.id\n\n for method in [n for n in item.body if isinstance(n, ast.FunctionDef)]:\n method_start_line = method.lineno - 1\n method_end_line = method.end_lineno\n method_source = \"\\n\".join(source.splitlines()[method_start_line:method_end_line])\n methods.append({'key': f\"{item.name}.{method.name}\", 'name': method.name, 'class': 'method', 'color': 'lightgreen', 'source': method_source})\n links.append({'from': item.name, 'to': f\"{item.name}.{method.name}\", 'color': 'blue'})\n\n method_calls[f\"{item.name}.{method.name}\"] = []\n for stmt in ast.walk(method):\n if isinstance(stmt, ast.Call):\n if isinstance(stmt.func, ast.Name):\n method_calls[f\"{item.name}.{method.name}\"].append(stmt.func.id)\n elif isinstance(stmt.func, ast.Attribute):\n method_calls[f\"{item.name}.{method.name}\"].append(stmt.func.attr)\n\n # Handle inheritance relationships\n for child, parent in class_inheritance.items():\n links.append({'from': child, 'to': parent, 'category': 'dashed', 'color': 'gray'})\n\n # Extract global functions\n for item in node.body:\n if isinstance(item, ast.FunctionDef):\n function_start_line = item.lineno - 1\n function_end_line = item.end_lineno\n function_source = \"\\n\".join(source.splitlines()[function_start_line:function_end_line])\n functions.append({'key': item.name, 'name': item.name, 'class': 'function', 'color': 'lightcoral', 'source': function_source})\n\n function_calls[item.name] = []\n for stmt in ast.walk(item):\n if isinstance(stmt, ast.Call):\n if isinstance(stmt.func, ast.Name):\n function_calls[item.name].append(stmt.func.id)\n elif isinstance(stmt.func, ast.Attribute):\n function_calls[item.name].append(stmt.func.attr)\n\n # Extract imported modules, classes, functions, or variables\n for item in node.body:\n if isinstance(item, ast.Import):\n for alias in item.names:\n imports.append({'key': alias.name, 'name': alias.name, 'class': 'import', 'color': 'lightyellow', 'source': f\"import {alias.name}\"})\n elif isinstance(item, ast.ImportFrom):\n module = item.module\n for alias in item.names:\n import_name = f\"{module}.{alias.name}\"\n imports.append({'key': alias.name, 'name': alias.name, 'class': 'import', 'color': 'lightyellow', 'source': f\"from {module} import {alias.name}\"})\n import_calls[alias.name] = import_name\n\n # Check which imports are actually called\n used_imports = set()\n for calls in method_calls.values():\n used_imports.update(calls)\n for calls in function_calls.values():\n used_imports.update(calls)\n\n # Filter out unused imports\n imports = [imp for imp in imports if imp['key'] in used_imports]\n\n # Add method and global function call links\n for caller, callees in method_calls.items():\n for callee in callees:\n if callee in method_calls or callee in function_calls or callee in import_calls:\n links.append({'from': caller, 'to': callee, 'category': 'dashed', 'color': 'green'})\n\n for caller, callees in function_calls.items():\n for callee in callees:\n if callee in method_calls or callee in function_calls or callee in import_calls:\n links.append({'from': caller, 'to': callee, 'category': 'dashed', 'color': 'green'})\n\n for import_name, import_module in import_calls.items():\n if import_name in used_imports:\n for caller, callees in method_calls.items():\n if import_name in callees:\n links.append({'from': caller, 'to': import_name, 'category': 'dashed', 'color': 'orange'})\n for caller, callees in function_calls.items():\n if import_name in callees:\n links.append({'from': caller, 'to': import_name, 'category': 'dashed', 'color': 'orange'})\n\n return {'nodeDataArray': classes + methods + functions + imports, 'linkDataArray': links}\n\ndef read_current_repo_path(current_session):\n if current_session:\n print(\"=============> path: \", os.path.join(\"projects\", current_session['name']))\n return os.path.join(\"projects\", current_session['name'])\n return None\n\ndef build_file_tree(directory):\n \"\"\"Build the file tree for the given directory\"\"\"\n file_tree = []\n for root, dirs, files in os.walk(directory):\n for d in dirs:\n dir_path = os.path.join(root, d)\n file_tree.append({\n 'name': d,\n 'path': dir_path,\n 'type': 'directory',\n 'children': build_file_tree(dir_path)\n })\n for f in files:\n if f.endswith('.py'):\n file_path = os.path.join(root, f)\n file_tree.append({\n 'name': f,\n 'path': file_path,\n 'type': 'file'\n })\n break # Only traverse current directory, not recursively\n return file_tree" }, { "path": "utils/go_codegraph.py", "content": "import json\nimport os\nimport subprocess\nfrom typing import Dict, List, Any\nimport re\n\n\ndef process_nodes(nodes: Dict[str, Any]) -> Dict[str, Any]:\n nodeDataArray = []\n linkDataArray = []\n methods = {}\n types = {}\n\n for key, node in nodes.items():\n print(f\"Processing node: {key} -> {node}\")\n\n if node[\"Type\"] == \"import\":\n node_data = {\n \"key\": key,\n \"name\": node[\"Name\"],\n \"class\": node[\"Type\"],\n \"source\": \"import \" + node[\"Code\"]\n }\n nodeDataArray.append(node_data)\n elif node[\"Type\"] == \"type\":\n types[key] = node\n node_data = {\n \"key\": key,\n \"name\": node[\"Name\"],\n \"class\": \"struct\",\n \"source\": \"type \" + node[\"Code\"]\n }\n nodeDataArray.append(node_data)\n elif node[\"Type\"] == \"method\":\n receiver_type = extract_receiver_type(key)\n method_name = extract_method_name(key)\n node_data = {\n \"key\": f\"{receiver_type}.{method_name}\",\n \"name\": method_name,\n \"class\": node[\"Type\"],\n \"source\": node[\"Code\"]\n }\n nodeDataArray.append(node_data)\n\n if receiver_type not in methods:\n methods[receiver_type] = []\n methods[receiver_type].append(node_data)\n elif node[\"Type\"] == \"func\":\n node_data = {\n \"key\": key,\n \"name\": node[\"Name\"],\n \"class\": \"function\",\n \"source\": node[\"Code\"]\n }\n nodeDataArray.append(node_data)\n else:\n node_data = {\n \"key\": key,\n \"name\": node[\"Name\"],\n \"class\": node[\"Type\"],\n \"source\": node[\"Code\"]\n }\n nodeDataArray.append(node_data)\n\n for receiver_type, method_list in methods.items():\n if receiver_type in types:\n for method in method_list:\n link = {\n \"from\": receiver_type,\n \"to\": method[\"key\"],\n \"category\": \"dashed\",\n \"color\": \"gray\"\n }\n linkDataArray.append(link)\n\n for node_key, node in nodes.items():\n for call in node[\"Calls\"]:\n print(f\"Checking call from {node_key} to {call}\")\n if call in nodes:\n link = {\n \"from\": node_key,\n \"to\": call,\n \"category\": \"dashed\",\n \"color\": \"green\"\n }\n linkDataArray.append(link)\n print(f\"Link added: {link}\")\n else:\n print(f\"Call {call} not found in nodes\")\n\n print(f\"Final nodeDataArray: {nodeDataArray}\")\n print(f\"Final linkDataArray: {linkDataArray}\")\n\n return {\"nodeDataArray\": nodeDataArray, \"linkDataArray\": linkDataArray}\n\n\ndef extract_receiver_type(key: str) -> str:\n if key.startswith(\"&\"):\n parts = re.split(r'[{}]', key)\n receiver_type = parts[1].strip().split(' ')[-1]\n else:\n receiver_type = key.split(\".\")[0]\n\n # Remove generic type parameters if present\n receiver_type = re.sub(r'\\[.*?\\]', '', receiver_type).strip()\n return receiver_type\n\n\ndef extract_method_name(key: str) -> str:\n return key.split(\".\")[-1]\n\n\ndef parse_go_code(filepath: str) -> Dict[str, Any]:\n try:\n result = subprocess.run(['./parser', filepath], capture_output=True, text=True, check=True)\n output = result.stdout.strip()\n nodes = json.loads(output)\n return process_nodes(nodes)\n except subprocess.CalledProcessError as e:\n print(f\"Error running parser: {e.stderr}\")\n return {\"nodeDataArray\": [], \"linkDataArray\": []}\n except json.JSONDecodeError as e:\n print(f\"Error decoding JSON: {e}\")\n return {\"nodeDataArray\": [], \"linkDataArray\": []}\n\n\ndef go_build_file_tree(directory: str) -> List[Dict[str, Any]]:\n file_tree = []\n for root, dirs, files in os.walk(directory):\n for d in dirs:\n dir_path = os.path.join(root, d)\n file_tree.append({\n 'name': d,\n 'path': dir_path,\n 'type': 'directory',\n 'children': go_build_file_tree(dir_path)\n })\n for f in files:\n if f.endswith('.go'):\n file_path = os.path.join(root, f)\n file_tree.append({\n 'name': f,\n 'path': file_path,\n 'type': 'file'\n })\n break # Only traverse current directory, not recursively\n return file_tree\n" }, { "path": "utils/helper.py", "content": "from langchain_community.document_loaders import TextLoader\nfrom langchain.text_splitter import CharacterTextSplitter\nfrom langchain_community.vectorstores import Chroma\nimport git\nimport os\nfrom queue import Queue\nimport shutil\nfrom urllib.parse import urlparse\nimport configparser\nfrom langchain.chains import ConversationalRetrievalChain\nfrom langchain_core.prompts.prompt import PromptTemplate\nfrom langchain_core.prompts import ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate\nfrom langchain.chains import ConversationChain\nfrom cachetools import cached, TTLCache\nfrom langchain.retrievers import ContextualCompressionRetriever\nfrom langchain.retrievers.document_compressors import FlashrankRerank\n\n# read from the config.ini\nconfig_path = os.path.join('config', 'config.ini')\nprompt_templates_path = 'config/prompt_templates.ini'\nconfig = configparser.ConfigParser()\nconfig.read(config_path)\nvectorstore_dir = config.get('the_project_dirs', 'vectorstore_dir')\nsessions_dir = config.get('the_project_dirs', 'sessions_dir')\nproject_dir = config.get('the_project_dirs', 'project_dir')\nmax_dir_depth = config.get('for_loop_dirs_depth', 'max_dir_depth')\nchunk_size = config.get('chunk_setting', 'chunk_size')\nchunk_overlap = config.get('chunk_setting', 'chunk_overlap')\nbase_url = config.get('ollama_llm_models', 'base_url')\nencode_kwargs = {\"normalize_embeddings\": False}\nmodel_kwargs = {\"device\": \"cuda:0\"} \nallowed_extensions = ['.py', '.md', '.js',\n '.html', '.css', '.ts', '.sh',\n '.go', '.java', '.svelte']\n\n# remove the directories for the download/upload projects\ndef remove_directory(dir_path):\n if os.path.exists(dir_path):\n # check and update file permisiion first\n for root, dirs, files in os.walk(dir_path, topdown=False):\n for name in files:\n filepath = os.path.join(root, name)\n try:\n os.chmod(filepath, 0o777) # modify permission\n except PermissionError:\n pass\n for name in dirs:\n dirpath = os.path.join(root, name)\n os.chmod(dirpath, 0o777) \n \n # remove it\n shutil.rmtree(dir_path, ignore_errors=True)\n\n\ndef document_to_string(document):\n # Get the document content\n content = document.page_content\n # Get the metadata\n metadata = document.metadata\n # Convert the metadata dictionary to a string\n metadata_str = ', '.join(f'{key}: {value}' for key, value in metadata.items())\n # Combine the content and metadata into a single string\n result = f'Content: {content}\\nMetadata: {metadata_str}'\n return result\n\ndef documents_to_string(documents):\n # Convert each Document object to a string and add to the list\n doc_strings = [document_to_string(doc) for doc in documents]\n # Join all strings into a single large string\n return '\\n\\n'.join(doc_strings)\n\n\ndef load_prompt_templates(path):\n config = configparser.ConfigParser()\n config.read(path)\n return {section: dict(config.items(section)) for section in config.sections()}\n\ncache = TTLCache(maxsize=100, ttl=300)\n\nclass DataHandler:\n def __init__(self, git_url, chat_model, embedding_model) -> None:\n self.git_url = git_url\n last_part = git_url.split('/')[-1]\n self.repo_name = last_part.rsplit('.', 1)[0]\n # create the store db and project dir\n if not os.path.exists(vectorstore_dir):\n os.makedirs(vectorstore_dir)\n if not os.path.exists(project_dir):\n os.makedirs(project_dir)\n # config the path\n self.db_dir = os.path.join(vectorstore_dir, self.repo_name)\n self.download_path = os.path.join(project_dir, self.repo_name) \n self.model = chat_model\n self.embedding_model = embedding_model \n self.ChatQueue = Queue(maxsize=2)\n\n # check the db dir exist or not\n def db_exists(self):\n return os.path.exists(self.db_dir)\n\n # update the chat message queue\n def update_chat_queue(self, value):\n if self.ChatQueue.full():\n self.ChatQueue.get()\n self.ChatQueue.put(value)\n\n # download or upload the project\n def git_clone_repo(self):\n url_parts = urlparse(self.git_url)\n\n # upload situation\n if not url_parts.scheme:\n print(\"Local repository detected, skipping cloning process.\")\n else:\n # git clone\n if not os.path.exists(self.download_path):\n print(f\"Cloning from Git URL: {self.git_url}\")\n try:\n git.Repo.clone_from(self.git_url, self.download_path)\n print(\"Repository cloned successfully.\")\n except Exception as e:\n print(f\"Failed to clone repository. Error: {e}\")\n\n # load the projects\n def load_files(self, root_dir=None, current_depth=0, base_depth=0):\n if root_dir is None:\n root_dir = self.download_path\n self.docs = []\n \n print(\"Loading files from:\", root_dir)\n \n # github projects\n if \"UploadedRepo\" not in self.git_url:\n for dirpath, _, filenames in os.walk(root_dir):\n for filename in filenames:\n if any(filename.endswith(ext) for ext in allowed_extensions):\n file_path = os.path.join(dirpath, filename)\n try:\n loader = TextLoader(file_path, encoding='utf-8')\n self.docs.extend(loader.load_and_split())\n except Exception as e:\n print(f\"Error loading file {file_path}: {e}\")\n else:\n # sosreport project or directories upload\n if current_depth - base_depth > int(max_dir_depth):\n return # over the dir depth, then stop\n \n for entry in os.scandir(root_dir):\n if entry.is_symlink():\n continue # skip the soft link(should be for windows)\n elif entry.is_dir():\n if entry.name == 'boot':\n base_depth = current_depth # start from boot\n self.load_files(entry.path, current_depth + 1, base_depth)\n elif entry.is_file():\n # not limit for the file extension\n try:\n loader = TextLoader(entry.path, encoding='utf-8')\n self.docs.extend(loader.load_and_split())\n except Exception as e:\n print(f\"Error loading file {entry.path}: {e}\")\n\n # split all the files\n def split_files(self):\n text_splitter = CharacterTextSplitter(chunk_size=int(chunk_size), chunk_overlap=int(chunk_overlap))\n self.texts = text_splitter.split_documents(self.docs)\n\n # store the all file chunk into chromadb\n def store_chroma(self): \n if not os.path.exists(self.db_dir):\n os.makedirs(self.db_dir)\n print(\"eb:\", self.embedding_model)\n db = Chroma.from_documents(self.texts, self.embedding_model, persist_directory=self.db_dir) \n db.persist() \n return db \n \n # load \n def load_into_db(self):\n if not os.path.exists(self.db_dir): \n ## Create and load\n self.load_files()\n self.split_files()\n self.db = self.store_chroma()\n else:\n print(\"start-->chromadb\")\n # Just load the DB\n self.db = Chroma(persist_directory=self.db_dir, embedding_function=self.embedding_model)\n print(\"end-->chromadb\")\n \n self.retriever = self.db.as_retriever()\n self.retriever.search_kwargs['k'] = 3\n self.retriever.search_type = 'similarity'\n\n # create a chain, send the message into llm and ouput the answer\n @cached(cache)\n def retrieval_qa(self, query, rsd=False, rr=False):\n config = configparser.ConfigParser()\n config.read(config_path)\n the_selected_provider = config.get('model_providers', 'selected_provider')\n chat_history = list(self.ChatQueue.queue)\n\n try:\n templates = load_prompt_templates(prompt_templates_path)\n qa_template_name = config.get('prompt_templates', 'qa_selected_prompt')\n qa_template = templates['qa_prompt_templates'][qa_template_name]\n except Exception as e:\n print(f\"Error reading config or templates: {e}\")\n raise\n\n custom_prompt = ChatPromptTemplate.from_messages([\n SystemMessagePromptTemplate.from_template(qa_template), \n HumanMessagePromptTemplate.from_template(\"{question}\")])\n \n if the_selected_provider != 'localai':\n # add reranker\n if rr:\n compressor = FlashrankRerank()\n compression_retriever = ContextualCompressionRetriever(\n base_compressor=compressor, base_retriever=self.retriever\n )\n the_retriever = compression_retriever\n else:\n the_retriever = self.retriever\n\n \n qa = ConversationalRetrievalChain.from_llm(\n self.model, \n chain_type=\"stuff\", \n retriever=the_retriever, \n condense_question_llm = self.model,\n return_source_documents=True,\n combine_docs_chain_kwargs={\"prompt\": custom_prompt})\n \n result = qa({\"question\": query, \"chat_history\": chat_history})\n\n self.update_chat_queue((query, result[\"answer\"]))\n\n # add the search source documents\n docs_strings = document_to_string(result['source_documents'][0])\n # docs_strings = documents_to_string(result['source_documents'])\n\n if rsd:\n return docs_strings\n else:\n return result['answer']\n \n elif the_selected_provider == 'localai':\n docs = self.retriever.get_relevant_documents(query)\n\n ds2s = documents_to_string(docs)\n\n the_question = \"\"\" \n the question: {question}\"\"\" \n\n # build the prompt with string\n combine_strings = qa_template + the_question\n prompt = combine_strings.format(context=ds2s, question=query)\n result = self.model.complete(prompt)\n self.update_chat_queue((query, result.text))\n if rsd:\n return ds2s\n return result.text\n \n else:\n return \"Wrong provider!!!\"\n \n @cached(cache)\n def restrieval_qa_for_code(self, query):\n config = configparser.ConfigParser()\n config.read(config_path)\n the_selected_provider = config.get('model_providers', 'selected_provider')\n\n try:\n templates = load_prompt_templates(prompt_templates_path)\n code_template_name = config.get('prompt_templates', 'code_selected_prompt')\n localai_template_name = config.get('prompt_templates', 'localai_selected_prompt')\n \n code_template = templates['qa_prompt_templates'][code_template_name]\n code_template_localai = templates['qa_prompt_templates'][localai_template_name]\n except Exception as e:\n print(f\"Error reading config or templates: {e}\")\n raise\n\n if the_selected_provider != 'localai':\n PROMPT = PromptTemplate(input_variables=[\"input\", \"history\"], template=code_template)\n # print(\"-->\", datetime.now())\n conversation = ConversationChain(\n prompt=PROMPT,\n llm=self.model,\n )\n # print(\"-->\", datetime.now())\n code_anaylsis = conversation.predict(input=query)\n return code_anaylsis\n elif the_selected_provider == 'localai':\n prompt = code_template_localai.format(input=query)\n result = self.model.complete(prompt)\n return result.text\n else:\n return \"Wrong provider!!!\"" } ]