[ { "path": ".gitignore", "content": ".venv/\n__pycache__/\nagents/__pycache__/\napplication_default_credentials.json\ndatabases/__pycache__/\nembeddings/__pycache__/\nutils/__pycache__/\n*/__pycache__/\n.DS_Store\npoetry.lock\ndist/\ntest-pypi-token.txt\nfirebase.json\n.firebaserc\nconfig_copy.ini\neval/" }, { "path": "CODE_OF_CONDUCT.md", "content": "\n# Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, gender identity and expression, level of\nexperience, education, socio-economic status, nationality, personal appearance,\nrace, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, or to ban temporarily or permanently any\ncontributor for other behaviors that they deem inappropriate, threatening,\noffensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\nThis Code of Conduct also applies outside the project spaces when the Project\nSteward has a reasonable belief that an individual's behavior may have a\nnegative impact on the project or its community.\n\n## Conflict Resolution\n\nWe do not believe that all conflict is bad; healthy debate and disagreement\noften yield positive results. However, it is never okay to be disrespectful or\nto engage in behavior that violates the project’s code of conduct.\n\nIf you see someone violating the code of conduct, you are encouraged to address\nthe behavior directly with those involved. Many issues can be resolved quickly\nand easily, and this gives people more control over the outcome of their\ndispute. If you are unable to resolve the matter for any reason, or if the\nbehavior is threatening or harassing, report it. We are dedicated to providing\nan environment where participants feel welcome and safe.\n\n\nReports should be directed to *googleapis-stewards@google.com*, the\nProject Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to\nreceive and address reported violations of the code of conduct. They will then\nwork with a committee consisting of representatives from the Open Source\nPrograms Office and the Google Open Source Strategy team. If for any reason you\nare uncomfortable reaching out to the Project Steward, please email\nopensource@google.com.\n\nWe will investigate every complaint, but you may not receive a direct response.\nWe will use our discretion in determining when and how to follow up on reported\nincidents, which may range from not taking action to permanent expulsion from\nthe project and project-sponsored spaces. We will notify the accused of the\nreport and provide them an opportunity to discuss it before any action is taken.\nThe identity of the reporter will be omitted from the details of the report\nsupplied to the accused. In potentially harmful situations, such as ongoing\nharassment or threats to anyone's safety, we may take action without notice.\n\n## Attribution\n\nThis Code of Conduct is adapted from the Contributor Covenant, version 1.4,\navailable at\nhttps://www.contributor-covenant.org/version/1/4/code-of-conduct.html" }, { "path": "CONTRIBUTING.md", "content": "# How to contribute\n\nWe'd love to accept your patches and contributions to this project.\n\n## Before you begin\n\n### Sign our Contributor License Agreement\n\nContributions to this project must be accompanied by a\n[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).\nYou (or your employer) retain the copyright to your contribution; this simply\ngives us permission to use and redistribute your contributions as part of the\nproject.\n\nIf you or your current employer have already signed the Google CLA (even if it\nwas for a different project), you probably don't need to do it again.\n\nVisit to see your current agreements or to\nsign a new one.\n\n### Review our community guidelines\n\nThis project follows\n[Google's Open Source Community Guidelines](https://opensource.google/conduct/).\n\n## Contribution process\n\n### Code reviews\n\nAll submissions, including submissions by project members, require review. We\nuse GitHub pull requests for this purpose. Consult\n[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more\ninformation on using pull requests." }, { "path": "Dockerfile", "content": "# Use the official lightweight Python image.\n# https://hub.docker.com/_/python\nFROM python:3.9-slim\n# Allow statements and log messages to immediately appear in the Knative logs\nENV PYTHONUNBUFFERED True\n\n# Copy local code to the container image.\nENV APP_HOME /app\nWORKDIR $APP_HOME\n\nCOPY . .\n\n# Install production dependencies.\nRUN pip install poetry\nRUN poetry install\n\n# Run the web service on container startup. Here we use the gunicorn\n# webserver, with one worker process and 8 threads.\n# For environments with multiple CPU cores, increase the number of workers\n# to be equal to the cores available.\n# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.\n# CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app\nCMD HOME=/root poetry run gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 backend-apis.main:app\n\n" }, { "path": "LICENSE", "content": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "MANIFEST.in", "content": "# -*- coding: utf-8 -*-\n#\n# Copyright 2023 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Generated by synthtool. DO NOT EDIT!\ninclude README.rst LICENSE\nrecursive-include third_party *\nrecursive-include bigframes *.json *.proto py.typed\nrecursive-include tests *\nglobal-exclude *.py[co]\nglobal-exclude __pycache__\n\n# Exclude scripts for samples readmegen\nprune scripts/readme-gen\n" }, { "path": "OWNERS", "content": "msubasioglu@google.com\nsteveswalker@google.com\nkpatlolla@google.com\nsrilakshmil@google.com\nmokshazna@google.com\n" }, { "path": "README.md", "content": "

\n \n \"aaie\n \n

\n

\n \n \"logo\"\n \n

\n

Open Data QnA - Chat with your SQL Database

\n\n_______________\n\n
\n 🚨 Version 2.0.0 is now live! Refer to the Release Notes for detailed information on updates and fixes. 🚨\n
\n\n_______________\n\n✨ Overview\n-------------\nThe **Open Data QnA** python library enables you to chat with your databases by leveraging LLM Agents on Google Cloud.\n\nOpen Data QnA enables a conversational approach to interacting with your data. Ask questions about your PostgreSQL or BigQuery databases in natural language and receive informative responses, without needing to write SQL. Open Data QnA leverages Large Language Models (LLMs) to bridge the gap between human language and database queries, streamlining data analysis and decision-making.\n\n![Alt Text](utilities/imgs/Teaser.gif)\n\n**Key Features:**\n\n* **Conversational Querying with Multiturn Support:** Ask questions naturally, without requiring SQL knowledge and ask follow up questions.\n* **Table Grouping:** Group tables under one usecase/user grouping name which can help filtering your large number tables for LLMs to understand about.\n* **Multi Schema/Dataset Support:** You can group tables from different schemas/datasets for embedding and asking questions against.\n* **Prompt Customization and Additional Context:** The prompts that are being used are loaded from a yaml file and it also give you ability to add extra context as well\n* **SQL Generation:** Automatically generates SQL queries based on your questions.\n* **Query Refinement:** Validates and debugs queries to ensure accuracy.\n* **Natural Language Responses:** DRun queries and present results in clear, easy-to-understand language.\n* **Visualizations (Optional):** Explore data visually with generated charts.\n* **Extensible:** Customize and integrate with your existing workflows(API, UI, Notebooks).\n\n\nIt is built on a modular design and currently supports the following components: \n\n### Database Connectors\n* **Google Cloud SQL for PostgreSQL**\n* **Google BigQuery**\n* **Google Firestore(for storing session logs)**\n\n### Vector Stores \n* **PGVector on Google Cloud SQL for PostgreSQL**\n* **BigQuery Vector Store**\n\n### Agents \n* **BuildSQLAgent:** An agent specialized in generating SQL queries for BigQuery or PostgreSQL databases. It analyzes user questions, available table schemas, and column descriptions to construct syntactically and semantically correct SQL queries, adapting its process based on the target database type.\n* **ValidateSQLAgent:** An agent that validates the syntax and semantic correctness of SQL queries. It uses a language model to analyze queries against a database schema and returns a JSON response indicating validity and potential errors.\n* **DebugSQLAgent:** An agent designed to debug and refine SQL queries for BigQuery or PostgreSQL databases. It interacts with a chat-based language model to iteratively troubleshoot queries, using error messages to generate alternative, correct queries.\n* **DescriptionAgent:** An agent specialized in generating descriptions for database tables and columns. It leverages a large language model to create concise and informative descriptions that aid in understanding data structures and facilitate SQL query generation.\n* **EmbedderAgent:** An agent specialized in generating text embeddings using Large Language Models (LLMs). It supports direct interaction with Vertex AI's TextEmbeddingModel or uses LangChain's VertexAIEmbeddings for a simplified interface.\n* **ResponseAgent:** An agent that generates natural language responses to user questions based on SQL query results. It acts as a data assistant, interpreting SQL results and transforming them into user-friendly answers using a language model.\n* **VisualizeAgent:** An agent that generates JavaScript code for Google Charts based on user questions and SQL results. It suggests suitable chart types and constructs the JavaScript code to create visualizations of the data.\n\n**Note:** the library was formerly named Talk2Data. You may still find artifacts with the old naming in this repository. \n\n📏 Architecture\n-------------\n

\n \n \"aaie\n \n

\n\nA detailed description of the Architecture can be found [`here`](/docs/architecture.md) in the docs. \n\n\n🧬 Repository Structure \n-------------\nDetails on the Repository Structure can be found [`here`](/docs/repo_structure.md) in the docs. \n\n\n\n\n\n🏁 Getting Started: Main Repository \n-------------\n\n### Clone the repository and switch to the correct directory \n \n git clone git@github.com:GoogleCloudPlatform/Open_Data_QnA.git\n cd Open_Data_QnA\n\n### 🚧 **Prerequisites**\n\nMake sure that Google Cloud CLI and Python >= 3.10 are installed before moving ahead! You can refer to the link below for guidance\n\nInstallation Guide: https://cloud.google.com/sdk/docs/install\n\nDownload Python: https://www.python.org/downloads/\n\nℹ️ **You can setup this solution with three approaches. Choose one based on your requirements:**\n - **A)** Using [Jupyter Notebooks](#a-jupyter-notebook-based-approach) (For better view at what is happening at each stage of the solution)\n - **B)** Using [CLI](#b-command-line-interface-cli-based-approach) (For ease of use and running with simple python commands, without the need to understand every step of the solution)\n - **C)** Using [terraform deployment](#c-using-terraform-to-deploy-the-solution) including your backend APIs with UI\n\n\n### A) Jupyter Notebook Based Approach\n\n#### 💻 **Install Code Dependencies (Create and setup venv)**\n\n#### **All commands in this cell to be run on the terminal (typically Ctrl+Shift+`) where your notebooks are running**\nInstall the dependencies by running the poetry commands below \n\n```\n# Install poetry\npip uninstall poetry -y\npip install poetry --quiet\n\n#Run the poetry commands below to set up the environment\npoetry lock #resolve dependecies (also auto create poetry venv if not exists)\npoetry install --quiet #installs dependencies\npoetry env info #Displays the evn just created and the path to it\n\npoetry shell #this command should activate your venv and you should see it enters into the venv\n\n##inside the activated venv shell []\n\n#If you are running on Worbench instance where the service account used has required permissions to run this solution you can skip the below gcloud auth commands and get to next kernel creation section\n\ngcloud auth login # Use this or below command to authenticate\n\ngcloud auth application-default login\n\ngcloud services enable \\\n serviceusage.googleapis.com \\\n cloudresourcemanager.googleapis.com --project <>\n\n```\n\nChose the relevant instructions based on where you are running the notebook\n\n**For IDEs like Cloud Shell Editor, VS Code**\n\nFor IDEs adding Juypter Extensions will automatically give you option to change the kernel. If not, manually select the python interpreter in your IDE (The exact is shown in the above cell. Path would look like e.g. /home/admin_/opendata/.venv/bin/python or ~cache/user/opendataqna/.venv/bin/python)\n\nProceed to the Step 1 below\n\n\n**For Jupyter Lab or Jupyter Environments on Workbench etc**\n\nCreate Kernel for with the envrionment created\n\n```\npip install jupyter\n\nipython kernel install --name \"openqna-venv\" --user \n\n```\n\nRestart your kernel or close the exsiting notebook and open again, you should now see the \"openqna-venv\" in the kernel drop down\n\n**What did we do here?**\n\n* Created Application Default Credentials to use for the code\n* Added venv to kernel to select for running the notebooks (For standalone Jupyter setups like Workbench etc)\n\n#### 1. Run the [1_Setup_OpenDataQnA](/notebooks/1_Setup_OpenDataQnA.ipynb) (Run Once for Initial Setup) \n\nThis notebook guides you through the setup and execution of the Open Data QnA application. It provides comprehensive instructions for setup the solution.\n\n\n#### 2. Run the [2_Run_OpenDataQnA](/notebooks/2_Run_OpenDataQnA.ipynb)\n\nThis notebook guides you by reading the configuration you setup with [1_Setup_OpenDataQnA](/1_Setup_OpenDataQnA) and running the pipeline to answer questions about your data.\n\n#### 3. [Loading Known Good SQL Examples](/notebooks/3_LoadKnownGoodSQL.ipynb)\n \n In case you want to separately load Known Good SQLs please run this notebook once the config variables are setup in config.ini file. This can be run multiple times just to load the known good sql queries and create embeddings for it.\n\n___________\n\n### B) Command Line Interface (CLI) Based Approach\n\n#### 1. Add Configuration values for the solution in [config.ini](/config.ini)\n\nFor setup we require details for vector store, source database etc. Edit the [config.ini](/config.ini) file and add values for the parameters based of below information.\n\nℹ️ Follow the guidelines from the [config guide document](/docs/config_guide.md) to populate your [config.ini](/config.ini) file.\n\n**Sources to connect**\n\n- This solution lets you setup multiple data source at same time.\n- You can group multiple tables from different datasets or schema into a grouping and provide the details\n- If your dataset/schema has many tables and you want to run the solution against few you should specifically choose a group for that tables only\n\n**Format for data_source_list.csv**\n\n**source | user_grouping | schema | table**\n\n**source** - Supported Data Sources. #Options: bigquery , cloudsql-pg\n\n**user_grouping** - Logical grouping or use case name for tables from same or different schema/dataset. When left black it default to the schema value in the next column\n\n**schema** - schema name for postgres or dataset name in bigquery \n\n**table** - name of the tables to run the solutions against. Leave this column blank after filling schema/dataset if you want to run solution for whole dataset/schema\n\nUpdate the [data_source_list.csv](/scripts/data_source_list.csv) according for your requirement.\n\nNote that the source details filled in the csv should have already be present. If not please use the Copy Notebooks if you want the demo source setup.\n\nEnabled Data Sources:\n* PostgreSQL on Google Cloud SQL (Copy Sample Data: [0_CopyDataToCloudSqlPG.ipynb](0_CopyDataToCloudSqlPG.ipynb))\n* BigQuery (Copy Sample Data: [0_CopyDataToBigQuery.ipynb](0_CopyDataToBigQuery.ipynb))\n\n#### 2. Creating Virtual Environment and Install Dependencies\n\n```\npip install poetry --quiet\npoetry lock\npoetry install --quiet\npoetry env info\npoetry shell\n```\nAuthenticate your credentials\n\n```\ngcloud auth login\n\nor \n\ngcloud auth application-default login\n```\n```\ngcloud services enable \\\n serviceusage.googleapis.com \\\n cloudresourcemanager.googleapis.com --project <>\n```\n```\ngcloud auth application-default set-quota-project <>\n```\n\nEnable APIs for the solution setup\n\n```\ngcloud services enable \\\n cloudapis.googleapis.com \\\n compute.googleapis.com \\\n iam.googleapis.com \\\n run.googleapis.com \\\n sqladmin.googleapis.com \\\n aiplatform.googleapis.com \\\n bigquery.googleapis.com \\\n firestore.googleapis.com --project <>\n\n```\n\n#### 3. Run [env_setup.py](/env_setup.py) to create vector store based on the configuration you did in Step 1\n\n```\npython env_setup.py\n```\n\n#### 4. Run [opendataqna.py](/opendataqna.py) to run the pipeline you just setup\n\nThe Open Data QnA SQL Generation tool can be conveniently used from your terminal or command prompt using a simple CLI interface. Here's how:\n\n```\npython opendataqna.py --session_id \"122133131f--ade-eweq\" --user_question \"What is most 5 common genres we have?\" --user_grouping \"MovieExplorer-bigquery\"\n```\n\nWhere\n\n*session_id* : Keep this unique unique same for follow up questions.\n\n*user_question* : Enter your question in string\n\n*user_grouping* : Enter the BQ_DATASET_NAME for BigQuery sources or PG_SCHEMA for PostgreSQL sources (refer your [data_source_list.csv](/scripts/data_source_list.csv) file)\n\n\n**Optional Parameters**\n\nYou can customize the pipeline's behavior using optional parameters. Here are some common examples:\n```\n# Enable the SQL debugger:\npython opendataqna.py --session_id=\"...\" --user_question \"...\" --user_grouping \"...\" --run_debugger\n\n# Execute the final generated SQL:\npython opendataqna.py --session_id=\"...\" --user_question \"...\" --user_grouping \"...\" --execute_final_sql\n\n# Change the number of debugging rounds:\npython opendataqna.py --session_id=\"...\" --user_question \"...\" --user_grouping \"...\" --debugging_rounds 5\n\n# Adjust similarity thresholds:\npython opendataqna.py --session_id=\"...\" --user_question \"...\" --user_grouping \"...\" --table_similarity_threshold 0.25 --column_similarity_threshold 0.4\n\n```\n\nYou can find a full list of available options and their descriptions by running:\n\n```\npython opendataqna.py --help\n```\n\n### C) Using Terraform to deploy the solution\n\nThe provided terraform streamlines the setup of this solution and serves as a blueprint for deployment. The script provides a one-click, one-time deployment option. However, it doesn't include CI/CD capabilities and is intended solely for initial setup.\n\n> [!NOTE]\n> Current version of the Terraform Google Cloud provider does not support deployment of a few resources, this solution uses null_resource to create those resources using Google Cloud SDK.\n\nPrior to executing terraform, ensure that the below mentioned steps have been completed.\n\n#### Data Sources Set Up\n\n1. Source data should already be available. If you do not have readily available source data, use the notebooks [0_CopyDataToBigQuery.ipynb](/notebooks/0_CopyDataToBigQuery.ipynb) or [0_CopyDataToCloudSqlPG.ipynb](/notebooks/0_CopyDataToCloudSqlPG.ipynb) based on the preferred source to populate sample data.\n2. Ensure that the [data_source_list.csv](/scripts/data_source_list.csv) is populated with the list of datasources to be used in this solution. Terraform will take care of creating the embeddings in the destination. Use [data_source_list_sample.csv](/scripts/data_source_list_sample.csv) to fill the [data_source_list.csv](/scripts/data_source_list.csv)\n3. If you want to use known good sqls for few shot prompting, ensure that the [known_good_sql.csv](/scripts/known_good_sql.csv) is populated with the required data. Terraform will take care of creating the embeddings in the destination.\n\n#### Enable Firebase\nFirebase will be used to host the frontend of the application.\n\n1. Go to https://console.firebase.google.com/\n1. Select add project and load your Google Cloud Platform project\n1. Add Firebase to one of your existing Google Cloud projects\n1. Confirm Firebase billing plan\n1. Continue and complete\n\n\n#### Terraform deployment\n> [!NOTE] \n> Terraform apply command for this application uses gcloud config to fetch & pass the set project id to the scripts. Please ensure that gcloud config has been set to your intended project id before proceeding.\n\n> [!IMPORTANT] \n> The Terraform scripts require specific IAM permissions to function correctly. The user needs either the broad `roles/resourcemanager.projectIamAdmin` role or a custom role with tailored permissions to manage IAM policies and roles.\n> Additionally, one script TEMPORARILY disables Domain Restricted Sharing Org Policies to enable the creation of a public endpoint. This requires the user to also have the `roles/orgpolicy.policyAdmin` role.\n\n1. Install [terraform 1.7 or higher](https://developer.hashicorp.com/terraform/install).\n1. [OPTIONAL] Update default values of variables in [variables.tf](/terraform/variables.tf) according to your preferences. You can find the description for each variable inside the file. This file will be used by terraform to get information about the resources it needs to deploy. If you do not update these, terraform will use the already specified default values in the file.\n1. Move to the terraform directory in the terminal\n\n```\ncd Open_Data_QnA/terraform\n\n#If you are running this outside Cloud Shell you need to set up your Google Cloud SDK Credentials\n\ngcloud config set project \ngcloud auth application-default set-quota-project \n\ngcloud services enable \\\n serviceusage.googleapis.com \\\n cloudresourcemanager.googleapis.com --project <>\n\nsh ./scripts/deploy-all.sh\n\n```\nThis script will perform the following steps:\n1. **Run terraform scripts** - These terraform scripts will generate all the GCP resources and configuration files required for the frontend & backend. It will also generate embeddings and store it in the destination vector db.\n1. **Deploy cloud run backend service with latest backend code** - The terraform in the previous step uses a dummy container image to deploy the initial version of cloud run service. This is the step where the actual backend code gets deployed.\n1. **Deploy frontend app** - All the config files, web app etc required to create the frontend are deployed via terraform. However, the actual UI deployment takes place in this step.\n\n### After deployment\n***Auth Provider***\n\nYou need to enable at least one authentication provider in Firebase, you can enable it using the following steps:\n1. Go to https://console.firebase.google.com/project/your_project_id/authentication/providers (change the `your_project_id` value)\n2. Click on Get Started (if needed)\n3. Select Google and enable it\n4. Set the name for the project and support email for project\n5. Save\n\nThis should deploy you end to end solution in the project with firebase web url\n\nFor detailed steps and known issues refer to README.md under [`/terraform`](/terraform/)\n\n\n🖥️ Build a angular based frontend for this solution \n---------------------------------------------------\nDeploy backend apis for the solution, refer to the README.md under [`/backend-apis`](/backend-apis/). This APIs are designed with work with the frontend and provide access to run the solution.\n\nOnce the backend APIs deployed successfully deploy the frontend for the solution, refer to the README.md under [`/frontend`](/frontend/).\n\n\n📗 FAQs and Best Practices \n-------------\nIf you successfully set up the solution accelerator and want to start optimizing to your needs, you can follow the tips in the [`Best Practice doc`](/docs/best_practices.md).\nAdditionally, if you stumble across any problems, take a look into the [`FAQ`](/docs/faq.md).\n\nIf neither of these resources helps, feel free to reach out to us directly by raising an Issue. \n\n🧹 CleanUp Resources \n-------------\nTo clean up the resources provisioned in this solution, use commands below to remove them using gcloud/bq: \n\nFor cloudsql-pgvector as vector store : [Delete SQL Instance]()\n\n```\ngcloud sql instances delete -q\n```\n\nDelete BigQuery Dataset Created for Logs and Vector Store : [Remove BQ Dataset]()\n\n```\nbq rm -r -f -d \n```\n\n(For Backend APIs)Remove the Cloud Run service : [Delete Service]()\n\n```\ngcloud run services delete \n```\n\nFor frontend, based on firebase: [Remove the firebase app]()\n\n📄 Documentation\n-------------\n\n* [Open Data QnA Source Code (GitHub)]()\n* [Open Data QnA usage notebooks](/notebooks)\n* [`Architecture`](/docs/architecture.md)\n* [`FAQ`](/docs/faq.md)\n* [`Best Practice doc`](/docs/best_practices.md)\n\n\n\n🚧 Quotas and limits\n------------------\n\n[BigQuery quotas]() including hardware, software, and network components.\n\n[Gemini quotas]().\n\n\n🪪 License\n-------\n\nOpen Data QnA is distributed with the [Apache-2.0 license]().\n\nIt also contains code derived from the following third-party packages:\n\n* [pandas]()\n* [Python]()\n \n\n🧪 Disclaimer\n----------\n\nThis repository provides an open-source solution accelerator designed to streamline your development process. Please be aware that all resources associated with this accelerator will be deployed within your own Google Cloud Platform (GCP) instances.\n\nIt is imperative that you thoroughly test all components and configurations in a non-production environment before integrating any part of this accelerator with your production data or systems.\n\nWhile we strive to provide a secure and reliable solution, we cannot be held responsible for any data loss, service disruptions, or other issues that may arise from the use of this accelerator.\n\nBy utilizing this repository, you acknowledge that you are solely responsible for the deployment, management, and security of the resources deployed within your GCP environment.\n\nIf you encounter any issues or have concerns about potential risks, please refrain from using this accelerator in a production setting.\n\nWe encourage responsible and informed use of this open-source solution.\n\n\n🙋 Getting Help\n----------\n\nIf you have any questions or if you found any problems with this repository, please report through GitHub issues.\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\nTo report a security issue, please use [g.co/vulnz](https://g.co/vulnz).\n\nThe Google Security Team will respond within 5 working days of your report on g.co/vulnz.\n\nWe use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.\n" }, { "path": "agents/BuildSQLAgent.py", "content": "from abc import ABC\nfrom vertexai.language_models import CodeChatModel\nfrom vertexai.generative_models import GenerativeModel, Content, Part, GenerationConfig\nfrom .core import Agent \nimport pandas as pd\nimport json\nfrom datetime import datetime\nfrom dbconnectors import pgconnector,bqconnector,firestoreconnector\nfrom utilities import PROMPTS, format_prompt\nfrom google.cloud.aiplatform import telemetry\nimport vertexai \nfrom utilities import PROJECT_ID, PG_REGION\nfrom vertexai.generative_models import GenerationConfig\n\nvertexai.init(project=PROJECT_ID, location=PG_REGION)\n\n\nclass BuildSQLAgent(Agent, ABC):\n\n agentType: str = \"BuildSQLAgent\"\n\n def __init__(self, model_id = 'gemini-1.5-pro'): \n super().__init__(model_id=model_id)\n\n\n def build_sql(self,source_type,user_grouping, user_question,session_history,tables_schema,columns_schema, similar_sql, max_output_tokens=2048, temperature=0.4, top_p=1, top_k=32):\n not_related_msg=f'''select 'Question is not related to the dataset' as unrelated_answer;'''\n \n if source_type=='bigquery':\n\n from dbconnectors import bq_specific_data_types\n specific_data_types = bq_specific_data_types() \n \n else:\n \n from dbconnectors import pg_specific_data_types\n specific_data_types = pg_specific_data_types()\n\n if f'usecase_{source_type}_{user_grouping}' in PROMPTS:\n usecase_context = PROMPTS[f'usecase_{source_type}_{user_grouping}']\n else:\n usecase_context = \"No extra context for the usecase is provided\"\n \n context_prompt = PROMPTS[f'buildsql_{source_type}']\n\n\n context_prompt = format_prompt(context_prompt,\n specific_data_types = specific_data_types,\n not_related_msg = not_related_msg, \n usecase_context = usecase_context,\n similar_sql=similar_sql, \n tables_schema=tables_schema, \n columns_schema = columns_schema)\n\n # print(f\"Prompt to Build SQL: \\n{context_prompt}\") \n\n \n # Chat history Retrieval\n\n chat_history=[]\n for entry in session_history:\n \n timestamp = entry[\"timestamp\"]\n timestamp_str = timestamp.isoformat(timespec='auto')\n\n user_message = Content(\n parts=[Part.from_text(entry[\"user_question\"])], \n role=\"user\"\n )\n\n bot_message = Content(\n parts=[Part.from_text(entry[\"bot_response\"])],\n role=\"assistant\"\n )\n chat_history.extend([user_message, bot_message]) # Add both to the history\n \n\n # print(\"Chat History Retrieved\")\n\n if self.model_id == 'codechat-bison-32k':\n with telemetry.tool_context_manager('opendataqna-buildsql-v2'):\n \n chat_session = self.model.start_chat(context=context_prompt)\n elif 'gemini' in self.model_id:\n with telemetry.tool_context_manager('opendataqna-buildsql-v2'):\n\n # print(\"SQL Builder Agent : \" + str(self.model_id))\n config = GenerationConfig(\n max_output_tokens=max_output_tokens, temperature=temperature, top_p=top_p, top_k=top_k\n )\n chat_session = self.model.start_chat(history=chat_history,response_validation=False)\n chat_session.send_message(context_prompt)\n else:\n raise ValueError('Invalid Model Specified')\n \n\n if session_history is None or not session_history:\n concated_questions = None\n re_written_qe = None\n previous_question = None\n previous_sql = None\n\n else:\n concated_questions,re_written_qe=self.rewrite_question(user_question,session_history)\n previous_question, previous_sql = self.get_last_sql(session_history)\n\n\n build_context_prompt=f\"\"\"\n\n Below is the previous user question from this conversation and its generated sql. \n\n Previous Question: {previous_question} \n\n Previous Generated SQL : {previous_sql}\n\n Respond with \n\n Generate SQL for User Question : {user_question}\n\n\n \"\"\"\n\n # print(\"BUILD CONTEXT ::: \"+str(build_context_prompt))\n\n\n with telemetry.tool_context_manager('opendataqna-buildsql-v2'):\n\n response = chat_session.send_message(build_context_prompt, stream=False)\n generated_sql = (str(response.text)).replace(\"```sql\", \"\").replace(\"```\", \"\")\n\n generated_sql = (str(response.text)).replace(\"```sql\", \"\").replace(\"```\", \"\")\n # print(generated_sql)\n return generated_sql\n\n def rewrite_question(self,question,session_history):\n formatted_history=''\n concat_questions=''\n for i, _row in enumerate(session_history,start=1):\n user_question = _row['user_question']\n sql_query = _row['bot_response']\n # print(user_question)\n formatted_history += f\"User Question - Turn :: {i} : {user_question}\\n\"\n formatted_history += f\"Generated SQL - Turn :: {i}: {sql_query}\\n\\n\"\n concat_questions += f\"{user_question} \"\n\n # print(formatted_history)\n\n\n context_prompt = f\"\"\"\n Your main objective is to rewrite and refine the question passed based on the session history of question and sql generated.\n\n Refine the given question using the provided session history to produce a queryable statement. The refined question should be self-contained, requiring no additional context for accurate SQL generation.\n\n Make sure all the information is included in the re-written question\n\n Below is the previous session history:\n\n {formatted_history}\n\n Question to rewrite:\n\n {question}\n \"\"\"\n re_written_qe = str(self.generate_llm_response(context_prompt))\n \n\n print(\"*\"*25 +\"Re-written question for the follow up:: \"+\"*\"*25+\"\\n\"+str(re_written_qe))\n\n return str(concat_questions),str(re_written_qe)\n\n def get_last_sql(self,session_history):\n\n for entry in reversed(session_history): \n if entry.get(\"bot_response\"): \n return entry[\"user_question\"],entry[\"bot_response\"] \n\n return None\n" }, { "path": "agents/DebugSQLAgent.py", "content": "from abc import ABC\n\nimport vertexai\nfrom vertexai.language_models import CodeChatModel\nfrom vertexai.generative_models import GenerativeModel,GenerationConfig\nfrom google.cloud.aiplatform import telemetry\nfrom dbconnectors import pgconnector, bqconnector\nfrom utilities import PROMPTS, format_prompt\nfrom .core import Agent\nimport pandas as pd\nimport json \n\nfrom utilities import PROJECT_ID, PG_REGION\nvertexai.init(project=PROJECT_ID, location=PG_REGION)\n\n\nclass DebugSQLAgent(Agent, ABC):\n \"\"\"\n An agent designed to debug and refine SQL queries for BigQuery or PostgreSQL databases.\n\n This agent interacts with a chat-based language model (CodeChat or Gemini) to iteratively troubleshoot SQL queries. It receives feedback in the form of error messages and uses the model's capabilities to generate alternative queries that address the identified issues. The agent strives to maintain the original intent of the query while ensuring its syntactic and semantic correctness.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"DebugSQLAgent\".\n model_id (str): The ID of the chat model to use for debugging. Valid options are:\n - \"codechat-bison-32k\"\n - \"gemini-1.0-pro\" \n - \"gemini-ultra\"\n\n Methods:\n init_chat(source_type, tables_schema, columns_schema, sql_example) -> ChatSession:\n Initializes a chat session with the chosen chat model.\n\n Args:\n source_type (str): The database type (\"bigquery\" or \"postgresql\").\n tables_schema (str): A description of the available tables and their columns.\n columns_schema (str): Detailed descriptions of the columns in the tables.\n sql_example (str, optional): An example SQL query for reference. Defaults to \"-No examples provided..-\".\n\n Returns:\n ChatSession: The initiated chat session object.\n\n rewrite_sql_chat(chat_session, question, error_df) -> str:\n Generates an alternative SQL query based on the chat session, original query, and error message.\n\n Args:\n chat_session (ChatSession): The active chat session.\n question (str): The original SQL query.\n error_df (pandas.DataFrame): The error message as a DataFrame.\n\n Returns:\n str: The rewritten SQL query.\n\n start_debugger(source_type, query, user_question, SQLChecker, tables_schema, columns_schema, AUDIT_TEXT, similar_sql, DEBUGGING_ROUNDS, LLM_VALIDATION) -> Tuple[str, bool, str]:\n\n Args:\n source_type (str): The database type (\"bigquery\" or \"postgresql\").\n query (str): The initial SQL query to debug.\n user_question (str): The user's original question for reference.\n SQLChecker: An object to validate the SQL syntax.\n tables_schema (str): Table schema information.\n columns_schema (str): Detailed column descriptions.\n AUDIT_TEXT (str): Textual audit trail of the debugging process.\n similar_sql (str, optional): Example SQL queries. Defaults to \"-No examples provided..-\".\n DEBUGGING_ROUNDS (int, optional): Maximum debugging attempts. Defaults to 2.\n LLM_VALIDATION (bool, optional): Whether to use LLM for syntax validation. Defaults to True.\n\n Returns:\n Tuple[str, bool, str]:\n - The final refined SQL query (or the original if unchanged).\n - A boolean indicating if the final query is considered invalid.\n - The updated AUDIT_TEXT with debugging steps.\n \"\"\"\n\n\n agentType: str = \"DebugSQLAgent\"\n\n def __init__(self, model_id = 'gemini-1.5-pro'):\n super().__init__(model_id=model_id) \n\n\n def init_chat(self,source_type,user_grouping, tables_schema,columns_schema,similar_sql=\"-No examples provided..-\"):\n\n if f'usecase_{source_type}_{user_grouping}' in PROMPTS:\n usecase_context = PROMPTS[f'usecase_{source_type}_{user_grouping}']\n else:\n usecase_context = \"No extra context for the usecase is provided\"\n \n context_prompt = PROMPTS[f'debugsql_{source_type}']\n\n context_prompt = format_prompt(context_prompt,\n usecase_context = usecase_context,\n similar_sql=similar_sql, \n tables_schema=tables_schema, \n columns_schema = columns_schema)\n\n # print(f\"Prompt to Debug SQL after formatting: \\n{context_prompt}\")\n \n if self.model_id == 'codechat-bison-32k':\n with telemetry.tool_context_manager('opendataqna-debugsql-v2'):\n\n chat_session = self.model.start_chat(context=context_prompt)\n elif 'gemini' in self.model_id:\n with telemetry.tool_context_manager('opendataqna-debugsql-v2'):\n\n chat_session = self.model.start_chat(response_validation=False)\n chat_session.send_message(context_prompt)\n else:\n raise ValueError('Invalid Chat Model Specified')\n \n return chat_session\n\n\n def rewrite_sql_chat(self, chat_session, sql, question, error_df):\n\n\n context_prompt = f\"\"\"\n What is an alternative SQL statement to address the error mentioned below?\n Present a different SQL from previous ones. It is important that the query still answer the original question.\n All columns selected must be present on tables mentioned on the join section.\n Avoid repeating suggestions.\n\n \n {sql}\n \n\n \n {question}\n \n\n \n {error_df}\n \n\n \"\"\"\n\n if self.model_id =='codechat-bison-32k':\n with telemetry.tool_context_manager('opendataqna-debugsql-v2'):\n response = chat_session.send_message(context_prompt)\n resp_return = (str(response.candidates[0])).replace(\"```sql\", \"\").replace(\"```\", \"\")\n elif 'gemini' in self.model_id:\n with telemetry.tool_context_manager('opendataqna-debugsql-v2'):\n response = chat_session.send_message(context_prompt, stream=False)\n resp_return = (str(response.text)).replace(\"```sql\", \"\").replace(\"```\", \"\")\n else:\n raise ValueError('Invalid Model Id')\n\n return resp_return\n\n\n def start_debugger (self,\n source_type,\n user_grouping,\n query,\n user_question, \n SQLChecker,\n tables_schema, \n columns_schema,\n AUDIT_TEXT, \n similar_sql=\"-No examples provided..-\", \n DEBUGGING_ROUNDS = 2,\n LLM_VALIDATION=False):\n i = 0 \n STOP = False \n invalid_response = False \n chat_session = self.init_chat(source_type,user_grouping,tables_schema,columns_schema,similar_sql)\n sql = query.replace(\"```sql\",\"\").replace(\"```\",\"\").replace(\"EXPLAIN ANALYZE \",\"\")\n\n AUDIT_TEXT=AUDIT_TEXT+\"\\n\\nEntering the debugging steps!\"\n while (not STOP):\n\n json_syntax_result={ \"valid\":True, \"errors\":\"None\"}\n # Check if LLM Validation is enabled \n if LLM_VALIDATION: \n # sql = query.replace(\"```sql\",\"\").replace(\"```\",\"\").replace(\"EXPLAIN ANALYZE \",\"\")\n json_syntax_result = SQLChecker.check(source_type,user_question,tables_schema,columns_schema, sql) \n\n else: \n json_syntax_result['valid'] = True\n AUDIT_TEXT=AUDIT_TEXT+\"\\nLLM Validation is deactivated. Jumping directly to dry run execution.\"\n \n\n if json_syntax_result['valid'] is True:\n AUDIT_TEXT=AUDIT_TEXT+\"\\nGenerated SQL is syntactically correct as per LLM Validation!\"\n \n # print(AUDIT_TEXT)\n if source_type=='bigquery':\n connector=bqconnector\n else:\n connector=pgconnector\n \n correct_sql, exec_result_df = connector.test_sql_plan_execution(sql)\n \n if not correct_sql:\n AUDIT_TEXT=AUDIT_TEXT+\"\\nGenerated SQL failed on execution! Here is the feedback from bigquery dryrun/ explain plan: \\n\" + str(exec_result_df)\n rewrite_result = self.rewrite_sql_chat(chat_session, sql, user_question, exec_result_df)\n print('\\n Rewritten and Cleaned SQL: ' + str(rewrite_result))\n AUDIT_TEXT=AUDIT_TEXT+\"\\nRewritten and Cleaned SQL: \\n' + str({rewrite_result})\"\n sql = str(rewrite_result).replace(\"```sql\",\"\").replace(\"```\",\"\").replace(\"EXPLAIN ANALYZE \",\"\")\n\n else: STOP = True\n else:\n print(f'\\nGenerated qeury failed on syntax check as per LLM Validation!\\nError Message from LLM: {json_syntax_result} \\nRewriting the query...')\n AUDIT_TEXT=AUDIT_TEXT+'\\nGenerated qeury failed on syntax check as per LLM Validation! \\nError Message from LLM: '+ str(json_syntax_result) + '\\nRewriting the query...'\n \n syntax_err_df = pd.read_json(json.dumps(json_syntax_result))\n rewrite_result=self.rewrite_sql_chat(chat_session, sql, user_question, syntax_err_df)\n print(rewrite_result)\n AUDIT_TEXT=AUDIT_TEXT+'\\n Rewritten SQL: ' + str(rewrite_result)\n sql=str(rewrite_result).replace(\"```sql\",\"\").replace(\"```\",\"\").replace(\"EXPLAIN ANALYZE \",\"\")\n i+=1\n if i > DEBUGGING_ROUNDS:\n AUDIT_TEXT=AUDIT_TEXT+ \"Exceeded the number of iterations for correction!\"\n AUDIT_TEXT=AUDIT_TEXT+ \"The generated SQL can be invalid!\"\n STOP = True\n invalid_response=True\n # After the while is completed\n if i > DEBUGGING_ROUNDS:\n invalid_response=True\n return sql, invalid_response, AUDIT_TEXT" }, { "path": "agents/DescriptionAgent.py", "content": "from abc import ABC\nfrom .core import Agent \n\n\nclass DescriptionAgent(Agent, ABC):\n \"\"\"\n An agent specialized in generating descriptions for database tables and columns.\n\n This agent leverages a large language model to create concise and informative descriptions that aid in understanding the structure and content of database elements. The generated descriptions can be valuable for documenting schemas, enhancing data exploration, and facilitating SQL query generation.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"DescriptionAgent\".\n\n Methods:\n generate_llm_response(prompt) -> str:\n Generates a response from the underlying language model based on the given prompt.\n\n Args:\n prompt (str): The prompt to feed into the language model.\n\n Returns:\n str: The generated text response, cleaned of any SQL-related formatting artifacts.\n\n generate_missing_descriptions(source, table_desc_df, column_name_df) -> Tuple[pd.DataFrame, pd.DataFrame]:\n Generates missing table and column descriptions using the language model.\n\n Args:\n source (str): The source of the database schema (\"bigquery\"). \n table_desc_df (pd.DataFrame): A DataFrame containing table metadata with potential missing descriptions.\n column_name_df (pd.DataFrame): A DataFrame containing column metadata with potential missing descriptions.\n\n Returns:\n Tuple[pd.DataFrame, pd.DataFrame]: \n - The updated `table_desc_df` with generated table descriptions.\n - The updated `column_name_df` with generated column descriptions.\n \"\"\"\n\n\n agentType: str = \"DescriptionAgent\"\n\n def generate_llm_response(self,prompt):\n context_query = self.model.generate_content(prompt,safety_settings=self.safety_settings,stream=False)\n return str(context_query.candidates[0].text).replace(\"```sql\", \"\").replace(\"```\", \"\")\n\n\n def generate_missing_descriptions(self,source,table_desc_df, column_name_df):\n llm_generated=0\n print(\"\\n\\n\")\n for index, row in table_desc_df.iterrows():\n if row['table_description'] is None or row['table_description']=='NA':\n q=f\"table_name == '{row['table_name']}' and table_schema == '{row['table_schema']}'\"\n if source=='bigquery':\n context_prompt = f\"\"\"\n Generate short and crisp description for the table {row['project_id']}.{row['table_schema']}.{row['table_name']}\n Remember that this desciprtion should help LLMs to help build better SQL for any quries related to this table.\n Parameters:\n - column metadata: {column_name_df.query(q).to_markdown(index = False)}\n - table metadata: {table_desc_df.query(q).to_markdown(index = False)}\n \n DO NOT generate description that is more than two lines\n \"\"\"\n else:\n context_prompt = f\"\"\"\n Generate short and crisp description for the table {row['table_schema']}.{row['table_name']}\n Remember that this desciprtions should help LLMs to help build better SQL for any quries related to this table.\n Parameters:\n - column metadata: {column_name_df.query(q).to_markdown(index = False)}\n - table metadata: {table_desc_df.query(q).to_markdown(index = False)}\n DO NOT generate description that is more than two lines\n \"\"\"\n\n table_desc_df.at[index,'table_description']=self.generate_llm_response(context_prompt)\n print(f\"Generated table description for {row['table_schema']}.{row['table_name']}\")\n llm_generated=llm_generated+1\n print(\"LLM generated \"+ str(llm_generated) + \" Table Descriptions\")\n llm_generated = 0\n print(\"\\n\\n\")\n for index, row in column_name_df.iterrows():\n # print(row['column_description'])\n if row['column_description'] is None or row['column_description']=='':\n q=f\"table_name == '{row['table_name']}' and table_schema == '{row['table_schema']}'\"\n if source=='bigquery':\n context_prompt = f\"\"\"\n Generate short and crisp description for the column {row['project_id']}.{row['table_schema']}.{row['table_name']}.{row['column_name']}\n Remember that this description should help LLMs to help generate better SQL for any queries related to these columns.\n\n Consider the below information while generating the description\n Name of the column : {row['column_name']}\n Data type of the column is : {row['data_type']}\n Details of the table of this column are below:\n {table_desc_df.query(q).to_markdown(index=False)}\n Column Contrainst of this column are : {row['column_constraints']}\n\n DO NOT generate description that is more than two lines\n \"\"\"\n\n else:\n context_prompt = f\"\"\"\n Generate short and crisp description for the column {row['table_schema']}.{row['table_name']}.{row['column_name']}\n Remember that this description should help LLMs to help generate better SQL for any queries related to these columns.\n\n Consider the below information while generating the description\n\n Name of the column : {row['column_name']}\n Data type of the column is : {row['data_type']}\n Details of the table of this column are below:\n {table_desc_df.query(q).to_markdown(index=False)}\n Column Contrainst of this column are : {row['column_constraints']}\n\n DO NOT generate description that is more than two lines\n \"\"\"\n column_name_df.at[index,'column_description']=self.generate_llm_response(prompt=context_prompt)\n print(f\"Generated column description for {row['table_schema']}.{row['table_name']}.{row['column_name']}\")\n llm_generated=llm_generated+1\n \n print(\"LLM generated \"+ str(llm_generated) + \" Column Descriptions\")\n return table_desc_df,column_name_df" }, { "path": "agents/EmbedderAgent.py", "content": "from abc import ABC\nfrom .core import Agent \nfrom vertexai.language_models import TextEmbeddingModel\n\n\n\nclass EmbedderAgent(Agent, ABC):\n \"\"\"\n An agent specialized in generating text embeddings using Large Language Models (LLMs).\n\n This agent supports two modes for generating embeddings:\n\n 1. \"vertex\": Directly interacts with the Vertex AI TextEmbeddingModel.\n 2. \"lang-vertex\": Uses LangChain's VertexAIEmbeddings for a streamlined interface.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"EmbedderAgent\".\n mode (str): The embedding generation mode (\"vertex\" or \"lang-vertex\").\n model: The underlying embedding model (Vertex AI TextEmbeddingModel or LangChain's VertexAIEmbeddings).\n\n Methods:\n create(question) -> list:\n Generates text embeddings for the given question(s).\n\n Args:\n question (str or list): The text input for which embeddings are to be generated. Can be a single string or a list of strings.\n\n Returns:\n list: A list of embedding vectors. Each embedding vector is represented as a list of floating-point numbers.\n\n Raises:\n ValueError: If the input `question` is not a string or list, or if the specified `mode` is invalid.\n \"\"\"\n\n\n agentType: str = \"EmbedderAgent\"\n\n def __init__(self, mode, embeddings_model='text-embedding-004'): \n if mode == 'vertex': \n self.mode = mode \n self.model = TextEmbeddingModel.from_pretrained(embeddings_model)\n\n elif mode == 'lang-vertex': \n self.mode = mode \n from langchain.embeddings import VertexAIEmbeddings\n self.model = VertexAIEmbeddings() \n\n else: raise ValueError('EmbedderAgent mode must be either vertex or lang-vertex')\n\n\n\n def create(self, question): \n \"\"\"Text embedding with a Large Language Model.\"\"\"\n\n if self.mode == 'vertex': \n if isinstance(question, str): \n embeddings = self.model.get_embeddings([question])\n for embedding in embeddings:\n vector = embedding.values\n return vector\n \n elif isinstance(question, list): \n vector = list() \n for q in question: \n embeddings = self.model.get_embeddings([q])\n\n for embedding in embeddings:\n vector.append(embedding.values) \n return vector\n \n else: raise ValueError('Input must be either str or list')\n\n elif self.mode == 'lang-vertex': \n vector = self.embeddings_service.embed_documents(question)\n return vector " }, { "path": "agents/ResponseAgent.py", "content": "import json \nfrom abc import ABC\nfrom .core import Agent\nfrom utilities import PROMPTS, format_prompt \nfrom vertexai.generative_models import HarmCategory, HarmBlockThreshold\nfrom google.cloud.aiplatform import telemetry\nimport vertexai \nfrom utilities import PROJECT_ID, PG_REGION\nvertexai.init(project=PROJECT_ID, location=PG_REGION)\n\n\nclass ResponseAgent(Agent, ABC):\n \"\"\"\n An agent that generates natural language responses to user questions based on SQL query results.\n\n This agent acts as a data assistant, interpreting SQL query results and transforming them into user-friendly, natural language answers. It utilizes a language model (currently Gemini) to craft responses that effectively convey the information derived from the data.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"ResponseAgent\".\n\n Methods:\n run(user_question, sql_result) -> str:\n Generates a natural language response to the user's question based on the SQL query result.\n\n Args:\n user_question (str): The question asked by the user in natural language.\n sql_result (str): The result of the SQL query executed to answer the question.\n\n Returns:\n str: The generated natural language response.\n \"\"\"\n\n\n agentType: str = \"ResponseAgent\"\n\n def run(self, user_question, sql_result):\n\n context_prompt = PROMPTS['nl_reponse']\n\n\n\n context_prompt = format_prompt(context_prompt,\n user_question = user_question,\n sql_result = sql_result)\n \n # print(f\"Prompt for Natural Language Response: \\n{context_prompt}\")\n\n\n if 'gemini' in self.model_id:\n with telemetry.tool_context_manager('opendataqna-response-v2'):\n context_query = self.model.generate_content(context_prompt,safety_settings=self.safety_settings, stream=False)\n generated_sql = str(context_query.candidates[0].text)\n\n else:\n with telemetry.tool_context_manager('opendataqna-response-v2'):\n context_query = self.model.predict(context_prompt, max_output_tokens = 8000, temperature=0)\n generated_sql = str(context_query.candidates[0])\n \n return generated_sql\n\n " }, { "path": "agents/ValidateSQLAgent.py", "content": "import json \nfrom abc import ABC\nfrom .core import Agent\nfrom utilities import PROMPTS, format_prompt \n\n\n\nclass ValidateSQLAgent(Agent, ABC):\n \"\"\"\n An agent that validates the syntax and semantic correctness of SQL queries.\n\n This agent leverages a language model (currently Gemini) to analyze a given SQL query against a provided database schema. It assesses whether the query is valid according to a set of predefined guidelines and generates a JSON response indicating the validity status and any potential errors.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"ValidateSQLAgent\".\n\n Methods:\n check(user_question, tables_schema, columns_schema, generated_sql) -> dict:\n Determines the validity of an SQL query and identifies potential errors.\n\n Args:\n user_question (str): The original question posed by the user (used for context).\n tables_schema (str): A description of the database tables and their relationships.\n columns_schema (str): Detailed descriptions of the columns within the tables.\n generated_sql (str): The SQL query to be validated.\n\n Returns:\n dict: A JSON-formatted dictionary with the following keys:\n - \"valid\": A boolean value indicating whether the query is valid or not.\n - \"errors\": A string describing any errors found in the query (empty if valid).\n \"\"\"\n\n\n agentType: str = \"ValidateSQLAgent\"\n\n def check(self,source_type, user_question, tables_schema, columns_schema, generated_sql):\n \n context_prompt = PROMPTS['validatesql']\n context_prompt = format_prompt(context_prompt,\n source_type = source_type,\n user_question = user_question,\n tables_schema = tables_schema, \n columns_schema = columns_schema,\n generated_sql=generated_sql)\n\n # print(f\"Prompt to Validate SQL after formatting: \\n{context_prompt}\")\n \n if \"gemini\" in self.model_id:\n context_query = self.model.generate_content(context_prompt, stream=False)\n generated_sql = str(context_query.candidates[0].text)\n\n else:\n context_query = self.model.predict(context_prompt, max_output_tokens = 8000, temperature=0)\n generated_sql = str(context_query.candidates[0])\n\n\n json_syntax_result = json.loads(str(generated_sql).replace(\"```json\",\"\").replace(\"```\",\"\"))\n\n # print('\\n SQL Syntax Validity:' + str(json_syntax_result['valid']))\n # print('\\n SQL Syntax Error Description:' +str(json_syntax_result['errors']) + '\\n')\n \n return json_syntax_result" }, { "path": "agents/VisualizeAgent.py", "content": "#This agent generates google charts code for displaying charts on web application\n\n#Generates two charts with elements \"chart-div\" and \"chart-div-1\"\n\n#Code is in javascript\n\nfrom abc import ABC\nfrom vertexai.language_models import CodeChatModel\nfrom vertexai.generative_models import GenerativeModel,HarmCategory,HarmBlockThreshold\nfrom .core import Agent\nfrom utilities import PROMPTS, format_prompt \nfrom agents import ValidateSQLAgent \nimport pandas as pd\nimport json \nfrom google.cloud.aiplatform import telemetry\nimport vertexai \nfrom utilities import PROJECT_ID, PG_REGION\nvertexai.init(project=PROJECT_ID, location=PG_REGION)\n\nclass VisualizeAgent(Agent, ABC):\n \"\"\"\n An agent that generates JavaScript code for Google Charts based on user questions and SQL results.\n\n This agent analyzes the user's question and the corresponding SQL query results to determine suitable chart types. It then constructs JavaScript code that uses Google Charts to create visualizations based on the data.\n\n Attributes:\n agentType (str): Indicates the type of agent, fixed as \"VisualizeAgent\".\n model_id (str): The ID of the language model used for chart type suggestion and code generation.\n model: The language model instance.\n\n Methods:\n getChartType(user_question, generated_sql) -> str:\n Suggests the two most suitable chart types based on the user's question and the generated SQL query.\n\n Args:\n user_question (str): The natural language question asked by the user.\n generated_sql (str): The SQL query generated to answer the question.\n\n Returns:\n str: A JSON string containing two keys, \"chart_1\" and \"chart_2\", each representing a suggested chart type.\n\n getChartPrompt(user_question, generated_sql, chart_type, chart_div, sql_results) -> str:\n Creates a prompt for the language model to generate the JavaScript code for a specific chart.\n\n Args:\n user_question (str): The user's question.\n generated_sql (str): The generated SQL query.\n chart_type (str): The desired chart type (e.g., \"Bar Chart\", \"Pie Chart\").\n chart_div (str): The HTML element ID where the chart will be rendered.\n sql_results (str): The results of the SQL query in JSON format.\n\n Returns:\n str: The prompt for the language model to generate the chart code.\n\n generate_charts(user_question, generated_sql, sql_results) -> dict:\n Generates JavaScript code for two Google Charts based on the given inputs.\n\n Args:\n user_question (str): The user's question.\n generated_sql (str): The generated SQL query.\n sql_results (str): The results of the SQL query in JSON format.\n\n Returns:\n dict: A dictionary containing two keys, \"chart_div\" and \"chart_div_1\", each holding the generated JavaScript code for a chart.\n \"\"\"\n\n\n agentType: str =\"VisualizeAgent\"\n\n def __init__(self):\n self.model_id = 'gemini-1.5-pro'\n self.model = GenerativeModel(\"gemini-1.5-pro-001\")\n\n def getChartType(self,user_question, generated_sql):\n\n chart_type_prompt = PROMPTS['visualize_chart_type']\n\n chart_type_prompt = format_prompt(chart_type_prompt,\n user_question = user_question,\n generated_sql = generated_sql)\n\n chart_type=self.model.generate_content(chart_type_prompt, stream=False).candidates[0].text\n # print(chart_type)\n # chart_type = model.predict(map_prompt, max_output_tokens = 1024, temperature= 0.2).candidates[0].text\n return chart_type.replace(\"\\n\", \"\").replace(\"```\", \"\").replace(\"json\", \"\").replace(\"```html\", \"\").replace(\"```\", \"\").replace(\"js\\n\",\"\").replace(\"json\\n\",\"\").replace(\"python\\n\",\"\").replace(\"javascript\",\"\")\n\n def getChartPrompt(self,user_question, generated_sql, chart_type, chart_div, sql_results):\n chart_prompt = PROMPTS['visualize_generate_chart_code']\n\n chart_prompt = format_prompt(chart_prompt,\n user_question = user_question,\n generated_sql = generated_sql,\n chart_type = chart_type,\n chart_div = chart_div,\n sql_results = sql_results)\n # print(f\"Prompt to generate code for google charts visualization after formatting: \\n{chart_prompt}\")\n return chart_prompt\n\n def generate_charts(self,user_question,generated_sql,sql_results):\n chart_type = self.getChartType(user_question,generated_sql)\n # chart_type = chart_type.split(\",\")\n # chart_list = [x.strip() for x in chart_type]\n chart_json = json.loads(chart_type)\n chart_list =[chart_json['chart_1'],chart_json['chart_2']]\n print(\"Charts Suggested : \" + str(chart_list))\n context_prompt=self.getChartPrompt(user_question,generated_sql,chart_list[0],\"chart_div\",sql_results)\n context_prompt_1=self.getChartPrompt(user_question,generated_sql,chart_list[1],\"chart_div_1\",sql_results)\n safety_settings: Optional[dict] = {\n HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,\n }\n with telemetry.tool_context_manager('opendataqna-visualize-v2'):\n context_query = self.model.generate_content(context_prompt,safety_settings=safety_settings, stream=False)\n context_query_1 = self.model.generate_content(context_prompt_1,safety_settings=safety_settings, stream=False)\n \n google_chart_js={\"chart_div\":context_query.candidates[0].text.replace(\"```json\", \"\").replace(\"```\", \"\").replace(\"json\", \"\").replace(\"```html\", \"\").replace(\"```\", \"\").replace(\"js\",\"\").replace(\"json\",\"\").replace(\"python\",\"\").replace(\"javascript\",\"\"),\n \"chart_div_1\":context_query_1.candidates[0].text.replace(\"```json\", \"\").replace(\"```\", \"\").replace(\"json\", \"\").replace(\"```html\", \"\").replace(\"```\", \"\").replace(\"js\",\"\").replace(\"json\",\"\").replace(\"python\",\"\").replace(\"javascript\",\"\")}\n\n return google_chart_js" }, { "path": "agents/__init__.py", "content": "from .BuildSQLAgent import BuildSQLAgent\nfrom .ValidateSQLAgent import ValidateSQLAgent\nfrom .DebugSQLAgent import DebugSQLAgent\nfrom .EmbedderAgent import EmbedderAgent\nfrom .ResponseAgent import ResponseAgent\nfrom .VisualizeAgent import VisualizeAgent\nfrom .DescriptionAgent import DescriptionAgent\n\n\n\n__all__ = [\"BuildSQLAgent\", \"ValidateSQLAgent\", \"DebugSQLAgent\", \"EmbedderAgent\", \"ResponseAgent\",\"VisualizeAgent\", \"DescriptionAgent\"]\n\n\n" }, { "path": "agents/core.py", "content": "\"\"\"\nProvides the base class for all Agents \n\"\"\"\n\nfrom abc import ABC\nimport vertexai\nfrom google.cloud.aiplatform import telemetry\nfrom vertexai.language_models import TextGenerationModel\nfrom vertexai.language_models import CodeGenerationModel\nfrom vertexai.language_models import CodeChatModel\nfrom vertexai.generative_models import GenerativeModel\nfrom vertexai.generative_models import HarmCategory,HarmBlockThreshold\n\n\n\nfrom utilities import PROJECT_ID, PG_REGION\nvertexai.init(project=PROJECT_ID, location=PG_REGION)\n\n\n\nclass Agent(ABC):\n \"\"\"\n The core class for all Agents\n \"\"\"\n\n agentType: str = \"Agent\"\n\n def __init__(self,\n model_id:str):\n \"\"\"\n model_id is the Model ID for initialization\n \"\"\"\n\n self.model_id = model_id \n\n if model_id == 'code-bison-32k':\n with telemetry.tool_context_manager('opendataqna'):\n self.model = CodeGenerationModel.from_pretrained('code-bison-32k')\n\n elif model_id == 'text-bison-32k':\n with telemetry.tool_context_manager('opendataqna'):\n self.model = TextGenerationModel.from_pretrained('text-bison-32k')\n \n elif model_id == 'codechat-bison-32k':\n with telemetry.tool_context_manager('opendataqna'):\n self.model = CodeChatModel.from_pretrained(\"codechat-bison-32k\")\n \n elif model_id == 'gemini-1.0-pro':\n with telemetry.tool_context_manager('opendataqna'):\n # print(\"Model is gemini 1.0 pro\")\n self.model = GenerativeModel(\"gemini-1.0-pro-001\")\n self.safety_settings: Optional[dict] = {\n HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,\n }\n \n elif model_id == 'gemini-1.5-flash':\n with telemetry.tool_context_manager('opendataqna'):\n # print(\"Model is gemini 1.5 flash\")\n self.model = GenerativeModel(\"gemini-1.5-flash-preview-0514\")\n self.safety_settings: Optional[dict] = {\n HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,\n }\n\n elif model_id == 'gemini-1.5-pro':\n with telemetry.tool_context_manager('opendataqna'):\n # print(\"Model is gemini 1.5 Pro\")\n self.model = GenerativeModel(\"gemini-1.5-pro-001\")\n self.safety_settings: Optional[dict] = {\n HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,\n HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,\n }\n \n else:\n raise ValueError(\"Please specify a compatible model.\")\n\n def generate_llm_response(self,prompt):\n context_query = self.model.generate_content(prompt,safety_settings=self.safety_settings,stream=False)\n return str(context_query.candidates[0].text).replace(\"```sql\", \"\").replace(\"```\", \"\").rstrip(\"\\n\")\n\n\n def rewrite_question(self,question,session_history):\n formatted_history=''\n concat_questions=''\n for i, _row in enumerate(session_history,start=1):\n user_question = _row['user_question']\n # print(user_question)\n formatted_history += f\"User Question - Turn :: {i} : {user_question}\\n\"\n concat_questions += f\"{user_question} \"\n\n # print(formatted_history)\n\n\n context_prompt = f\"\"\"\n Your main objective is to rewrite and refine the question based on the previous questions that has been asked.\n\n Refine the given question using the provided questions history to produce a standalone question with full context. The refined question should be self-contained, requiring no additional context for answering it.\n\n Make sure all the information is included in the re-written question. You just need to respond with the re-written question.\n\n Below is the previous questions history:\n\n {formatted_history}\n\n Question to rewrite:\n\n {question}\n \"\"\"\n re_written_qe = str(self.generate_llm_response(context_prompt))\n \n\n print(\"*\"*25 +\"Re-written question:: \"+\"*\"*25+\"\\n\"+str(re_written_qe))\n\n return str(concat_questions),str(re_written_qe)\n\n" }, { "path": "app.py", "content": "import streamlit as st\nimport pandas as pd\nimport json\nfrom streamlit.components.v1 import html\nfrom streamlit.logger import get_logger\nfrom opendataqna import generate_uuid, get_all_databases, run_pipeline, get_kgq\nimport asyncio\n\nlogger = get_logger(__name__)\n\n# Initialize session state variables if they don't exist\nif \"session_id\" not in st.session_state:\n st.session_state.session_id = generate_uuid()\n st.session_state.kgq = []\n st.session_state.user_grouping = None\n logger.info(f\"New Session Created - {st.session_state.session_id}\")\n\n\ndef get_known_databases():\n \"\"\"Retrieves a list of available database schemas from the backend.\n\n This function fetches a list of database schemas from the backend API.\n These schemas represent the available datasets that users can query.\n\n Returns:\n list: A list of database schema names.\n \"\"\"\n logger.info(\"Getting list of all user databases\")\n json_groupings, _ = get_all_databases()\n json_groupings = json.loads(json_groupings)\n groupings = [item[\"table_schema\"] for item in json_groupings if isinstance(item, dict)]\n logger.info(f\"user_groupings - {str(groupings)}\")\n return groupings\n\ndef get_known_sql(selected_schema):\n \"\"\"Retrieves known good SQL queries (KGQs) for a specific database schema.\n\n This function fetches a DataFrame containing KGQs for the given schema.\n KGQs are pre-defined SQL queries that can be used as examples or suggestions.\n\n Args:\n selected_schema (str): The name of the database schema.\n\n Returns:\n pd.DataFrame: A DataFrame containing KGQs for the specified schema.\n \"\"\"\n data = get_kgq(selected_schema)\n parsed_data = list(eval(data[0]))\n df = pd.DataFrame(parsed_data)\n return df\n\ndef generate_sql_results(selected_schema,user_question):\n \"\"\"Generates SQL query, executes it, and returns results and response.\n\n This function orchestrates the process of generating an SQL query based on\n the user's question and selected schema, executing the query, and generating\n a natural language response based on the results.\n\n Args:\n selected_schema (str): The name of the selected database schema.\n user_question (str): The user's natural language question.\n\n Returns:\n tuple: A tuple containing the generated SQL query (str), the query results\n as a Pandas DataFrame, and the generated natural language response (str).\n \"\"\"\n logger.info(f\"generating response for user question - {user_question}\")\n logger.info(f\"selected user groouping - {selected_schema}\")\n final_sql, results_df, response = asyncio.run(\n run_pipeline(\n st.session_state.session_id,\n user_question,\n selected_schema,\n RUN_DEBUGGER=True,\n EXECUTE_FINAL_SQL=True,\n DEBUGGING_ROUNDS=2,\n LLM_VALIDATION=False,\n Embedder_model='vertex', # Options: 'vertex' or 'vertex-lang'\n SQLBuilder_model='gemini-1.5-pro',\n SQLChecker_model='gemini-1.5-pro',\n SQLDebugger_model='gemini-1.5-pro',\n Responder_model='gemini-1.5-pro',\n num_table_matches=5,\n num_column_matches=10,\n table_similarity_threshold=0.1,\n column_similarity_threshold=0.1,\n example_similarity_threshold=0.1,\n num_sql_matches=3\n )\n )\n return(final_sql, results_df, response)\n\ndef generate_response(prompt):\n \"\"\"Generates and displays a response to the user's prompt.\n\n This function takes a user prompt as input, generates an SQL query and\n response using the `generate_sql_results` function, and displays the\n results in a conversational format using Streamlit's chat message feature.\n\n Args:\n prompt (str): The user's input prompt.\n \"\"\"\n for msg in st.session_state.messages:\n st.chat_message(msg[\"role\"]).write(msg[\"content\"])\n st.chat_message(\"user\").write(prompt)\n st.session_state.messages.append({\"role\": \"assistant\", \"content\": msg})\n msg = \"Generating Response\"\n st.session_state.messages.append({\"role\": \"assistant\", \"content\": msg})\n st.chat_message(\"assistant\").write(msg)\n query, results, response = generate_sql_results(st.session_state.user_grouping, prompt)\n msg = query\n st.session_state.messages.append({\"role\": \"assistant\", \"content\": msg})\n st.chat_message(\"assistant\").write(msg)\n msg = response\n st.session_state.messages.append({\"role\": \"assistant\", \"content\": msg})\n st.chat_message(\"assistant\").write(msg)\n with st.chat_message(\"assistant\"): \n st.dataframe(results)\n st.session_state.messages.append({\"role\": \"assistant\", \"content\": results})\n\n \nst.set_page_config(page_title='Open Data QnA', page_icon=\"📊\", initial_sidebar_state=\"expanded\", layout='wide')\nst.markdown(\"\"\"\n \n \"\"\", unsafe_allow_html=True)\n\nst.title(\"Open Data QnA\")\n\nwith st.sidebar:\n st.session_state.user_grouping = st.selectbox(\n 'Select Table Groupings',\n get_known_databases())\n if st.button(\"New Query\"):\n st.session_state.session_id = generate_uuid()\n st.session_state.messages.clear()\n st.rerun() \n \nif \"messages\" not in st.session_state:\n st.session_state[\"messages\"] = [{\"role\": \"assistant\", \"content\": \"Frequently Asked Questions\"}]\nif st.session_state.user_grouping is not None:\n df = get_known_sql(st.session_state.user_grouping)\n for index, row in df.iterrows():\n url = text = row[\"example_user_question\"]\n st.session_state.kgq.append(text)\n\nif prompt := st.chat_input():\n generate_response(prompt)\n" }, { "path": "backend-apis/README.md", "content": "\n\n\n\n

Create Endpoints

\n\n Here we are going to create publicly accessible endpoints (no authentication) .\n\n If you're working on a managed GCP project, it is common that there would be Domain Restricted Sharing Org Policies that will not allow the creation of a public facing endpoint.\n\n So we can allow all the domains and re-enable the same policy so that we don’t change the existing policy.\n\n Please run the below command before proceeding ahead. You need to have Organization Policy Admin rights to run the below commands.\n```\nexport PROJECT_ID=\n```\n\n```\ncd Open_Data_QnA/backend-apis\n\ngcloud resource-manager org-policies set-policy --project=$PROJECT_ID policy.yaml #This command will create policy that overrides to allow all domain\n\n```\n\nCreate the service account and add roles to run the solution backend for the APIs\n\n```\ngcloud iam service-accounts create opendataqna --project=$PROJECT_ID\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:opendataqna@$PROJECT_ID.iam.gserviceaccount.com --role='roles/cloudsql.client' --project=$PROJECT_ID --quiet\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:opendataqna@$PROJECT_ID.iam.gserviceaccount.com --role='roles/bigquery.admin' --project=$PROJECT_ID --quiet\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:opendataqna@$PROJECT_ID.iam.gserviceaccount.com --role='roles/aiplatform.user' --project=$PROJECT_ID --quiet\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:opendataqna@$PROJECT_ID.iam.gserviceaccount.com --role='roles/datastore.owner' --project=$PROJECT_ID --quiet\n\n```\n\n\n\n**Technologies**\n\n* **Programming language:** Python\n* **Framework:** Flask\n\n**Before you start :** Ensure all variables in your config.ini file are correct, especially those for your Postgres instance and BigQuery dataset. If you need to change the Postgres instance or BigQuery dataset values, update the config.ini file before proceeding. \n\n\n The endpoints deployed here are completely customized for the UI built in this demo solution. Feel free to customize the endpoint if needed for different UI/frontend. The gcloud run deploy command create a cloud build that uses the Dockerfile in the OpenDataQnA folder\n \n ***Deploy endpoints to Cloud Run***\n\n```\nexport PROJECT_ID=\n ```\n ```\nexport SERVICE_NAME=opendataqna #change the name if needed \nexport DEPLOY_REGION=us-central1 #change the cloud run deployment region if needed \n```\n\nEnable the cloud build API to deploy the endpoints\n```\ngcloud services enable cloudbuild.googleapis.com --project $PROJECT_ID\n```\n\nGet default service account for compute engine and cloud build to deploy the cloud run and add IAM Roles for deployment\n```\nexport PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format=\"value(projectNumber)\")\n\nexport DEFAULT_CE_SA=$(gcloud iam service-accounts list --project=$PROJECT_ID --format=\"value(EMAIL)\" --filter=\"EMAIL ~ $PROJECT_NUMBER-compute@developer.gserviceaccount.com\")\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CE_SA --role='roles/storage.admin' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CE_SA --role='roles/artifactregistry.admin' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CE_SA --role='roles/firebase.admin' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CE_SA --role='roles/cloudbuild.builds.builder' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CE_SA --role='roles/logging.logWriter' --project=$PROJECT_ID --quiet\n\n\nexport DEFAULT_CB_SA=$PROJECT_NUMBER'@cloudbuild.gserviceaccount.com'\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CB_SA --role='roles/firebase.admin' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CB_SA --role='roles/serviceusage.apiKeysAdmin' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CB_SA --role='roles/cloudbuild.builds.builder' --project=$PROJECT_ID --quiet\n\ngcloud projects add-iam-policy-binding $PROJECT_ID --member=serviceAccount:$DEFAULT_CB_SA --role='roles/artifactregistry.admin' --project=$PROJECT_ID --quiet\n\n```\n\n```\n cd Open_Data_QnA\n\n gcloud beta run deploy $SERVICE_NAME --region $DEPLOY_REGION --source . --service-account=opendataqna@$PROJECT_ID.iam.gserviceaccount.com --service-min-instances=1 --allow-unauthenticated --project=$PROJECT_ID \n \n #if you are deploying cloud run application for the first time in the project you will be prompted for a couple of settings. Go ahead and type Yes.\n\n\n```\n\n Once the deployment is done successfully you should be able to see the Service URL (endpoint point) link as shown below. Please keep this handy to add this in the frontend or you can get this uri from the cloud run page in the GCP Console. e.g. *https://OpenDataQnA-aeiouAEI-uc.a.run.app*\n\n Test if the endpoints are working with below command. This should return the dataset your created in the source env setup notebook.\n```\n curl /available_databases\n\n```\n\n\n\n

\n \n \"aaie\n \n

\n\n\n Delete the Org Policy on the Project that's created above. Do not run this if you haven’t created the org policy above\n\n```\ngcloud resource-manager org-policies delete iam.allowedPolicyMemberDomains --project=$PROJECT_ID\n```\n\n\n\n**API Details**\n\n All the payloads are in JSON format\n\n1. List Databases : Get the available databases in the vector store that solution can run against\n\n URI: {Service URL}/available_databases \n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/available_databases\n\n Method: GET\n\n Request Payload : NONE\n\n Request response:\n ```\n {\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"table_schema\\\":\\\"imdb-postgres\\\"},{\\\"table_schema\\\":\\\"retail-postgres\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n\n2. Known SQL : Get suggestive questions (previously asked/examples added) for selected database\n\n URI: /get_known_sql\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/get_known_sql \n\n Method: POST\n\n Request Payload :\n\n ```\n {\n \"user_grouping\":\"retail\"\n }\n ```\n\n Request response:\n\n ```\n {\n \"Error\": \"\",\n \"KnownSQL\": \"[{\\\"example_user_question\\\":\\\"Which city had maximum number of sales and what was the count?\\\",\\\"example_generated_sql\\\":\\\"select st.city_id, count(st.city_id) as city_sales_count from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by city_sales_count desc limit 1;\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n\n\n3. SQL Generation : Generate the SQL for the input question asked against a database\n\n URI: /generate_sql\n\n\n Method: POST\n\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/get_known_sql\n\n\n Request payload:\n\n ```\n {\n \"session_id\":\"\",\n \"user_id\":\"harry@hogwarts.com\",\n \"user_question\":\"Which city had maximum number of sales?\",\n \"user_grouping\":\"retail\"\n }\n ```\n\n\n Request response:\n ```\n {\n \"Error\": \"\",\n \"GeneratedSQL\": \" select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"ResponseCode\": 200,\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n\n\n4. Execute SQL : Run the SQL statement against provided database source\n\n URI:/run_query\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/run_query\n\n Method: POST\n\n Request payload:\n ```\n { \"user_grouping\": \"retail\",\n \"generated_sql\":\"select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n\n Request response:\n ```\n {\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"city_id\\\":\\\"C014\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n5. Embedd SQL : To embed known good SQLs to your example embeddings\n\n URI:/embed_sql\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/embed_sql\n\n METHOD: POST\n\n Request Payload:\n\n ```\n {\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"user_question\":\"Which city had maximum number of sales?\",\n \"generated_sql\":\"select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"user_grouping\":\"retail\"\n }\n ```\n\n Request response:\n ```\n {\n \"ResponseCode\" : 201, \n \"Message\" : \"Example SQL has been accepted for embedding\",\n \"Error\":\"\",\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n6. Generate Visualization Code : To generated javascript Google Charts code based on the SQL Results and display them on the UI\n\n As per design we have two visualizations suggested showing up when the user clicks the visualize button. Hence two divs are send as part of the response “chart_div”, “chart_div_1” to bind them to that element in the UI\n \n\n If you are only looking to setup enpoint you can stop here. In case you require the demo app (frontend UI) built in the solution, proceed to the next step.\n\n URI:/generate_viz\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/generate_viz\n \n METHOD: POST\n\n Request Payload:\n ```\n {\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\" ,\n \"user_question\": \"What are top 5 product skus that are ordered?\",\n \"sql_generated\": \"SELECT productSKU as ProductSKUCode, sum(total_ordered) as TotalOrderedItems FROM `inbq1-joonix.demo.sales_sku` group by productSKU order by sum(total_ordered) desc limit 5\",\n \"sql_results\": [\n {\n \"ProductSKUCode\": \"GGOEGOAQ012899\",\n \"TotalOrderedItems\": 456\n },\n {\n \"ProductSKUCode\": \"GGOEGDHC074099\",\n \"TotalOrderedItems\": 334\n },\n {\n \"ProductSKUCode\": \"GGOEGOCB017499\",\n \"TotalOrderedItems\": 319\n },\n {\n \"ProductSKUCode\": \"GGOEGOCC077999\",\n \"TotalOrderedItems\": 290\n },\n {\n \"ProductSKUCode\": \"GGOEGFYQ016599\",\n \"TotalOrderedItems\": 253\n }\n ]\n }\n \n ```\n\n Request response:\n ```\n {\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"Error\": \"\",\n \"GeneratedChartjs\": {\n \"chart_div\": \"google.charts.load('current', {\\n packages: ['corechart']\\n});\\ngoogle.charts.setOnLoadCallback(drawChart);\\n\\nfunction drawChart() {\\n var data = google.visualization.arrayToDataTable([\\n ['Product SKU', 'Total Ordered Items'],\\n ['GGOEGOAQ012899', 456],\\n ['GGOEGDHC074099', 334],\\n ['GGOEGOCB017499', 319],\\n ['GGOEGOCC077999', 290],\\n ['GGOEGFYQ016599', 253],\\n ]);\\n\\n var options = {\\n title: 'Top 5 Product SKUs Ordered',\\n width: 600,\\n height: 300,\\n hAxis: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n vAxis: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n legend: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n bar: {\\n groupWidth: '50%'\\n }\\n };\\n\\n var chart = new google.visualization.BarChart(document.getElementById('chart_div'));\\n\\n chart.draw(data, options);\\n}\\n\",\n \n \"chart_div_1\": \"google.charts.load('current', {'packages':['corechart']});\\ngoogle.charts.setOnLoadCallback(drawChart);\\nfunction drawChart() {\\n var data = google.visualization.arrayToDataTable([\\n ['ProductSKUCode', 'TotalOrderedItems'],\\n ['GGOEGOAQ012899', 456],\\n ['GGOEGDHC074099', 334],\\n ['GGOEGOCB017499', 319],\\n ['GGOEGOCC077999', 290],\\n ['GGOEGFYQ016599', 253]\\n ]);\\n\\n var options = {\\n title: 'Top 5 Product SKUs that are Ordered',\\n width: 600,\\n height: 300,\\n hAxis: {\\n textStyle: {\\n fontSize: 5\\n }\\n },\\n vAxis: {\\n textStyle: {\\n fontSize: 5\\n }\\n },\\n legend: {\\n textStyle: {\\n fontSize: 10\\n }\\n },\\n bar: {\\n groupWidth: \\\"60%\\\"\\n }\\n };\\n\\n var chart = new google.visualization.ColumnChart(document.getElementById('chart_div_1'));\\n\\n chart.draw(data, options);\\n}\\n\"\n },\n \"ResponseCode\": 200\n }\n\n ```\n7. Get Results : To directly get the sql results in JSON format\n\n URI:/get_results\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/embed_sql\n\n METHOD: POST\n\n Request Payload:\n\n ```\n {\n \"user_question\":\"Which city had maximum number of sales?\",\n \"user_database\":\"retail\"\n }\n ```\n\n Request response:\n ```\n {\n \"Error\": \"\",\n \"GeneratedResults\": \"[{\\\"city_id\\\":\\\"C014\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n\n\n### For setting up the demo UI with these endpoints please refer to README.md under [`/frontend`](/frontend/)\n" }, { "path": "backend-apis/__init__.py", "content": "" }, { "path": "backend-apis/main.py", "content": "# -*- coding: utf-8 -*-\n\n\n# Copyright 2024 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# https://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\nfrom flask import Flask, request, jsonify, render_template, Response\nimport asyncio\nfrom collections.abc import Callable\nimport logging as log\nimport json\nimport datetime\nimport urllib\nimport re\nimport time\nimport textwrap\nimport pandas as pd\nfrom flask_cors import CORS\nimport os\nimport sys\nimport firebase_admin\nfrom firebase_admin import credentials, auth\nfrom functools import wraps\n\nfirebase_admin.initialize_app()\n\nfrom opendataqna import get_all_databases,get_kgq,generate_sql,embed_sql,get_response,get_results,visualize\n\n\nmodule_path = os.path.abspath(os.path.join('.'))\nsys.path.append(module_path)\n\n\ndef jwt_authenticated(func: Callable[..., int]) -> Callable[..., int]:\n @wraps(func)\n async def decorated_function(*args, **kwargs):\n header = request.headers.get(\"Authorization\", None)\n if header:\n token = header.split(\" \")[1]\n try:\n \n print(\"TOKEN::\"+str(token))\n decoded_token = firebase_admin.auth.verify_id_token(token)\n except Exception as e:\n log.exception(e)\n return Response(status=403, response=f\"Error with authentication: {e}\")\n else:\n return Response(status=401)\n \n request.uid = decoded_token[\"uid\"]\n print(\"USER:: \"+str(request.uid))\n return await func(*args, **kwargs) if asyncio.iscoroutinefunction(func) else func(*args, **kwargs)\n \n return decorated_function\n\nRUN_DEBUGGER = True\nDEBUGGING_ROUNDS = 2 \nLLM_VALIDATION = False\nEXECUTE_FINAL_SQL = True\nEmbedder_model = 'vertex'\nSQLBuilder_model = 'gemini-1.5-pro'\nSQLChecker_model = 'gemini-1.5-pro'\nSQLDebugger_model = 'gemini-1.5-pro'\nnum_table_matches = 5\nnum_column_matches = 10\ntable_similarity_threshold = 0.3\ncolumn_similarity_threshold = 0.3\nexample_similarity_threshold = 0.3\nnum_sql_matches = 3\n\napp = Flask(__name__) \ncors = CORS(app, resources={r\"/*\": {\"origins\": \"*\"}})\n\n\n\n@app.route(\"/available_databases\", methods=[\"GET\"])\n# @jwt_authenticated\ndef getBDList():\n\n result,invalid_response=get_all_databases()\n \n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"KnownDB\" : result,\n \"Error\":\"\"\n }\n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownDB\" : \"\",\n \"Error\":result\n } \n return jsonify(responseDict)\n\n\n\n\n@app.route(\"/embed_sql\", methods=[\"POST\"])\n# @jwt_authenticated\nasync def embedSql():\n\n envelope = str(request.data.decode('utf-8'))\n envelope=json.loads(envelope)\n user_grouping=envelope.get('user_grouping')\n generated_sql = envelope.get('generated_sql')\n user_question = envelope.get('user_question')\n session_id = envelope.get('session_id')\n\n embedded, invalid_response=await embed_sql(session_id,user_grouping,user_question,generated_sql)\n\n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 201, \n \"Message\" : \"Example SQL has been accepted for embedding\",\n \"SessionID\" : session_id,\n \"Error\":\"\"\n } \n return jsonify(responseDict)\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownDB\" : \"\",\n \"SessionID\" : session_id,\n \"Error\":embedded\n } \n return jsonify(responseDict)\n\n\n\n\n@app.route(\"/run_query\", methods=[\"POST\"])\n# @jwt_authenticated\ndef getSQLResult():\n \n envelope = str(request.data.decode('utf-8'))\n envelope=json.loads(envelope)\n\n user_question = envelope.get('user_question')\n user_grouping = envelope.get('user_grouping')\n generated_sql = envelope.get('generated_sql')\n session_id = envelope.get('session_id')\n\n result_df,invalid_response=get_results(user_grouping,generated_sql)\n\n\n if not invalid_response:\n _resp,invalid_response=get_response(session_id,user_question,result_df.to_json(orient='records'))\n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"KnownDB\" : result_df.to_json(orient='records'),\n \"NaturalResponse\" : _resp,\n \"SessionID\" : session_id,\n \"Error\":\"\"\n }\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownDB\" : result_df.to_json(orient='records'),\n \"NaturalResponse\" : _resp,\n \"SessionID\" : session_id,\n \"Error\":\"\"\n }\n\n else:\n _resp=result_df\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownDB\" : \"\",\n \"NaturalResponse\" : _resp,\n \"SessionID\" : session_id,\n \"Error\":result_df\n } \n return jsonify(responseDict)\n\n\n\n\n@app.route(\"/get_known_sql\", methods=[\"POST\"])\n# @jwt_authenticated\ndef getKnownSQL():\n print(\"Extracting the known SQLs from the example embeddings.\")\n envelope = str(request.data.decode('utf-8'))\n envelope=json.loads(envelope)\n \n user_grouping = envelope.get('user_grouping')\n\n\n result,invalid_response=get_kgq(user_grouping)\n \n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"KnownSQL\" : result,\n \"Error\":\"\"\n }\n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownSQL\" : \"\",\n \"Error\":result\n } \n return jsonify(responseDict)\n\n\n\n@app.route(\"/generate_sql\", methods=[\"POST\"])\n# @jwt_authenticated\nasync def generateSQL():\n print(\"Here is the request payload \")\n envelope = str(request.data.decode('utf-8'))\n print(\"Here is the request payload \" + envelope)\n envelope=json.loads(envelope)\n\n user_question = envelope.get('user_question')\n user_grouping = envelope.get('user_grouping')\n session_id = envelope.get('session_id')\n user_id = envelope.get('user_id')\n generated_sql,session_id,invalid_response = await generate_sql(session_id,\n user_question,\n user_grouping, \n RUN_DEBUGGER,\n DEBUGGING_ROUNDS, \n LLM_VALIDATION,\n Embedder_model,\n SQLBuilder_model,\n SQLChecker_model,\n SQLDebugger_model,\n num_table_matches,\n num_column_matches,\n table_similarity_threshold,\n column_similarity_threshold,\n example_similarity_threshold,\n num_sql_matches,\n user_id=user_id)\n\n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"GeneratedSQL\" : generated_sql,\n \"SessionID\" : session_id,\n \"Error\":\"\"\n }\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedSQL\" : \"\",\n \"SessionID\" : session_id,\n \"Error\":generated_sql\n } \n\n return jsonify(responseDict)\n\n\n@app.route(\"/generate_viz\", methods=[\"POST\"])\n# @jwt_authenticated\nasync def generateViz():\n envelope = str(request.data.decode('utf-8'))\n # print(\"Here is the request payload \" + envelope)\n envelope=json.loads(envelope)\n\n user_question = envelope.get('user_question')\n generated_sql = envelope.get('generated_sql')\n sql_results = envelope.get('sql_results')\n session_id = envelope.get('session_id')\n chart_js=''\n\n try:\n chart_js, invalid_response = visualize(session_id,user_question,generated_sql,sql_results)\n \n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"GeneratedChartjs\" : chart_js,\n \"Error\":\"\",\n \"SessionID\":session_id\n }\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedSQL\" : \"\",\n \"SessionID\":session_id,\n \"Error\": chart_js\n } \n\n\n return jsonify(responseDict)\n\n except Exception as e:\n # util.write_log_entry(\"Cannot generate the Visualization!!!, please check the logs!\" + str(e))\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedSQL\" : \"\",\n \"SessionID\":session_id,\n \"Error\":\"Issue was encountered while generating the Google Chart, please check the logs!\" + str(e)\n } \n return jsonify(responseDict)\n\n@app.route(\"/summarize_results\", methods=[\"POST\"])\n# @jwt_authenticated\nasync def getSummary():\n envelope = str(request.data.decode('utf-8'))\n envelope=json.loads(envelope)\n \n user_question = envelope.get('user_question')\n sql_results = envelope.get('sql_results')\n\n result,invalid_response=get_response(user_question,sql_results)\n \n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"summary_response\" : result,\n \"Error\":\"\"\n } \n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"summary_response\" : \"\",\n \"Error\":result\n } \n return jsonify(responseDict)\n\n\n\n\n@app.route(\"/natural_response\", methods=[\"POST\"])\n# @jwt_authenticated\nasync def getNaturalResponse():\n envelope = str(request.data.decode('utf-8'))\n #print(\"Here is the request payload \" + envelope)\n envelope=json.loads(envelope)\n \n user_question = envelope.get('user_question')\n user_grouping = envelope.get('user_grouping')\n \n generated_sql,session_id,invalid_response = await generate_sql(user_question,\n user_grouping, \n RUN_DEBUGGER,\n DEBUGGING_ROUNDS, \n LLM_VALIDATION,\n Embedder_model,\n SQLBuilder_model,\n SQLChecker_model,\n SQLDebugger_model,\n num_table_matches,\n num_column_matches,\n table_similarity_threshold,\n column_similarity_threshold,\n example_similarity_threshold,\n num_sql_matches)\n \n if not invalid_response:\n\n result_df,invalid_response=get_results(user_grouping,generated_sql)\n \n if not invalid_response:\n result,invalid_response=get_response(user_question,result_df.to_json(orient='records'))\n\n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"summary_response\" : result,\n \"Error\":\"\"\n } \n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"summary_response\" : \"\",\n \"Error\":result\n } \n\n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"KnownDB\" : \"\",\n \"Error\":result_df\n } \n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedSQL\" : \"\",\n \"Error\":generated_sql\n }\n\n return jsonify(responseDict) \n\n\n@app.route(\"/get_results\", methods=[\"POST\"])\nasync def getResultsResponse():\n envelope = str(request.data.decode('utf-8'))\n #print(\"Here is the request payload \" + envelope)\n envelope=json.loads(envelope)\n \n user_question = envelope.get('user_question')\n user_database = envelope.get('user_database')\n \n generated_sql,invalid_response = await generate_sql(user_question,\n user_database, \n RUN_DEBUGGER,\n DEBUGGING_ROUNDS, \n LLM_VALIDATION,\n Embedder_model,\n SQLBuilder_model,\n SQLChecker_model,\n SQLDebugger_model,\n num_table_matches,\n num_column_matches,\n table_similarity_threshold,\n column_similarity_threshold,\n example_similarity_threshold,\n num_sql_matches)\n \n if not invalid_response:\n\n result_df,invalid_response=get_results(user_database,generated_sql)\n \n if not invalid_response:\n responseDict = { \n \"ResponseCode\" : 200, \n \"GeneratedResults\" : result_df.to_json(orient='records'),\n \"Error\":\"\"\n } \n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedResults\" : \"\",\n \"Error\":result_df\n } \n\n else:\n responseDict = { \n \"ResponseCode\" : 500, \n \"GeneratedResults\" : \"\",\n \"Error\":generated_sql\n }\n\n return jsonify(responseDict) \n \nif __name__ == \"__main__\":\n app.run(debug=True, host=\"0.0.0.0\", port=int(os.environ.get(\"PORT\", 8080)))" }, { "path": "backend-apis/policy.yaml", "content": "constraint: constraints/iam.allowedPolicyMemberDomains\nlistPolicy:\n allValues: ALLOW" }, { "path": "config.ini", "content": "[CONFIG]\nembedding_model = vertex\ndescription_model = gemini-1.5-pro\nvector_store = bigquery-vector\ndebugging = yes\nlogging = yes\nkgq_examples = yes\nfirestore_region = us-central1\nuse_session_history = yes\nuse_column_samples = no\n\n[GCP]\nproject_id = three-p-o\n\n[PGCLOUDSQL]\npg_region = us-central1\npg_instance = pg15-opendataqna\npg_database = opendataqna-db\npg_user = pguser\npg_password = pg123\n\n[BIGQUERY]\nbq_dataset_region = us-central1\nbq_opendataqna_dataset_name = opendataqna\nbq_log_table_name = audit_log_table\n\n\n\n\n" }, { "path": "dbconnectors/BQConnector.py", "content": "\"\"\"\nBigQuery Connector Class\n\"\"\"\nfrom google.cloud import bigquery\nfrom google.cloud import bigquery_connection_v1 as bq_connection\nfrom dbconnectors import DBConnector\nfrom abc import ABC\nfrom datetime import datetime\nimport google.auth\nimport pandas as pd\nfrom google.cloud.exceptions import NotFound\n\ndef get_auth_user():\n credentials, project_id = google.auth.default()\n\n if hasattr(credentials, 'service_account_email'):\n return credentials.service_account_email\n else:\n return \"Not Determined\"\n\ndef bq_specific_data_types(): \n return '''\n BigQuery offers a wide variety of datatypes to store different types of data effectively. Here's a breakdown of the available categories:\n Numeric Types -\n INTEGER (INT64): Stores whole numbers within the range of -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. Ideal for non-fractional values.\n FLOAT (FLOAT64): Stores approximate floating-point numbers with a range of -1.7E+308 to 1.7E+308. Suitable for decimals with a degree of imprecision.\n NUMERIC: Stores exact fixed-precision decimal numbers, with up to 38 digits of precision and 9 digits to the right of the decimal point. Useful for precise financial and accounting calculations.\n BIGNUMERIC: Similar to NUMERIC but with even larger scale and precision. Designed for extreme precision in calculations.\n \n Character Types -\n STRING: Stores variable-length Unicode character sequences. Enclosed using single, double, or triple quotes.\n \n Boolean Type -\n BOOLEAN: Stores logical values of TRUE or FALSE (case-insensitive).\n \n Date and Time Types -\n DATE: Stores dates without associated time information.\n TIME: Stores time information independent of a specific date.\n DATETIME: Stores both date and time information (without timezone information).\n TIMESTAMP: Stores an exact moment in time with microsecond precision, including a timezone component for global accuracy.\n \n Other Types\n BYTES: Stores variable-length binary data. Distinguished from strings by using 'B' or 'b' prefix in values.\n GEOGRAPHY: Stores points, lines, and polygons representing locations on the Earth's surface.\n ARRAY: Stores an ordered collection of zero or more elements of the same (non-ARRAY) data type.\n STRUCT: Stores an ordered collection of fields, each with its own name and data type (can be nested).\n \n This list covers the most common datatypes in BigQuery.\n '''\n\n\nclass BQConnector(DBConnector, ABC):\n \"\"\"\n A connector class for interacting with BigQuery databases.\n\n This class provides methods for connecting to BigQuery, executing queries, retrieving results as DataFrames, logging interactions, and managing embeddings.\n\n Attributes:\n project_id (str): The Google Cloud project ID where the BigQuery dataset resides.\n region (str): The region where the BigQuery dataset is located.\n dataset_name (str): The name of the BigQuery dataset to interact with.\n opendataqna_dataset (str): Name of the dataset to use for OpenDataQnA functionalities.\n audit_log_table_name (str): Name of the table to store audit logs.\n client (bigquery.Client): The BigQuery client instance for executing queries.\n\n Methods:\n getconn() -> bigquery.Client:\n Establishes a connection to BigQuery and returns a client object.\n\n retrieve_df(query) -> pd.DataFrame:\n Executes a SQL query and returns the results as a pandas DataFrame.\n\n make_audit_entry(source_type, user_grouping, model, question, generated_sql, found_in_vector, need_rewrite, failure_step, error_msg, FULL_LOG_TEXT) -> str:\n Logs an audit entry to BigQuery, recording details of the interaction and the generated SQL query.\n\n create_vertex_connection(connection_id) -> None:\n Creates a Vertex AI connection for remote model usage in BigQuery.\n\n create_embedding_model(connection_id, embedding_model) -> None:\n Creates or replaces an embedding model in BigQuery using a Vertex AI connection.\n\n retrieve_matches(mode, user_grouping, qe, similarity_threshold, limit) -> list:\n Retrieves the most similar table schemas, column schemas, or example queries based on the given mode and parameters.\n\n getSimilarMatches(mode, user_grouping, qe, num_matches, similarity_threshold) -> str:\n Returns a formatted string containing similar matches found for tables, columns, or examples.\n\n getExactMatches(query) -> str or None:\n Checks if the exact question is present in the example SQL set and returns the corresponding SQL query if found.\n\n test_sql_plan_execution(generated_sql) -> Tuple[bool, str]:\n Tests the execution plan of a generated SQL query in BigQuery. Returns a tuple indicating success and a message.\n\n return_table_schema_sql(dataset, table_names=None) -> str:\n Returns a SQL query to retrieve table schema information from a BigQuery dataset.\n\n return_column_schema_sql(dataset, table_names=None) -> str:\n Returns a SQL query to retrieve column schema information from a BigQuery dataset.\n \"\"\"\n\n\n def __init__(self,\n project_id:str,\n region:str,\n opendataqna_dataset:str,\n audit_log_table_name:str):\n\n self.project_id = project_id\n self.region = region\n self.opendataqna_dataset = opendataqna_dataset\n self.audit_log_table_name = audit_log_table_name\n self.client=self.getconn()\n\n def getconn(self):\n client = bigquery.Client(project=self.project_id)\n return client\n \n def retrieve_df(self,query):\n return self.client.query_and_wait(query).to_dataframe()\n\n def make_audit_entry(self, source_type, user_grouping, model, question, generated_sql, found_in_vector, need_rewrite, failure_step, error_msg, FULL_LOG_TEXT):\n # global FULL_LOG_TEXT\n auth_user=get_auth_user()\n\n PROJECT_ID = self.project_id\n\n table_id= PROJECT_ID+ '.' + self.opendataqna_dataset + '.' + self.audit_log_table_name\n now = datetime.now()\n\n table_exists=False\n client = self.getconn()\n\n df1 = pd.DataFrame(columns=[\n 'source_type',\n 'project_id',\n 'user',\n 'user_grouping',\n 'model_used',\n 'question',\n 'generated_sql',\n 'found_in_vector',\n 'need_rewrite',\n 'failure_step',\n 'error_msg',\n 'execution_time',\n 'full_log'\n ])\n\n new_row = {\n \"source_type\":source_type,\n \"project_id\":str(PROJECT_ID),\n \"user\":str(auth_user),\n \"user_grouping\": user_grouping,\n \"model_used\": model,\n \"question\": question,\n \"generated_sql\": generated_sql,\n \"found_in_vector\":found_in_vector,\n \"need_rewrite\":need_rewrite,\n \"failure_step\":failure_step,\n \"error_msg\":error_msg,\n \"execution_time\": now,\n \"full_log\": FULL_LOG_TEXT\n }\n\n df1.loc[len(df1)] = new_row\n\n db_schema=[\n # Specify the type of columns whose type cannot be auto-detected. For\n # example the \"title\" column uses pandas dtype \"object\", so its\n # data type is ambiguous.\n bigquery.SchemaField(\"source_type\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"project_id\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"user\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"user_grouping\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"model_used\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"question\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"generated_sql\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"found_in_vector\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"need_rewrite\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"failure_step\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"error_msg\", bigquery.enums.SqlTypeNames.STRING),\n bigquery.SchemaField(\"execution_time\", bigquery.enums.SqlTypeNames.TIMESTAMP),\n bigquery.SchemaField(\"full_log\", bigquery.enums.SqlTypeNames.STRING),\n ]\n\n try:\n client.get_table(table_id) # Make an API request.\n # print(\"Table {} already exists.\".format(table_id))\n table_exists=True\n except NotFound:\n print(\"Table {} is not found. Will create this log table\".format(table_id))\n table_exists=False\n\n if table_exists is True:\n # print('Performing streaming insert')\n errors = client.insert_rows_from_dataframe(table=table_id, dataframe=df1, selected_fields=db_schema) # Make an API request.\n if errors == [[]]:\n print(\"Logged the run\")\n else:\n print(\"Encountered errors while inserting rows: {}\".format(errors))\n else:\n job_config = bigquery.LoadJobConfig(schema=db_schema,write_disposition=\"WRITE_TRUNCATE\")\n # pandas_gbq.to_gbq(df1, table_id, project_id=PROJECT_ID) # replace to replace table; append to append to a table\n client.load_table_from_dataframe(df1,table_id,job_config=job_config) # replace to replace table; append to append to a table\n\n\n # df1.loc[len(df1)] = new_row\n # pandas_gbq.to_gbq(df1, table_id, project_id=PROJECT_ID, if_exists='append') # replace to replace table; append to append to a table\n # print('\\n Query added to BQ log table \\n')\n return 'Completed the logging step'\n\n def create_vertex_connection(self, connection_id : str):\n client=bq_connection.ConnectionServiceClient()\n \n cloud_resource_properties = bq_connection.types.CloudResourceProperties()\n new_connection=bq_connection.Connection(cloud_resource=cloud_resource_properties)\n response= client.create_connection(parent=f'projects/{self.project_id}/locations/{self.region}',connection=new_connection,connection_id=connection_id)\n\n \n def create_embedding_model(self,connection_id: str, embedding_model: str):\n client = self.getconn()\n client.query_and_wait(f'''CREATE OR REPLACE MODEL `{self.project_id}.{self.opendataqna_dataset}.EMBEDDING_MODEL`\n REMOTE WITH CONNECTION `{self.project_id}.{self.region}.{connection_id}`\n OPTIONS (ENDPOINT = '{embedding_model}');''')\n \n \n def retrieve_matches(self, mode, user_grouping, qe, similarity_threshold, limit): \n \"\"\"\n This function retrieves the most similar table_schema and column_schema.\n Modes can be either 'table', 'column', or 'example' \n \"\"\"\n matches = []\n\n if mode == 'table':\n sql = '''select base.content as tables_content from vector_search(\n (SELECT * FROM `{}.table_details_embeddings` WHERE user_grouping = '{}'), \"embedding\", \n (SELECT {} as qe), top_k=> {},distance_type=>\"COSINE\") where 1-distance > {} '''\n \n elif mode == 'column':\n sql='''select base.content as columns_content from vector_search(\n (SELECT * FROM `{}.tablecolumn_details_embeddings` WHERE user_grouping = '{}'), \"embedding\",\n (SELECT {} as qe), top_k=> {}, distance_type=>\"COSINE\") where 1-distance > {} '''\n\n elif mode == 'example': \n sql='''select base.example_user_question, base.example_generated_sql from vector_search ( \n (SELECT * FROM `{}.example_prompt_sql_embeddings` WHERE user_grouping = '{}'), \"embedding\",\n (select {} as qe), top_k=> {}, distance_type=>\"COSINE\") where 1-distance > {} '''\n \n else: \n ValueError(\"No valid mode. Must be either table, column, or example\")\n name_txt = ''\n\n results=self.client.query_and_wait(sql.format('{}.{}'.format(self.project_id,self.opendataqna_dataset),user_grouping,qe,limit,similarity_threshold)).to_dataframe()\n # CHECK RESULTS \n if len(results) == 0:\n print(f\"Did not find any results for {mode}. Adjust the query parameters.\")\n else:\n print(f\"Found {len(results)} similarity matches for {mode}.\")\n\n if mode == 'table': \n name_txt = ''\n for _ , r in results.iterrows():\n name_txt=name_txt+r[\"tables_content\"]+\"\\n\"\n\n elif mode == 'column': \n name_txt = '' \n for _ ,r in results.iterrows():\n name_txt=name_txt+r[\"columns_content\"]+\"\\n\"\n\n elif mode == 'example': \n name_txt = ''\n for _ , r in results.iterrows():\n example_user_question=r[\"example_user_question\"]\n example_sql=r[\"example_generated_sql\"]\n name_txt = name_txt + \"\\n Example_question: \"+example_user_question+ \"; Example_SQL: \"+example_sql\n\n else: \n ValueError(\"No valid mode. Must be either table, column, or example\")\n name_txt = ''\n\n matches.append(name_txt)\n \n\n return matches\n\n def getSimilarMatches(self, mode, user_grouping, qe, num_matches, similarity_threshold):\n\n if mode == 'table': \n match_result= self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n match_result = match_result[0]\n # print(match_result)\n\n elif mode == 'column': \n match_result= self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n match_result = match_result[0]\n \n elif mode == 'example': \n match_result= self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n if len(match_result) == 0:\n match_result = None\n else:\n match_result = match_result[0]\n\n return match_result\n\n def getExactMatches(self, query):\n \"\"\"Checks if the exact question is already present in the example SQL set\"\"\"\n check_history_sql=f\"\"\"SELECT example_user_question,example_generated_sql FROM `{self.project_id}.{self.opendataqna_dataset}.example_prompt_sql_embeddings`\n WHERE lower(example_user_question) = lower(\"{query}\") LIMIT 1; \"\"\"\n\n exact_sql_history = self.client.query_and_wait(check_history_sql).to_dataframe()\n\n\n if exact_sql_history[exact_sql_history.columns[0]].count() != 0:\n sql_example_txt = ''\n exact_sql = ''\n for index, row in exact_sql_history.iterrows():\n example_user_question=row[\"example_user_question\"]\n example_sql=row[\"example_generated_sql\"]\n exact_sql=example_sql\n sql_example_txt = sql_example_txt + \"\\n Example_question: \"+example_user_question+ \"; Example_SQL: \"+example_sql\n\n # print(\"Found a matching question from the history!\" + str(sql_example_txt))\n final_sql=exact_sql\n\n else: \n print(\"No exact match found for the user prompt\")\n final_sql = None\n\n return final_sql\n\n def test_sql_plan_execution(self, generated_sql):\n try:\n exec_result_df=\"\"\n job_config=bigquery.QueryJobConfig(dry_run=True, use_query_cache=False)\n query_job = self.client.query(generated_sql,job_config=job_config)\n # print(query_job)\n exec_result_df=(\"This query will process {} bytes.\".format(query_job.total_bytes_processed))\n correct_sql = True\n print(exec_result_df)\n return correct_sql, exec_result_df\n except Exception as e:\n return False,str(e)\n\n\n def return_table_schema_sql(self, dataset, table_names=None): \n \"\"\"\n Returns the SQL query to be run on 'Source DB' to get the Table Schema\n The SQL query below returns a df containing the cols table_schema, table_name, table_description, table_columns (with cols in the table)\n for the schema specified above, e.g. 'retail'\n - table_schema: e.g. retail \n - table_name: name of the table inside the schema, e.g. products \n - table_description: text descriptor, can be empty \n - table_columns: aggregate of the col names inside the table \n \"\"\"\n\n user_dataset = self.project_id + '.' + dataset\n\n table_filter_clause = \"\"\n\n if table_names:\n # Extract individual table names from the input string\n #table_names = [name.strip() for name in table_names[1:-1].split(\",\")] # Handle the string as a list\n formatted_table_names = [f\"'{name}'\" for name in table_names]\n table_filter_clause = f\"\"\"AND TABLE_NAME IN ({', '.join(formatted_table_names)})\"\"\"\n\n\n table_schema_sql = f\"\"\"\n (SELECT\n TABLE_CATALOG as project_id, TABLE_SCHEMA as table_schema , TABLE_NAME as table_name, OPTION_VALUE as table_description,\n (SELECT STRING_AGG(column_name, ', ') from `{user_dataset}.INFORMATION_SCHEMA.COLUMNS` where TABLE_NAME= t.TABLE_NAME and TABLE_SCHEMA=t.TABLE_SCHEMA) as table_columns\n FROM\n `{user_dataset}.INFORMATION_SCHEMA.TABLE_OPTIONS` as t\n WHERE\n OPTION_NAME = \"description\"\n {table_filter_clause}\n ORDER BY\n project_id, table_schema, table_name)\n\n UNION ALL\n\n (SELECT\n TABLE_CATALOG as project_id, TABLE_SCHEMA as table_schema , TABLE_NAME as table_name, \"NA\" as table_description,\n (SELECT STRING_AGG(column_name, ', ') from `{user_dataset}.INFORMATION_SCHEMA.COLUMNS` where TABLE_NAME= t.TABLE_NAME and TABLE_SCHEMA=t.TABLE_SCHEMA) as table_columns\n FROM\n `{user_dataset}.INFORMATION_SCHEMA.TABLES` as t \n WHERE \n NOT EXISTS (SELECT 1 FROM\n `{user_dataset}.INFORMATION_SCHEMA.TABLE_OPTIONS` \n WHERE\n OPTION_NAME = \"description\" AND TABLE_NAME= t.TABLE_NAME and TABLE_SCHEMA=t.TABLE_SCHEMA)\n {table_filter_clause}\n ORDER BY\n project_id, table_schema, table_name)\n \"\"\"\n return table_schema_sql\n \n\n\n def return_column_schema_sql(self, dataset, table_names=None): \n \"\"\"\n Returns the SQL query to be run on 'Source DB' to get the column schema \n \n The SQL query below returns a df containing the cols table_schema, table_name, column_name, data_type, column_description, table_description, primary_key, column_constraints\n for the schema specified above, e.g. 'retail'\n - table_schema: e.g. retail \n - table_name: name of the tables inside the schema, e.g. products \n - column_name: name of each col in each table in the schema, e.g. id_product \n - data_type: data type of each col \n - column_description: col descriptor, can be empty \n - table_description: text descriptor, can be empty \n - primary_key: whether the col is PK; if yes, the field contains the col_name \n - column_constraints: e.g. \"Primary key for this table\"\n \"\"\"\n\n user_dataset = self.project_id + '.' + dataset\n\n table_filter_clause = \"\"\n\n if table_names:\n \n # table_names = [name.strip() for name in table_names[1:-1].split(\",\")] # Handle the string as a list\n formatted_table_names = [f\"'{name}'\" for name in table_names]\n table_filter_clause = f\"\"\"AND C.TABLE_NAME IN ({', '.join(formatted_table_names)})\"\"\"\n \n column_schema_sql = f\"\"\"\n SELECT\n C.TABLE_CATALOG as project_id, C.TABLE_SCHEMA as table_schema, C.TABLE_NAME as table_name, C.COLUMN_NAME as column_name,\n C.DATA_TYPE as data_type, C.DESCRIPTION as column_description, CASE WHEN T.CONSTRAINT_TYPE=\"PRIMARY KEY\" THEN \"This Column is a Primary Key for this table\" WHEN \n T.CONSTRAINT_TYPE = \"FOREIGN_KEY\" THEN \"This column is Foreign Key\" ELSE NULL END as column_constraints\n FROM\n `{user_dataset}.INFORMATION_SCHEMA.COLUMN_FIELD_PATHS` C \n LEFT JOIN \n `{user_dataset}.INFORMATION_SCHEMA.TABLE_CONSTRAINTS` T \n ON C.TABLE_CATALOG = T.TABLE_CATALOG AND\n C.TABLE_SCHEMA = T.TABLE_SCHEMA AND \n C.TABLE_NAME = T.TABLE_NAME AND \n T.ENFORCED ='YES'\n LEFT JOIN \n `{user_dataset}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE` K\n ON K.CONSTRAINT_NAME=T.CONSTRAINT_NAME AND C.COLUMN_NAME = K.COLUMN_NAME \n WHERE\n 1=1\n {table_filter_clause} \n ORDER BY\n project_id, table_schema, table_name, column_name;\n \"\"\"\n\n return column_schema_sql\n\n def get_column_samples(self,columns_df):\n sample_column_list=[]\n\n for index, row in columns_df.iterrows():\n get_column_sample_sql=f'''SELECT STRING_AGG(CAST(value AS STRING)) as sample_values FROM UNNEST((SELECT APPROX_TOP_COUNT({row[\"column_name\"]},5) as osn \n FROM `{row[\"project_id\"]}.{row[\"table_schema\"]}.{row[\"table_name\"]}`\n ))'''\n\n column_samples_df=self.retrieve_df(get_column_sample_sql)\n # display(column_samples_df)\n sample_column_list.append(column_samples_df['sample_values'].to_string(index=False))\n\n columns_df[\"sample_values\"]=sample_column_list\n return columns_df\n" }, { "path": "dbconnectors/FirestoreConnector.py", "content": "from google.cloud import firestore \nfrom google.cloud.exceptions import NotFound\nimport time\nfrom dbconnectors import DBConnector\nfrom abc import ABC\nimport uuid\n\ndef create_unique_id():\n \"\"\"Creates a unique ID using the UUID4 algorithm.\n\n Returns:\n A string representing a unique ID.\n \"\"\"\n\n return str(uuid.uuid1())\n\n\nclass FirestoreConnector(DBConnector, ABC):\n def __init__(self, \n project_id:str, \n firestore_database:str):\n \"\"\"Initializes the Firestore connection and authentication.\"\"\"\n self.db = firestore.Client(project=project_id,database=firestore_database)\n\n def log_chat(self,session_id, user_question, bot_response,user_id=\"TEST\",):\n \"\"\"Logs a chat message to Firestore.\n Args:\n session_id (str): The ID of the chat session.\n user_id (str): The ID of the user who sent the message.\n user_question (str): The question the user asked.\n bot_response (str): The response from the bot.\n \"\"\"\n\n log_chat = {\n \"session_id\": session_id,\n \"user_id\": user_id,\n \"user_question\": user_question,\n \"bot_response\": bot_response,\n \"timestamp\": firestore.SERVER_TIMESTAMP,\n }\n\n self.db.collection(\"session_logs\").document().set(log_chat) \n \n def get_chat_logs_for_session(self,session_id):\n \"\"\"Gets all chat logs for a given session.\n Args:\n session_id (str): The ID of the chat session.\n \"\"\"\n\n sessions_log_ref = self.db.collection(\"session_logs\")\n\n # sessions_log_ref=sessions_log_ref.order_by(\"timestamp\")\n query= sessions_log_ref.where(filter=firestore.FieldFilter(\"session_id\",\"==\",session_id))\n \n # query = sessions_log_ref.where(\"session_id\", \"==\", session_id).order_by(\"timestamp\") \n\n # Note: Use of CollectionRef stream() is prefered to get()\n docs = query.stream()\n\n session_history=[]\n for doc in docs:\n session_history.append(doc.to_dict()) # Add values to the list\n sorted_session_history=sorted(session_history,key=lambda x: x[\"timestamp\"])\n\n return [{'user_question': item['user_question'], 'bot_response': item['bot_response'],'timestamp':item['timestamp']} for item in sorted_session_history]\n \n" }, { "path": "dbconnectors/PgConnector.py", "content": "\"\"\"\nPostgreSQL Connector Class \n\"\"\"\nimport asyncpg\nfrom google.cloud.sql.connector import Connector\nfrom sqlalchemy import create_engine\nimport pandas as pd \nfrom sqlalchemy.sql import text\nfrom pgvector.asyncpg import register_vector\nimport asyncio\nfrom pg8000.exceptions import DatabaseError \n\nfrom utilities import root_dir\nfrom google.cloud.sql.connector import Connector\n\nfrom dbconnectors import DBConnector\nfrom abc import ABC\n\n\n\ndef pg_specific_data_types(): \n return '''\n PostgreSQL offers a wide variety of datatypes to store different types of data effectively. Here's a breakdown of the available categories:\n\n Numeric datatypes -\n SMALLINT: Stores small-range integers between -32768 and 32767.\n INTEGER: Stores typical integers between -2147483648 and 2147483647.\n BIGINT: Stores large-range integers between -9223372036854775808 and 9223372036854775807.\n DECIMAL(p,s): Stores arbitrary precision numbers with a maximum of p digits and s digits to the right of the decimal point.\n NUMERIC: Similar to DECIMAL but with additional features like automatic scaling.\n REAL: Stores single-precision floating-point numbers with an approximate range of -3.4E+38 to 3.4E+38.\n DOUBLE PRECISION: Stores double-precision floating-point numbers with an approximate range of -1.7E+308 to 1.7E+308.\n\n\n Character datatypes -\n CHAR(n): Fixed-length character string with a specified length of n characters.\n VARCHAR(n): Variable-length character string with a maximum length of n characters.\n TEXT: Variable-length string with no maximum size limit.\n CHARACTER VARYING(n): Alias for VARCHAR(n).\n CHARACTER: Alias for CHAR.\n\n Monetary datatypes -\n MONEY: Stores monetary amounts with two decimal places.\n\n Date/Time datatypes -\n DATE: Stores dates without time information.\n TIME: Stores time of day without date information (optionally with time zone).\n TIMESTAMP: Stores both date and time information (optionally with time zone).\n INTERVAL: Stores time intervals between two points in time.\n\n Binary types -\n BYTEA: Stores variable-length binary data.\n BIT: Stores single bits.\n BIT VARYING: Stores variable-length bit strings.\n\n Other types -\n BOOLEAN: Stores true or false values.\n UUID: Stores universally unique identifiers.\n XML: Stores XML data.\n JSON: Stores JSON data.\n ENUM: Stores user-defined enumerated values.\n RANGE: Stores ranges of data values.\n\n This list covers the most common datatypes in PostgreSQL.\n '''\n\n\n\n\nclass PgConnector(DBConnector, ABC):\n \"\"\"\n A connector class for interacting with PostgreSQL databases.\n\n This class provides methods for establishing connections to PostgreSQL instances, executing SQL queries, retrieving results as DataFrames, caching known SQL queries, and managing embeddings. It utilizes the `pg8000` library for connections and the `asyncpg` library for asynchronous operations.\n\n Attributes:\n project_id (str): The Google Cloud project ID where the PostgreSQL instance resides.\n region (str): The region where the PostgreSQL instance is located.\n instance_name (str): The name of the PostgreSQL instance.\n database_name (str): The name of the database to connect to.\n database_user (str): The username for authentication.\n database_password (str): The password for authentication.\n pool (Engine): A SQLAlchemy engine object for managing database connections.\n\n Methods:\n getconn() -> connection:\n Establishes a connection to the PostgreSQL instance and returns a connection object.\n\n retrieve_df(query) -> pd.DataFrame:\n Executes a SQL query and returns the results as a pandas DataFrame. Handles potential database errors.\n\n cache_known_sql() -> None:\n Caches known good SQL queries into a PostgreSQL table for future reference.\n\n retrieve_matches(mode, user_grouping, qe, similarity_threshold, limit) -> list:\n Retrieves similar matches (table schemas, column schemas, or example queries) from the database based on the given mode, query embedding (`qe`), similarity threshold, and limit.\n\n getSimilarMatches(mode, user_grouping, qe, num_matches, similarity_threshold) -> str:\n Gets similar matches for tables, columns, or examples asynchronously, formatting the results into a string.\n\n test_sql_plan_execution(generated_sql) -> Tuple[bool, pd.DataFrame]:\n Tests the execution plan of a generated SQL query in PostgreSQL. Returns a tuple indicating success and the result DataFrame.\n\n getExactMatches(query) -> str or None:\n Checks if the exact question is present in the example SQL set and returns the corresponding SQL query if found.\n\n return_column_schema_sql(schema) -> str:\n Returns a SQL query to retrieve column schema information from a PostgreSQL schema.\n\n return_table_schema_sql(schema) -> str:\n Returns a SQL query to retrieve table schema information from a PostgreSQL schema.\n \"\"\"\n\n\n def __init__(self,\n project_id:str, \n region:str, \n instance_name:str,\n database_name:str, \n database_user:str, \n database_password:str): \n\n self.project_id = project_id\n self.region = region \n self.instance_name = instance_name \n self.database_name = database_name\n self.database_user = database_user\n self.database_password = database_password\n\n self.pool = create_engine(\n \"postgresql+pg8000://\",\n creator=self.getconn,\n )\n\n\n def getconn(self): \n \"\"\"\n function to return the database connection object\n \"\"\"\n # initialize Connector object\n connector = Connector()\n conn = connector.connect(\n f\"{self.project_id}:{self.region}:{self.instance_name}\",\n \"pg8000\",\n user=f\"{self.database_user}\",\n password=f\"{self.database_password}\",\n db=f\"{self.database_name}\"\n )\n\n return conn \n\n\n def retrieve_df(self, query):\n \"\"\" \n TODO: Description \n \"\"\"\n\n result_df=pd.DataFrame()\n try: \n with self.pool.connect() as db_conn:\n \n df = pd.read_sql(text(query), con=db_conn)\n result_df = df\n # print('\\n Return from code execution: ' + str(result_df) )\n return result_df\n \n except Exception as e: \n print(f\"Database Error: {e}\")\n df = pd.DataFrame({'Error. Message': e}, index=[0])\n return df \n \n \n async def cache_known_sql(self):\n \n df = pd.read_csv(f\"{root_dir}/{scripts}/known_good_sql.csv\")\n df = df.loc[:, [\"prompt\", \"sql\", \"database_name\"]]\n df = df.dropna()\n\n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # # Create connection to Cloud SQL database.\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{self.project_id}:{self.region}:{self.instance_name}\", \n \"asyncpg\",\n user=f\"{self.database_user}\",\n password=f\"{self.database_password}\",\n db=f\"{self.database_name}\",\n )\n\n\n await register_vector(conn)\n\n # Delete the table if it exists.\n await conn.execute(\"DROP TABLE IF EXISTS query_example_embeddings CASCADE\")\n \n # Create the `query_example_embeddings` table.\n await conn.execute(\n \"\"\"CREATE TABLE query_example_embeddings(\n prompt TEXT,\n sql TEXT,\n user_grouping TEXT)\"\"\"\n )\n\n # Copy the dataframe to the 'query_example_embeddings' table.\n tuples = list(df.itertuples(index=False))\n await conn.copy_records_to_table(\n \"query_example_embeddings\", records=tuples, columns=list(df), timeout=10000\n )\n \n await conn.close()\n\n\n async def retrieve_matches(self, mode, user_groupinguping, qe, similarity_threshold, limit): \n \"\"\"\n This function retrieves the most similar table_schema and column_schema.\n Modes can be either 'table', 'column', or 'example' \n \"\"\"\n matches = [] \n\n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # # Create connection to Cloud SQL database.\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{self.project_id}:{self.region}:{self.instance_name}\", \n \"asyncpg\",\n user=f\"{self.database_user}\",\n password=f\"{self.database_password}\",\n db=f\"{self.database_name}\",\n )\n\n\n await register_vector(conn)\n\n\n # Prepare the SQL depending on 'mode' \n if mode == 'table': \n sql = \"\"\"\n SELECT content as tables_content,\n 1 - (embedding <=> $1) AS similarity\n FROM table_details_embeddings\n WHERE 1 - (embedding <=> $1) > $2\n AND user_grouping = $4\n ORDER BY similarity DESC LIMIT $3\n \"\"\"\n \n\n elif mode == 'column': \n sql = \"\"\"\n SELECT content as columns_content,\n 1 - (embedding <=> $1) AS similarity\n FROM tablecolumn_details_embeddings\n WHERE 1 - (embedding <=> $1) > $2\n AND user_grouping = $4\n ORDER BY similarity DESC LIMIT $3\n \"\"\"\n\n elif mode == 'example': \n sql = \"\"\"\n SELECT user_grouping, example_user_question, example_generated_sql,\n 1 - (embedding <=> $1) AS similarity\n FROM example_prompt_sql_embeddings\n WHERE 1 - (embedding <=> $1) > $2\n AND user_grouping = $4\n ORDER BY similarity DESC LIMIT $3\n \"\"\"\n\n else: \n ValueError(\"No valid mode. Must be either table, column, or example\")\n name_txt = ''\n \n # print(sql,qe,similarity_threshold,limit,user_grouping)\n # FETCH RESULTS FROM POSTGRES DB \n results = await conn.fetch(\n sql,\n qe,\n similarity_threshold,\n limit,\n user_groupinguping\n )\n\n # CHECK RESULTS \n if len(results) == 0:\n print(f\"Did not find any results for {mode}. Adjust the query parameters.\")\n else:\n print(f\"Found {len(results)} similarity matches for {mode}.\")\n\n if mode == 'table': \n name_txt = ''\n for r in results:\n name_txt=name_txt+r[\"tables_content\"]+\"\\n\\n\"\n\n elif mode == 'column': \n name_txt = '' \n for r in results:\n name_txt=name_txt+r[\"columns_content\"]+\"\\n\\n \"\n\n elif mode == 'example': \n name_txt = ''\n for r in results:\n example_user_question=r[\"example_user_question\"]\n example_sql=r[\"example_generated_sql\"]\n # print(example_user_question+\"\\nThreshold::\"+str(r[\"similarity\"]))\n name_txt = name_txt + \"\\n Example_question: \"+example_user_question+ \"; Example_SQL: \"+example_sql\n\n else: \n ValueError(\"No valid mode. Must be either table, column, or example\")\n name_txt = ''\n\n matches.append(name_txt)\n\n # Close the connection to the database.\n await conn.close()\n\n return matches \n\n\n\n async def getSimilarMatches(self, mode, user_grouping, qe, num_matches, similarity_threshold):\n\n if mode == 'table': \n match_result=await self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n match_result = match_result[0]\n\n elif mode == 'column': \n match_result=await self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n match_result = match_result[0]\n \n elif mode == 'example': \n match_result=await self.retrieve_matches(mode, user_grouping, qe, similarity_threshold, num_matches)\n if len(match_result) == 0:\n match_result = None\n else:\n match_result = match_result[0]\n\n return match_result\n\n\n def test_sql_plan_execution(self, generated_sql):\n try:\n exec_result_df = pd.DataFrame()\n sql = f\"\"\"EXPLAIN ANALYZE {generated_sql}\"\"\"\n exec_result_df = self.retrieve_df(sql)\n\n if not exec_result_df.empty:\n if str(exec_result_df.iloc[0]).startswith('Error. Message'):\n correct_sql = False \n \n else:\n print('\\n No need to rewrite the query. This seems to work fine and returned rows...')\n correct_sql = True\n else:\n print('\\n No need to rewrite the query. This seems to work fine but no rows returned...')\n correct_sql = True\n \n return correct_sql, exec_result_df\n\n except Exception as e:\n return False,str(e)\n \n\n\n\n def getExactMatches(self, query): \n \"\"\" \n Checks if the exact question is already present in the example SQL set \n \"\"\"\n check_history_sql=f\"\"\"SELECT example_user_question,example_generated_sql\n FROM example_prompt_sql_embeddings\n WHERE lower(example_user_question) = lower('{query}') LIMIT 1; \"\"\"\n\n exact_sql_history = self.retrieve_df(check_history_sql)\n\n if exact_sql_history[exact_sql_history.columns[0]].count() != 0:\n sql_example_txt = ''\n exact_sql = ''\n for index, row in exact_sql_history.iterrows():\n example_user_question=row[\"example_user_question\"]\n example_sql=row[\"example_generated_sql\"]\n exact_sql=example_sql\n sql_example_txt = sql_example_txt + \"\\n Example_question: \"+example_user_question+ \"; Example_SQL: \"+example_sql\n\n # print(\"Found a matching question from the history!\" + str(sql_example_txt))\n final_sql=exact_sql\n\n else: \n print(\"No exact match found for the user prompt\")\n final_sql = None\n\n return final_sql \n \n\n\n\n\n def return_column_schema_sql(self, schema, table_names=None): \n \"\"\"\n This SQL returns a df containing the cols table_schema, table_name, column_name, data_type, column_description, table_description, primary_key, column_constraints\n for the schema specified above, e.g. 'retail'\n - table_schema: e.g. retail \n - table_name: name of the table inside the schema, e.g. products \n - column_name: name of each col in each table in the schema, e.g. id_product \n - data_type: data type of each col \n - column_description: col descriptor, can be empty \n - table_description: text descriptor, can be empty \n - primary_key: whether the col is PK; if yes, the field contains the col_name \n - column_constraints: e.g. \"Primary key for this table\"\n \"\"\"\n table_filter_clause = \"\"\n if table_names:\n \n # table_names = [name.strip() for name in table_names[1:-1].split(\",\")] # Handle the string as a list\n formatted_table_names = [f\"'{name}'\" for name in table_names]\n table_filter_clause = f\"\"\"and table_name in ({', '.join(formatted_table_names)})\"\"\"\n \n\n column_schema_sql = f'''\n WITH\n columns_schema\n AS\n (select c.table_schema,c.table_name,c.column_name,c.data_type,d.description as column_description, obj_description(c1.oid) as table_description\n from information_schema.columns c\n inner join pg_class c1\n on c.table_name=c1.relname\n inner join pg_catalog.pg_namespace n\n on c.table_schema=n.nspname\n and c1.relnamespace=n.oid\n left join pg_catalog.pg_description d\n on d.objsubid=c.ordinal_position\n and d.objoid=c1.oid\n where\n c.table_schema='{schema}' {table_filter_clause}) ,\n pk_schema as\n (SELECT table_name, column_name AS primary_key\n FROM information_schema.key_column_usage\nWHERE TABLE_SCHEMA='{schema}' {table_filter_clause}\n AND CONSTRAINT_NAME like '%_pkey%'\n ORDER BY table_name, primary_key),\n fk_schema as\n (SELECT table_name, column_name AS foreign_key\n FROM information_schema.key_column_usage\n WHERE TABLE_SCHEMA='{schema}' {table_filter_clause}\n AND CONSTRAINT_NAME like '%_fkey%'\n ORDER BY table_name, foreign_key)\n\n select lr.*,\n case\n when primary_key is not null then 'Primary key for this table'\n when foreign_key is not null then CONCAT('Foreign key',column_description)\n else null\n END as column_constraints\n from\n (select l.*,r.primary_key\n from\n columns_schema l\n left outer join\n pk_schema r\n on\n l.table_name=r.table_name\n and\n l.column_name=r.primary_key) lr\n left outer join\n fk_schema rt\n on\n lr.table_name=rt.table_name\n and\n lr.column_name=rt.foreign_key\n ;\n '''\n\n return column_schema_sql\n\n\n\n \n def return_table_schema_sql(self, schema, table_names=None): \n \"\"\"\n This SQL returns a df containing the cols table_schema, table_name, table_description, table_columns (with cols in the table)\n for the schema specified above, e.g. 'retail'\n - table_schema: e.g. retail \n - table_name: name of the table inside the schema, e.g. products \n - table_description: text descriptor, can be empty \n - table_columns: aggregate of the col names inside the table \n \"\"\"\n\n table_filter_clause = \"\"\n\n if table_names:\n # Extract individual table names from the input string\n #table_names = [name.strip() for name in table_names[1:-1].split(\",\")] # Handle the string as a list\n formatted_table_names = [f\"'{name}'\" for name in table_names]\n table_filter_clause = f\"\"\"and table_name in ({', '.join(formatted_table_names)})\"\"\"\n\n\n table_schema_sql = f'''\n SELECT table_schema, table_name,table_description, array_to_string(array_agg(column_name), ' , ') as table_columns\n FROM\n (select c.table_schema,c.table_name,c.column_name,c.ordinal_position,c.column_default,c.data_type,d.description, obj_description(c1.oid) as table_description\n from information_schema.columns c\n inner join pg_class c1\n on c.table_name=c1.relname\n inner join pg_catalog.pg_namespace n\n on c.table_schema=n.nspname\n and c1.relnamespace=n.oid\n left join pg_catalog.pg_description d\n on d.objsubid=c.ordinal_position\n and d.objoid=c1.oid\n where\n c.table_schema='{schema}' {table_filter_clause} ) data\n GROUP BY table_schema, table_name, table_description\n ORDER BY table_name;\n '''\n\n return table_schema_sql \n \n\n def get_column_samples(self,columns_df):\n sample_column_list=[]\n\n for index, row in columns_df.iterrows():\n get_column_sample_sql=f'''SELECT most_common_vals AS sample_values FROM pg_stats WHERE tablename = '{row[\"table_name\"]}' AND schemaname = '{row[\"table_schema\"]}' AND attname = '{row[\"column_name\"]}' '''\n\n column_samples_df=self.retrieve_df(get_column_sample_sql)\n # display(column_samples_df)\n sample_column_list.append(column_samples_df['sample_values'].to_string(index=False).replace(\"{\",\"\").replace(\"}\",\"\"))\n\n columns_df[\"sample_values\"]=sample_column_list\n return columns_df\n" }, { "path": "dbconnectors/__init__.py", "content": "from .core import DBConnector\nfrom .PgConnector import PgConnector, pg_specific_data_types\nfrom .BQConnector import BQConnector, bq_specific_data_types\nfrom .FirestoreConnector import FirestoreConnector\nfrom utilities import (PROJECT_ID, \n PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION,BQ_REGION,\n BQ_OPENDATAQNA_DATASET_NAME,BQ_LOG_TABLE_NAME)\n\npgconnector = PgConnector(PROJECT_ID, PG_REGION, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD)\nbqconnector = BQConnector(PROJECT_ID,BQ_REGION,BQ_OPENDATAQNA_DATASET_NAME,BQ_LOG_TABLE_NAME)\nfirestoreconnector = FirestoreConnector(PROJECT_ID,\"opendataqna-session-logs\")\n\n__all__ = [\"pgconnector\", \"pg_specific_data_types\", \"bqconnector\",\"firestoreconnector\"]" }, { "path": "dbconnectors/core.py", "content": "\"\"\"\nProvides the base class for all Connectors \n\"\"\"\n\n\nfrom abc import ABC\n\n\nclass DBConnector(ABC):\n \"\"\"\n The core class for all Connectors\n \"\"\"\n\n connectorType: str = \"Base\"\n\n def __init__(self,\n project_id:str, \n region:str, \n instance_name:str,\n database_name:str, \n database_user:str, \n database_password:str,\n dataset_name:str):\n \"\"\"\n Args:\n project_id (str | None): GCP Project Id.\n dataset_name (str): \n TODO\n \"\"\"\n self.project_id = project_id\n self.region = region \n self.instance_name = instance_name \n self.database_name = database_name\n self.database_user = database_user\n self.database_password = database_password\n self.dataset_name = dataset_name\n " }, { "path": "docs/README.md", "content": "This directory contains documentation and resources to help you understand and use the Open Data QnA library effectively.\n\n## Contents\n\n* **README.md:** This file. Provides an overview of the documentation in this directory.\n* **best_practices.md:** Best practices and guidelines for using the library, including recommended configurations, tips for improving performance, and common pitfalls to avoid.\n* **faq.md:** Frequently asked questions about the library, covering common issues, troubleshooting tips, and general usage guidance.\n* **repo_structure.md:** A detailed explanation of the library's repository structure, including the purpose of each file and directory, and how to navigate the codebase.\n\n\n## How to Use This Documentation\n**Start with the README.md on the root dir:** This file provides a high-level overview and guides you to the relevant resources.\n**Consult the FAQ:** If you have any questions or encounter issues, check the FAQ section for possible solutions and answers.\n**Explore Best Practices:** For optimizing your usage and getting the most out of the library, review the best practices document.\n**Understand the Codebase:** If you want to dive deeper into the library's code, refer to the repository structure document for a detailed explanation of how the code is organized.\n" }, { "path": "docs/architecture.md", "content": "Architecture\n-------------\n

\n \n \"aaie\n \n

\n\n\n\nArchitecture Summary\n-------------\nOpen Data QnA operates in a sequence of well-defined steps, orchestrating various agents to process user queries and generate informative responses:\n\n* **Vector Store Creation:** The vector store is initialized, storing embeddings of known good SQL queries, table schemas, and column details. This serves as a knowledge base for retrieval-augmented generation (RAG).\n\n* **RAG (Retrieval-Augmented Generation):** User queries are embedded and compared to the vector store to retrieve relevant context (table/column details and similar past queries) for improved query generation.\n\n* **SQL Generation (BuildSQLAgent):** The BuildSQLAgent leverages the retrieved context and the user's natural language question to generate an initial SQL query.\n\n* **Optional Validation (ValidateSQLAgent):** If enabled, the ValidateSQLAgent assesses the generated SQL for syntactic and semantic correctness.\n\n* **Optional Debugging (DebugSQLAgent):** If the initial SQL is invalid and debugging is enabled, the DebugSQLAgent iteratively refines the query based on error feedback.\n\n* **SQL Execution (Dry Run/Explain):** The refined SQL query is tested with a dry run (BigQuery) or explain plan (PostgreSQL) to estimate resource usage and identify potential errors.\n\n* **SQL Execution (Full Run):** If the query is deemed valid, it's executed against the database to fetch the results.\n\n* **Response Generation (ResponseAgent):** The ResponseAgent analyzes the SQL results and the user's question to generate a natural language response, providing a clear and concise answer.\n\n* **Optional Visualization (VisualizeAgent):** If enabled, the VisualizeAgent suggests suitable chart types and generates JavaScript code for Google Charts to display the SQL results in a visually appealing manner.\n\n\n**Key Points:**\n\n* **Modularity:** Each step is handled by a specialized agent, allowing for flexibility and customization.\n* **RAG Enhancement:** The use of retrieval-augmented generation leverages existing knowledge for better query formulation.\n* **Validation and Debugging:** Optional agents enhance the reliability and accuracy of generated queries.\n* **Informative Responses:** The ResponseAgent aims to provide meaningful and contextually relevant answers.\n* **Visual Appeal:** The optional visualization adds an interactive layer to the user experience.\n" }, { "path": "docs/best_practices.md", "content": "# Open Data QnA: Best Practices\r\n\r\n## General Usage \r\n\r\n### Select the Right Database Connector: \r\nChoose between `PgConnector`(Google Cloud SQL PostgreSQL) and `BQConnector`(BigQuery) to match your specific database. \r\n\r\n### Prepare your data: \r\nEnsure your database tables are structured logically with appropriate column names and data types. We further recommend adding concise descriptions to tables and columns to provide the LLM agents with the necessary context. \r\nAdditionally, please ensure that the overall data quality of your database is good - if you have pattern mismatches or missing values, these will impact the performance of the Open Data QnA solution. \r\n\r\n### Start simple: \r\nBegin with straightforward questions and fewer tables and progressively experiment with more complex queries and adding more tables. \r\n\r\n### Leverage the ‘Known Good SQL’ Cache\r\nThe `Known Good SQL` cache can (and should) be populated with example user question <-> SQL query pairs relating to your use case. This benefits the solution in two ways: \r\nCaching layer reduces latency: if a known user question is found in the cache that exactly matches (meaning, each char is matching, down to punctuation) the new input question, the known good SQL query is fetched and SQL generation will be skipped. \r\nIn Context Learning: if a known user question is found to be similar to one of the existing queries in the cache, the similar user question is retrieved along with the corresponding SQL query and used as a few-shot example in the prompt for the SQL Generation agent. The user can specify how many example values should be retrieved to use as few-shot examples. We recommend using 3-5 examples, but this further depends on the variations of user questions you expect in your use case. \r\n\r\n### Explore Visualizations\r\nUtilize the `VisualizeAgent`to generate charts and graphs for a more intuitive understanding of your data. However, make sure to only run the agent on queries that the pipeline has flagged as ‘valid’. \r\n\r\n\r\n\r\n## Customization & Optimization\r\n### Agent Modification \r\nThe `core`Agent class (agents/core.py) specifies the models supported for the different agents in the Open Data QnA solution. \r\n\r\nIn version 1, these are: \r\n- Code Bison ('code-bison-32k')\r\n- Text Bison ('text-bison-32k')\r\n- Codechat Bison ('codechat-bison-32k') \r\n- Gemini 1.0 pro ('gemini-1.0-pro')\r\n\r\nYou can set the different models for each agent when calling the pipeline_run function (see below under `Pipeline Run Configurations`). \r\n\r\n### Prompt Engineering \r\nEach of the defined agents has their own prompt specified in its agent class file. \r\nBuildSQLAgent.py: prompts for BigQuery and PostgreSQL SQL Generation. \r\nDebugSQLAgent.py: prompts for debugging for either BQ or PG queries. \r\nDescriptionAgent.py: prompts for generating missing table and column descriptions. \r\nResponseAgent.py: prompt to generate a natural language response, answering the user question by using the output of the generated SQL query. \r\nValidateSQLAgent.py: prompt to classify a given SQL as valid or invalid. \r\nVisualizeAgent.py two prompts; one for proposing a fitting graph / plot for a given question <-> SQL pair; the other for generating the visualization. \r\n\r\n\r\n### Pipeline Run Configurations \r\nAdditionally to changing the base models and the prompts, it is advisable to experiment with different configuration settings of the pipeline run function: \r\n```\r\nasync def run_pipeline(user_question,\r\n RUN_DEBUGGER=True,\r\n EXECUTE_FINAL_SQL=True,\r\n DEBUGGING_ROUNDS = 2,\r\n LLM_VALIDATION=True,\r\n SQLBuilder_model= 'gemini-1.0-pro',\r\n SQLChecker_model= 'gemini-1.0-pro',\r\n SQLDebugger_model= 'gemini-1.0-pro',\r\n Responder_model= 'gemini-1.0-pro',\r\n num_table_matches = 5,\r\n num_column_matches = 10,\r\n table_similarity_threshold = 0.3,\r\n column_similarity_threshold = 0.3,\r\n example_similarity_threshold = 0.3,\r\n num_sql_matches=3)\r\n```\r\n\r\n\r\nArgs:\r\n\r\n* **user_question (str):** The natural language question to answer.\r\n* **RUN_DEBUGGER (bool, optional):** Whether to run the SQL debugger. Defaults to True.\r\nIt is recommended to use the debugger for improved SQL Generation accuracy.\r\n* **DEBUGGING_ROUNDS (int, optional):** The number of debugging rounds. Defaults to 2.\r\nWe suggest using a value between 2-5, depending on your accuracy and latency requirements. \r\n* **EXECUTE_FINAL_SQL (bool, optional):** Whether to execute the final SQL query. Defaults to True.\r\nYou can disable the SQL execution. This will leave you with the generated SQL query as a response, skipping the retrieval of the execution result and the response generation. \r\n* **LLM_VALIDATION (bool, optional):** Whether to use LLM for SQL validation during debugging. Defaults to True.\r\nYou can disable the SQL Validator if you have specific latency requirements. When disabled, the Debugger will execute a dry run to retrieve any errors from the database call and debug accordingly. \r\n* **SQLBuilder_model (str, optional):** The name of the SQL building model. Defaults to 'gemini-1.0-pro'.\r\n* **SQLChecker_model (str, optional):** The name of the SQL validation model. Defaults to 'gemini-1.0-pro'.\r\n* **SQLDebugger_model (str, optional):** The name of the SQL debugging model. Defaults to 'gemini-1.0-pro'.\r\n* **Responder_model (str, optional):** The name of the response generation model. Defaults to 'gemini-1.0-pro'.\r\n* **num_table_matches (int, optional):** The number of similar tables to retrieve. Defaults to 5.\r\nThese will be used when calling the SQL Generation Agent. \r\nWe recommend setting this higher if you have high variations in your database and user queries. \r\n* **num_column_matches (int, optional):** The number of similar columns to retrieve. Defaults to 10.\r\nThese will be used when calling the SQL Generation Agent. \r\nWe recommend setting this higher if you have high variations in your database and user queries. \r\n* **table_similarity_threshold (float, optional):** The similarity threshold for tables. Defaults to 0.3.\r\nStart with higher values and gradually decrease them if you’re not getting enough relevant results. \r\n* **column_similarity_threshold (float, optional):** The similarity threshold for columns. Defaults to 0.3.\r\nStart with higher values and gradually decrease them if you’re not getting enough relevant results. \r\n* **example_similarity_threshold (float, optional):** The similarity threshold for example questions. Defaults to 0.3.\r\nStart with higher values and gradually decrease them if you’re not getting enough relevant results. \r\n* **num_sql_matches (int, optional):** The number of similar SQL queries to retrieve. Defaults to 3.\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n\r\n" }, { "path": "docs/changelog.md", "content": "# Release Notes - Open Data QnA v2.0.0\nThis major release brings significant improvements and new features to Open Data QnA.\n\n## Multi turn capabilities\nAbility to interact back and forth with the database in a context. Initial v1 was established with a single turn query. In this release, we have created a multi turn architecture that saves the session info, previous query information and can answer accordingly. For more information on the architecture: link \n\n## Table Grouping\n\nInitial v1 was tied to single dataset processing and all the tables under this dataset. In reality, users most likely want to restrict the tables and add other datasets if needed. This table grouping provides a way for users to be able to define their scope \n

\n \n \"aaie\n \n

\n\n## Data Sampling\nWe provide a sampling of data values in a column to provide contextual information to the SQL Generation agent. For this, top 5 values are retrieved for every column in the specified tables. \nThis information is aggregated and stored back into the vector store, and is retrieved during the retrieval process. \n

\n \n \"aaie\n \n

\n

\n \n \"aaie\n \n

\n\n\n## Data summarization\nIn the initial V1 release, the results were in tabular format. With this release , we provide summarized answers in a natural language format that can be integrated into a chatbot. User does have an option to still get the tabular and visualized results based on their settings.\n

\n \n \"aaie\n \n

\n\n## Resolving ambiguities\nThe multi-turn approach helps to resolve ambiguities in the questions, by allowing the user to provide follow-up questions and clarifications. \n\nFurthermore, it is possible to provide additional context in the instruction prompt to let the LLM resolve ambiguities before triggering the pipeline. This can be achieved with the help of a LLM router added as a first layer before the Open Data QnA pipeline. \nThese clarification questions can help provide more context to the SQL creation.\n\nAmbiguities can be categorized into semantic, application, business and database context. With this release we look for semantic and business level context and resolve such ambiguities through the chat interface. \n\n## UX through Flutter and Streamlit\n\nIn addition to the AngularJS, we have added support through Flutter as part of the release which can be found under the front end code folder. \nFurthermore, to enable more efficient development, we have added support for streamlit, so users can quickly iterate and test in a dev frontend before deploying to Angular or Flutter. \n\n\n\n\n\n# Release Notes - Open Data QnA v1.2.0\nThis release brings significant improvements and new features to enhance the stability, functionality, and user experience of the Open Data QnA.\n\n## 🗝️ Key Enhancements:\n* **Enhanced Functionality:** Added the ability to specify a list of table names to be processed in BQ, instead of parsing all tables in a dataset. \n* **Improved Debugging:** The SQL debugger now incorporates the user's question into its prompts, leading to more accurate and relevant debugging suggestions.\n* **Simplified Setup:** Streamlined notebook setup and environment variable management for a smoother user experience.\n* **Quickstart**: Added a standalone notebook for quick experimentation with the overall approach, limited to BQ. \n* **Flexible Configuration:** Introduced optional arguments for the CLI pipeline, allowing users to customize various parameters like table and column similarity thresholds.\n* **Code Refinements:** Removed hardcoded embedding models and added a save_config function for cleaner configuration management.\n* **Bug Fixes:** Resolved various bugs, including issues with root directory checking, utility initialization, source type determination, and safety settings.\n* **Expanded Documentation:** Added comprehensive docstrings to functions for better clarity and understanding.\n\n## 📈 Additional Improvements:\n* **Code Cleanup:** Removed unnecessary files and redundant code, improving overall code maintainability.\n* **Updated README:** Improved the README file with clearer instructions and updated information.\n* **Enhanced User Interface:** Introduced a CLI approach (experimental) for more streamlined interaction.\n\n## 🐜 Bug Fixes:\n* Fixed bugs in standalone notebook functionality.\n* Removed telemetry test code.\n* Corrected embedding distances in BigQuery.\n* Resolved various typos and inconsistencies in the codebase.\n\nThis release marks a significant step forward in the development of the Open Data QnA SQL Generation tool, making it more reliable, flexible, and user-friendly. We encourage you to upgrade and explore the new features!\n" }, { "path": "docs/config_guide.md", "content": "## Follow the below guide to populate your config.ini file: \n\n______________\n\n**[CONFIG]**\n\n**embedding_model = vertex** *;Options: 'vertex' or 'vertex-lang'*\n\n**description_model = gemini-1.0-pro** *;Options 'gemini-1.0-pro', 'gemini-1.5-pro', 'text-bison-32k', 'gemini-1.5-flash'*\n\n**vector_store = cloudsql-pgvector** *;Options: 'bigquery-vector', 'cloudsql-pgvector'*\n\n**debugging = yes** *;if debugging is enabled. yes or no*\n\n**logging = yes** *;if logging is enabled. yes or no* \n\n**kgq_examples = yes** *;if known-good-queries are provided. yes or no.* \n\n**use_session_history = yes** *;if you want to use current session's questions without re-evaluating them*\n\n**use_column_samples = yes** *;if you want the solution to collect some samples values from the data source columns to imporve understanding of values. yes or no*\n\n**[GCP]**\n\n**project_id = my_project** *;your GCP project* \n\n\n*; fill out the values below if you want to use PG as your vector database:*\n\n**[PGCLOUDSQL]**\n\n**pg_region = us-central1** \n\n**pg_instance = pg15-opendataqna**\n\n**pg_database = opendataqna-db**\n\n**pg_user = pguser**\n\n**pg_password = pg123**\n\n\n*; fill out the values below if you want to use BQ as your vector database:* \n\n**[BIGQUERY]**\n\n\n*; the remaining values are the settings for the BQ vector store / log dataset and table created by the solution:* \n\n**bq_dataset_region = us-central1**\n\n**bq_opendataqna_dataset_name = opendataqna**\n\n**bq_log_table_name = audit_log_table**\n\n**firestore_region = us-central** *;region for NoSQL DB firestore region to deploy*\n\n\n________________\n" }, { "path": "docs/faq.md", "content": "# Open Data QnA: FAQ\r\n\r\n## Source and Vector Store Setup\r\n**Q: If new to the vector store concept, which vector store would you recommend?**\r\n\r\nA: Both the vector stores (pgvector and bigquery vector) are created using embedding model as you specify and also the vector search for both the vector stores are using cosine similarity to find the nearest matches. You can choose bigquery vector as that avoids any extra resource like cloudsql. \r\nVector Embeddings and Search\r\n\r\n________\r\n\r\n**Q: Why are my example SQLs not being pulled as few-shot examples for the question asked even though the question is almost similar?**\r\n\r\nA: Verify if the embedding of the example question has happened successfully.\r\nCheck the retrieval SQL written to pull the similar sqls for a few shot examples. If the cosine similarity logic is wrong that might be the reason for the issue. Correct the SQL to pull required similarity based SQLs\r\n\r\n## Accuracy and Latency\r\n\r\n**Q: How accurate are the results?**\r\n\r\nA: Depending on the context, the more accurate these are helpful with accuracy.\r\nBuilding blocks such as known good sql, validation all help with accuracy\r\n\r\n________\r\n**Q: How is the latency overall?**\r\nA: Ambiguous questions have increased latency. If latency is a factor, would suggest adding caching layer and reducing validation steps\r\nV2 is also coming up with resolving ambiguity\r\n\r\n\r\n\r\n## Overall Solution\r\n**Q: How do I get started quickly?**\r\n\r\nA: The quickest way is to follow the \"Quickstart with Open Data QnA: Standalone BigQuery Notebook.\" It provides a simplified experience using BigQuery. If you need more customization, follow the instructions for setting up the main repository.\r\n\r\n________\r\n**Q: Which databases does Open Data QnA currently support?**\r\n\r\nA: Currently, it supports Google Cloud SQL for PostgreSQL and Google BigQuery.\r\n\r\n________\r\n**Q: What are the requirements to use Open Data QnA?**\r\n\r\nA: You'll need:\r\nA Google Cloud Project\r\nAn active database (PostgreSQL or BigQuery)\r\nPython 3.9 or higher\r\nRequired Python packages (listed in requirements.txt)\r\n\r\n________\r\n**Q: Can I customize the behavior of the agents?**\r\n\r\nA: Yes, the agents are designed to be modular and extensible. You can modify their code or create your own custom agents.\r\n\r\n________\r\n**Q: How do I incorporate my own known good SQL queries into the system?**\r\n\r\nA: Follow the setup instructions or use the \"3. Loading Known Good SQL Examples\" notebook to add your own SQL queries to the vector store. This will improve the accuracy of query generation through RAG.\r\n________\r\n\r\n**Q: How do I set the table, column, and example similarity thresholds?**\r\n\r\nA: These thresholds are used during the Retrieval-Augmented Generation (RAG) process to determine how similar your query is to the stored embeddings.\r\nTable Similarity Threshold: Determines how closely a user's query needs to match a table name in the vector store to be considered relevant. Higher values make the matching stricter.\r\nColumn Similarity Threshold: Similar to the table threshold, but for column names.\r\nExample Similarity Threshold: Controls how closely a user's query needs to match a known good SQL query example to be considered similar.\r\nYou can adjust these thresholds when running the pipeline_run function. Start with the default values and experiment to find what works best for your specific data and queries. Generally, start with higher values and gradually decrease them if you're not getting enough relevant results.\r\n\r\n________\r\n**Q: Can I visualize the results of my queries?**\r\n\r\nA: Yes, the VisualizeAgent can generate JavaScript code for Google Charts to create visualizations of your data.\r\n________\r\n\r\n**Q: Are all building blocks mandatory?**\r\n\r\nA: No. They can be replaced\r\n________\r\n\r\n**Q: Can this be tested against any database?**\r\n\r\nA: Tested against Oracle and Snowflake\r\n________\r\n**Q: How are the competitors doing?**\r\n\r\nA: Few langchain labs, some experimenting with agents\r\n________\r\n\r\n**Q: I created a test colab with langchain and a simple implementation. Why complicate it?**\r\n\r\nA: If your environment is not complex, we would suggest to leverage your simplified approach, or look into the [standalone notebook](/notebooks/(standalone)Run_OpenDataQnA.ipynb) \r\n\r\n\r\n" }, { "path": "docs/repo_structure.md", "content": "Repository Structure \n-------------\n\n```\n.\n├── agents\n └── __init__.py\n └── core.py\n └── BuildSQLAgent.py\n └── DebugSQLAgent.py\n └── DescriptionAgent.py\n └── EmbedderAgent.py\n └── ResponseAgent.py\n └── ValidateSQLAgent.py\n └── VisualizeAgent.py\n└── Dockerfile\n└── backend-apis\n └── __init__.py\n └── policy.yaml\n └── main.py\n└── dbconnectors\n └── __init__.py\n └── core.py\n └── PgConnector.py\n └── BQConnector.py\n└── docs\n └── best_practices.md\n └── faq.md\n └── repo_structure.md\n└── embeddings\n └── __init__.py\n └── retrieve_embeddings.py\n └── store_embeddings.py\n └── kgq_embeddings.py\n└── frontend\n└── notebooks\n └── 0_CopyDataToBigQuery.ipynb\n └── 0_CopyDataToCloudSqlPG.ipynb\n └── 1_Setup_OpenDataQnA.ipynb\n └── 2_Run_OpenDataQnA.ipynb\n └── 3_LoadKnownGoodSQL.ipynb\n└── scripts\n └── tables_columns_descriptions.csv\n └── copy_select_table_column_bigquery.csv\n └── data_source_list.csv\n └── known_good_sql.csv\n └── save_config.py\n └── Scenarios Sample.csv\n└── utilities\n └── __init__.py\n└── prompts.yaml\n└── pyproject.toml\n└── config.ini\n└── env_setup.py\n└── opendataqna.py\n```\n\n- [`/agents`](/agents): Source code for the LLM Agents. \n- [`/backend-apis`](/backend-apis/) : Cloud Run based api deployement for frontend to demo the solution on a UI\n- [`/dbconnectors`](/dbconnectors): Source code for database connectors.\n- [`/docs`](/docs): Documentations, including FAQ & Best Practices for using this library. \n- [`/embeddings`](/embeddings): Source code for creating and storing embeddings.\n - [`/retrieve_embeddings.py`](/embeddings/retrieve_embeddings.py): Source code for retrieving table schema and embedding creation. \n - [`/store_embeddings.py`](/embeddings/store_embeddings.py): Source code for storing table schema embeddings in Vector Store.\n - [`/kgq_embeddings.py`](/embeddings/kgq_embeddings.py): Source code for loading good sqls and creating embeddings in the Vector Store) \n- [`/frontend`](/frontend) : Angular based frontend code to deploy demo app using the API developed with [`/main.py`](backend-apis/main.py)\n- [`/notebooks`](/notebooks): Sample notebooks demonstrating the usage of this library. \n- [`/scripts`](/scripts): Additional scripts for initial setup.\n - [`/Sample Scenarios.csv`](/scripts/Scenarios%20Sample.csv): Sample Scenarios file that can used to load them on the frontend UI for demos\n - [`/copy_select_table_column_bigquery.py`](/scripts/copy_select_table_column_bigquery.py): Code Sample to copy select tables and columns from one BQ table to another; add table and column descriptions from csv file.\n - [`/tables_columns_descriptions.csv`](/scripts/tables_columns_descriptions.csv): CSV file containing table and column names and descriptions to be copied \n - [`/known_good_sql.csv`](/scripts/known_good_sql.csv): CSV files\n - [`/data_source_list.csv`](/scripts/data_source_list.csv): Data Source CSV File to mention the list of tables and source type etc.\n- [`/Dockerfile`](/Dockerfile): Dockerfile for deployment of backend apis. It is placed at the root folder to give it right context and access to the files.\n- [`/env_setup.py`](/env_setup.py): Python file for initial setup. \n- [`/opendataqna.py`](/opendataqna.py): Python file for running the main pipeline. \n- [`/prompts.yaml`](/prompts.yaml): Yaml file that contains the prompts used by the solution. It also provides users the ability to prompt extra context for the use case if any.\n" }, { "path": "embeddings/__init__.py", "content": "from .retrieve_embeddings import retrieve_embeddings\nfrom .store_embeddings import store_schema_embeddings\nfrom .kgq_embeddings import store_kgq_embeddings, setup_kgq_table, load_kgq_df\n\n\n\n__all__ = [\"retrieve_embeddings\", \"store_schema_embeddings\",\"store_kgq_embeddings\", \"setup_kgq_table\", \"load_kgq_df\"]" }, { "path": "embeddings/kgq_embeddings.py", "content": "import os\nimport asyncio\nimport asyncpg\nimport pandas as pd\nimport numpy as np\nfrom pgvector.asyncpg import register_vector\nfrom google.cloud.sql.connector import Connector\nfrom langchain_community.embeddings import VertexAIEmbeddings\nfrom google.cloud import bigquery\nfrom dbconnectors import pgconnector\nfrom agents import EmbedderAgent\nfrom sqlalchemy.sql import text\nfrom utilities import PROJECT_ID, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION, BQ_OPENDATAQNA_DATASET_NAME, BQ_REGION\n\nembedder = EmbedderAgent('vertex')\n\n\nasync def setup_kgq_table( project_id,\n instance_name,\n database_name,\n schema,\n database_user,\n database_password,\n region,\n VECTOR_STORE = \"cloudsql-pgvector\"):\n \"\"\" \n This function sets up or refreshes the Vector Store for Known Good Queries (KGQ)\n \"\"\"\n if VECTOR_STORE=='bigquery-vector':\n\n # Create BQ Client\n client=bigquery.Client(project=project_id)\n\n # Delete an old table\n # client.query_and_wait(f'''DROP TABLE IF EXISTS `{project_id}.{schema}.example_prompt_sql_embeddings`''')\n # Create a new emptry table\n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{project_id}.{schema}.example_prompt_sql_embeddings` (\n user_grouping string NOT NULL, example_user_question string NOT NULL, example_generated_sql string NOT NULL,\n embedding ARRAY)''')\n \n\n elif VECTOR_STORE=='cloudsql-pgvector':\n\n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # Create connection to Cloud SQL database\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{project_id}:{region}:{instance_name}\", # Cloud SQL instance connection name\n \"asyncpg\",\n user=f\"{database_user}\",\n password=f\"{database_password}\",\n db=f\"{database_name}\",\n )\n\n # Drop on old table\n # await conn.execute(\"DROP TABLE IF EXISTS example_prompt_sql_embeddings\")\n # Create a new emptry table\n await conn.execute(\n \"\"\"CREATE TABLE IF NOT EXISTS example_prompt_sql_embeddings(\n user_grouping VARCHAR(1024) NOT NULL,\n example_user_question text NOT NULL,\n example_generated_sql text NOT NULL,\n embedding vector(768))\"\"\"\n )\n\n else: raise ValueError(\"Not a valid parameter for a vector store.\")\n\nasync def store_kgq_embeddings(df_kgq, \n project_id,\n instance_name,\n database_name,\n schema,\n database_user,\n database_password,\n region,\n VECTOR_STORE = \"cloudsql-pgvector\"\n ):\n \"\"\" \n Create and save the Known Good Query Embeddings to Vector Store \n \"\"\"\n if VECTOR_STORE=='bigquery-vector':\n\n client=bigquery.Client(project=project_id)\n \n example_sql_details_chunked = []\n\n for _, row_aug in df_kgq.iterrows():\n\n example_user_question = str(row_aug['prompt'])\n example_generated_sql = str(row_aug['sql'])\n example_grouping = str(row_aug['user_grouping'])\n emb = embedder.create(example_user_question)\n \n\n r = {\"example_grouping\":example_grouping,\"example_user_question\": example_user_question,\"example_generated_sql\": example_generated_sql,\"embedding\": emb}\n example_sql_details_chunked.append(r)\n\n example_prompt_sql_embeddings = pd.DataFrame(example_sql_details_chunked)\n\n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{project_id}.{schema}.example_prompt_sql_embeddings` (\n user_grouping string NOT NULL, example_user_question string NOT NULL, example_generated_sql string NOT NULL,\n embedding ARRAY)''')\n\n for _, row in example_prompt_sql_embeddings.iterrows():\n client.query_and_wait(f'''DELETE FROM `{project_id}.{schema}.example_prompt_sql_embeddings`\n WHERE user_grouping= '{row[\"example_grouping\"]}' and example_user_question= \"{row[\"example_user_question\"]}\" '''\n )\n # embedding=np.array(row[\"embedding\"])\n cleaned_sql = row[\"example_generated_sql\"].replace(\"\\r\", \" \").replace(\"\\n\", \" \")\n client.query_and_wait(f'''INSERT INTO `{project_id}.{schema}.example_prompt_sql_embeddings` \n VALUES (\"{row[\"example_grouping\"]}\",\"{row[\"example_user_question\"]}\" , \n \"{cleaned_sql}\",{row[\"embedding\"]} )''')\n \n \n\n\n elif VECTOR_STORE=='cloudsql-pgvector':\n\n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # Create connection to Cloud SQL database\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{project_id}:{region}:{instance_name}\", # Cloud SQL instance connection name\n \"asyncpg\",\n user=f\"{database_user}\",\n password=f\"{database_password}\",\n db=f\"{database_name}\",\n )\n\n\n example_sql_details_chunked = []\n \n for _, row_aug in df_kgq.iterrows():\n\n example_user_question = str(row_aug['prompt'])\n example_generated_sql = str(row_aug['sql'])\n example_grouping = str(row_aug['user_grouping'])\n\n emb = embedder.create(example_user_question)\n\n r = {\"example_grouping\":example_grouping,\"example_user_question\": example_user_question,\"example_generated_sql\": example_generated_sql,\"embedding\": emb}\n example_sql_details_chunked.append(r)\n\n example_prompt_sql_embeddings = pd.DataFrame(example_sql_details_chunked)\n \n for _, row in example_prompt_sql_embeddings.iterrows():\n await conn.execute(\n \"DELETE FROM example_prompt_sql_embeddings WHERE user_grouping= $1 and example_user_question=$2\",\n row[\"example_grouping\"],\n row[\"example_user_question\"])\n cleaned_sql = row[\"example_generated_sql\"].replace(\"\\r\", \" \").replace(\"\\n\", \" \")\n await conn.execute(\n \"INSERT INTO example_prompt_sql_embeddings (user_grouping, example_user_question, example_generated_sql, embedding) VALUES ($1, $2, $3, $4)\",\n row[\"example_grouping\"],\n row[\"example_user_question\"],\n cleaned_sql,\n str(row[\"embedding\"]),\n )\n\n await conn.close()\n\n else: raise ValueError(\"Not a valid parameter for a vector store.\")\n\n\ndef load_kgq_df():\n import pandas as pd\n \n def is_root_dir():\n current_dir = os.getcwd()\n notebooks_path = os.path.join(current_dir, \"notebooks\")\n agents_path = os.path.join(current_dir, \"agents\")\n \n return os.path.exists(notebooks_path) or os.path.exists(agents_path)\n\n if is_root_dir():\n current_dir = os.getcwd()\n root_dir = current_dir\n else:\n root_dir = os.path.abspath(os.path.join(os.getcwd(), '..'))\n\n file_path = root_dir + \"/scripts/known_good_sql.csv\"\n\n # Load the file\n df_kgq = pd.read_csv(file_path)\n df_kgq = df_kgq.loc[:, [\"prompt\", \"sql\", \"user_grouping\"]]\n df_kgq = df_kgq.dropna()\n\n return df_kgq\n\n\n\nif __name__ == '__main__': \n from utilities import PROJECT_ID, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION\n VECTOR_STORE = \"cloudsql-pgvector\"\n \n current_dir = os.getcwd()\n root_dir = os.path.expanduser('~') # Start at the user's home directory\n\n while current_dir != root_dir:\n for dirpath, dirnames, filenames in os.walk(current_dir):\n config_path = os.path.join(dirpath, 'known_good_sql.csv')\n if os.path.exists(config_path):\n file_path = config_path # Update root_dir to the found directory\n break # Stop outer loop once found\n\n current_dir = os.path.dirname(current_dir)\n\n print(\"Known Good SQL Found at Path :: \"+file_path)\n\n # Load the file\n df_kgq = pd.read_csv(file_path)\n df_kgq = df_kgq.loc[:, [\"prompt\", \"sql\", \"database_name\"]]\n df_kgq = df_kgq.dropna()\n\n print('Known Good SQLs Loaded into a Dataframe')\n\n asyncio.run(setup_kgq_table(PROJECT_ID,\n PG_INSTANCE,\n PG_DATABASE,\n PG_USER,\n PG_PASSWORD,\n PG_REGION,\n VECTOR_STORE))\n\n asyncio.run(store_kgq_embeddings(df_kgq,\n PROJECT_ID,\n PG_INSTANCE,\n PG_DATABASE,\n PG_USER,\n PG_PASSWORD,\n PG_REGION,\n VECTOR_STORE))\n" }, { "path": "embeddings/retrieve_embeddings.py", "content": "import re\nimport io\nimport sys \nimport pandas as pd\nfrom dbconnectors import pgconnector,bqconnector\nfrom agents import EmbedderAgent, ResponseAgent, DescriptionAgent\nfrom utilities import EMBEDDING_MODEL, DESCRIPTION_MODEL, USE_COLUMN_SAMPLES\n\nembedder = EmbedderAgent(EMBEDDING_MODEL)\n# responder = ResponseAgent('gemini-1.0-pro')\ndescriptor = DescriptionAgent(DESCRIPTION_MODEL)\n\n\ndef get_embedding_chunked(textinput, batch_size): \n for i in range(0, len(textinput), batch_size):\n request = [x[\"content\"] for x in textinput[i : i + batch_size]]\n response = embedder.create(request) # Vertex Textmodel Embedder \n\n # Store the retrieved vector embeddings for each chunk back.\n for x, e in zip(textinput[i : i + batch_size], response):\n x[\"embedding\"] = e\n\n # Store the generated embeddings in a pandas dataframe.\n out_df = pd.DataFrame(textinput)\n return out_df\n\n\ndef retrieve_embeddings(SOURCE, SCHEMA=\"public\", table_names = None): \n \"\"\" Augment all the DB schema blocks to create document for embedding \"\"\"\n\n if SOURCE == \"cloudsql-pg\":\n \n table_schema_sql = pgconnector.return_table_schema_sql(SCHEMA,table_names=table_names)\n table_desc_df = pgconnector.retrieve_df(table_schema_sql)\n \n column_schema_sql = pgconnector.return_column_schema_sql(SCHEMA,table_names=table_names)\n column_name_df = pgconnector.retrieve_df(column_schema_sql)\n \n\n #GENERATE MISSING DESCRIPTIONS\n table_desc_df,column_name_df= descriptor.generate_missing_descriptions(SOURCE,table_desc_df,column_name_df)\n\n #ADD SAMPLES VALUES FOR COLUMNS\n column_name_df[\"sample_values\"]=None\n \n if USE_COLUMN_SAMPLES:\n column_name_df = pgconnector.get_column_samples(column_name_df)\n \n \n \n ### TABLE EMBEDDING ###\n \"\"\"\n This SQL returns a df containing the cols table_schema, table_name, table_description, table_columns (with cols in the table)\n for the schema specified above, e.g. 'retail'\n \"\"\"\n table_details_chunked = []\n\n for index_aug, row_aug in table_desc_df.iterrows():\n\n cur_table_name = str(row_aug['table_name'])\n cur_table_schema = str(row_aug['table_schema'])\n curr_col_names = str(row_aug['table_columns'])\n curr_tbl_desc = str(row_aug['table_description'])\n\n\n table_detailed_description=f\"\"\"\n Table Name: {cur_table_name} |\n Schema Name: {cur_table_schema} |\n Table Description - {curr_tbl_desc}) | \n Columns List: [{curr_col_names}]\"\"\"\n\n r = {\"table_schema\": cur_table_schema,\"table_name\": cur_table_name,\"content\": table_detailed_description}\n table_details_chunked.append(r)\n\n table_details_embeddings = get_embedding_chunked(table_details_chunked, 10)\n\n\n ### COLUMN EMBEDDING ###\n \"\"\"\n This SQL returns a df containing the cols table_schema, table_name, column_name, data_type, column_description, table_description, primary_key, column_constraints\n for the schema specified above, e.g. 'retail'\n \"\"\"\n \n column_details_chunked = []\n\n for index_aug, row_aug in column_name_df.iterrows():\n\n cur_table_name = str(row_aug['table_name'])\n cur_table_owner = str(row_aug['table_schema'])\n curr_col_name = str(row_aug['table_schema'])+'.'+str(row_aug['table_name'])+'.'+str(row_aug['column_name'])\n curr_col_datatype = str(row_aug['data_type'])\n curr_col_description = str(row_aug['column_description'])\n curr_col_constraints = str(row_aug['column_constraints'])\n curr_column_name = str(row_aug['column_name'])\n curr_column_samples = str(row_aug['sample_values'])\n\n\n column_detailed_description=f\"\"\"Schema Name:{cur_table_owner} | Column Name: {curr_col_name} (Data type: {curr_col_datatype}) | Table Name: {cur_table_name} | (column description: {curr_col_description})(constraints: {curr_col_constraints}) | (Sample Values in the Column: {curr_column_samples})\"\"\"\n\n r = {\"table_schema\": cur_table_owner,\"table_name\": cur_table_name,\"column_name\":curr_column_name, \"content\": column_detailed_description}\n column_details_chunked.append(r)\n\n column_details_embeddings = get_embedding_chunked(column_details_chunked, 10)\n\n\n elif SOURCE=='bigquery':\n\n table_schema_sql = bqconnector.return_table_schema_sql(SCHEMA, table_names=table_names)\n table_desc_df = bqconnector.retrieve_df(table_schema_sql)\n\n column_schema_sql = bqconnector.return_column_schema_sql(SCHEMA, table_names=table_names)\n column_name_df = bqconnector.retrieve_df(column_schema_sql)\n #GENERATE MISSING DESCRIPTIONS\n table_desc_df,column_name_df= descriptor.generate_missing_descriptions(SOURCE,table_desc_df,column_name_df)\n \n #ADD SAMPLES VALUES FOR COLUMNS\n column_name_df[\"sample_values\"]=None\n \n if USE_COLUMN_SAMPLES:\n column_name_df = bqconnector.get_column_samples(column_name_df)\n\n #TABLE EMBEDDINGS\n table_details_chunked = []\n\n for index_aug, row_aug in table_desc_df.iterrows():\n cur_project_name =str(row_aug['project_id'])\n cur_table_name = str(row_aug['table_name'])\n cur_table_schema = str(row_aug['table_schema'])\n curr_col_names = str(row_aug['table_columns'])\n curr_tbl_desc = str(row_aug['table_description'])\n \n\n\n table_detailed_description=f\"\"\"\n Full Table Name : {cur_project_name}.{cur_table_schema}.{cur_table_name} |\n Table Columns List: [{curr_col_names}] |\n Table Description: {curr_tbl_desc} \"\"\"\n\n r = {\"table_schema\": cur_table_schema,\"table_name\": cur_table_name,\"content\": table_detailed_description}\n table_details_chunked.append(r)\n\n table_details_embeddings = get_embedding_chunked(table_details_chunked, 10)\n\n\n ### COLUMN EMBEDDING ###\n \"\"\"\n This SQL returns a df containing the cols table_schema, table_name, column_name, data_type, column_description, table_description, primary_key, column_constraints\n for the schema specified above, e.g. 'retail'\n \"\"\"\n\n column_details_chunked = []\n\n for index_aug, row_aug in column_name_df.iterrows():\n cur_project_name =str(row_aug['project_id'])\n cur_table_name = str(row_aug['table_name'])\n cur_table_owner = str(row_aug['table_schema'])\n curr_col_name = str(row_aug['table_schema'])+'.'+str(row_aug['table_name'])+'.'+str(row_aug['column_name'])\n curr_col_datatype = str(row_aug['data_type'])\n curr_col_description = str(row_aug['column_description'])\n curr_col_constraints = str(row_aug['column_constraints'])\n curr_column_name = str(row_aug['column_name'])\n curr_column_samples = str(row_aug['sample_values'])\n\n\n column_detailed_description=f\"\"\"\n Column Name: {curr_col_name}|\n Full Table Name : {cur_project_name}.{cur_table_schema}.{cur_table_name} |\n Data type: {curr_col_datatype}|\n Column description: {curr_col_description}|\n Column Constraints: {curr_col_constraints}|\n Sample Values in the Column : {curr_column_samples}\"\"\"\n\n r = {\"table_schema\": cur_table_owner,\"table_name\": cur_table_name,\"column_name\":curr_column_name, \"content\": column_detailed_description}\n column_details_chunked.append(r)\n\n column_details_embeddings = get_embedding_chunked(column_details_chunked, 10)\n\n\n return table_details_embeddings, column_details_embeddings\n \n\n\nif __name__ == '__main__':\n SOURCE = 'cloudsql-pg'\n t, c = retrieve_embeddings(SOURCE, SCHEMA=\"public\") " }, { "path": "embeddings/store_embeddings.py", "content": "import asyncio\nimport asyncpg\nimport pandas as pd\nimport numpy as np\nfrom pgvector.asyncpg import register_vector\nfrom google.cloud.sql.connector import Connector\nfrom langchain_community.embeddings import VertexAIEmbeddings\nfrom google.cloud import bigquery\nfrom dbconnectors import pgconnector\nfrom agents import EmbedderAgent\nfrom sqlalchemy.sql import text\nfrom utilities import VECTOR_STORE, PROJECT_ID, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION, BQ_OPENDATAQNA_DATASET_NAME, BQ_REGION, EMBEDDING_MODEL\n\nembedder = EmbedderAgent(EMBEDDING_MODEL)\n\nasync def store_schema_embeddings(table_details_embeddings, \n tablecolumn_details_embeddings, \n project_id,\n instance_name,\n database_name,\n schema,\n database_user,\n database_password,\n region,\n VECTOR_STORE):\n \"\"\" \n Store the vectorised table and column details in the DB table.\n This code may run for a few minutes. \n \"\"\"\n\n if VECTOR_STORE == \"cloudsql-pgvector\":\n \n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # Create connection to Cloud SQL database.\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{project_id}:{region}:{instance_name}\", # Cloud SQL instance connection name\n \"asyncpg\",\n user=f\"{database_user}\",\n password=f\"{database_password}\",\n db=f\"{database_name}\",\n )\n\n await conn.execute(\"CREATE EXTENSION IF NOT EXISTS vector\")\n await register_vector(conn)\n\n # await conn.execute(f\"DROP SCHEMA IF EXISTS {pg_schema} CASCADE\") \n\n # await conn.execute(f\"CREATE SCHEMA {pg_schema}\") \n\n # await conn.execute(\"DROP TABLE IF EXISTS table_details_embeddings\")\n # Create the `table_details_embeddings` table to store vector embeddings.\n await conn.execute(\n \"\"\"CREATE TABLE IF NOT EXISTS table_details_embeddings(\n source_type VARCHAR(100) NOT NULL,\n user_grouping VARCHAR(100) NOT NULL,\n table_schema VARCHAR(1024) NOT NULL,\n table_name VARCHAR(1024) NOT NULL,\n content TEXT,\n embedding vector(768))\"\"\"\n )\n\n # Store all the generated embeddings back into the database.\n for index, row in table_details_embeddings.iterrows():\n await conn.execute(\n f\"\"\"\n MERGE INTO table_details_embeddings AS target\n USING (SELECT $1::text AS source_type, $2::text AS user_grouping, $3::text AS table_schema, $4::text AS table_name, $5::text AS content, $6::vector AS embedding) AS source\n ON target.user_grouping = source.user_grouping AND target.table_name = source.table_name\n WHEN MATCHED THEN \n UPDATE SET source_type = source.source_type, table_schema = source.table_schema, content = source.content, embedding = source.embedding\n WHEN NOT MATCHED THEN\n INSERT (source_type, user_grouping, table_schema, table_name, content, embedding) \n VALUES (source.source_type, source.user_grouping, source.table_schema, source.table_name, source.content, source.embedding);\n \"\"\",\n row[\"source_type\"],\n row[\"user_grouping\"],\n row[\"table_schema\"],\n row[\"table_name\"],\n row[\"content\"],\n np.array(row[\"embedding\"]),\n )\n\n # await conn.execute(\"DROP TABLE IF EXISTS tablecolumn_details_embeddings\")\n # Create the `table_details_embeddings` table to store vector embeddings.\n await conn.execute(\n \"\"\"CREATE TABLE IF NOT EXISTS tablecolumn_details_embeddings(\n source_type VARCHAR(100) NOT NULL,\n user_grouping VARCHAR(100) NOT NULL,\n table_schema VARCHAR(1024) NOT NULL,\n table_name VARCHAR(1024) NOT NULL,\n column_name VARCHAR(1024) NOT NULL,\n content TEXT,\n embedding vector(768))\"\"\"\n )\n\n # Store all the generated embeddings back into the database.\n for index, row in tablecolumn_details_embeddings.iterrows():\n await conn.execute(\n f\"\"\"\n MERGE INTO tablecolumn_details_embeddings AS target\n USING (SELECT $1::text AS source_type, $2::text AS user_grouping, $3::text AS table_schema, \n $4::text AS table_name, $5::text AS column_name, $6::text AS content, $7::vector AS embedding) AS source\n ON target.user_grouping = source.user_grouping \n AND target.table_name = source.table_name \n AND target.column_name = source.column_name\n WHEN MATCHED THEN \n UPDATE SET source_type = source.source_type, table_schema = source.table_schema, content = source.content, embedding = source.embedding\n WHEN NOT MATCHED THEN\n INSERT (source_type, user_grouping, table_schema, table_name, column_name, content, embedding) \n VALUES (source.source_type, source.user_grouping, source.table_schema, source.table_name, source.column_name, source.content, source.embedding);\n \"\"\",\n row[\"source_type\"],\n row[\"user_grouping\"],\n row[\"table_schema\"],\n row[\"table_name\"],\n row[\"column_name\"],\n row[\"content\"],\n np.array(row[\"embedding\"]),\n )\n await conn.execute(\"CREATE EXTENSION IF NOT EXISTS vector\")\n await register_vector(conn)\n\n # await conn.execute(\"DROP TABLE IF EXISTS example_prompt_sql_embeddings\")\n await conn.execute(\n \"\"\"CREATE TABLE IF NOT EXISTS example_prompt_sql_embeddings(\n user_grouping VARCHAR(1024) NOT NULL,\n example_user_question text NOT NULL,\n example_generated_sql text NOT NULL,\n embedding vector(768))\"\"\"\n )\n\n await conn.close()\n\n\n elif VECTOR_STORE == \"bigquery-vector\": \n \n client=bigquery.Client(project=project_id)\n\n #Store table embeddings\n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{project_id}.{schema}.table_details_embeddings` (\n source_type string NOT NULL, user_grouping string NOT NULL, table_schema string NOT NULL, table_name string NOT NULL, content string, embedding ARRAY)''')\n #job_config = bigquery.LoadJobConfig(write_disposition=\"WRITE_TRUNCATE\")\n\n delete_conditions = table_details_embeddings[['user_grouping', 'table_name']].apply(tuple, axis=1).tolist()\n where_clause = \" OR \".join([f\"(user_grouping = '{cond[0]}' AND table_name = '{cond[1]}')\" for cond in delete_conditions])\n\n delete_query = f\"\"\"\n DELETE FROM `{project_id}.{schema}.table_details_embeddings`\n WHERE {where_clause}\n \"\"\"\n client.query_and_wait(delete_query)\n \n client.load_table_from_dataframe(table_details_embeddings,f'{project_id}.{schema}.table_details_embeddings')\n\n\n #Store column embeddings\n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{project_id}.{schema}.tablecolumn_details_embeddings` (\n source_type string NOT NULL,user_grouping string NOT NULL, table_schema string NOT NULL, table_name string NOT NULL, column_name string NOT NULL,\n content string, embedding ARRAY)''')\n #job_config = bigquery.LoadJobConfig(write_disposition=\"WRITE_TRUNCATE\")\n\n delete_conditions = tablecolumn_details_embeddings[['user_grouping', 'table_name', 'column_name']].apply(tuple, axis=1).tolist()\n where_clause = \" OR \".join([f\"(user_grouping = '{cond[0]}' AND table_name = '{cond[1]}' AND column_name = '{cond[2]}')\" for cond in delete_conditions])\n\n delete_query = f\"\"\"\n DELETE FROM `{project_id}.{schema}.tablecolumn_details_embeddings`\n WHERE {where_clause}\n \"\"\"\n client.query_and_wait(delete_query)\n\n client.load_table_from_dataframe(tablecolumn_details_embeddings,f'{project_id}.{schema}.tablecolumn_details_embeddings')\n\n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{project_id}.{schema}.example_prompt_sql_embeddings` (\n user_grouping string NOT NULL, example_user_question string NOT NULL, example_generated_sql string NOT NULL,\n embedding ARRAY)''')\n \n else: raise ValueError(\"Please provide a valid Vector Store.\")\n return \"Embeddings are stored successfully\"\n\nasync def add_sql_embedding(user_question, generated_sql, database):\n \n \n emb=embedder.create(user_question)\n\n if VECTOR_STORE == \"cloudsql-pgvector\":\n # sql= f'''MERGE INTO example_prompt_sql_embeddings as tgt\n # using (SELECT '{user_question}' as example_user_question) as src \n # on tgt.example_user_question=src.example_user_question \n # when not matched then\n # insert (table_schema, example_user_question,example_generated_sql,embedding) \n # values('{database}','{user_question}','{generated_sql}','{(emb)}')\n # when matched then update set\n # table_schema = '{database}',\n # example_generated_sql = '{generated_sql}',\n # embedding = '{(emb)}' '''\n\n # # print(sql)\n # conn=pgconnector.pool.connect()\n # await conn.execute(text(sql))\n # pgconnector.retrieve_df(sql)\n loop = asyncio.get_running_loop()\n async with Connector(loop=loop) as connector:\n # Create connection to Cloud SQL database.\n conn: asyncpg.Connection = await connector.connect_async(\n f\"{PROJECT_ID}:{PG_REGION}:{PG_INSTANCE}\", # Cloud SQL instance connection name\n \"asyncpg\",\n user=f\"{PG_USER}\",\n password=f\"{PG_PASSWORD}\",\n db=f\"{PG_DATABASE}\",\n )\n\n await conn.execute(\"CREATE EXTENSION IF NOT EXISTS vector\")\n await register_vector(conn)\n\n await conn.execute(\"DELETE FROM example_prompt_sql_embeddings WHERE user_grouping= $1 and example_user_question=$2\",\n database,\n user_question)\n cleaned_sql =generated_sql.replace(\"\\r\", \" \").replace(\"\\n\", \" \")\n await conn.execute(\n \"INSERT INTO example_prompt_sql_embeddings (user_grouping, example_user_question, example_generated_sql, embedding) VALUES ($1, $2, $3, $4)\",\n database,\n user_question,\n cleaned_sql,\n np.array(emb),\n )\n\n elif VECTOR_STORE == \"bigquery-vector\":\n\n client=bigquery.Client(project=PROJECT_ID)\n \n client.query_and_wait(f'''CREATE TABLE IF NOT EXISTS `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.example_prompt_sql_embeddings` (\n user_grouping string NOT NULL, example_user_question string NOT NULL, example_generated_sql string NOT NULL,\n embedding ARRAY)''')\n client.query_and_wait(f'''DELETE FROM `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.example_prompt_sql_embeddings`\n WHERE user_grouping= '{database}' and example_user_question= \"{user_question}\" '''\n )\n # embedding=np.array(row[\"embedding\"])\n cleaned_sql = generated_sql.replace(\"\\r\", \" \").replace(\"\\n\", \" \")\n client.query_and_wait(f'''INSERT INTO `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.example_prompt_sql_embeddings` \n VALUES (\"{database}\",\"{user_question}\" , \n \"{cleaned_sql}\",{emb})''')\n return 1\n\n\n\nif __name__ == '__main__': \n from retrieve_embeddings import retrieve_embeddings\n from utilities import PG_SCHEMA, PROJECT_ID, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION\n VECTOR_STORE = \"cloudsql-pgvector\"\n t, c = retrieve_embeddings(VECTOR_STORE, PG_SCHEMA) \n asyncio.run(store_schema_embeddings(t, \n c, \n PROJECT_ID,\n PG_INSTANCE,\n PG_DATABASE,\n PG_SCHEMA,\n PG_USER,\n PG_PASSWORD,\n PG_REGION,\n VECTOR_STORE = VECTOR_STORE))" }, { "path": "env_setup.py", "content": "\nimport asyncio\nfrom google.cloud import bigquery\nimport google.api_core \n\nfrom embeddings import retrieve_embeddings, store_schema_embeddings, setup_kgq_table, load_kgq_df, store_kgq_embeddings\n\nfrom utilities import ( PG_REGION, PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, \n BQ_REGION, \n EXAMPLES, LOGGING, VECTOR_STORE, PROJECT_ID, \n BQ_OPENDATAQNA_DATASET_NAME,FIRESTORE_REGION) \nimport subprocess\nimport time\n\n\nif VECTOR_STORE == 'bigquery-vector':\n DATASET_REGION = BQ_REGION\n\nelif VECTOR_STORE == 'cloudsql-pgvector':\n DATASET_REGION = PG_REGION\n \ndef setup_postgresql(pg_instance, pg_region, pg_database, pg_user, pg_password):\n \"\"\"Sets up a PostgreSQL Cloud SQL instance with a database and user.\n\n Args:\n pg_instance (str): Name of the Cloud SQL instance.\n pg_region (str): Region where the instance should be located.\n pg_database (str): Name of the database to create.\n pg_user (str): Name of the user to create.\n pg_password (str): Password for the user.\n \"\"\"\n \n # Check if Cloud SQL instance exists\n describe_cmd = [\"gcloud\", \"sql\", \"instances\", \"describe\", pg_instance, \"--format=value(databaseVersion)\"]\n describe_process = subprocess.run(describe_cmd, capture_output=True, text=True)\n\n if describe_process.returncode == 0:\n if describe_process.stdout.startswith(\"POSTGRES\"):\n print(\"Found existing Postgres Cloud SQL Instance!\")\n else:\n raise RuntimeError(\"Existing Cloud SQL instance is not PostgreSQL\")\n else:\n print(\"Creating new Cloud SQL instance...\")\n create_cmd = [\n \"gcloud\", \"sql\", \"instances\", \"create\", pg_instance,\n \"--database-version=POSTGRES_15\", \"--region\", pg_region,\n \"--cpu=1\", \"--memory=4GB\", \"--root-password\", pg_password,\n \"--database-flags=cloudsql.iam_authentication=On\"\n ]\n subprocess.run(create_cmd, check=True) # Raises an exception if creation fails\n\n # Wait for instance to be ready\n print(\"Waiting for instance to be ready...\")\n time.sleep(9999) # You might need to adjust this depending on how long it takes\n\n # Create the database\n list_cmd = [\"gcloud\", \"sql\", \"databases\", \"list\", \"--instance\", pg_instance]\n list_process = subprocess.run(list_cmd, capture_output=True, text=True)\n\n if pg_database in list_process.stdout:\n print(\"Found existing Postgres Cloud SQL database!\")\n else:\n print(\"Creating new Cloud SQL database...\")\n create_db_cmd = [\"gcloud\", \"sql\", \"databases\", \"create\", pg_database, \"--instance\", pg_instance]\n subprocess.run(create_db_cmd, check=True) \n\n # Create the user\n create_user_cmd = [\n \"gcloud\", \"sql\", \"users\", \"create\", pg_user,\n \"--instance\", pg_instance, \"--password\", pg_password\n ]\n subprocess.run(create_user_cmd, check=True)\n\n print(f\"PG Database {pg_database} in instance {pg_instance} is ready.\")\n\n\n\n\ndef create_vector_store():\n \"\"\"\n Initializes the environment and sets up the vector store for Open Data QnA.\n\n This function performs the following steps:\n \n 1. Loads configurations from the \"config.ini\" file.\n 2. Determines the data source (BigQuery or CloudSQL PostgreSQL) and sets the dataset region accordingly.\n 3. If the vector store is \"cloudsql-pgvector\" and the data source is not CloudSQL PostgreSQL, it creates a new PostgreSQL dataset for the vector store.\n 4. If logging is enabled or the vector store is \"bigquery-vector\", it creates a BigQuery dataset for the vector store and logging table.\n 5. It creates a Vertex AI connection for the specified model and embeds the table schemas and columns into the vector database.\n 6. If embeddings are stored in BigQuery, creates a table column_details_embeddings in the BigQuery Dataset.\n 7. It generates the embeddings for the table schemas and column descriptions, and then inserts those embeddings into the BigQuery table.\n \n\n Configuration:\n - Requires the following environment variables to be set in \"config.ini\":\n - `DATA_SOURCE`: The data source (e.g., \"bigquery\" or \"cloudsql-pg\").\n - `VECTOR_STORE`: The type of vector store (e.g., \"bigquery-vector\" or \"cloudsql-pgvector\").\n - `BQ_REGION`: The BigQuery region.\n - `PROJECT_ID`: The Google Cloud project ID.\n - `BQ_OPENDATAQNA_DATASET_NAME`: The name of the BigQuery dataset for Open Data QnA.\n - `LOGGING`: Whether logging is enabled.\n\n - If `VECTOR_STORE` is \"cloudsql-pgvector\" and `DATA_SOURCE` is not \"cloudsql-pg\":\n - Requires additional environment variables for PostgreSQL instance setup.\n\n Returns:\n None\n\n Raises:\n RuntimeError: If there are errors during the setup process (e.g., dataset creation failure).\n \"\"\"\n\n\n print(\"Initializing environment setup.\")\n print(\"Loading configurations from config.ini file.\")\n\n\n\n print(\"Vector Store source set to: \", VECTOR_STORE)\n\n # Create PostgreSQL Instance is data source is different from PostgreSQL Instance\n if VECTOR_STORE == 'cloudsql-pgvector' :\n print(\"Generating pg dataset for vector store.\")\n # Parameters for PostgreSQL Instance\n pg_region = DATASET_REGION\n pg_instance = \"pg15-opendataqna\"\n pg_database = \"opendataqna-db\"\n pg_user = \"pguser\"\n pg_password = \"pg123\"\n pg_schema = 'pg-vector-store' \n\n setup_postgresql(pg_instance, pg_region, pg_database, pg_user, pg_password)\n\n\n # Create a new data set on Bigquery to use for the logs table\n if LOGGING or VECTOR_STORE == 'bigquery-vector':\n if LOGGING: \n print(\"Logging is enabled\")\n\n if VECTOR_STORE == 'bigquery-vector':\n print(\"Vector store set to 'bigquery-vector'\")\n\n print(f\"Generating Big Query dataset {BQ_OPENDATAQNA_DATASET_NAME}\")\n client=bigquery.Client(project=PROJECT_ID)\n dataset_ref = f\"{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}\"\n\n\n # Create the dataset if it does not exist already\n try:\n client.get_dataset(dataset_ref)\n print(\"Destination Dataset exists\")\n except google.api_core.exceptions.NotFound:\n print(\"Cannot find the dataset hence creating.......\")\n dataset=bigquery.Dataset(dataset_ref)\n dataset.location=DATASET_REGION\n client.create_dataset(dataset)\n print(str(dataset_ref)+\" is created\")\n\n\n\n\ndef get_embeddings():\n \"\"\"Generates and returns embeddings for table schemas and column descriptions.\n\n This function performs the following steps:\n\n 1. Retrieves table schema and column description data based on the specified data source (BigQuery or PostgreSQL).\n 2. Generates embeddings for the retrieved data using the configured embedding model.\n 3. Returns the generated embeddings for both tables and columns.\n\n Returns:\n Tuple[pd.DataFrame, pd.DataFrame]: A tuple containing two pandas DataFrames:\n - table_schema_embeddings: Embeddings for the table schemas.\n - col_schema_embeddings: Embeddings for the column descriptions.\n\n Configuration:\n This function relies on the following configuration variables:\n - DATA_SOURCE: The source database (\"bigquery\" or \"cloudsql-pg\").\n - BQ_DATASET_NAME (if DATA_SOURCE is \"bigquery\"): The BigQuery dataset name.\n - BQ_TABLE_LIST (if DATA_SOURCE is \"bigquery\"): The list of BigQuery tables to process.\n - PG_SCHEMA (if DATA_SOURCE is \"cloudsql-pg\"): The PostgreSQL schema name.\n \"\"\"\n\n\n print(\"Generating embeddings from source db schemas\")\n\n import pandas as pd\n import os\n\n current_dir = os.getcwd()\n root_dir = os.path.expanduser('~') # Start at the user's home directory\n\n while current_dir != root_dir:\n for dirpath, dirnames, filenames in os.walk(current_dir):\n config_path = os.path.join(dirpath, 'data_source_list.csv')\n if os.path.exists(config_path):\n file_path = config_path # Update root_dir to the found directory\n break # Stop outer loop once found\n\n current_dir = os.path.dirname(current_dir)\n\n print(\"Source Found at Path :: \"+file_path)\n\n # Load the file\n df_src = pd.read_csv(file_path)\n df_src = df_src.loc[:, [\"source\", \"user_grouping\", \"schema\",\"table\"]]\n df_src = df_src.sort_values(by=[\"source\",\"user_grouping\",\"schema\",\"table\"])\n \n #If no schema Error Out\n if df_src['schema'].astype(str).str.len().min()==0 or df_src['schema'].isna().any():\n raise ValueError(\"Schema column cannot be empty\")\n\n\n #Group by for all the tables filtered\n df=df_src.groupby(['source','schema'])['table'].agg(lambda x: list(x.dropna().unique())).reset_index()\n\n df['table']=df['table'].apply(lambda x: None if pd.isna(x).any() else x)\n \n print(\"The Embeddings are extracted for the below combinations\")\n print(df)\n table_schema_embeddings=pd.DataFrame(columns=['source_type','join_by','table_schema', 'table_name', 'content','embedding'])\n col_schema_embeddings=pd.DataFrame(columns=['source_type','join_by','table_schema', 'table_name', 'column_name', 'content','embedding'])\n\n for _, row in df.iterrows():\n DATA_SOURCE = row['source']\n SCHEMA = row['schema']\n TABLE_LIST = row['table']\n _t, _c = retrieve_embeddings(DATA_SOURCE, SCHEMA=SCHEMA, table_names=TABLE_LIST)\n _t[\"source_type\"]=DATA_SOURCE\n _c[\"source_type\"]=DATA_SOURCE\n if not TABLE_LIST:\n _t[\"join_by\"]=DATA_SOURCE+\"_\"+SCHEMA+\"_\"+SCHEMA\n _c[\"join_by\"]=DATA_SOURCE+\"_\"+SCHEMA+\"_\"+SCHEMA\n table_schema_embeddings = pd.concat([table_schema_embeddings,_t],ignore_index=True)\n col_schema_embeddings = pd.concat([col_schema_embeddings,_c],ignore_index=True)\n\n df_src['join_by'] = df_src.apply(\n lambda row: f\"{row['source']}_{row['schema']}_{row['schema']}\" if pd.isna(row['table']) else f\"{row['source']}_{row['schema']}_{row['table']}\",axis=1)\n\n table_schema_embeddings['join_by'] = table_schema_embeddings['join_by'].fillna(table_schema_embeddings['source_type'] + \"_\" + table_schema_embeddings['table_schema'] + \"_\" + table_schema_embeddings['table_name'])\n\n\n col_schema_embeddings['join_by'] = col_schema_embeddings['join_by'].fillna(col_schema_embeddings['source_type'] + \"_\" + col_schema_embeddings['table_schema'] + \"_\" + col_schema_embeddings['table_name'])\n\n\n\n table_schema_embeddings = table_schema_embeddings.merge(df_src[['join_by', 'user_grouping']], on='join_by', how='left')\n\n table_schema_embeddings.drop(columns=[\"join_by\"],inplace=True)\n #Replace NaN values in group to default to the schema\n\n \n table_schema_embeddings['user_grouping'] = table_schema_embeddings['user_grouping'].fillna(table_schema_embeddings['table_schema']+\"-\"+table_schema_embeddings['source_type'])\n\n\n col_schema_embeddings = col_schema_embeddings.merge(df_src[['join_by', 'user_grouping']], on='join_by', how='left')\n\n col_schema_embeddings.drop(columns=[\"join_by\"],inplace=True)\n\n #Replace NaN values in group to default to the schema\n col_schema_embeddings['user_grouping'] = col_schema_embeddings['user_grouping'].fillna(col_schema_embeddings['table_schema']+\"-\"+col_schema_embeddings['source_type'])\n\n print(\"Table and Column embeddings are created\")\n\n\n return table_schema_embeddings, col_schema_embeddings\n\n\nasync def store_embeddings(table_schema_embeddings, col_schema_embeddings):\n \"\"\"\n Stores table and column embeddings into the specified vector store.\n\n This asynchronous function saves precomputed embeddings for table schemas and column descriptions \n into either BigQuery or PostgreSQL (with pgvector extension) based on the VECTOR_STORE configuration.\n\n Args:\n table_schema_embeddings (pd.DataFrame): Embeddings for the table schemas.\n col_schema_embeddings (pd.DataFrame): Embeddings for the column descriptions.\n\n Configuration:\n This function relies on the following configuration variables:\n - VECTOR_STORE: Determines the target vector store (\"bigquery-vector\" or \"cloudsql-pgvector\").\n - PROJECT_ID, BQ_REGION, BQ_OPENDATAQNA_DATASET_NAME (if VECTOR_STORE is \"bigquery-vector\"):\n Configuration for BigQuery storage.\n - PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION (if VECTOR_STORE is \"cloudsql-pgvector\"):\n Configuration for PostgreSQL storage.\n\n Returns:\n None\n \"\"\"\n\n print(\"Storing embeddings back to the vector store.\")\n if VECTOR_STORE=='bigquery-vector':\n await(store_schema_embeddings(table_details_embeddings=table_schema_embeddings, \n tablecolumn_details_embeddings=col_schema_embeddings, \n project_id=PROJECT_ID,\n instance_name=None,\n database_name=None,\n schema=BQ_OPENDATAQNA_DATASET_NAME,\n database_user=None,\n database_password=None,\n region=BQ_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n\n elif VECTOR_STORE=='cloudsql-pgvector':\n await(store_schema_embeddings(table_details_embeddings=table_schema_embeddings, \n tablecolumn_details_embeddings=col_schema_embeddings, \n project_id=PROJECT_ID,\n instance_name=PG_INSTANCE,\n database_name=PG_DATABASE,\n schema=None,\n database_user=PG_USER,\n database_password=PG_PASSWORD,\n region=PG_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n\n print(\"Table and Column embeddings are saved to vector store\")\n\n\n\nasync def create_kgq_sql_table():\n \"\"\"\n Creates a table for storing Known Good Query (KGQ) embeddings in the vector store.\n\n This asynchronous function conditionally sets up a table to store known good SQL queries and their embeddings, \n which are used to provide examples to the LLM during query generation. The table is created only \n if the `EXAMPLES` configuration variable is set to 'yes'. If not, it prints a warning message encouraging \n the user to create a query cache for better results.\n\n Configuration:\n This function relies on the following configuration variables:\n - EXAMPLES: Determines whether to create the KGQ table ('yes' to create).\n - VECTOR_STORE: Specifies the target vector store (\"bigquery-vector\" or \"cloudsql-pgvector\").\n - PROJECT_ID, BQ_REGION, BQ_OPENDATAQNA_DATASET_NAME (if VECTOR_STORE is \"bigquery-vector\"):\n Configuration for BigQuery storage.\n - PG_INSTANCE, PG_DATABASE, PG_USER, PG_PASSWORD, PG_REGION (if VECTOR_STORE is \"cloudsql-pgvector\"):\n Configuration for PostgreSQL storage.\n \n Returns:\n None\n \"\"\"\n if EXAMPLES:\n print(\"Creating kgq table in vector store.\")\n # Delete any old tables and create a new table to KGQ embeddings\n if VECTOR_STORE=='bigquery-vector':\n await(setup_kgq_table(project_id=PROJECT_ID,\n instance_name=None,\n database_name=None,\n schema=BQ_OPENDATAQNA_DATASET_NAME,\n database_user=None,\n database_password=None,\n region=BQ_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n\n elif VECTOR_STORE=='cloudsql-pgvector':\n await(setup_kgq_table(project_id=PROJECT_ID,\n instance_name=PG_INSTANCE,\n database_name=PG_DATABASE,\n schema=None,\n database_user=PG_USER,\n database_password=PG_PASSWORD,\n region=PG_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n else:\n print(\"⚠️ WARNING: No Known Good Queries are provided to create query cache for Few shot examples!\")\n print(\"Creating a query cache is highly recommended for best outcomes\")\n print(\"If no Known Good Queries for the dataset are availabe at this time, you can use 3_LoadKnownGoodSQL.ipynb to load them later!!\")\n\n\n\n\nasync def store_kgq_sql_embeddings():\n \"\"\"\n Stores known good query (KGQ) embeddings into the specified vector store.\n\n This asynchronous function reads known good SQL queries from the \"known_good_sql.csv\" file\n and stores their embeddings in either BigQuery or PostgreSQL (with pgvector) depending on the\n `VECTOR_STORE` configuration. This process is only performed if the `EXAMPLES` configuration \n variable is set to 'yes'. Otherwise, a warning message is displayed, highlighting the \n importance of creating a query cache.\n\n Configuration:\n - Requires the \"known_good_sql.csv\" file to be present in the project directory.\n - Relies on the following configuration variables:\n - `EXAMPLES`: Determines whether to store KGQ embeddings ('yes' to store).\n - `VECTOR_STORE`: Specifies the target vector store (\"bigquery-vector\" or \"cloudsql-pgvector\").\n - `PROJECT_ID`, `BQ_REGION`, `BQ_OPENDATAQNA_DATASET_NAME` (if VECTOR_STORE is \"bigquery-vector\"):\n Configuration for BigQuery storage.\n - `PG_INSTANCE`, `PG_DATABASE`, `PG_USER`, `PG_PASSWORD`, `PG_REGION` (if VECTOR_STORE is \"cloudsql-pgvector\"):\n Configuration for PostgreSQL storage.\n\n Returns:\n None\n \"\"\"\n if EXAMPLES:\n print(\"Reading contents of known_good_sql.csv\")\n # Load the contents of the known_good_sql.csv file into a dataframe\n df_kgq = load_kgq_df()\n\n\n\n print(\"Storing kgq embeddings in vector store table.\")\n # Add KGQ to the vector store\n if VECTOR_STORE=='bigquery-vector':\n await(store_kgq_embeddings(df_kgq,\n project_id=PROJECT_ID,\n instance_name=None,\n database_name=None,\n schema=BQ_OPENDATAQNA_DATASET_NAME,\n database_user=None,\n database_password=None,\n region=BQ_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n\n elif VECTOR_STORE=='cloudsql-pgvector':\n await(store_kgq_embeddings(df_kgq,\n project_id=PROJECT_ID,\n instance_name=PG_INSTANCE,\n database_name=PG_DATABASE,\n schema=None,\n database_user=PG_USER,\n database_password=PG_PASSWORD,\n region=PG_REGION,\n VECTOR_STORE = VECTOR_STORE\n ))\n print('kgq embeddings stored.')\n\n else:\n print(\"⚠️ WARNING: No Known Good Queries are provided to create query cache for Few shot examples!\")\n print(\"Creating a query cache is highly recommended for best outcomes\")\n print(\"If no Known Good Queries for the dataset are availabe at this time, you can use 3_LoadKnownGoodSQL.ipynb to load them later!!\")\n\n\ndef create_firestore_db(firestore_region=FIRESTORE_REGION,firestore_database=\"opendataqna-session-logs\"):\n\n # Check if Firestore database exists\n database_exists_cmd = [\n \"gcloud\", \"firestore\", \"databases\", \"list\", \n \"--filter\", f\"name=projects/{PROJECT_ID}/databases/{firestore_database}\", \n \"--format\", \"value(name)\" # Extract just the name if found\n ]\n\n database_exists_process = subprocess.run(\n database_exists_cmd, capture_output=True, text=True\n )\n \n if database_exists_process.returncode == 0 and database_exists_process.stdout:\n if database_exists_process.stdout.startswith(f\"projects/{PROJECT_ID}/databases/{firestore_database}\"):\n print(\"Found existing Firestore database with this name already!\")\n else:\n raise RuntimeError(\"Issue with checking if the firestore db exists or not\")\n else:\n # Create Firestore database\n print(\"Creating new Firestore database...\")\n create_db_cmd = [\n \"gcloud\", \"firestore\", \"databases\", \"create\", \n \"--database\", firestore_database,\n \"--location\", firestore_region,\n \"--project\", PROJECT_ID\n ]\n subprocess.run(create_db_cmd, check=True) # Raise exception on failure\n\n # Potential wait for database readiness (optional)\n time.sleep(30) # May not be strictly necessary for basic use\n\n\n\n\n\nif __name__ == '__main__':\n # Setup vector store for embeddings\n create_vector_store() \n\n # Generate embeddings for tables and columns\n table_schema_embeddings, col_schema_embeddings = get_embeddings() \n\n # Store table/column embeddings (asynchronous)\n asyncio.run(store_embeddings(table_schema_embeddings, col_schema_embeddings)) \n\n # Create table for known good queries (if enabled)\n asyncio.run(create_kgq_sql_table()) \n\n # Store known good query embeddings (if enabled)\n asyncio.run(store_kgq_sql_embeddings())\n\n create_firestore_db() \n\n" }, { "path": "frontend/.gitignore", "content": "# See http://help.github.com/ignore-files/ for more about ignoring files.\n\n# Compiled output\n/tmp\n/out-tsc\n/bazel-out\n\n# Node\n/node_modules\nnpm-debug.log\nyarn-error.log\n\n# IDEs and editors\n.idea/\n.project\n.classpath\n.c9/\n*.launch\n.settings/\n*.sublime-workspace\n\n# Visual Studio Code\n.vscode/*\n!.vscode/settings.json\n!.vscode/tasks.json\n!.vscode/launch.json\n!.vscode/extensions.json\n.history/*\n\n# Miscellaneous\n/.angular/cache\n.sass-cache/\n/connect.lock\n/coverage\n/libpeerconnection.log\ntestem.log\n/typings\n\n# System files\n.DS_Store\nThumbs.db" }, { "path": "frontend/README.md", "content": "

Deploy Frontend Demo UI

\n\n**Technologies and Components**\n\n* **Framework:** Angular\n* **Hosting Platform:** Firebase\n\n**Note** : This UI demo doesn't configure any domain restrictions. If you choose to build one refer to this link https://firebase.google.com/docs/functions/auth-blocking-events?gen=2nd#only_allowing_registration_from_a_specific_domain\n\n1. Install the firebase tools to run CLI commands\n ```\n cd Open_Data_QnA\n \n gcloud services enable firebase.googleapis.com --project=$PROJECT_ID # Enable firebase management API\n\n npm install -g firebase-tools\n\n ```\n ```\n export PROJECT_ID=\n export REGION=\n ```\n\n\n2. Build the firebase community builder image\n\n Cloud Build provides a Firebase community builder image that you can use to invoke firebase commands in Cloud Build. To use this builder in a Cloud Build config file, you must first build the image and push it to the Container Registry in your project.\n\n **Note**:*Please complete the steps carely and use the same project which you are going to host the app*\n\n Follow detailed instructions:\n\n \n 1. Navigate to your project root directory.\n 2. Clone the cloud-builders-community repository:\n\n ```\n git clone https://github.com/GoogleCloudPlatform/cloud-builders-community.git\n ```\n 3. Navigate to the firebase builder image:\n\n ```\n cd cloud-builders-community/firebase \n ```\n 4. Submit the builder to your project, where REGION is one of the supported build regions: \n\n ```\n gcloud builds submit --region=$REGION . --project=$PROJECT_ID\n ```\n\n 5. Navigate back to your project root directory:\n \n ```\n cd ../..\n ```\n \n 6. Remove the repository from your root directory:\n ```\n rm -rf cloud-builders-community/\n ```\n\n\n3. Create and Initialize Firebase\n\n ```\n cd Open_Data_QnA/frontend\n\n rm firebase.json .firebaserc\n\n firebase login --no-localhost\n\n ## Below command can be used re authenticate in case of authentication errors\n\n firebase login --reauth --no-localhost \n\n #If incase there are old firebase files \n\n ```\n\n ```\n\n firebase init hosting \n\n ## Select \"Add Firebase to an existing Google Cloud Platfrom Project\"\n\n ## For the public directory prompt provide >> /dist/frontend/browser\n\n ## Rewrite all URLs to index prompt enter >> Yes (Enter No for any other options)\n\n ## You should now see firebase.json created in the folder\n\n ```\n ```\n ## To modify the contents for this solution update it using the cp command as below\n\n cp firebase_setup.json firebase.json\n ```\n ```\n \n ## Run below command to create a webapp to host your application\n\n firebase apps:create --project $PROJECT_ID\n\n ## Select Web and Provide name : \"opendataqna\"\n ```\n ```\n\n ## Below command provides the initialization code to add to your constant file\n\n firebase apps:sdkconfig --project $PROJECT_ID\n\n ```\n4. Enable Google Authentication in Firebase Console\n\n - Go to the Firebase console (https://console.firebase.google.com/).\n - Select your project.\n - Navigate to \"Authentication\" -> \"Sign-in method\".\n - Click \"Add new provider\" and select \"Google\".\n - Provide a support email and click \"Enable\". This will enable Google authentication for your project.\n\n5. Update the Config Code and Endpoint URLs for the frontend\n\n In the file [`/frontend/src/assets/constants.ts`](/frontend/src/assets/constants.ts) \n \n * Replace the config object with the one you copied in the above step \n * Replace the ENDPOINT_OPENDATAQNA value with the Service URL from the Endpoint Deployment section in the backend-apis README.md\n\n ***Note that these variables need to be exported using \"export\" keyword. So make sure export is mentioned for both the variables***\n\n

\n \n \"aaie\n \n

\n\n6. Deploy the app\n \n Run the below commands on the terminal\n\n ```\n\n cd Open_Data_QnA/frontend\n ```\n ```\n gcloud builds submit . --config frontend.yaml --substitutions _FIREBASE_PROJECT_ID=$PROJECT_ID --project=$PROJECT_ID\n\n ```\n \n---------\n---------\n\nYou can see the app URL at the end of successful deployment\n\n> Once deployed login if your Google Account > Select Business User > Select a database in the dropdown (top right) > Type in the Query > Hit Query\n\nA successful SQL generated will be show as below with result as below\n\n

\n \n \"aaie\n \n

\n\n

\n \n \"aaie\n \n

\n\nHit Visualize to see the results and charts as below\n\n

\n \n \"aaie\n \n

\n\n----\n----\n\n\n**API Details**\n\n All the payloads are in JSON format\n\n1. List Databases : Get the available databases in the vector store that solution can run against\n\n URI: {Service URL}/available_databases \n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/available_databases\n\n Method: GET\n\n Request Payload : NONE\n\n Request response:\n ```\n {\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"table_schema\\\":\\\"imdb-postgres\\\"},{\\\"table_schema\\\":\\\"retail-postgres\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n\n2. Known SQL : Get suggestive questions (previously asked/examples added) for selected database\n\n URI: /get_known_sql\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/get_known_sql \n\n Method: POST\n\n Request Payload :\n\n ```\n {\n \"user_grouping\":\"retail\"\n }\n ```\n\n Request response:\n\n ```\n {\n \"Error\": \"\",\n \"KnownSQL\": \"[{\\\"example_user_question\\\":\\\"Which city had maximum number of sales and what was the count?\\\",\\\"example_generated_sql\\\":\\\"select st.city_id, count(st.city_id) as city_sales_count from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by city_sales_count desc limit 1;\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n\n\n3. SQL Generation : Generate the SQL for the input question asked against a database\n\n URI: /generate_sql\n\n\n Method: POST\n\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/get_known_sql\n\n\n Request payload:\n\n ```\n {\n \"session_id\":\"\",\n \"user_id\":\"harry@hogwarts.com\",\n \"user_question\":\"Which city had maximum number of sales?\",\n \"user_grouping\":\"retail\"\n }\n ```\n\n\n Request response:\n ```\n {\n \"Error\": \"\",\n \"GeneratedSQL\": \" select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"ResponseCode\": 200,\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n\n\n4. Execute SQL : Run the SQL statement against provided database source\n\n URI:/run_query\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/run_query\n\n Method: POST\n\n Request payload:\n ```\n { \"user_grouping\": \"retail\",\n \"generated_sql\":\"select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n\n Request response:\n ```\n {\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"city_id\\\":\\\"C014\\\"}]\",\n \"ResponseCode\": 200\n }\n ```\n5. Embedd SQL : To embed known good SQLs to your example embeddings\n\n URI:/embed_sql\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/embed_sql\n\n METHOD: POST\n\n Request Payload:\n\n ```\n {\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"user_question\":\"Which city had maximum number of sales?\",\n \"generated_sql\":\"select st.city_id from retail.sales as s join retail.stores as st on s.id_store = st.id_store group by st.city_id order by count(*) desc limit 1;\",\n \"user_grouping\":\"retail\"\n }\n ```\n\n Request response:\n ```\n {\n \"ResponseCode\" : 201, \n \"Message\" : \"Example SQL has been accepted for embedding\",\n \"Error\":\"\",\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\"\n }\n ```\n6. Generate Visualization Code : To generated javascript Google Charts code based on the SQL Results and display them on the UI\n\n As per design we have two visualizations suggested showing up when the user clicks the visualize button. Hence two divs are send as part of the response “chart_div”, “chart_div_1” to bind them to that element in the UI\n \n\n If you are only looking to setup enpoint you can stop here. In case you require the demo app (frontend UI) built in the solution, proceed to the next step.\n\n URI:/generate_viz\n Complete URL Sample : https://OpenDataQnA-aeiouAEI-uc.a.run.app/generate_viz\n \n METHOD: POST\n\n Request Payload:\n ```\n {\n \"session_id\":\"1iuu2u-k1ij2-kkkhhj12131\" ,\n \"user_question\": \"What are top 5 product skus that are ordered?\",\n \"sql_generated\": \"SELECT productSKU as ProductSKUCode, sum(total_ordered) as TotalOrderedItems FROM `inbq1-joonix.demo.sales_sku` group by productSKU order by sum(total_ordered) desc limit 5\",\n \"sql_results\": [\n {\n \"ProductSKUCode\": \"GGOEGOAQ012899\",\n \"TotalOrderedItems\": 456\n },\n {\n \"ProductSKUCode\": \"GGOEGDHC074099\",\n \"TotalOrderedItems\": 334\n },\n {\n \"ProductSKUCode\": \"GGOEGOCB017499\",\n \"TotalOrderedItems\": 319\n },\n {\n \"ProductSKUCode\": \"GGOEGOCC077999\",\n \"TotalOrderedItems\": 290\n },\n {\n \"ProductSKUCode\": \"GGOEGFYQ016599\",\n \"TotalOrderedItems\": 253\n }\n ]\n }\n \n ```\n\n Request response:\n ```\n {\n \"SessionID\":\"1iuu2u-k1ij2-kkkhhj12131\",\n \"Error\": \"\",\n \"GeneratedChartjs\": {\n \"chart_div\": \"google.charts.load('current', {\\n packages: ['corechart']\\n});\\ngoogle.charts.setOnLoadCallback(drawChart);\\n\\nfunction drawChart() {\\n var data = google.visualization.arrayToDataTable([\\n ['Product SKU', 'Total Ordered Items'],\\n ['GGOEGOAQ012899', 456],\\n ['GGOEGDHC074099', 334],\\n ['GGOEGOCB017499', 319],\\n ['GGOEGOCC077999', 290],\\n ['GGOEGFYQ016599', 253],\\n ]);\\n\\n var options = {\\n title: 'Top 5 Product SKUs Ordered',\\n width: 600,\\n height: 300,\\n hAxis: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n vAxis: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n legend: {\\n textStyle: {\\n fontSize: 12\\n }\\n },\\n bar: {\\n groupWidth: '50%'\\n }\\n };\\n\\n var chart = new google.visualization.BarChart(document.getElementById('chart_div'));\\n\\n chart.draw(data, options);\\n}\\n\",\n \n \"chart_div_1\": \"google.charts.load('current', {'packages':['corechart']});\\ngoogle.charts.setOnLoadCallback(drawChart);\\nfunction drawChart() {\\n var data = google.visualization.arrayToDataTable([\\n ['ProductSKUCode', 'TotalOrderedItems'],\\n ['GGOEGOAQ012899', 456],\\n ['GGOEGDHC074099', 334],\\n ['GGOEGOCB017499', 319],\\n ['GGOEGOCC077999', 290],\\n ['GGOEGFYQ016599', 253]\\n ]);\\n\\n var options = {\\n title: 'Top 5 Product SKUs that are Ordered',\\n width: 600,\\n height: 300,\\n hAxis: {\\n textStyle: {\\n fontSize: 5\\n }\\n },\\n vAxis: {\\n textStyle: {\\n fontSize: 5\\n }\\n },\\n legend: {\\n textStyle: {\\n fontSize: 10\\n }\\n },\\n bar: {\\n groupWidth: \\\"60%\\\"\\n }\\n };\\n\\n var chart = new google.visualization.ColumnChart(document.getElementById('chart_div_1'));\\n\\n chart.draw(data, options);\\n}\\n\"\n },\n \"ResponseCode\": 200\n }\n\n ```" }, { "path": "frontend/angular.json", "content": "{\n \"$schema\": \"./node_modules/@angular/cli/lib/config/schema.json\",\n \"version\": 1,\n \"newProjectRoot\": \"projects\",\n \"projects\": {\n \"frontend\": {\n \"projectType\": \"application\",\n \"schematics\": {\n \"@schematics/angular:component\": {\n \"style\": \"scss\"\n }\n },\n \"root\": \"\",\n \"sourceRoot\": \"src\",\n \"prefix\": \"app\",\n \"architect\": {\n \"build\": {\n \"builder\": \"@angular-devkit/build-angular:application\",\n \"options\": {\n \"outputPath\": \"dist/frontend\",\n \"index\": \"src/index.html\",\n \"browser\": \"src/main.ts\",\n \"polyfills\": [\n \"zone.js\"\n ],\n \"tsConfig\": \"tsconfig.app.json\",\n \"inlineStyleLanguage\": \"scss\",\n \"assets\": [\n \"src/favicon.ico\",\n \"src/assets\"\n ],\n \"styles\": [\n \"node_modules/bootstrap/dist/css/bootstrap.min.css\",\n \"src/styles.scss\"\n ],\n \"scripts\": [],\n \"server\": \"src/main.server.ts\",\n \"prerender\": true,\n \"ssr\": {\n \"entry\": \"server.ts\"\n }\n },\n \"configurations\": {\n \"production\": {\n \"budgets\": [\n {\n \"type\": \"initial\",\n \"maximumWarning\": \"1mb\",\n \"maximumError\": \"2mb\"\n },\n {\n \"type\": \"anyComponentStyle\",\n \"maximumWarning\": \"2kb\",\n \"maximumError\": \"4kb\"\n }\n ],\n \"outputHashing\": \"all\"\n },\n \"development\": {\n \"optimization\": false,\n \"extractLicenses\": false,\n \"sourceMap\": true\n }\n },\n \"defaultConfiguration\": \"production\"\n },\n \"serve\": {\n \"builder\": \"@angular-devkit/build-angular:dev-server\",\n \"options\": {\n \"buildTarget\": \"frontend:build\"\n },\n \"configurations\": {\n \"production\": {\n \"buildTarget\": \"frontend:build:production\"\n },\n \"development\": {\n \"buildTarget\": \"frontend:build:development\"\n }\n },\n \"defaultConfiguration\": \"development\"\n },\n \"extract-i18n\": {\n \"builder\": \"@angular-devkit/build-angular:extract-i18n\",\n \"options\": {\n \"buildTarget\": \"frontend:build\"\n }\n },\n \"test\": {\n \"builder\": \"@angular-devkit/build-angular:karma\",\n \"options\": {\n \"polyfills\": [\n \"zone.js\",\n \"zone.js/testing\"\n ],\n \"tsConfig\": \"tsconfig.spec.json\",\n \"inlineStyleLanguage\": \"scss\",\n \"assets\": [\n \"src/favicon.ico\",\n \"src/assets\"\n ],\n \"styles\": [\n \"src/styles.scss\"\n ],\n \"scripts\": []\n }\n }\n }\n }\n },\n \"cli\": {\n \"analytics\": false\n }\n}\n" }, { "path": "frontend/database.indexes.json", "content": "{\n \"indexes\": [\n {\n \"collectionGroup\": \"session_logs\",\n \"queryScope\": \"COLLECTION\",\n \"fields\": [\n {\n \"fieldPath\": \"user_id\",\n \"order\": \"ASCENDING\"\n },\n {\n \"fieldPath\": \"timestamp\",\n \"order\": \"DESCENDING\"\n }\n ]\n }\n ],\n \"fieldOverrides\": []\n}" }, { "path": "frontend/database.rules.json", "content": "rules_version = '2';\nservice cloud.firestore {\n match /databases/{database}/documents {\n match /{document=**} {\n allow read: if true;\n allow write: if false;\n }\n }\n}" }, { "path": "frontend/firebase_setup.json", "content": "{\n \"hosting\": {\n \"public\": \"/dist/frontend/browser\",\n \"ignore\": [\n \"firebase.json\",\n \"**/.*\",\n \"**/node_modules/**\"\n ],\n \"rewrites\": [\n {\n \"source\": \"**\",\n \"destination\": \"/index.html\"\n }\n ]\n },\n \"firestore\": [\n {\n \"database\": \"opendataqna-session-logs\",\n \"rules\": \"database.rules.json\",\n \"indexes\": \"database.indexes.json\"\n }\n ]\n}\n" }, { "path": "frontend/frontend-flutter/.flutter-plugins", "content": "# This is a generated file; do not edit or check into version control.\naudio_waveforms=/Users/raimeur/.pub-cache/hosted/pub.dev/audio_waveforms-1.0.5/\ncloud_firestore=/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore-5.4.0/\ncloud_firestore_web=/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore_web-4.2.0/\nemoji_picker_flutter=/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\nfile_picker=/Users/raimeur/.pub-cache/hosted/pub.dev/file_picker-8.0.6/\nfile_selector_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_linux-0.9.2+1/\nfile_selector_macos=/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_macos-0.9.4/\nfile_selector_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_windows-0.9.3+2/\nfirebase_auth=/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth-5.1.3/\nfirebase_auth_web=/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth_web-5.12.5/\nfirebase_core=/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core-3.4.0/\nfirebase_core_web=/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core_web-2.17.5/\nflutter_inappwebview=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview-6.0.0/\nflutter_inappwebview_android=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_android-1.0.13/\nflutter_inappwebview_ios=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_ios-1.0.13/\nflutter_inappwebview_macos=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_macos-1.0.11/\nflutter_inappwebview_web=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_web-1.0.8/\nflutter_keyboard_visibility=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility-6.0.0/\nflutter_keyboard_visibility_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_linux-1.0.0/\nflutter_keyboard_visibility_macos=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_macos-1.0.0/\nflutter_keyboard_visibility_web=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_web-2.0.0/\nflutter_keyboard_visibility_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_windows-1.0.0/\nflutter_plugin_android_lifecycle=/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_plugin_android_lifecycle-2.0.21/\nimage_picker=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker-1.1.2/\nimage_picker_android=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_android-0.8.12+10/\nimage_picker_for_web=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_for_web-3.0.4/\nimage_picker_ios=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_ios-0.8.12/\nimage_picker_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_linux-0.2.1+1/\nimage_picker_macos=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_macos-0.2.1+1/\nimage_picker_web=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_web-4.0.0/\nimage_picker_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_windows-0.2.1+1/\nlibphonenumber_plugin=/Users/raimeur/.pub-cache/hosted/pub.dev/libphonenumber_plugin-0.3.3/\nlibphonenumber_web=/Users/raimeur/.pub-cache/hosted/pub.dev/libphonenumber_web-0.3.2/\npath_provider=/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider-2.1.4/\npath_provider_android=/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_android-2.2.9/\npath_provider_foundation=/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.0/\npath_provider_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/\npath_provider_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/\npointer_interceptor=/Users/raimeur/.pub-cache/hosted/pub.dev/pointer_interceptor-0.10.1+1/\npointer_interceptor_ios=/Users/raimeur/.pub-cache/hosted/pub.dev/pointer_interceptor_ios-0.10.1/\npointer_interceptor_web=/Users/raimeur/.pub-cache/hosted/pub.dev/pointer_interceptor_web-0.10.2/\nshared_preferences=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences-2.3.0/\nshared_preferences_android=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_android-2.3.0/\nshared_preferences_foundation=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_foundation-2.5.0/\nshared_preferences_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_linux-2.4.0/\nshared_preferences_web=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_web-2.4.0/\nshared_preferences_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_windows-2.4.0/\nurl_launcher=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher-6.3.0/\nurl_launcher_android=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_android-6.3.8/\nurl_launcher_ios=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_ios-6.3.1/\nurl_launcher_linux=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_linux-3.1.1/\nurl_launcher_macos=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_macos-3.2.0/\nurl_launcher_web=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_web-2.3.1/\nurl_launcher_windows=/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_windows-3.1.2/\n" }, { "path": "frontend/frontend-flutter/.flutter-plugins-dependencies", "content": "{\"info\":\"This is a generated file; do not edit or check into version control.\",\"plugins\":{\"ios\":[{\"name\":\"audio_waveforms\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/audio_waveforms-1.0.5/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"cloud_firestore\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore-5.4.0/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"file_picker\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_picker-8.0.6/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"firebase_auth\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth-5.1.3/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"firebase_core\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core-3.4.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_inappwebview_ios\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_ios-1.0.13/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility-6.0.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"image_picker_ios\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_ios-0.8.12/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"libphonenumber_plugin\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/libphonenumber_plugin-0.3.3/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"path_provider_foundation\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.0/\",\"shared_darwin_source\":true,\"native_build\":true,\"dependencies\":[]},{\"name\":\"pointer_interceptor_ios\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/pointer_interceptor_ios-0.10.1/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"shared_preferences_foundation\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_foundation-2.5.0/\",\"shared_darwin_source\":true,\"native_build\":true,\"dependencies\":[]},{\"name\":\"url_launcher_ios\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_ios-6.3.1/\",\"native_build\":true,\"dependencies\":[]}],\"android\":[{\"name\":\"audio_waveforms\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/audio_waveforms-1.0.5/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"cloud_firestore\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore-5.4.0/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"file_picker\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_picker-8.0.6/\",\"native_build\":true,\"dependencies\":[\"flutter_plugin_android_lifecycle\"]},{\"name\":\"firebase_auth\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth-5.1.3/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"firebase_core\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core-3.4.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_inappwebview_android\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_android-1.0.13/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility-6.0.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_plugin_android_lifecycle\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_plugin_android_lifecycle-2.0.21/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"image_picker_android\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_android-0.8.12+10/\",\"native_build\":true,\"dependencies\":[\"flutter_plugin_android_lifecycle\"]},{\"name\":\"libphonenumber_plugin\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/libphonenumber_plugin-0.3.3/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"path_provider_android\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_android-2.2.9/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"shared_preferences_android\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_android-2.3.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"url_launcher_android\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_android-6.3.8/\",\"native_build\":true,\"dependencies\":[]}],\"macos\":[{\"name\":\"cloud_firestore\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore-5.4.0/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"file_selector_macos\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_macos-0.9.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"firebase_auth\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth-5.1.3/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"firebase_core\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core-3.4.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_inappwebview_macos\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_macos-1.0.11/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_macos\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_macos-1.0.0/\",\"native_build\":false,\"dependencies\":[]},{\"name\":\"image_picker_macos\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_macos-0.2.1+1/\",\"native_build\":false,\"dependencies\":[\"file_selector_macos\"]},{\"name\":\"path_provider_foundation\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_foundation-2.4.0/\",\"shared_darwin_source\":true,\"native_build\":true,\"dependencies\":[]},{\"name\":\"shared_preferences_foundation\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_foundation-2.5.0/\",\"shared_darwin_source\":true,\"native_build\":true,\"dependencies\":[]},{\"name\":\"url_launcher_macos\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_macos-3.2.0/\",\"native_build\":true,\"dependencies\":[]}],\"linux\":[{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"file_selector_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_linux-0.9.2+1/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_linux-1.0.0/\",\"native_build\":false,\"dependencies\":[]},{\"name\":\"image_picker_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_linux-0.2.1+1/\",\"native_build\":false,\"dependencies\":[\"file_selector_linux\"]},{\"name\":\"path_provider_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_linux-2.2.1/\",\"native_build\":false,\"dependencies\":[]},{\"name\":\"shared_preferences_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_linux-2.4.0/\",\"native_build\":false,\"dependencies\":[\"path_provider_linux\"]},{\"name\":\"url_launcher_linux\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_linux-3.1.1/\",\"native_build\":true,\"dependencies\":[]}],\"windows\":[{\"name\":\"cloud_firestore\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore-5.4.0/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"file_selector_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_selector_windows-0.9.3+2/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"firebase_auth\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth-5.1.3/\",\"native_build\":true,\"dependencies\":[\"firebase_core\"]},{\"name\":\"firebase_core\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core-3.4.0/\",\"native_build\":true,\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_windows-1.0.0/\",\"native_build\":false,\"dependencies\":[]},{\"name\":\"image_picker_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_windows-0.2.1+1/\",\"native_build\":false,\"dependencies\":[\"file_selector_windows\"]},{\"name\":\"path_provider_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/path_provider_windows-2.3.0/\",\"native_build\":false,\"dependencies\":[]},{\"name\":\"shared_preferences_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_windows-2.4.0/\",\"native_build\":false,\"dependencies\":[\"path_provider_windows\"]},{\"name\":\"url_launcher_windows\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_windows-3.1.2/\",\"native_build\":true,\"dependencies\":[]}],\"web\":[{\"name\":\"cloud_firestore_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/cloud_firestore_web-4.2.0/\",\"dependencies\":[\"firebase_core_web\"]},{\"name\":\"emoji_picker_flutter\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/emoji_picker_flutter-1.6.4/\",\"dependencies\":[]},{\"name\":\"file_picker\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/file_picker-8.0.6/\",\"dependencies\":[]},{\"name\":\"firebase_auth_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_auth_web-5.12.5/\",\"dependencies\":[\"firebase_core_web\"]},{\"name\":\"firebase_core_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/firebase_core_web-2.17.5/\",\"dependencies\":[]},{\"name\":\"flutter_inappwebview_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_inappwebview_web-1.0.8/\",\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/flutter_keyboard_visibility_web-2.0.0/\",\"dependencies\":[]},{\"name\":\"image_picker_for_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_for_web-3.0.4/\",\"dependencies\":[]},{\"name\":\"image_picker_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/image_picker_web-4.0.0/\",\"dependencies\":[]},{\"name\":\"libphonenumber_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/libphonenumber_web-0.3.2/\",\"dependencies\":[]},{\"name\":\"pointer_interceptor_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/pointer_interceptor_web-0.10.2/\",\"dependencies\":[]},{\"name\":\"shared_preferences_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/shared_preferences_web-2.4.0/\",\"dependencies\":[]},{\"name\":\"url_launcher_web\",\"path\":\"/Users/raimeur/.pub-cache/hosted/pub.dev/url_launcher_web-2.3.1/\",\"dependencies\":[]}]},\"dependencyGraph\":[{\"name\":\"audio_waveforms\",\"dependencies\":[]},{\"name\":\"cloud_firestore\",\"dependencies\":[\"cloud_firestore_web\",\"firebase_core\"]},{\"name\":\"cloud_firestore_web\",\"dependencies\":[\"firebase_core\",\"firebase_core_web\"]},{\"name\":\"emoji_picker_flutter\",\"dependencies\":[\"shared_preferences\"]},{\"name\":\"file_picker\",\"dependencies\":[\"flutter_plugin_android_lifecycle\"]},{\"name\":\"file_selector_linux\",\"dependencies\":[]},{\"name\":\"file_selector_macos\",\"dependencies\":[]},{\"name\":\"file_selector_windows\",\"dependencies\":[]},{\"name\":\"firebase_auth\",\"dependencies\":[\"firebase_auth_web\",\"firebase_core\"]},{\"name\":\"firebase_auth_web\",\"dependencies\":[\"firebase_core\",\"firebase_core_web\"]},{\"name\":\"firebase_core\",\"dependencies\":[\"firebase_core_web\"]},{\"name\":\"firebase_core_web\",\"dependencies\":[]},{\"name\":\"flutter_inappwebview\",\"dependencies\":[\"flutter_inappwebview_android\",\"flutter_inappwebview_ios\",\"flutter_inappwebview_macos\",\"flutter_inappwebview_web\"]},{\"name\":\"flutter_inappwebview_android\",\"dependencies\":[]},{\"name\":\"flutter_inappwebview_ios\",\"dependencies\":[]},{\"name\":\"flutter_inappwebview_macos\",\"dependencies\":[]},{\"name\":\"flutter_inappwebview_web\",\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility\",\"dependencies\":[\"flutter_keyboard_visibility_linux\",\"flutter_keyboard_visibility_macos\",\"flutter_keyboard_visibility_web\",\"flutter_keyboard_visibility_windows\"]},{\"name\":\"flutter_keyboard_visibility_linux\",\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_macos\",\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_web\",\"dependencies\":[]},{\"name\":\"flutter_keyboard_visibility_windows\",\"dependencies\":[]},{\"name\":\"flutter_plugin_android_lifecycle\",\"dependencies\":[]},{\"name\":\"image_picker\",\"dependencies\":[\"image_picker_android\",\"image_picker_for_web\",\"image_picker_ios\",\"image_picker_linux\",\"image_picker_macos\",\"image_picker_windows\"]},{\"name\":\"image_picker_android\",\"dependencies\":[\"flutter_plugin_android_lifecycle\"]},{\"name\":\"image_picker_for_web\",\"dependencies\":[]},{\"name\":\"image_picker_ios\",\"dependencies\":[]},{\"name\":\"image_picker_linux\",\"dependencies\":[\"file_selector_linux\"]},{\"name\":\"image_picker_macos\",\"dependencies\":[\"file_selector_macos\"]},{\"name\":\"image_picker_web\",\"dependencies\":[]},{\"name\":\"image_picker_windows\",\"dependencies\":[\"file_selector_windows\"]},{\"name\":\"libphonenumber_plugin\",\"dependencies\":[\"libphonenumber_web\"]},{\"name\":\"libphonenumber_web\",\"dependencies\":[]},{\"name\":\"path_provider\",\"dependencies\":[\"path_provider_android\",\"path_provider_foundation\",\"path_provider_linux\",\"path_provider_windows\"]},{\"name\":\"path_provider_android\",\"dependencies\":[]},{\"name\":\"path_provider_foundation\",\"dependencies\":[]},{\"name\":\"path_provider_linux\",\"dependencies\":[]},{\"name\":\"path_provider_windows\",\"dependencies\":[]},{\"name\":\"pointer_interceptor\",\"dependencies\":[\"pointer_interceptor_ios\",\"pointer_interceptor_web\"]},{\"name\":\"pointer_interceptor_ios\",\"dependencies\":[]},{\"name\":\"pointer_interceptor_web\",\"dependencies\":[]},{\"name\":\"shared_preferences\",\"dependencies\":[\"shared_preferences_android\",\"shared_preferences_foundation\",\"shared_preferences_linux\",\"shared_preferences_web\",\"shared_preferences_windows\"]},{\"name\":\"shared_preferences_android\",\"dependencies\":[]},{\"name\":\"shared_preferences_foundation\",\"dependencies\":[]},{\"name\":\"shared_preferences_linux\",\"dependencies\":[\"path_provider_linux\"]},{\"name\":\"shared_preferences_web\",\"dependencies\":[]},{\"name\":\"shared_preferences_windows\",\"dependencies\":[\"path_provider_windows\"]},{\"name\":\"url_launcher\",\"dependencies\":[\"url_launcher_android\",\"url_launcher_ios\",\"url_launcher_linux\",\"url_launcher_macos\",\"url_launcher_web\",\"url_launcher_windows\"]},{\"name\":\"url_launcher_android\",\"dependencies\":[]},{\"name\":\"url_launcher_ios\",\"dependencies\":[]},{\"name\":\"url_launcher_linux\",\"dependencies\":[]},{\"name\":\"url_launcher_macos\",\"dependencies\":[]},{\"name\":\"url_launcher_web\",\"dependencies\":[]},{\"name\":\"url_launcher_windows\",\"dependencies\":[]}],\"date_created\":\"2024-09-10 18:23:23.537388\",\"version\":\"3.22.2\"}" }, { "path": "frontend/frontend-flutter/Open Data QnA - Working Sheet V2 - sample_questions_UI copy.csv", "content": "user_grouping,scenario,question\nMovieExplorer-bigquery,Genres,What are the top 5 most common movie genres in the dataset?\nMovieExplorer-bigquery,Genres,How many are musicals?\nMovieExplorer-bigquery,Genres,Romance?\nMovieExplorer-bigquery,Movie,What is the average user rating of the God Father movie?\nMovieExplorer-bigquery,Movie,Which year was it released?\nMovieExplorer-bigquery,Movie,How long is it?\nMovieExplorer-bigquery,Movie,Who is the lead actor?\nMovieExplorer-bigquery,Movie,Director\nMovieExplorer-bigquery,Movie,Cast\nWorldCensus-cloudsql-pg,Life Expectancy,What is the life expectancy for men and women in a United States in 2022?\nWorldCensus-cloudsql-pg,Life Expectancy,In India?\nWorldCensus-cloudsql-pg,Life Expectancy,Which country has highest male life expectancy?\nWorldCensus-cloudsql-pg,Life Expectancy,Female life expectancy?\nWorldCensus-cloudsql-pg,Population Density,What are the top 5 coutries with highest population density in 2024? \nWorldCensus-cloudsql-pg,Population Density,What are the birth and death rates in these counties?\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the sex ratio at birth in China in 2023?\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Which country has the highest?\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Whats the world average?" }, { "path": "frontend/frontend-flutter/Open_Data_QnA_sample_questions_v3 copy.csv", "content": "user_grouping,scenario,question,main_question\nMovieExplorer-bigquery,Genres,What are the top 5 most common movie genres in the dataset?,Y\nMovieExplorer-bigquery,Genres,How many are musicals?,N\nMovieExplorer-bigquery,Genres,Romance?,N\nMovieExplorer-bigquery,Movie,What is the average user rating of the God Father movie?,Y\nMovieExplorer-bigquery,Movie,Which year was it released?,N\nMovieExplorer-bigquery,Movie,How long is it?,N\nMovieExplorer-bigquery,Movie,Who is the lead actor?,N\nMovieExplorer-bigquery,Movie,Director,N\nMovieExplorer-bigquery,Movie,Cast,N\nMovieExplorer-bigquery,Movie,Who is the actor playing the role of the godfather in the Godfather movie?,Y\nMovieExplorer-bigquery,Movie,and the one playing the role of Sony?,N\nMovieExplorer-bigquery,Movie,How many people saw the Godfather?,y\nWorldCensus-cloudsql-pg,Life Expectancy,What is the life expectancy for men and women in a United States in 2022?,Y\nWorldCensus-cloudsql-pg,Life Expectancy,In India?,N\nWorldCensus-cloudsql-pg,Life Expectancy,Which country has highest male life expectancy?,N\nWorldCensus-cloudsql-pg,Life Expectancy,Female life expectancy?,N\nWorldCensus-cloudsql-pg,Population Density,What are the top 5 coutries with highest population density in 2024?,Y \nWorldCensus-cloudsql-pg,Population Density,What are the birth and death rates in these counties?,N\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the sex ratio at birth in China in 2023?,Y\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Which country has the highest?,N\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the world average?,N\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What country has the lowest male ration?,Y\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Since when?,N\n" }, { "path": "frontend/frontend-flutter/README.md", "content": "# Deploy the Flutter-based Frontend demo UI\n\n

\n \n \"aaie\n \n

\n\n## Technologies and Components\nIn order to use easily the Open Data QnA SDK and the backend you have just installed (please refer to README.md related to the backend on this repo for the details if not installed yet) you need to have a front end. \nThis page explains how to install the frontend provided by this solution gracefully so that you can jump start the use of the Open Data QnA solution. \nObviously, you can pretty much develop your own frontend and call the APIs available of the backend deployments.\n\nThere are 2 versions of the frontend, one written in Angular and one written in Flutter, an Open Souce UI framework written in dart language and backed up by Google. Functionality-wise, they are both equivalent even though some minor differences may exist. \n\nThis readme is about installing the Flutter-based frontend.\nFor more information on Flutter :\n- [Flutter documentation](https://docs.flutter.dev/?_gl=1*17csnxq*_ga*MTI3NTU2MjQxMC4xNzI1ODc2Njg5*_ga_04YGWK0175*MTcyNTg3NjY4OS4xLjAuMTcyNTg3NjY4OS4wLjAuMA..)\n\nThe frontend needs to be deployed to Firebase, a Mobile Backend as a Service. It has a free tier call \"Spark Plan\" allowing you to use the services required to run the frontend.\nFor more details on Firebase, plase have a look at the documentation below:\n- [Firebase pricing tiers](https://firebase.google.com/pricing)\n- [Firebase Documentation](https://firebase.google.com/docs)\n\n## Getting Started\n\n### Installing Flutter and dart SDK\n\n#### Installing Flutter\n\nThe first step is to install the Flutter framework.\n\nTo build the Flutter app, you can either use an IDE, like Visual Studio Code with the Flutter SDK (and plugin) and relevant extension, or just the Flutter's command-line tools to build the app manually.\nThis guide only explains how to use the Flutter's command-line tools via the installation of the Flutter SDK bundle (that also contains the dart SDK).\n\nPlease click on the link below.\n- [Flutter SDK installation](https://docs.flutter.dev/get-started/install)\n\nYou'll end-up on the the landing page below. \n\n1- Click on the platform corresponding to the OS of the desktop you'll install the frontend on.\nAs an example, let's use Windows :\n\n

\n \n \n \n

\n\n2- Click on the Web type of app :\n

\n \n \n \n

\n\n3- Click on the flutter_windows_3.24.2-stable.zip button (version will vary over time)\n\n4- Move the zip file into the target folder you want, let's say \"dev\"\n\n5- Extract the archive\n

\n \n \n \n

\n\n6- Update the PATH environment variable :\n%USERPROFILE%\\dev\\flutter\\bin\n\n7- Test the installation by typing the \"flutter doctor\" command :\n

\n \n \n \n

\n\n\n### Installing Firebase tools\n\n1- Install the Firebase CLI \nIn order to interact easily with Firebase, you have to install the Firebase CLI.\nYou need to install npm first as a prerequisite:\n\n```\nnpm install -g firebase-tools\n```\nWith these Firebase CLI commands, you're able to authenticate to Firebase and do the deployment of the frontend on the Firebase Hosting service.\n\n2- Test the Firebase CLI\n\nIn order to very the Firebase CLI are working fine, login to Firebase using the command below.\n\n```\nfirebase login\n```\n\nFor more details on the installation, please look at the link below:\n- [Firebase CLI installation documentation](https://firebase.google.com/docs/cli#windows-npm)\n\n### Installing Flutterfire CLI\n\nFlutter is tightly integrated to Firebase.\nThe flutterfire_cli command,which is part of the FlutterFire CLI, is specifically designed for Flutter projects. It streamlines the process of connecting your Flutter app to Firebase and generating the necessary configuration files (firebase_options.dart) for different platforms (Android, iOS, web, etc.).\n\nInstall it from any directory, we'll need it later on.\n\n```\ndart pub global activate flutterfire_cli\n```\n\nYou may need to modify the PATH environment variable to access the flutterfire CLI.\nOn Windows, the binary is stored in :\n%USERPROFILE%\\AppData\\Local\\Pub\\Cache\\bin\n\n### Creating and configuring the Firebase project\n\n1- Go to the Firebase console\n\nClick on \"Get started with a Firebase project\"\n```\nhttps://console.firebase.google.com/\n```\n\n

\n \n \n \n

\n\n2- Give a name to the project\n\nUse the same name you used during the backend installation. As an example, let's us \"opendataqna\"\n

\n \n \n \n

\n\n3- Google Analytics\n\nYou can choose to activate Google Analytics or not. This is not required for the app to work.\nFor the sake of completness, let's use it as it is proposed by default.\n\n

\n \n \n \n

\n\nJust click on the \"Continue\" button.\n\n

\n \n \n \n

\nThen click on the \"Create project\" button.\n\nOnce done, you have access to your newly Firebase project\n

\n \n \n \n

\n\nAlternatively, you can also use the \"flutterfire configure\" CLI to create the Firebase project. \n\n4- List the project using the Firebase CLI \n\nNow go back to your terminal and list this new Firebase project :\n```\nfirebase projects:list\n```\n

\n \n \n \n

\n\n### Creating the Firestore Database\n\nThe frontend requires a Firestore database (which is a Firebase service) to work.\nThe free tier only allows the creation of 1 database that has the default name (derived from the project name)\n\n1-On the Firebase console, click on the Firestore menu\n\n

\n \n \n \n

\n\n2- Select the location\n

\n \n \n \n

\n\n3- Select the production mode\n

\n \n \n \n

\n\n4- Modify the security rules\n\nThe front end needs to read and write on different collections in order to store the configuration, the know good SQL and the imported questions.\nFor that, you need to change the security rules like below (please take appropriate measures to protect access to the database):\n\n

\n \n \n \n

\n\n### Enabling Sign-In method\n\nThe app requires that you authenticate using your Google account, whether it is personal or professional.\n\n1- Go to the Firebase Authentication menu \nFor that, go to the Firebase console and click on the Authentication menu and then on the Sing-in method tab:\n\n\n

\n \n \n \n

\n\n\n2- Choose Google as the Provider\nClick on the Google icon to enable federated identity via Google:\n\n

\n \n \n \n

\n\n3- Update the project level settings with the info below\n

\n \n \n \n

\n\n\n### Installing the frontend\n\nNow that the Firebase project is created, let's get the source code of the Flutter frontend.\n\n1- Create a folder for your project\n\nCreat folder that will contain the source code and the Firebase configuration.\nLet's call it opendatadna as well:\n\n```\nmkdir OpenDataQnA\n```\n\n2- clone the source code from the repository\n\nGo to the opendataqna folder and clone the source code of the frontend app using the command below:\n```\ngit clone https://github.com/GoogleCloudPlatform/Open_Data_QnA.git\n\ncd frontend/frontend-flutter\n```\n\n3- Registering the frontend app to Firebase\n\nIn order to register the app to Firebase and selected the Firebase project it will leave in, use the flutterfire configure command:\n```\nflutterfire configure\n```\nYou'll be guided by the command via a couple of questions :\n

\n \n \n \n

\n\nYou can check on the Firebase console that app has been registered to the opendataqna project by clicking on \"Project Overview\":\n

\n \n \n \n

\n\n4- Deploy the app to Firebase Hosting\n\nFirebase Hosting is a Firebase service allowing to host a web app.\n\nIf not logged in yet to Firebase, do so with the firebase\n```\nfirebase login\n```\n\nYou also need to initialize the Firebase project. That will update the firebase.json file create by flutterfire configuration by adding the service you want to use (hosting).\nFor that make sure you're at the root of the project :\n```\ncd OpenData/frontend/frontend-flutter\n```\nthen add the command below to help Firebase hosting better understand and integrate with Flutter in order to optimize the build and deployment:\n```\nfirebase experiments:enable webframeworks\n```\n\nThen type the command below:\n\n```\nfirebase init hosting\n```\n

\n \n \n \n

\nAnswer the questions asked, and a firebase.json file will be generated at the root of the project that looks like the one below:\n\n

\n \n \n \n

\n\nWe're now ready to deploy the app on Firebase Hosting.\nFor that, still at the root level of the project (frontend-flutter folder) type the command below to deploy the app to Firebase Hosting service:\n```\nfirebase deploy\n```\n

\n \n \n \n

\n\nOnce the app has been built and deployed, it is available via the Hosting URL that shows up at the bottom of the output of the forebase deploy commad and has teh ofrm:\nhttps://.web.app\n\n\n### Using the frontend\n\n1- Setup the config_frontend config file\n\nThe app requires some configuration to work, like the URI of the endpoint created during the backend installation and other information.\nCreate a json file and name it config_frontend.json (the name does not matter) and copy paste the json object below:\n\n```\n{\n\"endpoint_opendataqnq\": \"https://opendataqna-ol22ywferse-uc.v.run.app\",\n\"firestore_database_id\": \"opendataqna-session-logs\",\n\"firestore_history_collection\": \"session_logs\",\n\"firestore_cfg_collection\": \"front_end_flutter_cfg\",\n\"expert_mode\": true,\n\"anonymized_data\": false,\n\"firebase_app_name\": \"opendataqna\",\n\"imported_questions\": \"imported_questions\"\n}\n```\nChange the values based on your setup.\n\n- endpoint_opendataqnq : contains the URI of the backend created (String)\n- firestore_database_id : name of the database you created earlier. If it is the default database, use \"default\" (String)\n- firestore_history_collection : name of the collection used to store all the known-good sql, questions, answers, user_id and timestamp (String)\n- firestore_cfg_collection : name of this file (String)\n- expert_mode : if true, activates the expert mode (boolean)\n- anonymized_data : if true, activates data anonymization\n- firebase_app_name : name of the app as registered to the Firebase project as it shows up on the Project overview on the Firebase console (String)\n- imported_questions : name of the collection used to store imported questions\n\n2- Access the app\n\nAccess the app using the https://.web.app link generated during the deployment of the app.\nAccept the terms and conditions and agree.\n\n

\n \n \n \n

\n\n\n3- Authenticate yourself using your Google account\n\n

\n \n \n \n

\n\n4- Landing page\n\nYou end up on the landing page below.\n\n

\n \n \n \n

\n\n- The stepper shows the status and the processing time of the request. It only shows up when the user is in Expert mode (more on that below)\n- The app is keeping track of the questions and answers (and SQL request) taht were successful in the Firestore database. When the app is launched, At most, the 4 last questions asked are displayed. Ckicking on any of them fills out automatically the input text field\n- the humberger menu is collapsed by default so that the app can provide more real estate for the canvas.\n\n\n5- Menu\n\n

\n \n \n \n

\n\n- New chat : Allows to reset the context so that the next query is not answered based on the previous answers\n- Import : This allows the user to import questions\n\n- Imported questions: import a csv file containing questions that are asked often. Ot only shows up in Expert mode. There are 2 falvors of this cvs file that are supported :\n - 3 columns that have to be : user_grouping,scenario,question\n\nAn example of such a file is provided in frontend-flutter/script/Open Data QnA - Working Sheet V2 - sample_questions_UI copy.csv)\n\n\n```\nuser_grouping,scenario,question\nMovieExplorer-bigquery,Genres,What are the top 5 most common movie genres in the dataset?\nMovieExplorer-bigquery,Genres,How many are musicals?\nMovieExplorer-bigquery,Genres,Romance?\nMovieExplorer-bigquery,Movie,What is the average user rating of the God Father movie?\nMovieExplorer-bigquery,Movie,Which year was it released?\nMovieExplorer-bigquery,Movie,How long is it?\nMovieExplorer-bigquery,Movie,Who is the lead actor?\nMovieExplorer-bigquery,Movie,Director\nMovieExplorer-bigquery,Movie,Cast\nWorldCensus-cloudsql-pg,Life Expectancy,What is the life expectancy for men and women in a United States in 2022?\nWorldCensus-cloudsql-pg,Life Expectancy,In India?\nWorldCensus-cloudsql-pg,Life Expectancy,Which country has highest male life expectancy?\nWorldCensus-cloudsql-pg,Life Expectancy,Female life expectancy?\nWorldCensus-cloudsql-pg,Population Density,What are the top 5 coutries with highest population density in 2024?\nWorldCensus-cloudsql-pg,Population Density,What are the birth and death rates in these counties?\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the sex ratio at birth in China in 2023?\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Which country has the highest?\n```\n-\n - 4 columns that have to be : user_grouping,scenario,question,main_question\n \n This format allows to add an indent to the follow-up question for the sake of clarity. An example of such a file is provided in frontend-flutter/script/Open_Data_QnA_sample_questions_v3.csv)\n \n```\n user_grouping,scenario,question,main_question\n MovieExplorer-bigquery,Genres,What are the top 5 most common movie genres in the dataset?,Y\n MovieExplorer-bigquery,Genres,How many are musicals?,N\n MovieExplorer-bigquery,Genres,Romance?,N\n MovieExplorer-bigquery,Movie,What is the average user rating of the God Father movie?,Y\n MovieExplorer-bigquery,Movie,Which year was it released?,N\n MovieExplorer-bigquery,Movie,How long is it?,N\n MovieExplorer-bigquery,Movie,Who is the lead actor?,N\n MovieExplorer-bigquery,Movie,Director,N\n MovieExplorer-bigquery,Movie,Cast,N\n MovieExplorer-bigquery,Movie,Who is the actor playing the role of the godfather in the Godfather movie?,Y\n MovieExplorer-bigquery,Movie,and the one playing the role of Sony?,N\n MovieExplorer-bigquery,Movie,How many people saw the Godfather?,y\n WorldCensus-cloudsql-pg,Life Expectancy,What is the life expectancy for men and women in a United States in 2022?,Y\n WorldCensus-cloudsql-pg,Life Expectancy,In India?,N\n WorldCensus-cloudsql-pg,Life Expectancy,Which country has highest male life expectancy?,N\n WorldCensus-cloudsql-pg,Life Expectancy,Female life expectancy?,N\n WorldCensus-cloudsql-pg,Population Density,What are the top 5 coutries with highest population density in 2024?,Y\n WorldCensus-cloudsql-pg,Population Density,What are the birth and death rates in these counties?,N\n WorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the sex ratio at birth in China in 2023?,Y\n WorldCensus-cloudsql-pg,Sex Ratio at Birth,Which country has the highest?,N\n WorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the world average?,N\n WorldCensus-cloudsql-pg,Sex Ratio at Birth,What country has the lowest male ration?,Y\n WorldCensus-cloudsql-pg,Sex Ratio at Birth,Since when?,N\n\n```\n - user_grouping : name of the dataset or database\n - scenario : used to tag your questions that are related to the same topic\n - question: question in natural language\n - main_question : when the value is Y, this is a fully understable and contextual question (let's call it main question). When the value is N, it is a follow-up question of the main question and does not have a full context\n - Once loaded, the questions show up like this (4 colums format)\n\n\n\n

\n \n \n \n

\n\n- History : Gives the list of questions types during the session\n- Most popular questions : Lists all the historical questions sorted by the number they have been typed\n- Settings : Allows to\n - upload the config_frontend.json file which is stored on Firestore so that fater the app is relaunched the configuration is kept\n - anonymize data (in case the user wants to do a demo but not show real data)\n - set the expert mode\n - All the other options are not implemented\n\n

\n \n \n \n

\n\n6- Suggestions\n\nWhen ever a question is clicked, whether from the 4 last question cards, the imported questions, the history or most popular questions, 3 suggestions are showing up at the bottom of the app. These suggestions are actually coming from the known good sql stored on the backend.\n

\n \n \n \n

\n\nClicking on one of these suggestions also triggers 3 other suggestions.\n\n\n7- Auto-completion\n\nThe app is proposing autocompletion each time a character is typed.\nThe questions suggested for the autocompletion are coming from the known good sql stored on the backed.\n\n

\n \n \n \n

\n\n\n8- Expert mode\n\nAs said before, when enabled, the Expert mode allows to display the imported questions and the stepper.\nClicking on a step, once completed, display details of this step :\n\n

\n \n \n \n

\n" }, { "path": "frontend/frontend-flutter/analysis_options.yaml", "content": "# This file configures the analyzer, which statically analyzes Dart code to\n# check for errors, warnings, and lints.\n#\n# The issues identified by the analyzer are surfaced in the UI of Dart-enabled\n# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be\n# invoked from the command line by running `flutter analyze`.\n\n# The following line activates a set of recommended lints for Flutter apps,\n# packages, and plugins designed to encourage good coding practices.\ninclude: package:flutter_lints/flutter.yaml\n\nlinter:\n # The lint rules applied to this project can be customized in the\n # section below to disable rules from the `package:flutter_lints/flutter.yaml`\n # included above or to enable additional rules. A list of all available lints\n # and their documentation is published at https://dart.dev/lints.\n #\n # Instead of disabling a lint rule for the entire project in the\n # section below, it can also be suppressed for a single line of code\n # or a specific dart file by using the `// ignore: name_of_lint` and\n # `// ignore_for_file: name_of_lint` syntax on the line or in the file\n # producing the lint.\n rules:\n # avoid_print: false # Uncomment to disable the `avoid_print` rule\n # prefer_single_quotes: true # Uncomment to enable the `prefer_single_quotes` rule\n\n# Additional information about this file can be found at\n# https://dart.dev/guides/language/analysis-options\n" }, { "path": "frontend/frontend-flutter/android/.gitignore", "content": "gradle-wrapper.jar\n/.gradle\n/captures/\n/gradlew\n/gradlew.bat\n/local.properties\nGeneratedPluginRegistrant.java\n\n# Remember to never publicly share your keystore.\n# See https://flutter.dev/docs/deployment/android#reference-the-keystore-from-the-app\nkey.properties\n**/*.keystore\n**/*.jks\n" }, { "path": "frontend/frontend-flutter/android/app/build.gradle", "content": "plugins {\n id \"com.android.application\"\n // START: FlutterFire Configuration\n id 'com.google.gms.google-services'\n // END: FlutterFire Configuration\n id \"kotlin-android\"\n id \"dev.flutter.flutter-gradle-plugin\"\n}\n\ndef localProperties = new Properties()\ndef localPropertiesFile = rootProject.file('local.properties')\nif (localPropertiesFile.exists()) {\n localPropertiesFile.withReader('UTF-8') { reader ->\n localProperties.load(reader)\n }\n}\n\ndef flutterVersionCode = localProperties.getProperty('flutter.versionCode')\nif (flutterVersionCode == null) {\n flutterVersionCode = '1'\n}\n\ndef flutterVersionName = localProperties.getProperty('flutter.versionName')\nif (flutterVersionName == null) {\n flutterVersionName = '1.0'\n}\n\nandroid {\n namespace \"com.pilotcap.ttmd\"\n compileSdkVersion flutter.compileSdkVersion\n ndkVersion flutter.ndkVersion\n\n compileOptions {\n sourceCompatibility JavaVersion.VERSION_1_8\n targetCompatibility JavaVersion.VERSION_1_8\n }\n\n kotlinOptions {\n jvmTarget = '1.8'\n }\n\n sourceSets {\n main.java.srcDirs += 'src/main/kotlin'\n }\n\n defaultConfig {\n // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).\n applicationId \"com.pilotcap.ttmd\"\n // You can update the following values to match your application needs.\n // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration.\n minSdkVersion flutter.minSdkVersion\n targetSdkVersion flutter.targetSdkVersion\n versionCode flutterVersionCode.toInteger()\n versionName flutterVersionName\n }\n\n buildTypes {\n release {\n // TODO: Add your own signing config for the release build.\n // Signing with the debug keys for now, so `flutter run --release` works.\n signingConfig signingConfigs.debug\n }\n }\n}\n\nflutter {\n source '../..'\n}\n\ndependencies {}\n" }, { "path": "frontend/frontend-flutter/android/app/google-services.json", "content": "{\n \"project_info\": {\n \"project_number\": \"7413174684\",\n \"project_id\": \"dic-caa-ra1\",\n \"storage_bucket\": \"dic-caa-ra1.appspot.com\"\n },\n \"client\": [\n {\n \"client_info\": {\n \"mobilesdk_app_id\": \"1:7413174684:android:f4ed287d5da149fce5f859\",\n \"android_client_info\": {\n \"package_name\": \"com.pilotcap.ttmd\"\n }\n },\n \"oauth_client\": [],\n \"api_key\": [\n {\n \"current_key\": \"AIzaSyCytznMTfW7u99AuEQRbdiqmfaCAoMh0Bc\"\n }\n ],\n \"services\": {\n \"appinvite_service\": {\n \"other_platform_oauth_client\": []\n }\n }\n }\n ],\n \"configuration_version\": \"1\"\n}" }, { "path": "frontend/frontend-flutter/android/app/src/debug/AndroidManifest.xml", "content": "\n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/AndroidManifest.xml", "content": "\n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/kotlin/com/pilotcap/ttmd/MainActivity.kt", "content": "package com.pilotcap.ttmd\n\nimport io.flutter.embedding.android.FlutterActivity\n\nclass MainActivity: FlutterActivity() {\n}\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/res/drawable/launch_background.xml", "content": "\n\n\n \n\n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/res/drawable-v21/launch_background.xml", "content": "\n\n\n \n\n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/res/values/styles.xml", "content": "\n\n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/main/res/values-night/styles.xml", "content": "\n\n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/android/app/src/profile/AndroidManifest.xml", "content": "\n \n \n\n" }, { "path": "frontend/frontend-flutter/android/build.gradle", "content": "buildscript {\n ext.kotlin_version = '1.7.10'\n repositories {\n google()\n mavenCentral()\n }\n\n dependencies {\n classpath \"org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version\"\n }\n}\n\nallprojects {\n repositories {\n google()\n mavenCentral()\n }\n}\n\nrootProject.buildDir = '../build'\nsubprojects {\n project.buildDir = \"${rootProject.buildDir}/${project.name}\"\n}\nsubprojects {\n project.evaluationDependsOn(':app')\n}\n\ntasks.register(\"clean\", Delete) {\n delete rootProject.buildDir\n}\n" }, { "path": "frontend/frontend-flutter/android/gradle/wrapper/gradle-wrapper.properties", "content": "distributionBase=GRADLE_USER_HOME\ndistributionPath=wrapper/dists\nzipStoreBase=GRADLE_USER_HOME\nzipStorePath=wrapper/dists\ndistributionUrl=https\\://services.gradle.org/distributions/gradle-7.5-all.zip\n" }, { "path": "frontend/frontend-flutter/android/gradle.properties", "content": "org.gradle.jvmargs=-Xmx4G\nandroid.useAndroidX=true\nandroid.enableJetifier=true\n" }, { "path": "frontend/frontend-flutter/android/nl2sql_oss_android.iml", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "frontend/frontend-flutter/android/settings.gradle", "content": "pluginManagement {\n def flutterSdkPath = {\n def properties = new Properties()\n file(\"local.properties\").withInputStream { properties.load(it) }\n def flutterSdkPath = properties.getProperty(\"flutter.sdk\")\n assert flutterSdkPath != null, \"flutter.sdk not set in local.properties\"\n return flutterSdkPath\n }\n settings.ext.flutterSdkPath = flutterSdkPath()\n\n includeBuild(\"${settings.ext.flutterSdkPath}/packages/flutter_tools/gradle\")\n\n repositories {\n google()\n mavenCentral()\n gradlePluginPortal()\n }\n\n plugins {\n id \"dev.flutter.flutter-gradle-plugin\" version \"1.0.0\" apply false\n }\n}\n\nplugins {\n id \"dev.flutter.flutter-plugin-loader\" version \"1.0.0\"\n id \"com.android.application\" version \"7.3.0\" apply false\n // START: FlutterFire Configuration\n id \"com.google.gms.google-services\" version \"4.3.15\" apply false\n // END: FlutterFire Configuration\n}\n\ninclude \":app\"\n" }, { "path": "frontend/frontend-flutter/build/web/.last_build_id", "content": "5389598c2a36d064db55a8edf57e320f" }, { "path": "frontend/frontend-flutter/ios/.gitignore", "content": "**/dgph\n*.mode1v3\n*.mode2v3\n*.moved-aside\n*.pbxuser\n*.perspectivev3\n**/*sync/\n.sconsign.dblite\n.tags*\n**/.vagrant/\n**/DerivedData/\nIcon?\n**/Pods/\n**/.symlinks/\nprofile\nxcuserdata\n**/.generated/\nFlutter/App.framework\nFlutter/Flutter.framework\nFlutter/Flutter.podspec\nFlutter/Generated.xcconfig\nFlutter/ephemeral/\nFlutter/app.flx\nFlutter/app.zip\nFlutter/flutter_assets/\nFlutter/flutter_export_environment.sh\nServiceDefinitions.json\nRunner/GeneratedPluginRegistrant.*\n\n# Exceptions to above rules.\n!default.mode1v3\n!default.mode2v3\n!default.pbxuser\n!default.perspectivev3\n" }, { "path": "frontend/frontend-flutter/ios/Flutter/AppFrameworkInfo.plist", "content": "\n\n\n\n CFBundleDevelopmentRegion\n en\n CFBundleExecutable\n App\n CFBundleIdentifier\n io.flutter.flutter.app\n CFBundleInfoDictionaryVersion\n 6.0\n CFBundleName\n App\n CFBundlePackageType\n FMWK\n CFBundleShortVersionString\n 1.0\n CFBundleSignature\n ????\n CFBundleVersion\n 1.0\n MinimumOSVersion\n 12.0\n\n\n" }, { "path": "frontend/frontend-flutter/ios/Flutter/Debug.xcconfig", "content": "#include? \"Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig\"\n#include \"Generated.xcconfig\"\n" }, { "path": "frontend/frontend-flutter/ios/Flutter/Release.xcconfig", "content": "#include? \"Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig\"\n#include \"Generated.xcconfig\"\n" }, { "path": "frontend/frontend-flutter/ios/Podfile", "content": "# Uncomment this line to define a global platform for your project\n# platform :ios, '12.0'\n\n# CocoaPods analytics sends network stats synchronously affecting flutter build latency.\nENV['COCOAPODS_DISABLE_STATS'] = 'true'\n\nproject 'Runner', {\n 'Debug' => :debug,\n 'Profile' => :release,\n 'Release' => :release,\n}\n\ndef flutter_root\n generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__)\n unless File.exist?(generated_xcode_build_settings_path)\n raise \"#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first\"\n end\n\n File.foreach(generated_xcode_build_settings_path) do |line|\n matches = line.match(/FLUTTER_ROOT\\=(.*)/)\n return matches[1].strip if matches\n end\n raise \"FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get\"\nend\n\nrequire File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root)\n\nflutter_ios_podfile_setup\n\ntarget 'Runner' do\n use_frameworks!\n use_modular_headers!\n\n flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__))\n target 'RunnerTests' do\n inherit! :search_paths\n end\nend\n\npost_install do |installer|\n installer.pods_project.targets.each do |target|\n flutter_additional_ios_build_settings(target)\n end\nend\n" }, { "path": "frontend/frontend-flutter/ios/Runner/AppDelegate.swift", "content": "import UIKit\nimport Flutter\n\n@UIApplicationMain\n@objc class AppDelegate: FlutterAppDelegate {\n override func application(\n _ application: UIApplication,\n didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?\n ) -> Bool {\n GeneratedPluginRegistrant.register(with: self)\n return super.application(application, didFinishLaunchingWithOptions: launchOptions)\n }\n}\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json", "content": "{\n \"images\" : [\n {\n \"size\" : \"20x20\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-20x20@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"20x20\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-20x20@3x.png\",\n \"scale\" : \"3x\"\n },\n {\n \"size\" : \"29x29\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-29x29@1x.png\",\n \"scale\" : \"1x\"\n },\n {\n \"size\" : \"29x29\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-29x29@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"29x29\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-29x29@3x.png\",\n \"scale\" : \"3x\"\n },\n {\n \"size\" : \"40x40\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-40x40@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"40x40\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-40x40@3x.png\",\n \"scale\" : \"3x\"\n },\n {\n \"size\" : \"60x60\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-60x60@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"60x60\",\n \"idiom\" : \"iphone\",\n \"filename\" : \"Icon-App-60x60@3x.png\",\n \"scale\" : \"3x\"\n },\n {\n \"size\" : \"20x20\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-20x20@1x.png\",\n \"scale\" : \"1x\"\n },\n {\n \"size\" : \"20x20\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-20x20@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"29x29\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-29x29@1x.png\",\n \"scale\" : \"1x\"\n },\n {\n \"size\" : \"29x29\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-29x29@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"40x40\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-40x40@1x.png\",\n \"scale\" : \"1x\"\n },\n {\n \"size\" : \"40x40\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-40x40@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"76x76\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-76x76@1x.png\",\n \"scale\" : \"1x\"\n },\n {\n \"size\" : \"76x76\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-76x76@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"83.5x83.5\",\n \"idiom\" : \"ipad\",\n \"filename\" : \"Icon-App-83.5x83.5@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"size\" : \"1024x1024\",\n \"idiom\" : \"ios-marketing\",\n \"filename\" : \"Icon-App-1024x1024@1x.png\",\n \"scale\" : \"1x\"\n }\n ],\n \"info\" : {\n \"version\" : 1,\n \"author\" : \"xcode\"\n }\n}\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Assets.xcassets/LaunchImage.imageset/Contents.json", "content": "{\n \"images\" : [\n {\n \"idiom\" : \"universal\",\n \"filename\" : \"LaunchImage.png\",\n \"scale\" : \"1x\"\n },\n {\n \"idiom\" : \"universal\",\n \"filename\" : \"LaunchImage@2x.png\",\n \"scale\" : \"2x\"\n },\n {\n \"idiom\" : \"universal\",\n \"filename\" : \"LaunchImage@3x.png\",\n \"scale\" : \"3x\"\n }\n ],\n \"info\" : {\n \"version\" : 1,\n \"author\" : \"xcode\"\n }\n}\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Assets.xcassets/LaunchImage.imageset/README.md", "content": "# Launch Screen Assets\n\nYou can customize the launch screen with your own desired assets by replacing the image files in this directory.\n\nYou can also do it by opening your Flutter project's Xcode project with `open ios/Runner.xcworkspace`, selecting `Runner/Assets.xcassets` in the Project Navigator and dropping in the desired images." }, { "path": "frontend/frontend-flutter/ios/Runner/Base.lproj/LaunchScreen.storyboard", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Base.lproj/Main.storyboard", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/ios/Runner/GoogleService-Info.plist", "content": "\n\n\n\n\tAPI_KEY\n\tAIzaSyB5RZFwdDakT3zrJ7-DMum0IHCSfyLqtBQ\n\tGCM_SENDER_ID\n\t7413174684\n\tPLIST_VERSION\n\t1\n\tBUNDLE_ID\n\tcom.pilotcap.ttmd\n\tPROJECT_ID\n\tdic-caa-ra1\n\tSTORAGE_BUCKET\n\tdic-caa-ra1.appspot.com\n\tIS_ADS_ENABLED\n\t\n\tIS_ANALYTICS_ENABLED\n\t\n\tIS_APPINVITE_ENABLED\n\t\n\tIS_GCM_ENABLED\n\t\n\tIS_SIGNIN_ENABLED\n\t\n\tGOOGLE_APP_ID\n\t1:7413174684:ios:9a2317f461e3a05ce5f859\n\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Info.plist", "content": "\n\n\n\n\tCFBundleDevelopmentRegion\n\t$(DEVELOPMENT_LANGUAGE)\n\tCFBundleDisplayName\n\tTtmd\n\tCFBundleExecutable\n\t$(EXECUTABLE_NAME)\n\tCFBundleIdentifier\n\t$(PRODUCT_BUNDLE_IDENTIFIER)\n\tCFBundleInfoDictionaryVersion\n\t6.0\n\tCFBundleName\n\tttmd\n\tCFBundlePackageType\n\tAPPL\n\tCFBundleShortVersionString\n\t$(FLUTTER_BUILD_NAME)\n\tCFBundleSignature\n\t????\n\tCFBundleVersion\n\t$(FLUTTER_BUILD_NUMBER)\n\tLSRequiresIPhoneOS\n\t\n\tUILaunchStoryboardName\n\tLaunchScreen\n\tUIMainStoryboardFile\n\tMain\n\tUISupportedInterfaceOrientations\n\t\n\t\tUIInterfaceOrientationPortrait\n\t\tUIInterfaceOrientationLandscapeLeft\n\t\tUIInterfaceOrientationLandscapeRight\n\t\n\tUISupportedInterfaceOrientations~ipad\n\t\n\t\tUIInterfaceOrientationPortrait\n\t\tUIInterfaceOrientationPortraitUpsideDown\n\t\tUIInterfaceOrientationLandscapeLeft\n\t\tUIInterfaceOrientationLandscapeRight\n\t\n\tCADisableMinimumFrameDurationOnPhone\n\t\n\tUIApplicationSupportsIndirectInputEvents\n\t\n\n\n" }, { "path": "frontend/frontend-flutter/ios/Runner/Runner-Bridging-Header.h", "content": "#import \"GeneratedPluginRegistrant.h\"\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcodeproj/project.pbxproj", "content": "// !$*UTF8*$!\n{\n\tarchiveVersion = 1;\n\tclasses = {\n\t};\n\tobjectVersion = 54;\n\tobjects = {\n\n/* Begin PBXBuildFile section */\n\t\t104157D68C004C47E46A0CC5 /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = FA7A5FA34E36E0E31106D75D /* Pods_RunnerTests.framework */; };\n\t\t1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; };\n\t\t331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; };\n\t\t3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; };\n\t\t74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; };\n\t\t97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; };\n\t\t97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; };\n\t\t97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; };\n\t\tB9A15A94E93BB2ECB5A0330B /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 232A579B11BD1BA63E216F6D /* Pods_Runner.framework */; };\n\t\tCD1D1C48D4206E8BEA39AABB /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = BA68F78AEC387609C8F55641 /* GoogleService-Info.plist */; };\n/* End PBXBuildFile section */\n\n/* Begin PBXContainerItemProxy section */\n\t\t331C8085294A63A400263BE5 /* PBXContainerItemProxy */ = {\n\t\t\tisa = PBXContainerItemProxy;\n\t\t\tcontainerPortal = 97C146E61CF9000F007C117D /* Project object */;\n\t\t\tproxyType = 1;\n\t\t\tremoteGlobalIDString = 97C146ED1CF9000F007C117D;\n\t\t\tremoteInfo = Runner;\n\t\t};\n/* End PBXContainerItemProxy section */\n\n/* Begin PBXCopyFilesBuildPhase section */\n\t\t9705A1C41CF9048500538489 /* Embed Frameworks */ = {\n\t\t\tisa = PBXCopyFilesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tdstPath = \"\";\n\t\t\tdstSubfolderSpec = 10;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tname = \"Embed Frameworks\";\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXCopyFilesBuildPhase section */\n\n/* Begin PBXFileReference section */\n\t\t0BC3F20A14D692495160648B /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-Runner.profile.xcconfig\"; path = \"Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig\"; sourceTree = \"\"; };\n\t\t1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = \"\"; };\n\t\t1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = \"\"; };\n\t\t232A579B11BD1BA63E216F6D /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = \"\"; };\n\t\t331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = \"\"; };\n\t\t7378EF21874F9A1DFF00F4F0 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-Runner.release.xcconfig\"; path = \"Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig\"; sourceTree = \"\"; };\n\t\t74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = \"Runner-Bridging-Header.h\"; sourceTree = \"\"; };\n\t\t74858FAE1ED2DC5600515810 /* AppDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = \"\"; };\n\t\t7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = Release.xcconfig; path = Flutter/Release.xcconfig; sourceTree = \"\"; };\n\t\t7D9D3E084497F437970DDDAA /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-Runner.debug.xcconfig\"; path = \"Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig\"; sourceTree = \"\"; };\n\t\t9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Debug.xcconfig; path = Flutter/Debug.xcconfig; sourceTree = \"\"; };\n\t\t9740EEB31CF90195004384FC /* Generated.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Generated.xcconfig; path = Flutter/Generated.xcconfig; sourceTree = \"\"; };\n\t\t97C146EE1CF9000F007C117D /* Runner.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Runner.app; sourceTree = BUILT_PRODUCTS_DIR; };\n\t\t97C146FB1CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = \"\"; };\n\t\t97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = \"\"; };\n\t\t97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = \"\"; };\n\t\t97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = \"\"; };\n\t\tB1C6DDD94FCC20EC357B6FDE /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-RunnerTests.release.xcconfig\"; path = \"Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig\"; sourceTree = \"\"; };\n\t\tBA68F78AEC387609C8F55641 /* GoogleService-Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; name = \"GoogleService-Info.plist\"; path = \"Runner/GoogleService-Info.plist\"; sourceTree = \"\"; };\n\t\tD6C5A2A3DB181063A7E47D6B /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-RunnerTests.profile.xcconfig\"; path = \"Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig\"; sourceTree = \"\"; };\n\t\tDF41D8FEA220BF2CA2DE0E0D /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = \"Pods-RunnerTests.debug.xcconfig\"; path = \"Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig\"; sourceTree = \"\"; };\n\t\tFA7A5FA34E36E0E31106D75D /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; };\n/* End PBXFileReference section */\n\n/* Begin PBXFrameworksBuildPhase section */\n\t\t97C146EB1CF9000F007C117D /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\tB9A15A94E93BB2ECB5A0330B /* Pods_Runner.framework in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\tC8A669C122AD2A677A6E05F1 /* Frameworks */ = {\n\t\t\tisa = PBXFrameworksBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t104157D68C004C47E46A0CC5 /* Pods_RunnerTests.framework in Frameworks */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXFrameworksBuildPhase section */\n\n/* Begin PBXGroup section */\n\t\t331C8082294A63A400263BE5 /* RunnerTests */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t331C807B294A618700263BE5 /* RunnerTests.swift */,\n\t\t\t);\n\t\t\tpath = RunnerTests;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t4609EEA3E5E4780CEBAC8AD1 /* Frameworks */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t232A579B11BD1BA63E216F6D /* Pods_Runner.framework */,\n\t\t\t\tFA7A5FA34E36E0E31106D75D /* Pods_RunnerTests.framework */,\n\t\t\t);\n\t\t\tname = Frameworks;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t9740EEB11CF90186004384FC /* Flutter */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */,\n\t\t\t\t9740EEB21CF90195004384FC /* Debug.xcconfig */,\n\t\t\t\t7AFA3C8E1D35360C0083082E /* Release.xcconfig */,\n\t\t\t\t9740EEB31CF90195004384FC /* Generated.xcconfig */,\n\t\t\t);\n\t\t\tname = Flutter;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t97C146E51CF9000F007C117D = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t9740EEB11CF90186004384FC /* Flutter */,\n\t\t\t\t97C146F01CF9000F007C117D /* Runner */,\n\t\t\t\t97C146EF1CF9000F007C117D /* Products */,\n\t\t\t\t331C8082294A63A400263BE5 /* RunnerTests */,\n\t\t\t\tDEFF93311464728E4415F42A /* Pods */,\n\t\t\t\t4609EEA3E5E4780CEBAC8AD1 /* Frameworks */,\n\t\t\t\tBA68F78AEC387609C8F55641 /* GoogleService-Info.plist */,\n\t\t\t);\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t97C146EF1CF9000F007C117D /* Products */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t97C146EE1CF9000F007C117D /* Runner.app */,\n\t\t\t\t331C8081294A63A400263BE5 /* RunnerTests.xctest */,\n\t\t\t);\n\t\t\tname = Products;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t97C146F01CF9000F007C117D /* Runner */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t97C146FA1CF9000F007C117D /* Main.storyboard */,\n\t\t\t\t97C146FD1CF9000F007C117D /* Assets.xcassets */,\n\t\t\t\t97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */,\n\t\t\t\t97C147021CF9000F007C117D /* Info.plist */,\n\t\t\t\t1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */,\n\t\t\t\t1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */,\n\t\t\t\t74858FAE1ED2DC5600515810 /* AppDelegate.swift */,\n\t\t\t\t74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */,\n\t\t\t);\n\t\t\tpath = Runner;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\tDEFF93311464728E4415F42A /* Pods */ = {\n\t\t\tisa = PBXGroup;\n\t\t\tchildren = (\n\t\t\t\t7D9D3E084497F437970DDDAA /* Pods-Runner.debug.xcconfig */,\n\t\t\t\t7378EF21874F9A1DFF00F4F0 /* Pods-Runner.release.xcconfig */,\n\t\t\t\t0BC3F20A14D692495160648B /* Pods-Runner.profile.xcconfig */,\n\t\t\t\tDF41D8FEA220BF2CA2DE0E0D /* Pods-RunnerTests.debug.xcconfig */,\n\t\t\t\tB1C6DDD94FCC20EC357B6FDE /* Pods-RunnerTests.release.xcconfig */,\n\t\t\t\tD6C5A2A3DB181063A7E47D6B /* Pods-RunnerTests.profile.xcconfig */,\n\t\t\t);\n\t\t\tname = Pods;\n\t\t\tpath = Pods;\n\t\t\tsourceTree = \"\";\n\t\t};\n/* End PBXGroup section */\n\n/* Begin PBXNativeTarget section */\n\t\t331C8080294A63A400263BE5 /* RunnerTests */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = 331C8087294A63A400263BE5 /* Build configuration list for PBXNativeTarget \"RunnerTests\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tA0EA34C1A77C0EA52B0275F1 /* [CP] Check Pods Manifest.lock */,\n\t\t\t\t331C807D294A63A400263BE5 /* Sources */,\n\t\t\t\t331C807F294A63A400263BE5 /* Resources */,\n\t\t\t\tC8A669C122AD2A677A6E05F1 /* Frameworks */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t\t331C8086294A63A400263BE5 /* PBXTargetDependency */,\n\t\t\t);\n\t\t\tname = RunnerTests;\n\t\t\tproductName = RunnerTests;\n\t\t\tproductReference = 331C8081294A63A400263BE5 /* RunnerTests.xctest */;\n\t\t\tproductType = \"com.apple.product-type.bundle.unit-test\";\n\t\t};\n\t\t97C146ED1CF9000F007C117D /* Runner */ = {\n\t\t\tisa = PBXNativeTarget;\n\t\t\tbuildConfigurationList = 97C147051CF9000F007C117D /* Build configuration list for PBXNativeTarget \"Runner\" */;\n\t\t\tbuildPhases = (\n\t\t\t\tE45AEE757B505F5B8C631A7D /* [CP] Check Pods Manifest.lock */,\n\t\t\t\t9740EEB61CF901F6004384FC /* Run Script */,\n\t\t\t\t97C146EA1CF9000F007C117D /* Sources */,\n\t\t\t\t97C146EB1CF9000F007C117D /* Frameworks */,\n\t\t\t\t97C146EC1CF9000F007C117D /* Resources */,\n\t\t\t\t9705A1C41CF9048500538489 /* Embed Frameworks */,\n\t\t\t\t3B06AD1E1E4923F5004D2608 /* Thin Binary */,\n\t\t\t\t93B12CA83BE59495BC4E20EC /* [CP] Embed Pods Frameworks */,\n\t\t\t);\n\t\t\tbuildRules = (\n\t\t\t);\n\t\t\tdependencies = (\n\t\t\t);\n\t\t\tname = Runner;\n\t\t\tproductName = Runner;\n\t\t\tproductReference = 97C146EE1CF9000F007C117D /* Runner.app */;\n\t\t\tproductType = \"com.apple.product-type.application\";\n\t\t};\n/* End PBXNativeTarget section */\n\n/* Begin PBXProject section */\n\t\t97C146E61CF9000F007C117D /* Project object */ = {\n\t\t\tisa = PBXProject;\n\t\t\tattributes = {\n\t\t\t\tBuildIndependentTargetsInParallel = YES;\n\t\t\t\tLastUpgradeCheck = 1510;\n\t\t\t\tORGANIZATIONNAME = \"\";\n\t\t\t\tTargetAttributes = {\n\t\t\t\t\t331C8080294A63A400263BE5 = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 14.0;\n\t\t\t\t\t\tTestTargetID = 97C146ED1CF9000F007C117D;\n\t\t\t\t\t};\n\t\t\t\t\t97C146ED1CF9000F007C117D = {\n\t\t\t\t\t\tCreatedOnToolsVersion = 7.3.1;\n\t\t\t\t\t\tLastSwiftMigration = 1100;\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t};\n\t\t\tbuildConfigurationList = 97C146E91CF9000F007C117D /* Build configuration list for PBXProject \"Runner\" */;\n\t\t\tcompatibilityVersion = \"Xcode 9.3\";\n\t\t\tdevelopmentRegion = en;\n\t\t\thasScannedForEncodings = 0;\n\t\t\tknownRegions = (\n\t\t\t\ten,\n\t\t\t\tBase,\n\t\t\t);\n\t\t\tmainGroup = 97C146E51CF9000F007C117D;\n\t\t\tproductRefGroup = 97C146EF1CF9000F007C117D /* Products */;\n\t\t\tprojectDirPath = \"\";\n\t\t\tprojectRoot = \"\";\n\t\t\ttargets = (\n\t\t\t\t97C146ED1CF9000F007C117D /* Runner */,\n\t\t\t\t331C8080294A63A400263BE5 /* RunnerTests */,\n\t\t\t);\n\t\t};\n/* End PBXProject section */\n\n/* Begin PBXResourcesBuildPhase section */\n\t\t331C807F294A63A400263BE5 /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\t97C146EC1CF9000F007C117D /* Resources */ = {\n\t\t\tisa = PBXResourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */,\n\t\t\t\t3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */,\n\t\t\t\t97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */,\n\t\t\t\t97C146FC1CF9000F007C117D /* Main.storyboard in Resources */,\n\t\t\t\tCD1D1C48D4206E8BEA39AABB /* GoogleService-Info.plist in Resources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXResourcesBuildPhase section */\n\n/* Begin PBXShellScriptBuildPhase section */\n\t\t3B06AD1E1E4923F5004D2608 /* Thin Binary */ = {\n\t\t\tisa = PBXShellScriptBuildPhase;\n\t\t\talwaysOutOfDate = 1;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tinputPaths = (\n\t\t\t\t\"${TARGET_BUILD_DIR}/${INFOPLIST_PATH}\",\n\t\t\t);\n\t\t\tname = \"Thin Binary\";\n\t\t\toutputPaths = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t\tshellPath = /bin/sh;\n\t\t\tshellScript = \"/bin/sh \\\"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\\\" embed_and_thin\";\n\t\t};\n\t\t93B12CA83BE59495BC4E20EC /* [CP] Embed Pods Frameworks */ = {\n\t\t\tisa = PBXShellScriptBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tinputFileListPaths = (\n\t\t\t\t\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-input-files.xcfilelist\",\n\t\t\t);\n\t\t\tname = \"[CP] Embed Pods Frameworks\";\n\t\t\toutputFileListPaths = (\n\t\t\t\t\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-output-files.xcfilelist\",\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t\tshellPath = /bin/sh;\n\t\t\tshellScript = \"\\\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\\\"\\n\";\n\t\t\tshowEnvVarsInLog = 0;\n\t\t};\n\t\t9740EEB61CF901F6004384FC /* Run Script */ = {\n\t\t\tisa = PBXShellScriptBuildPhase;\n\t\t\talwaysOutOfDate = 1;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tinputPaths = (\n\t\t\t);\n\t\t\tname = \"Run Script\";\n\t\t\toutputPaths = (\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t\tshellPath = /bin/sh;\n\t\t\tshellScript = \"/bin/sh \\\"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\\\" build\";\n\t\t};\n\t\tA0EA34C1A77C0EA52B0275F1 /* [CP] Check Pods Manifest.lock */ = {\n\t\t\tisa = PBXShellScriptBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tinputFileListPaths = (\n\t\t\t);\n\t\t\tinputPaths = (\n\t\t\t\t\"${PODS_PODFILE_DIR_PATH}/Podfile.lock\",\n\t\t\t\t\"${PODS_ROOT}/Manifest.lock\",\n\t\t\t);\n\t\t\tname = \"[CP] Check Pods Manifest.lock\";\n\t\t\toutputFileListPaths = (\n\t\t\t);\n\t\t\toutputPaths = (\n\t\t\t\t\"$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt\",\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t\tshellPath = /bin/sh;\n\t\t\tshellScript = \"diff \\\"${PODS_PODFILE_DIR_PATH}/Podfile.lock\\\" \\\"${PODS_ROOT}/Manifest.lock\\\" > /dev/null\\nif [ $? != 0 ] ; then\\n # print error to STDERR\\n echo \\\"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\\\" >&2\\n exit 1\\nfi\\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\\necho \\\"SUCCESS\\\" > \\\"${SCRIPT_OUTPUT_FILE_0}\\\"\\n\";\n\t\t\tshowEnvVarsInLog = 0;\n\t\t};\n\t\tE45AEE757B505F5B8C631A7D /* [CP] Check Pods Manifest.lock */ = {\n\t\t\tisa = PBXShellScriptBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t);\n\t\t\tinputFileListPaths = (\n\t\t\t);\n\t\t\tinputPaths = (\n\t\t\t\t\"${PODS_PODFILE_DIR_PATH}/Podfile.lock\",\n\t\t\t\t\"${PODS_ROOT}/Manifest.lock\",\n\t\t\t);\n\t\t\tname = \"[CP] Check Pods Manifest.lock\";\n\t\t\toutputFileListPaths = (\n\t\t\t);\n\t\t\toutputPaths = (\n\t\t\t\t\"$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt\",\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t\tshellPath = /bin/sh;\n\t\t\tshellScript = \"diff \\\"${PODS_PODFILE_DIR_PATH}/Podfile.lock\\\" \\\"${PODS_ROOT}/Manifest.lock\\\" > /dev/null\\nif [ $? != 0 ] ; then\\n # print error to STDERR\\n echo \\\"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\\\" >&2\\n exit 1\\nfi\\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\\necho \\\"SUCCESS\\\" > \\\"${SCRIPT_OUTPUT_FILE_0}\\\"\\n\";\n\t\t\tshowEnvVarsInLog = 0;\n\t\t};\n/* End PBXShellScriptBuildPhase section */\n\n/* Begin PBXSourcesBuildPhase section */\n\t\t331C807D294A63A400263BE5 /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n\t\t97C146EA1CF9000F007C117D /* Sources */ = {\n\t\t\tisa = PBXSourcesBuildPhase;\n\t\t\tbuildActionMask = 2147483647;\n\t\t\tfiles = (\n\t\t\t\t74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */,\n\t\t\t\t1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */,\n\t\t\t);\n\t\t\trunOnlyForDeploymentPostprocessing = 0;\n\t\t};\n/* End PBXSourcesBuildPhase section */\n\n/* Begin PBXTargetDependency section */\n\t\t331C8086294A63A400263BE5 /* PBXTargetDependency */ = {\n\t\t\tisa = PBXTargetDependency;\n\t\t\ttarget = 97C146ED1CF9000F007C117D /* Runner */;\n\t\t\ttargetProxy = 331C8085294A63A400263BE5 /* PBXContainerItemProxy */;\n\t\t};\n/* End PBXTargetDependency section */\n\n/* Begin PBXVariantGroup section */\n\t\t97C146FA1CF9000F007C117D /* Main.storyboard */ = {\n\t\t\tisa = PBXVariantGroup;\n\t\t\tchildren = (\n\t\t\t\t97C146FB1CF9000F007C117D /* Base */,\n\t\t\t);\n\t\t\tname = Main.storyboard;\n\t\t\tsourceTree = \"\";\n\t\t};\n\t\t97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */ = {\n\t\t\tisa = PBXVariantGroup;\n\t\t\tchildren = (\n\t\t\t\t97C147001CF9000F007C117D /* Base */,\n\t\t\t);\n\t\t\tname = LaunchScreen.storyboard;\n\t\t\tsourceTree = \"\";\n\t\t};\n/* End PBXVariantGroup section */\n\n/* Begin XCBuildConfiguration section */\n\t\t249021D3217E4FDB00AE95B9 /* Profile */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++0x\";\n\t\t\t\tCLANG_CXX_LIBRARY = \"libc++\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\t\"CODE_SIGN_IDENTITY[sdk=iphoneos*]\" = \"iPhone Developer\";\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu99;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 12.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tSDKROOT = iphoneos;\n\t\t\t\tSUPPORTED_PLATFORMS = iphoneos;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t\tVALIDATE_PRODUCT = YES;\n\t\t\t};\n\t\t\tname = Profile;\n\t\t};\n\t\t249021D4217E4FDB00AE95B9 /* Profile */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = \"$(FLUTTER_BUILD_NUMBER)\";\n\t\t\t\tDEVELOPMENT_TEAM = RRL2F46W5P;\n\t\t\t\tENABLE_BITCODE = NO;\n\t\t\t\tINFOPLIST_FILE = Runner/Info.plist;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_OBJC_BRIDGING_HEADER = \"Runner/Runner-Bridging-Header.h\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t};\n\t\t\tname = Profile;\n\t\t};\n\t\t331C8088294A63A400263BE5 /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = DF41D8FEA220BF2CA2DE0E0D /* Pods-RunnerTests.debug.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tBUNDLE_LOADER = \"$(TEST_HOST)\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd.RunnerTests;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTEST_HOST = \"$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t331C8089294A63A400263BE5 /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = B1C6DDD94FCC20EC357B6FDE /* Pods-RunnerTests.release.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tBUNDLE_LOADER = \"$(TEST_HOST)\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd.RunnerTests;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTEST_HOST = \"$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\t331C808A294A63A400263BE5 /* Profile */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = D6C5A2A3DB181063A7E47D6B /* Pods-RunnerTests.profile.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tBUNDLE_LOADER = \"$(TEST_HOST)\";\n\t\t\t\tCODE_SIGN_STYLE = Automatic;\n\t\t\t\tCURRENT_PROJECT_VERSION = 1;\n\t\t\t\tGENERATE_INFOPLIST_FILE = YES;\n\t\t\t\tMARKETING_VERSION = 1.0;\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd.RunnerTests;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tTEST_HOST = \"$(BUILT_PRODUCTS_DIR)/Runner.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Runner\";\n\t\t\t};\n\t\t\tname = Profile;\n\t\t};\n\t\t97C147031CF9000F007C117D /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++0x\";\n\t\t\t\tCLANG_CXX_LIBRARY = \"libc++\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\t\"CODE_SIGN_IDENTITY[sdk=iphoneos*]\" = \"iPhone Developer\";\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = dwarf;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tENABLE_TESTABILITY = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu99;\n\t\t\t\tGCC_DYNAMIC_NO_PIC = NO;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_OPTIMIZATION_LEVEL = 0;\n\t\t\t\tGCC_PREPROCESSOR_DEFINITIONS = (\n\t\t\t\t\t\"DEBUG=1\",\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t);\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 12.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = YES;\n\t\t\t\tONLY_ACTIVE_ARCH = YES;\n\t\t\t\tSDKROOT = iphoneos;\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t97C147041CF9000F007C117D /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbuildSettings = {\n\t\t\t\tALWAYS_SEARCH_USER_PATHS = NO;\n\t\t\t\tCLANG_ANALYZER_NONNULL = YES;\n\t\t\t\tCLANG_CXX_LANGUAGE_STANDARD = \"gnu++0x\";\n\t\t\t\tCLANG_CXX_LIBRARY = \"libc++\";\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCLANG_ENABLE_OBJC_ARC = YES;\n\t\t\t\tCLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;\n\t\t\t\tCLANG_WARN_BOOL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_COMMA = YES;\n\t\t\t\tCLANG_WARN_CONSTANT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;\n\t\t\t\tCLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;\n\t\t\t\tCLANG_WARN_EMPTY_BODY = YES;\n\t\t\t\tCLANG_WARN_ENUM_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_INFINITE_RECURSION = YES;\n\t\t\t\tCLANG_WARN_INT_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;\n\t\t\t\tCLANG_WARN_OBJC_LITERAL_CONVERSION = YES;\n\t\t\t\tCLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;\n\t\t\t\tCLANG_WARN_RANGE_LOOP_ANALYSIS = YES;\n\t\t\t\tCLANG_WARN_STRICT_PROTOTYPES = YES;\n\t\t\t\tCLANG_WARN_SUSPICIOUS_MOVE = YES;\n\t\t\t\tCLANG_WARN_UNREACHABLE_CODE = YES;\n\t\t\t\tCLANG_WARN__DUPLICATE_METHOD_MATCH = YES;\n\t\t\t\t\"CODE_SIGN_IDENTITY[sdk=iphoneos*]\" = \"iPhone Developer\";\n\t\t\t\tCOPY_PHASE_STRIP = NO;\n\t\t\t\tDEBUG_INFORMATION_FORMAT = \"dwarf-with-dsym\";\n\t\t\t\tENABLE_NS_ASSERTIONS = NO;\n\t\t\t\tENABLE_STRICT_OBJC_MSGSEND = YES;\n\t\t\t\tGCC_C_LANGUAGE_STANDARD = gnu99;\n\t\t\t\tGCC_NO_COMMON_BLOCKS = YES;\n\t\t\t\tGCC_WARN_64_TO_32_BIT_CONVERSION = YES;\n\t\t\t\tGCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;\n\t\t\t\tGCC_WARN_UNDECLARED_SELECTOR = YES;\n\t\t\t\tGCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;\n\t\t\t\tGCC_WARN_UNUSED_FUNCTION = YES;\n\t\t\t\tGCC_WARN_UNUSED_VARIABLE = YES;\n\t\t\t\tIPHONEOS_DEPLOYMENT_TARGET = 12.0;\n\t\t\t\tMTL_ENABLE_DEBUG_INFO = NO;\n\t\t\t\tSDKROOT = iphoneos;\n\t\t\t\tSUPPORTED_PLATFORMS = iphoneos;\n\t\t\t\tSWIFT_COMPILATION_MODE = wholemodule;\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-O\";\n\t\t\t\tTARGETED_DEVICE_FAMILY = \"1,2\";\n\t\t\t\tVALIDATE_PRODUCT = YES;\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n\t\t97C147061CF9000F007C117D /* Debug */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = 9740EEB21CF90195004384FC /* Debug.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = \"$(FLUTTER_BUILD_NUMBER)\";\n\t\t\t\tDEVELOPMENT_TEAM = RRL2F46W5P;\n\t\t\t\tENABLE_BITCODE = NO;\n\t\t\t\tINFOPLIST_FILE = Runner/Info.plist;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_OBJC_BRIDGING_HEADER = \"Runner/Runner-Bridging-Header.h\";\n\t\t\t\tSWIFT_OPTIMIZATION_LEVEL = \"-Onone\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t};\n\t\t\tname = Debug;\n\t\t};\n\t\t97C147071CF9000F007C117D /* Release */ = {\n\t\t\tisa = XCBuildConfiguration;\n\t\t\tbaseConfigurationReference = 7AFA3C8E1D35360C0083082E /* Release.xcconfig */;\n\t\t\tbuildSettings = {\n\t\t\t\tASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;\n\t\t\t\tCLANG_ENABLE_MODULES = YES;\n\t\t\t\tCURRENT_PROJECT_VERSION = \"$(FLUTTER_BUILD_NUMBER)\";\n\t\t\t\tDEVELOPMENT_TEAM = RRL2F46W5P;\n\t\t\t\tENABLE_BITCODE = NO;\n\t\t\t\tINFOPLIST_FILE = Runner/Info.plist;\n\t\t\t\tLD_RUNPATH_SEARCH_PATHS = (\n\t\t\t\t\t\"$(inherited)\",\n\t\t\t\t\t\"@executable_path/Frameworks\",\n\t\t\t\t);\n\t\t\t\tPRODUCT_BUNDLE_IDENTIFIER = com.pilotcap.ttmd;\n\t\t\t\tPRODUCT_NAME = \"$(TARGET_NAME)\";\n\t\t\t\tSWIFT_OBJC_BRIDGING_HEADER = \"Runner/Runner-Bridging-Header.h\";\n\t\t\t\tSWIFT_VERSION = 5.0;\n\t\t\t\tVERSIONING_SYSTEM = \"apple-generic\";\n\t\t\t};\n\t\t\tname = Release;\n\t\t};\n/* End XCBuildConfiguration section */\n\n/* Begin XCConfigurationList section */\n\t\t331C8087294A63A400263BE5 /* Build configuration list for PBXNativeTarget \"RunnerTests\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t331C8088294A63A400263BE5 /* Debug */,\n\t\t\t\t331C8089294A63A400263BE5 /* Release */,\n\t\t\t\t331C808A294A63A400263BE5 /* Profile */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\t97C146E91CF9000F007C117D /* Build configuration list for PBXProject \"Runner\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t97C147031CF9000F007C117D /* Debug */,\n\t\t\t\t97C147041CF9000F007C117D /* Release */,\n\t\t\t\t249021D3217E4FDB00AE95B9 /* Profile */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n\t\t97C147051CF9000F007C117D /* Build configuration list for PBXNativeTarget \"Runner\" */ = {\n\t\t\tisa = XCConfigurationList;\n\t\t\tbuildConfigurations = (\n\t\t\t\t97C147061CF9000F007C117D /* Debug */,\n\t\t\t\t97C147071CF9000F007C117D /* Release */,\n\t\t\t\t249021D4217E4FDB00AE95B9 /* Profile */,\n\t\t\t);\n\t\t\tdefaultConfigurationIsVisible = 0;\n\t\t\tdefaultConfigurationName = Release;\n\t\t};\n/* End XCConfigurationList section */\n\t};\n\trootObject = 97C146E61CF9000F007C117D /* Project object */;\n}\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcodeproj/project.xcworkspace/contents.xcworkspacedata", "content": "\n\n \n \n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist", "content": "\n\n\n\n\tIDEDidComputeMac32BitWarning\n\t\n\n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings", "content": "\n\n\n\n\tPreviewsEnabled\n\t\n\n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcworkspace/contents.xcworkspacedata", "content": "\n\n \n \n \n \n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist", "content": "\n\n\n\n\tIDEDidComputeMac32BitWarning\n\t\n\n\n" }, { "path": "frontend/frontend-flutter/ios/Runner.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings", "content": "\n\n\n\n\tPreviewsEnabled\n\t\n\n\n" }, { "path": "frontend/frontend-flutter/ios/RunnerTests/RunnerTests.swift", "content": "import Flutter\nimport UIKit\nimport XCTest\n\nclass RunnerTests: XCTestCase {\n\n func testExample() {\n // If you add code to the Runner application, consider adding tests here.\n // See https://developer.apple.com/documentation/xctest for more information about using XCTest.\n }\n\n}\n" }, { "path": "frontend/frontend-flutter/lib/firebase_options.dart", "content": "// File generated by FlutterFire CLI.\n// ignore_for_file: type=lint\nimport 'package:firebase_core/firebase_core.dart' show FirebaseOptions;\nimport 'package:flutter/foundation.dart'\n show defaultTargetPlatform, kIsWeb, TargetPlatform;\n\n/// Default [FirebaseOptions] for use with your Firebase apps.\n///\n/// Example:\n/// ```dart\n/// import 'firebase_options.dart';\n/// // ...\n/// await Firebase.initializeApp\n/// options: DefaultFirebaseOptions.currentPlatform,\n/// );\n/// ```\nclass DefaultFirebaseOptions {\n static FirebaseOptions get currentPlatform {\n if (kIsWeb) {\n return web;\n }\n switch (defaultTargetPlatform) {\n case TargetPlatform.android:\n return android;\n case TargetPlatform.iOS:\n return ios;\n case TargetPlatform.macOS:\n throw UnsupportedError(\n 'DefaultFirebaseOptions have not been configured for macos - '\n 'you can reconfigure this by running the FlutterFire CLI again.',\n );\n case TargetPlatform.windows:\n throw UnsupportedError(\n 'DefaultFirebaseOptions have not been configured for windows - '\n 'you can reconfigure this by running the FlutterFire CLI again.',\n );\n case TargetPlatform.linux:\n throw UnsupportedError(\n 'DefaultFirebaseOptions have not been configured for linux - '\n 'you can reconfigure this by running the FlutterFire CLI again.',\n );\n default:\n throw UnsupportedError(\n 'DefaultFirebaseOptions are not supported for this platform.',\n );\n }\n }\n\n static const FirebaseOptions web = FirebaseOptions(\n apiKey: 'AIzaSyDQJNMaGcmhjuqsyXDpsXtLj4YSvMr8jkI',\n appId: '1:603067936438:web:2d6b41a3c67c155651c017',\n messagingSenderId: '603067936438',\n projectId: 'odqna-ai-seminar',\n authDomain: 'odqna-ai-seminar.firebaseapp.com',\n storageBucket: 'odqna-ai-seminar.appspot.com',\n );\n\n static const FirebaseOptions android = FirebaseOptions(\n apiKey: 'AIzaSyCytznMTfW7u99AuEQRbdiqmfaCAoMh0Bc',\n appId: '1:7413174684:android:f4ed287d5da149fce5f859',\n messagingSenderId: '7413174684',\n projectId: 'dic-caa-ra1',\n storageBucket: 'dic-caa-ra1.appspot.com',\n );\n\n static const FirebaseOptions ios = FirebaseOptions(\n apiKey: 'AIzaSyB5RZFwdDakT3zrJ7-DMum0IHCSfyLqtBQ',\n appId: '1:7413174684:ios:9a2317f461e3a05ce5f859',\n messagingSenderId: '7413174684',\n projectId: 'dic-caa-ra1',\n storageBucket: 'dic-caa-ra1.appspot.com',\n iosBundleId: 'com.pilotcap.ttmd',\n );\n}" }, { "path": "frontend/frontend-flutter/lib/main.dart", "content": "import 'dart:ui' as ui;\nimport 'package:file_picker/file_picker.dart';\nimport 'package:flutter/cupertino.dart';\nimport 'package:flutter/services.dart' show rootBundle;\nimport 'package:flutter_bloc/flutter_bloc.dart';\nimport 'package:flutter/material.dart';\nimport 'package:font_awesome_flutter/font_awesome_flutter.dart';\nimport 'package:simple_gradient_text/simple_gradient_text.dart';\nimport 'package:ttmd/screens/bot.dart';\nimport 'package:ttmd/screens/disclaimer.dart';\nimport 'package:ttmd/screens/settings.dart' as ts;\nimport 'package:ttmd/services/display_stepper/display_stepper_cubit.dart';\nimport 'package:ttmd/services/display_stepper/display_stepper_state.dart';\nimport 'package:ttmd/services/first_question/first_question_cubit.dart';\nimport 'package:ttmd/services/first_question/first_question_state.dart';\nimport 'package:ttmd/services/load_question/load_question_cubit.dart';\nimport 'dart:async';\nimport 'dart:convert';\nimport 'package:easy_sidemenu/easy_sidemenu.dart';\nimport 'package:expandable_tree_menu/expandable_tree_menu.dart';\nimport 'package:ttmd/services/load_question/load_question_state.dart';\nimport 'package:intl/intl.dart';\nimport 'package:badges/badges.dart' as badges;\nimport 'package:ttmd/services/new_suggestions/new_suggestion_cubit.dart';\nimport 'package:ttmd/services/new_suggestions/new_suggestion_state.dart';\nimport 'package:ttmd/services/update_expert_mode/update_expert_mode_cubit.dart';\nimport 'package:ttmd/services/update_popular_questions/update_popular_questions_cubit.dart';\nimport 'package:ttmd/services/update_popular_questions/update_popular_questions_state.dart';\nimport 'package:ttmd/services/update_stepper/update_stepper_cubit.dart';\nimport 'package:ttmd/services/update_stepper/update_stepper_state.dart';\nimport 'package:ttmd/utils/TextToDocParameter.dart';\nimport 'package:ttmd/utils/most_popular_questions.dart';\nimport 'package:ttmd/utils/pdf_viewer.dart';\nimport 'package:ttmd/utils/stepper_expert_info.dart';\nimport 'package:expansion_tile_card/expansion_tile_card.dart';\nimport 'package:flutter/services.dart';\nimport 'package:flutter_json_viewer/flutter_json_viewer.dart';\nimport 'dart:html' as html;\nimport 'package:firebase_core/firebase_core.dart';\nimport 'firebase_options.dart';\nimport 'package:firebase_auth/firebase_auth.dart';\nimport 'package:csv/csv.dart';\nimport 'package:flutter_inappwebview/flutter_inappwebview.dart';\nimport 'package:cloud_firestore/cloud_firestore.dart';\nimport 'package:flutter/rendering.dart';\n\nlate final FirebaseApp app;\nlate final FirebaseAuth auth;\nlate User currentUser;\nlate String LastName;\nlate String userID;\nlate FirebaseFirestore db;\nfinal GlobalKey navigatorKey = GlobalKey();\n\nMap myRoutes = {\n '/landingPage': (context) => ContentTtmd(title: 'Open data QnA'),\n //'/pdfViewer': (context) => PdfViewer(),\n '/settings': (context) => ts.Settings(db),\n};\n\nvoid main() async {\n WidgetsFlutterBinding.ensureInitialized();\n /*app = await Firebase.initializeApp(\n name: 'opendataqna',\n options: DefaultFirebaseOptions.web, // currentPlatform,\n );*/\n app = await Firebase.initializeApp(options: DefaultFirebaseOptions.web);\n\n auth = FirebaseAuth.instanceFor(app: app);\n\n /*db = await FirebaseFirestore.instanceFor(\n app: app, databaseId: 'opendataqna-session-logs');*/\n db = await FirebaseFirestore.instanceFor(app: app);\n\n\n print('Main: main() : auth = $auth');\n print('Main: main() : db = $db');\n print('Main: main() : db.databaseId = ${db.databaseId}');\n\n //FirebaseAuth.instance\n auth.authStateChanges().listen((User? user) {\n if (user != null) {\n print(\"Main : user.uid = ${user.uid}\");\n currentUser = user;\n }\n });\n\n runApp(ttmd());\n}\n\nclass ttmd extends StatelessWidget {\n ttmd({super.key});\n\n // This widget is the root of your application.\n @override\n Widget build(BuildContext context) {\n return MultiBlocProvider(\n providers: [\n BlocProvider(\n create: (context) => LoadQuestionCubit(),\n ),\n BlocProvider(\n create: (context) => FirstQuestionCubit(),\n ),\n BlocProvider(\n create: (context) => UpdatePopularQuestionsCubit(),\n ),\n BlocProvider(\n create: (context) => UpdateStepperCubit(),\n ),\n BlocProvider(\n create: (context) => NewSuggestionCubit(),\n ),\n BlocProvider(\n create: (context) => DisplayStepperCubit(),\n ),\n BlocProvider(\n create: (context) => UpdateExpertModeCubit(),\n ),\n ],\n child: MaterialApp(\n debugShowCheckedModeBanner: false,\n //navigatorKey: navigatorKey,\n title: 'Open data QnA',\n theme: ThemeData(\n appBarTheme: AppBarTheme(\n backgroundColor: Colors.white,\n ),\n checkboxTheme: CheckboxThemeData(\n fillColor: WidgetStateProperty.resolveWith((states) {\n if (!states.contains(WidgetState.selected)) {\n return Colors.white;\n }\n return Colors.green;\n }), checkColor: WidgetStateProperty.resolveWith((states) {\n if (states.contains(WidgetState.selected)) {\n return Colors.red;\n }\n return null;\n }))),\n onGenerateRoute: (settings) {\n print('Main: ttmd : build() : onGenerateRoute : START');\n if (settings.name == '/landingPage' &&\n !TextToDocParameter.isAuthenticated) {\n print(\n 'Main: ttmd : build() : onGenerateRoute : attempting to access /landingPage without authentication');\n return MaterialPageRoute(\n //builder: (context) => LoginScreen(),\n builder: (context) => Disclaimer(auth),\n );\n } else if (settings.name == '/pdfViewer' &&\n !TextToDocParameter.isAuthenticated) {\n print(\n 'Main: ttmd : build() : onGenerateRoute : attempting to access /pdfViewer without authentication');\n return MaterialPageRoute(\n //builder: (context) => LoginScreen(),\n builder: (context) => Disclaimer(auth),\n );\n } else if (settings.name == '/settings' &&\n !TextToDocParameter.isAuthenticated) {\n print(\n 'Main: ttmd : build() : onGenerateRoute : attempting to access /settings without authentication');\n return MaterialPageRoute(\n //builder: (context) => LoginScreen(),\n builder: (context) => Disclaimer(auth),\n );\n } else {\n print(\n 'Main: ttmd : build() : onGenerateRoute : attempting to access ${settings.name!} with proper authentication');\n if (settings.name == '/pdfViewer') {\n final args = settings.arguments as Map;\n\n return MaterialPageRoute(\n builder: (context) {\n return PdfViewer(bytes: args[\"bytes\"]);\n },\n );\n } else {\n return MaterialPageRoute(\n builder: myRoutes[settings.name!]!,\n );\n }\n } // Let the default routing handle other routes\n },\n initialRoute: '/',\n home: Disclaimer(auth),\n ),\n );\n }\n}\n\nclass ContentTtmd extends StatefulWidget {\n const ContentTtmd({super.key, required this.title});\n\n final String title;\n\n @override\n State createState() => _ContentTtmdState();\n}\n\nclass _ContentTtmdState extends State {\n late GlobalKey _botKey;\n int _selectedIndex = 0;\n int _selectedIndexNavRail = 0;\n PageController pageController = PageController();\n SideMenuController sideMenu = SideMenuController();\n SideMenuExpansionItem? historySideMenuExpansionItem;\n List? childrenHistorySideMenuItem;\n Map? mostPopularQuestionsMap = {};\n int currentStep = 0;\n List stepperExpertInfoList = [];\n bool isFirstQuestionStatus = true;\n List> nodes = [];\n final List _destinations = [\n 'New Chat',\n 'History',\n 'Most popular questions',\n 'Summary Extract',\n //'Help',\n 'Settings'\n ];\n bool _isExpanded = false;\n bool _isQuestionExpanded = false;\n Size? screenSize;\n bool _isFirstQuestionNotAskedYet = true;\n TextEditingController textEditingController = TextEditingController();\n Bot? bot;\n double overallProcessingTime = 0;\n bool useExpertMode = false;\n bool isTextToDoC = false;\n bool isTextToDoC1 = false;\n bool light = true;\n Container? suggestionContainer;\n List> importedQuestionTreeNodeList = [\n TreeNode(\"No questions available\")\n ];\n InAppWebViewController? webViewController;\n //late FirebaseFirestore db;\n List userGroupingList = [];\n String selectedValue = \"\";\n final ValueNotifier updateSelection = ValueNotifier(false);\n final ValueNotifier importedQuestionNotifier = ValueNotifier(false);\n final ValueNotifier selectedValueNotifier =\n ValueNotifier(null);\n //final TextEditingController _dropdownController = TextEditingController();\n\n List> CreateNodesMostPopularQuestion(\n List mostPopularQuestionsList) {\n List> nodes = [];\n\n for (int i = 0; i < mostPopularQuestionsList.length; i++) {\n if (mostPopularQuestionsList[i].question.length != 0)\n nodes.add(TreeNode(mostPopularQuestionsList[i].time +\n \"|||\" +\n mostPopularQuestionsList[i].question +\n \"|||\" +\n mostPopularQuestionsList[i].count.toString()));\n }\n\n return nodes;\n }\n\n void _addData(String data, bool isHistory) {\n List? mostPopularQuestionsList = [];\n int lenghtTmp = 0;\n String timeString = \"\";\n\n timeString = displayDateTime();\n\n if (data.length == 0) return;\n print(\"Main: _addData : BEFORE ADD : nodes.length = ${nodes.length}\");\n print(\"Main: _addData : BEFORE ADD : data = ${data}\");\n print(\"Main: _addData : BEFORE ADD : nodes = ${nodes}\");\n if (isHistory)\n nodes.add(TreeNode(timeString + \"|||\" + data)); //displayDateTime()\n else\n nodes.add(TreeNode(data));\n\n //add questions to mostPopularQuestionsMap\n if (mostPopularQuestionsMap!.containsKey(data)) {\n print('Main: _addData : mostPopularQuestionsMap contains $data');\n print(\n 'Main: _addData : mostPopularQuestionsMap : BEFORE INCREMENT : (mostPopularQuestionsMap![data] as MostPopularQ).count = ${(mostPopularQuestionsMap![data] as MostPopularQ).count}');\n mostPopularQuestionsMap![data]!.count =\n mostPopularQuestionsMap![data]!.count + 1;\n mostPopularQuestionsMap![data]!.time = timeString;\n print(\n 'Main: _addData : mostPopularQuestionsMap : AFTER INCREMENT : (mostPopularQuestionsMap![data] as MostPopularQ).count = ${(mostPopularQuestionsMap![data] as MostPopularQ).count}');\n } else {\n print('Main: _addData : mostPopularQuestionsMap does not contain $data');\n mostPopularQuestionsMap![data] = MostPopularQ(data, 1, timeString);\n }\n\n var sortedByValueMap = Map.fromEntries(\n mostPopularQuestionsMap!.entries.toList()\n ..sort((e1, e2) => e2.value.count.compareTo(e1.value.count)));\n\n print(\n 'Main: _addData : sortedByValueMap.length = ${sortedByValueMap.length}');\n print('Main: _addData : sortedByValueMap = $sortedByValueMap');\n\n int countEntries = 0;\n\n for (var entry in sortedByValueMap.entries) {\n mostPopularQuestionsList!.add(entry.value);\n if (countEntries == 2 || countEntries == sortedByValueMap.length - 1)\n break;\n countEntries++;\n }\n\n //countEntries = 0;\n\n print(\n \"Main: _addData : BEFORE FILLING : mostPopularQuestionsList.length = ${mostPopularQuestionsList.length}\");\n print(\n \"Main: _addData : BEFORE FILLING : mostPopularQuestionsList = ${mostPopularQuestionsList}\");\n\n lenghtTmp = mostPopularQuestionsList.length;\n\n //Fill mostPopularQuestionsList with dummy entries if there are not 3 most popular questions\n for (int i = 0; i <= (2 - lenghtTmp); i++) {\n mostPopularQuestionsList!.add(MostPopularQ(\"\", 0, \"\"));\n }\n\n print(\n \"Main: _addData : AFTER FILLING : mostPopularQuestionsList.length = ${mostPopularQuestionsList.length}\");\n print(\n \"Main: _addData : AFTER FILLING : mostPopularQuestionsList = ${mostPopularQuestionsList}\");\n\n //Update the 3 most popular questions\n BlocProvider.of(context)\n .updateMostPopularQuestions(\n mostPopularQuestionsList: mostPopularQuestionsList!,\n time: timeString);\n\n print(\"Main: _addData : AFTER : nodes.length = ${nodes.length}\");\n print(\"Main: _addData : AFTER : nodes = ${nodes}\");\n }\n\n void _nodeSelected(context, nodeValue) {\n int scenarioNumber = 0;\n String questionTmp = \"\";\n print('Main : _nodeSelected() : START');\n print(\n 'Main : _nodeSelected() : nodeValue.toString() = ${nodeValue.toString()}');\n\n questionTmp = nodeValue.toString().split(\":\")[0];\n print('Main : _nodeSelected() : questionTmp = $questionTmp');\n\n if (questionTmp.length <= 2) {\n //we don't expect to have more than 99 questions, so 2 digits are enough\n scenarioNumber = int.parse(questionTmp);\n print(\n 'Main : _nodeSelected() : questionTmp.length <=2 : scenarioNumber = $scenarioNumber');\n }\n\n if (nodeValue.toString().contains(\":\")) {\n print('Main : _nodeSelected() : nodeValue.toString().contains(\":\")');\n\n //scenario#:question:genre:user_grouping in the future, may add a 5th element : main_question\n\n print(\n 'Main : _nodeSelected() : nodeValue.toString().contains(\":\") : nodeValue.toString().split(\":\")[2] = ${nodeValue.toString().split(\":\")[2]}');\n\n textEditingController.text = nodeValue.toString().split(\":\")[1];\n BlocProvider.of(context).generateNewSuggestions(\n int.parse(nodeValue.toString().split(\":\")[0]),\n textEditingController.text,\n isACannedQuestion: false,\n userGrouping: nodeValue.toString().split(\":\")[3]);\n print(\n \"Main: _nodeSelected() : textEditingController.text = = ${textEditingController.text}\");\n\n //setting on the UI the current user_grouping based on the question clicked assuming next question is for the same user_grouping\n selectedValueNotifier.value = nodeValue.toString().split(\":\")[3];\n TextToDocParameter.currentUserGrouping =\n nodeValue.toString().split(\":\")[3];\n TextToDocParameter.currentScenarioName =\n nodeValue.toString().split(\":\")[2];\n } else\n return; //textEditingController.text = nodeValue.toString();\n }\n\n void _nodeSelected1(context, nodeValue) {\n var val = nodeValue.toString().split(\"|||\");\n textEditingController.text = val[1];\n /*final route =\n MaterialPageRoute(builder: (context) => DetailPage(value: nodeValue));\n Navigator.of(context).push(route);*/\n }\n\n /// Build the Node widget at a specific node in the tree\n Widget _nodeBuilder(context, nodeValue) {\n return Card(\n margin: EdgeInsets.symmetric(vertical: 1),\n child: Padding(\n padding: const EdgeInsets.all(8.0),\n child: Text(nodeValue.toString()),\n ));\n }\n\n Widget _nodeBuilder2(context, nodeValue) {\n String? badgeCount;\n String scenarioTitle = \"\";\n String question = \"\";\n List tmpList = [];\n\n print(\"Main : _nodeBuilder2() : START\");\n print(\"Main : _nodeBuilder2() : nodeValue = $nodeValue\");\n\n if (nodeValue.toString().contains(\":\")) {\n //question#: question\n tmpList = nodeValue.toString().split(\":\");\n scenarioTitle = tmpList[2]; //\"Scenario \" + tmpList[0];\n question = tmpList[1];\n } else if (nodeValue.toString().contains(\" - \")) {\n //Scenario x - user_grouping -#of questions\n tmpList = nodeValue.toString().split(\" - \");\n badgeCount = tmpList[2];\n question = tmpList[0] + ' - ' + tmpList[1];\n }\n\n bool isScenario = nodeValue.toString().contains(\"Scenario\");\n\n print(\"Main : _nodeBuilder2() : scenarioTitle = $scenarioTitle\");\n print(\"Main : _nodeBuilder2() : question = $question\");\n\n return Card(\n margin: EdgeInsets.symmetric(vertical: 1),\n child: Column(\n mainAxisSize: MainAxisSize.min,\n children: [\n ListTile(\n leading: Padding(\n padding: const EdgeInsets.only(right: 9.0),\n child: !isScenario\n ? CircleAvatar(\n backgroundColor: Colors.green,\n radius: 12,\n child: Text(\"Q${TextToDocParameter.questionCount++}\",\n style: TextStyle(color: Colors.white, fontSize: 9)))\n : badges.Badge(\n badgeContent: Text(badgeCount!),\n child: Icon(Icons.account_tree_sharp),\n ),\n ),\n title: !isScenario\n ? Text(scenarioTitle,\n style: TextStyle(fontSize: 9, color: Colors.indigoAccent))\n : null,\n subtitle: !isScenario\n ? Text(question)\n : Text(\" \" + question,\n style: TextStyle(fontWeight: FontWeight.bold)),\n ),\n ],\n ));\n }\n\n Widget _nodeBuilder1(context, nodeValue) {\n var val = nodeValue.toString().split(\"|||\");\n return Card(\n margin: EdgeInsets.symmetric(vertical: 1),\n child: Column(\n mainAxisSize: MainAxisSize.min,\n children: [\n ListTile(\n leading: Padding(\n padding: const EdgeInsets.only(right: 10.0),\n child: CircleAvatar(\n backgroundColor: Colors.green,\n radius: 12,\n child: Icon(Icons.question_mark,\n size: 10, color: Colors.white)),\n ),\n title: Text(val[0],\n style: TextStyle(fontSize: 8, color: Colors.indigoAccent)),\n subtitle: Text(val[1]),\n ),\n ],\n ));\n }\n\n Widget _nodeBuilder3(context, nodeValue) {\n var val = nodeValue.toString().split(\"|||\");\n //val[0] => timestamp\n //val[1] => question\n //val[2] => Count\n\n return Card(\n margin: EdgeInsets.symmetric(vertical: 1),\n child: Column(\n mainAxisSize: MainAxisSize.min,\n children: [\n ListTile(\n leading: Padding(\n padding: const EdgeInsets.only(right: 20.0),\n child: /*2,\n child: Icon(Icons.question_mark,\n size: 10, color: Colors.white))*/\n badges.Badge(\n badgeContent: Text(val[2]),\n child: Icon(Icons.question_mark),\n ),\n ),\n title: Text(val[0],\n style: TextStyle(fontSize: 8, color: Colors.indigoAccent)),\n subtitle: Text(val[1]),\n ),\n ],\n ));\n }\n\n FutureBuilder>?> _createSideMenu() {\n print('Main : _createSideMenu() : START');\n TextToDocParameter.questionCount = 1;\n return FutureBuilder(\n future: loadQuestionsFromFirestore(),\n builder: (context, snapshot) {\n return SideMenu(\n controller: sideMenu,\n style: SideMenuStyle(\n // showTooltip: false,\n displayMode: SideMenuDisplayMode.open,\n showHamburger: true,\n hoverColor: Colors.blue[100],\n selectedHoverColor: Colors.blue[100],\n selectedColor: Colors.lightBlue,\n selectedTitleTextStyle: const TextStyle(color: Colors.black),\n selectedIconColor: Colors.white,\n ),\n title: Column(\n children: [\n ConstrainedBox(\n constraints: const BoxConstraints(\n maxHeight: 100,\n maxWidth: 250,\n ),\n child: Image.asset(\n 'assets/images/ttmd_logo1.png', //'assets/images/drawer_header1.png',\n ),\n ),\n const Divider(\n indent: 8.0,\n endIndent: 8.0,\n ),\n ],\n ),\n footer: Padding(\n padding: const EdgeInsets.only(top: 8.0),\n child: ConstrainedBox(\n constraints: const BoxConstraints(\n maxHeight: 80,\n ),\n child: Container(\n color: const Color.fromARGB(\n 255, 240, 240, 240), // Colors.grey.withOpacity(1),////\n child: Column(\n children: [\n const Divider(\n indent: 8.0,\n endIndent: 8.0,\n ),\n Row(\n mainAxisAlignment: MainAxisAlignment.spaceEvenly,\n mainAxisSize: MainAxisSize.max,\n children: [\n CircleAvatar(\n backgroundImage:\n //const AssetImage('assets/images/john_smith.jpeg'),\n NetworkImage(TextToDocParameter.picture),\n radius: 30,\n ),\n Text(\"${TextToDocParameter.email}\",\n style: TextStyle(\n color: Colors.blueAccent, fontSize: 12.0),\n textAlign: TextAlign.end),\n ],\n ),\n ],\n ),\n ),\n ),\n ),\n items: [\n SideMenuItem(\n title: 'Open data QnA',\n onTap: (index, _) {\n //sideMenu.changePage(index);\n },\n icon: const Icon(Icons.home),\n tooltipContent: \"Open data QnA\",\n ),\n SideMenuItem(\n title: 'New chat',\n onTap: (index, _) {\n TextToDocParameter.sessionId = \"\";\n print(\n \"Main: _createSideMenu() : New chat : : TextToDocParameter.sessionId = ${TextToDocParameter.sessionId}\");\n },\n icon: const Icon(Icons.restart_alt_outlined),\n ),\n if (snapshot.hasData) //if (TextToDocParameter.expert_mode)\n SideMenuExpansionItem(\n title: \"Imported questions\",\n icon: const Icon(Icons.manage_history_outlined),\n children: [\n SideMenuItem(\n builder: (context, displayMode) {\n return ExpandableTree(\n nodes: importedQuestionTreeNodeList,\n nodeBuilder: _nodeBuilder2,\n onSelect: (node) => _nodeSelected(context, node),\n );\n },\n onTap: (index, _) {\n //sideMenu.changePage(index);\n },\n )\n ],\n ),\n SideMenuExpansionItem(\n title: \"History\",\n icon: const Icon(Icons.manage_history_outlined),\n children: [\n SideMenuItem(\n builder: (context, displayMode) {\n return BlocBuilder(\n builder: (context, state) {\n print(\n \"Main: _createSideMenu() : BlocBuilder : state = $state\");\n print(\n \"Main: _createSideMenu() : BlocBuilder : state.question = ${state.question}\");\n print(\n \"Main: _createSideMenu() : BlocBuilder : state.time = ${state.time}\");\n _addData(state.question!, true);\n return ExpandableTree(\n nodes: nodes,\n nodeBuilder: _nodeBuilder1,\n onSelect: (node) => _nodeSelected1(context, node),\n );\n },\n );\n },\n onTap: (index, _) {\n //sideMenu.changePage(index);\n },\n )\n ],\n ),\n SideMenuExpansionItem(\n title: \"Most popular questions\",\n icon: const Icon(Icons.manage_history_outlined),\n children: [\n SideMenuItem(\n builder: (context, displayMode) {\n return BlocBuilder(\n builder: (context, state) {\n print(\n \"Main: _createSideMenu() : Most popular questions : BlocBuilder : state.status = ${state.status}\");\n print(\n \"Main: _createSideMenu() : Most popular questions : BlocBuilder : state.mostPopularQuestionsList = ${state.mostPopularQuestionsList}\");\n\n //_addData(state.question!, true);\n List> nodesMostPopularQuestions =\n CreateNodesMostPopularQuestion(\n state.mostPopularQuestionsList!);\n\n return ExpandableTree(\n nodes: nodesMostPopularQuestions,\n nodeBuilder: _nodeBuilder3,\n onSelect: (node) => _nodeSelected1(context, node),\n );\n },\n );\n },\n onTap: (index, _) {\n //sideMenu.changePage(index);\n },\n )\n ],\n ),\n SideMenuItem(\n title: 'Import',\n onTap: (index, _) async {\n var questionList = await importQuestions();\n\n if (questionList.length == 0) {\n print(\n \"Main: _createSideMenu() : SideMenuItem : Import : onTap() : questionList is empty\");\n return;\n }\n\n //sort the list of list based on the name of the database :\n //grouping, scenario, question\n //Remove the header\n questionList.removeAt(0);\n questionList.sort((a, b) => a.first.compareTo(b.first));\n\n //Save questions into\n SaveImportedQuestionsToFirestore(questionList);\n\n for (var entry in questionList)\n print(\n \"Main: _createSideMenu() : SideMenuItem : Import : onTap() : sorted questionList = ${entry[0]}, ${entry[2]}\");\n\n setState(() {\n importedQuestionTreeNodeList =\n createQuestionList(questionList);\n //print(\"Main: _createSideMenu() : SideMenuItem : Import : setState() : importedQuestionTreeNodeList = ${importedQuestionTreeNodeList}\");\n });\n\n const snackBar = SnackBar(\n content: Text('Importing questions.'),\n duration: Duration(seconds: 3),\n );\n ScaffoldMessenger.of(context).showSnackBar(snackBar);\n },\n icon: const Icon(Icons.file_upload_outlined),\n ),\n /* //Commented out because the export of the answers (images and tables) to a pdf is not implemented yet\n SideMenuItem(\n title: 'Export',\n onTap: (index, _) {\n _createPDF();\n\n const snackBar = SnackBar(\n content: Text('Exporting data to a pdf file.'),\n duration: Duration(seconds: 5),\n );\n ScaffoldMessenger.of(context).showSnackBar(snackBar);\n },\n icon: const Icon(Icons.import_export_outlined),\n ),*/\n /*SideMenuItem(\n // do not delete\n title: 'Clear Firestore history',\n onTap: (index, _) async {\n int count = 0;\n //get all the documents corresponding to the user id\n try {\n var querySnapshot = await db\n .collection(\"session_logs\")\n .where(\"user_id\", isEqualTo: TextToDocParameter.userID)\n .get();\n\n print(\n \"Main: _createSideMenu() : Clear Firestore history : querySnapshot.docs.length = ${querySnapshot.docs.length}\");\n print(\n \"Main: _createSideMenu() : Clear Firestore history : querySnapshot = ${querySnapshot}\");\n\n //delete all these documents\n //await db.collection(\"session_logs\").doc('FLoOKBwvVJ8mXkzu06He').delete();\n\n for (var docSnapshot in querySnapshot.docs) {\n print(\n 'Main: _createSideMenu() : Clear Firestore history : ${docSnapshot.id} => ${docSnapshot.data()}');\n print(\n 'Main: _createSideMenu() : Clear Firestore history : docSnapshot.reference ${docSnapshot.reference}');\n db\n .collection(\"session_logs\")\n .doc('${docSnapshot.id}')\n .delete();\n count++;\n }\n } catch (e) {\n print(\n 'Main: _createSideMenu() : Clear Firestore history : EXCEPTION : $e');\n }\n var snackBar = SnackBar(\n content: Text('Deleted $count questions from Firestore'),\n duration: Duration(seconds: 3),\n );\n ScaffoldMessenger.of(context).showSnackBar(snackBar);\n },\n icon: const Icon(Icons.auto_delete_outlined),\n ),*/\n SideMenuItem(\n title: 'Settings',\n onTap: (index, _) async {\n await Navigator.pushNamed(context, '/settings',\n arguments: {\"dummy\": \"\"});\n print(\n \"Main : createSideMenu() : TextToDocParameter.expert_mode = ${TextToDocParameter.expert_mode}\");\n setState(() {\n //useExpertMode = ts.Settings.isExpert;\n //useExpertMode = TextToDocParameter.expert_mode;\n TextToDocParameter.expert_mode;\n });\n\n //GalleryScreen\n },\n icon: const Icon(Icons.settings),\n ),\n ],\n );\n },\n )!;\n }\n\n @override\n void initState() {\n super.initState();\n setup();\n }\n\n Future setup() async {\n _botKey = GlobalKey();\n\n bot =\n Bot(key: _botKey, textEditingController: textEditingController, db: db);\n\n //Initialize stepper states\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.initial,\n message: \"Please enter a question.\",\n stateStepper: StepState.disabled,\n isActiveStepper: false);\n print(\n \"main: setup() : After BlocProvider.of(context).updateStepperStatusUploaded() : stepper initialized\");\n }\n\n Future initializeFirestore() async {\n await loadCfgFromFirestore();\n //await loadQuestionsFromFirestore();\n }\n\n Future>?> loadQuestionsFromFirestore() async {\n print('Main: loadQuestionsFromFirestore() : START');\n List>? questionList = [];\n\n print(\n 'Main: loadQuestionsFromFirestore() : TextToDocParameter.imported_questions = ${TextToDocParameter.imported_questions}');\n print(\n 'Main: loadQuestionsFromFirestore() : TextToDocParameter.userID = ${TextToDocParameter.userID}');\n\n try {\n var querySnapshot = await db!\n .collection(\"imported_questions\")\n .where(\"user_id\", isEqualTo: TextToDocParameter.userID)\n .orderBy('scenario', descending: false)\n .orderBy('order')\n .get();\n\n for (var docSnapshot in querySnapshot.docs) {\n final data = docSnapshot.data() as Map;\n print(\n 'Main: loadQuestionsFromFirestore() : data[\"user_grouping\"] = ${data[\"user_grouping\"]} : data[\"scenario\"] = ${data[\"scenario\"]} : data[\"question\"] = ${data[\"question\"]}');\n questionList.add([\n data[\"user_grouping\"],\n data[\"scenario\"],\n data[\"question\"],\n data[\"main_question\"]\n ]);\n }\n\n print(\n 'Main: loadQuestionsFromFirestore() : questionList.length = ${questionList.length}');\n print(\n 'Main: loadQuestionsFromFirestore() : questionList = ${questionList}');\n\n //create the questionList from Firestore\n if (questionList.length == 0)\n questionList = null;\n else\n importedQuestionTreeNodeList = createQuestionList(questionList);\n\n /*BlocProvider.of(context)\n .updateExpertMode(TextToDocParameter.expert_mode);*/\n } catch (e) {\n print('Main: loadQuestionsFromFirestore() : EXCEPTION : e = $e');\n } finally {\n return questionList;\n }\n }\n\n Future loadCfgFromFirestore() async {\n /*db = await FirebaseFirestore.instanceFor(\n app: app, databaseId: 'opendataqna-session-logs');*/\n\n print(\"main: loadCfgFromFirestore() : db = $db\");\n\n if (TextToDocParameter.userID.isEmpty) {\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.userID is empty = ${TextToDocParameter.userID}\");\n WidgetsBinding.instance.addPostFrameCallback((_) {\n noCfgStoredinFirestore();\n });\n return;\n }\n\n try {\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.userID = ${TextToDocParameter.userID}\");\n\n DocumentSnapshot doc = await db!\n .collection(\"front_end_flutter_cfg\")\n .doc('${TextToDocParameter.userID}')\n .get();\n\n if (doc != null) {\n final data = doc.data() as Map;\n\n TextToDocParameter.anonymized_data = data[\"anonymized_data\"];\n TextToDocParameter.expert_mode = data[\"expert_mode\"];\n TextToDocParameter.endpoint_opendataqnq = data[\"endpoint_opendataqnq\"];\n TextToDocParameter.firestore_database_id =\n data[\"firestore_database_id\"];\n TextToDocParameter.firebase_app_name = data[\"firebase_app_name\"];\n TextToDocParameter.firestore_history_collection =\n data[\"firestore_history_collection\"];\n TextToDocParameter.firestore_cfg_collection =\n data[\"firestore_cfg_collection\"];\n TextToDocParameter.imported_questions = data[\"imported_questions\"];\n\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.anonymized_data = ${TextToDocParameter.anonymized_data}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.expert_mode = ${TextToDocParameter.expert_mode}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.firestore_database_id = ${TextToDocParameter.firestore_database_id}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.endpoint_opendataqnq = ${TextToDocParameter.endpoint_opendataqnq}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.firebase_app_name = ${TextToDocParameter.firebase_app_name}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.firestore_history_collection = ${TextToDocParameter.firestore_history_collection}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.firestore_cfg_collection = ${TextToDocParameter.firestore_cfg_collection}\");\n print(\n \"main: loadCfgFromFirestore() : TextToDocParameter.imported_questions = ${TextToDocParameter.imported_questions}\");\n\n BlocProvider.of(context)\n .displayStepper(TextToDocParameter.expert_mode);\n } else {\n print(\"main: loadCfgFromFirestore() : doc == null\");\n WidgetsBinding.instance.addPostFrameCallback((_) {\n noCfgStoredinFirestore();\n });\n }\n } catch (e) {\n print(\"main: loadCfgFromFirestore() : EXCEPTION ON FIRESTORE : e = $e\");\n WidgetsBinding.instance.addPostFrameCallback((_) {\n noCfgStoredinFirestore();\n });\n }\n }\n\n Future> _getUserGrouping() async {\n print('Main : _getUserGrouping() : START');\n List resp = [];\n\n await initializeFirestore();\n\n print('Main : _getUserGrouping() : bot = $bot');\n\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n try {\n print(\n 'Main : _getUserGrouping() : url = ${TextToDocParameter.endpoint_opendataqnq}/available_databases');\n\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/available_databases',\n method: \"GET\");\n\n print('Main : _getUserGrouping() : response = ' + response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('Main : _getUserGrouping() : jsonData = $jsonData');\n\n /* Expected response :\n {\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"table_schema\\\":\\\"imdb-postgres\\\"},{\\\"table_schema\\\":\\\"retail-postgres\\\"}]\",\n \"ResponseCode\": 200\n }*/\n\n var knownSqlMap = jsonDecode(jsonData['KnownDB']);\n\n print('Main : _getUserGrouping() : knownSqlMap = ${knownSqlMap}');\n\n print(\n 'Main : _getUserGrouping() : knownSqlMap[0] = ${knownSqlMap[0].toString()}');\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"table_schema\") resp.add(entry.value);\n }\n }\n } else {\n resp.add(\"\");\n }\n } catch (e) {\n print('Main : _getUserGrouping() : EXCEPTION = $e');\n throw Exception('Failed to get earning calls question suggestions: $e');\n } finally {\n print('Main : _getUserGrouping() : resp = $resp');\n return resp;\n }\n }\n\n Future>> _getQuestions() async {\n var questionList = await _getLastQuestions();\n print(\"Main : _getQuestions() : questionList = $questionList\");\n return questionList;\n }\n\n @override\n Widget build(BuildContext context) {\n debugPaintSizeEnabled = false;\n screenSize = MediaQuery.of(context).size;\n bool _isEarningCalls1Hovered = false;\n bool _isEarningCalls2Hovered = false;\n bool _isBookingAnalysisHovered = false;\n bool _isFunelAnalysisHovered = false;\n\n print(\"Main : build() : START\");\n print(\"Main : build() : TuserGroupingList = ${userGroupingList}\");\n print(\n \"Main : build() : TextToDocParameter.currentUserGrouping = ${TextToDocParameter.currentUserGrouping}\");\n\n return Scaffold(\n appBar: AppBar(\n title: Image.asset('assets/images/cymbal_logo.png', scale: 2),\n centerTitle: false,\n actions: [\n Padding(\n padding: const EdgeInsets.only(right: 100),\n child: Container(\n width: 320,\n child: FutureBuilder(\n future: _getUserGrouping(),\n builder: (context, snapshot) {\n if (snapshot.connectionState == ConnectionState.waiting) {\n return Padding(\n padding: const EdgeInsets.only(right: 135.0, left: 135.0),\n child: Container(\n width: 50,\n height: 50,\n child: /*Text('TEST')*/ CircularProgressIndicator()),\n ); // Loading indicator\n } else if (snapshot.hasError) {\n return Text('-Error-: ${snapshot.error}');\n // Error handling\n } else {\n if (snapshot.data!.isNotEmpty) {\n TextToDocParameter.currentUserGrouping =\n snapshot.data![0];\n TextToDocParameter.userGroupingList = snapshot.data!;\n selectedValueNotifier.value = snapshot.data!.first;\n print(\n \"main: build() : TextToDocParameter.currentUserGrouping = ${TextToDocParameter.currentUserGrouping}\");\n print(\"main: build() : selectedValue = ${selectedValue}\");\n }\n return ValueListenableBuilder(\n valueListenable: selectedValueNotifier,\n builder: (context, value, child) {\n return Container(\n width: 300,\n child: Row(\n children: [\n Text(\"User Grouping: \",\n style: TextStyle(\n fontSize: 13.0, color: Colors.black)),\n Expanded(\n child: DropdownButton(\n value: value,\n //icon: const Icon(Icons.arrow_downward),\n elevation: 16,\n style: const TextStyle(\n fontSize: 13.0,\n color: Colors.deepPurple),\n onChanged: (String? value) {\n selectedValueNotifier.value = value!;\n TextToDocParameter.currentUserGrouping =\n value!;\n print(\n 'Main: build() : DropdownButton : onChanged() : TextToDocParameter.currentUserGrouping => ${TextToDocParameter.currentUserGrouping}');\n },\n items: snapshot.data!\n .map>(\n (String value) {\n return DropdownMenuItem(\n value: value,\n child: Text(value),\n );\n }).toList(),\n ),\n ),\n ],\n ),\n );\n });\n }\n },\n ),\n ),\n ),\n ],\n leading: Text(\"\"),\n ),\n body: Row(\n mainAxisSize: MainAxisSize.max,\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n _createSideMenu()!,\n const VerticalDivider(\n width: 0,\n ),\n Expanded(\n child: Stack(\n children: [\n Container(\n padding: EdgeInsets.only(left: 100, right: 100),\n child: Column(\n mainAxisAlignment: MainAxisAlignment.start,\n //mainAxisSize: MainAxisSize.min,\n children: [\n BlocBuilder(\n builder: (context, state) {\n print(\n \"Main: build() : BlocBuilder : START\");\n if (state.status ==\n displayStepperStatus.display_stepper) {\n print(\n \"Main: build() : BlocBuilder : state =${state.status}\");\n return Flexible(\n fit: FlexFit.loose,\n child: Padding(\n padding: const EdgeInsets.only(bottom: 5),\n child: _buildStepper(),\n ),\n flex: 2);\n } else if (state.status ==\n displayStepperStatus.remove_stepper) {\n print(\n \"Main: build() : BlocBuilder : state =${state.status}\");\n return Text(\"\");\n } else {\n print(\n \"Main: build() : BlocBuilder : ERROR : state =$state\");\n return Text(\"Unable to load user_grouping\");\n }\n },\n ),\n bot!,\n BlocBuilder(\n builder: (context, state) {\n if (state.status == NewSuggestionStateStatus.loaded) {\n print(\n \"Main: build() : BlocBuilder : START : state =$state\");\n return makeSuggestions(\n state.suggestionList!, state.scenarioNumber!);\n } else {\n return makeSuggestions([\"x:\", \"x:\", \"x:\"], 0);\n }\n },\n ),\n //const Spacer(flex: 1,),\n ],\n ),\n ),\n BlocBuilder(\n builder: (context, state) {\n print(\n \"Main : build() : BlocBuilder : START\");\n if (state.status ==\n firstQuestionStatus.display_welcome_message) {\n print(\n \"Main : build() : BlocBuilder : state.status == firstQuestionStatus.display_welcome_message\");\n return Positioned(\n //top: 0,\n //left: 200, //200,\n child: Column(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Container(height: 100),\n Image.asset(\n //UNCOMMENT\n \"assets/images/gemini.png\"!,\n height: 90,\n width: 90,\n fit: BoxFit.cover,\n ),\n Padding(\n padding: const EdgeInsets.only(bottom: 50),\n child: Column(\n mainAxisAlignment: MainAxisAlignment.center,\n crossAxisAlignment: CrossAxisAlignment.center,\n children: [\n Row(\n mainAxisAlignment: MainAxisAlignment.center,\n children: [\n Text(\n \"Hello ${TextToDocParameter.firstName} ! \",\n style: TextStyle(\n fontSize: 40.0,\n fontWeight: FontWeight.bold)),\n GradientText(\n 'How can I help you today ?',\n style: TextStyle(\n fontSize: 40.0,\n fontWeight: FontWeight.bold),\n colors: [\n Colors.blue,\n Colors.red,\n Colors.teal,\n ],\n ),\n ],\n ),\n const Row(\n mainAxisAlignment:\n MainAxisAlignment.center,\n children: [\n Text(\n 'Learn more by selecting a card below\\n',\n style: TextStyle(\n fontSize: 30.0,\n fontWeight: FontWeight.normal,\n color: Colors.black))\n ])\n ],\n ),\n ),\n FutureBuilder(\n future: _getQuestions(),\n builder: (context, snapshot) {\n if (snapshot.connectionState ==\n ConnectionState.waiting) {\n return CircularProgressIndicator(); // Loading indicator\n } else if (snapshot.hasError) {\n return Text('*Error*: ${snapshot.error}');\n // Error handling\n } else {\n return Container(\n width: screenSize!.width / 1.5,\n child: Column(\n children: [\n Row(\n crossAxisAlignment:\n CrossAxisAlignment.center,\n mainAxisAlignment:\n MainAxisAlignment.spaceEvenly,\n children: [\n if (snapshot.data!.length >= 1)\n createCardsSuggestion(\n title: snapshot.data![0]\n ['scenario_name'] ??\n \"Error - No title\",\n text: snapshot.data![0]\n ['user_question'] ??\n \"Error - No question\",\n scenarioNumber: 2,\n isHovered:\n _isEarningCalls1Hovered,\n imagePath:\n \"assets/images/last_questions.png\",\n timeStamp: snapshot.data![0]\n ['timestamp'] ??\n \"Error - No timestamp\",\n userGrouping: snapshot.data![0]\n ['user_grouping']),\n if (snapshot.data!.length >= 2)\n createCardsSuggestion(\n title: snapshot.data![1]\n ['scenario_name'] ??\n \"Error - No title\",\n text: snapshot.data![1]\n ['user_question'] ??\n \"Error - No question\",\n scenarioNumber: 2,\n isHovered:\n _isEarningCalls1Hovered,\n imagePath:\n \"assets/images/last_questions.png\",\n timeStamp: snapshot.data![1]\n ['timestamp'] ??\n \"Error - No timestamp\",\n userGrouping: snapshot.data![0]\n ['user_grouping']),\n ],\n ),\n Row(\n crossAxisAlignment:\n CrossAxisAlignment.center,\n mainAxisAlignment:\n MainAxisAlignment.spaceEvenly,\n children: [\n if (snapshot.data!.length >= 3)\n createCardsSuggestion(\n title: snapshot.data![2][\n 'scenario_name'] ??\n \"Error - No title\",\n text: snapshot.data![2][\n 'user_question'] ??\n \"Error - No question\",\n scenarioNumber: 2,\n isHovered:\n _isEarningCalls1Hovered,\n imagePath:\n \"assets/images/last_questions.png\",\n timeStamp: snapshot\n .data![2]\n ['timestamp'] ??\n \"Error - No timestamp\",\n userGrouping: snapshot.data![0]\n ['user_grouping']),\n if (snapshot.data!.length >= 4)\n createCardsSuggestion(\n title: snapshot.data![3][\n 'scenario_name'] ??\n \"Error - No title\",\n text: snapshot.data![3][\n 'user_question'] ??\n \"Error - No question\",\n scenarioNumber: 2,\n isHovered:\n _isEarningCalls1Hovered,\n imagePath:\n \"assets/images/last_questions.png\",\n timeStamp: snapshot\n .data![3]\n ['timestamp'] ??\n \"Error - No timestamp\",\n userGrouping: snapshot.data![0]\n ['user_grouping']),\n ])\n ],\n ),\n );\n }\n }), //UNCOMMENT\n ],\n ), //UNCOMMENT\n );\n } else {\n print(\n \"Main : build() : BlocBuilder : state.status != firstQuestionStatus.display_welcome_message\");\n isFirstQuestionStatus = false;\n return Text(\"\");\n }\n },\n ),\n ],\n ),\n ),\n ],\n ),\n // This trailing comma makes auto-formatting nicer for build methods.\n );\n }\n\n void _onItemTapped(int index) {\n setState(() {\n _selectedIndex = index;\n });\n }\n\n Future> _createPDF() async {\n //Method not called because the export menu has been commented out.\n //This will remain commented out until it is possible to screenshot web views\n List bytes = [];\n\n return bytes;\n }\n\n String displayDateTime() {\n String? dateTimeS;\n\n final now = DateTime.now();\n dateTimeS = DateFormat('yyyy-MM-dd HH:mm:ss').format(now);\n return dateTimeS!;\n }\n\n Widget createCards(\n {required String? imagePath,\n required String? title,\n required String subTitle,\n required String text,\n required int badgeCount}) {\n return Expanded(\n child: Container(\n width: screenSize!.width / 3.5,\n //height: screenSize!.height / 6,\n child: Card(\n elevation: 10,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.circular(10),\n ),\n // Define how the card's content should be clipped\n clipBehavior: Clip.antiAliasWithSaveLayer,\n // Define the child widget of the card\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n // Add padding around the row widget\n Padding(\n padding: const EdgeInsets.all(10),\n child: Row(\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n // Add an image widget to display an image\n Image.asset(\n imagePath!,\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n // Add some spacing between the image and the text\n Container(width: 20),\n // Add an expanded widget to take up the remaining horizontal space\n Expanded(\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n // Add some spacing between the top of the card and the title\n Container(height: 5),\n // Add a title widget\n Row(\n mainAxisAlignment: MainAxisAlignment.spaceBetween,\n children: [\n Text(\n title!,\n style: TextStyle(\n fontSize: 18,\n fontWeight: FontWeight.bold,\n color: Color(0xFF37474F)),\n ),\n badges.Badge(\n position: badges.BadgePosition.topEnd(\n top: -10, end: -12),\n showBadge: true,\n ignorePointer: false,\n onTap: () {},\n badgeContent: Text(\n badgeCount < 10\n ? \" \" + badgeCount.toString()\n : badgeCount.toString(),\n style: TextStyle(\n fontSize: 15,\n color: Colors\n .white)), //Icon(Icons.check, color: Colors.white, size: 14),\n badgeAnimation: badges.BadgeAnimation.rotation(\n animationDuration: Duration(seconds: 1),\n colorChangeAnimationDuration:\n Duration(seconds: 1),\n loopAnimation: false,\n curve: Curves.fastOutSlowIn,\n colorChangeAnimationCurve: Curves.easeInCubic,\n ),\n badgeStyle: badges.BadgeStyle(\n shape: badges.BadgeShape.instagram,\n badgeColor: Colors.red,\n //padding: EdgeInsets.all(5),\n borderRadius: BorderRadius.circular(4),\n borderSide:\n BorderSide(color: Colors.white, width: 2),\n elevation: 2,\n ),\n //child: Text('Badge'),\n ),\n ],\n ),\n // Add some spacing between the title and the subtitle\n Container(height: 5),\n // Add a subtitle widget\n Text(\n subTitle!,\n style: Theme.of(context)\n .textTheme\n .bodyMedium!\n .copyWith(\n color: Colors.grey[500],\n ),\n ),\n // Add some spacing between the subtitle and the text\n Container(height: 10),\n // Add a text widget to display some text\n Text(\n text.length <= 35\n ? text!\n : text!.substring(0, 35) + ' ...',\n style: Theme.of(context)\n .textTheme\n .titleMedium!\n .copyWith(\n color: Colors.grey[700],\n ),\n maxLines: 1,\n ),\n Row(children: [\n Spacer(),\n TextButton(\n style: TextButton.styleFrom(\n foregroundColor: Colors.transparent,\n ),\n child: const Text(\n \"LOAD\",\n style: TextStyle(color: Color(0xFFFF4081)),\n ),\n onPressed: () {\n print(\n 'Bot() : build() : TextButton : onPressed() : START : text = $text');\n //BlocProvider.of(context).loadQuestionToChat(\"test\");\n //setState(() {\n textEditingController.text = text;\n _isFirstQuestionNotAskedYet = false;\n //});\n },\n ),\n ])\n ],\n ),\n ),\n ],\n ),\n ),\n ],\n ),\n ),\n ),\n );\n }\n\n Padding makeSuggestions(\n List listRandomQuestions, int scenarioNumber) {\n print(\"Main: makeSuggestions() : START\");\n TextToDocParameter.lastScenarioNumber = scenarioNumber;\n\n return Padding(\n padding: const EdgeInsets.only(bottom: 20),\n child: Container(\n width: screenSize!.width,\n //height: screenSize!.height / 20,\n color: Colors.white,\n child: Row(\n mainAxisAlignment: MainAxisAlignment.end,\n mainAxisSize: MainAxisSize.min,\n children: [\n if (listRandomQuestions.length >= 1 &&\n listRandomQuestions[0] != \"x:\")\n Expanded(\n child: ConstrainedBox(\n constraints: BoxConstraints(\n minWidth: 100.0, // Set minimum width\n maxWidth: 400.0, // Set maximum width\n ),\n child: InkWell(\n onTap: () {\n //textEditingController.text = \"Can you provide the Top 10 pace platinum accounts with booking value and YOY growth %?\";\n textEditingController.text =\n listRandomQuestions[0].split(\":\")[1];\n _isFirstQuestionNotAskedYet = false;\n BlocProvider.of(context)\n .generateNewSuggestions(\n scenarioNumber, textEditingController.text,\n isACannedQuestion: false);\n print(\n \"Main: makeSuggestions() : textEditingController.text = = ${textEditingController.text}\");\n },\n child: Card(\n elevation: 5,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.circular(10),\n ),\n child: Padding(\n padding: const EdgeInsets.all(5.0),\n child: Row(\n children: [\n Image.asset(\n \"assets/images/suggestions.png\",\n height: 20,\n width: 20,\n fit: BoxFit.cover,\n ),\n Flexible(\n child: Text(\n //' Can you provide the Top 10 pace platinum accounts with booking value and YOY growth %? ',\n listRandomQuestions[0].split(\":\")[1],\n style: TextStyle(\n decoration: TextDecoration.underline,\n fontSize: 15,\n ),\n softWrap: true,\n overflow: TextOverflow.ellipsis,\n maxLines: 2),\n ),\n ],\n ),\n ),\n ),\n ),\n ),\n ),\n //Container(width: 10),\n if (listRandomQuestions.length >= 2 &&\n listRandomQuestions[1] != \"x:\")\n Expanded(\n child: ConstrainedBox(\n constraints: BoxConstraints(\n minWidth: 100.0, // Set minimum width\n maxWidth: 400.0, // Set maximum width\n ),\n child: InkWell(\n onTap: () {\n textEditingController.text =\n listRandomQuestions[1].split(\":\")[1];\n _isFirstQuestionNotAskedYet = false;\n BlocProvider.of(context)\n .generateNewSuggestions(\n scenarioNumber, textEditingController.text,\n isACannedQuestion: false);\n print(\n \"Main: makeSuggestions() : textEditingController.text = = ${textEditingController.text}\");\n },\n child: Card(\n elevation: 5,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.circular(10),\n ),\n child: Padding(\n padding: const EdgeInsets.all(5.0),\n child: Row(\n children: [\n Image.asset(\n \"assets/images/suggestions.png\",\n height: 20,\n width: 20,\n fit: BoxFit.cover,\n ),\n Flexible(\n child: Text(\n listRandomQuestions[1].split(\":\")[1],\n style: TextStyle(\n decoration: TextDecoration.underline,\n fontSize: 15,\n ),\n softWrap: true,\n overflow: TextOverflow.ellipsis,\n maxLines: 2),\n ),\n ],\n ),\n ),\n ),\n ),\n ),\n ),\n //Container(width: 10),\n if (listRandomQuestions.length >= 3 &&\n listRandomQuestions[2] != \"x:\")\n Expanded(\n child: ConstrainedBox(\n constraints: BoxConstraints(\n minWidth: 100.0, // Set minimum width\n maxWidth: 400.0, // Set maximum width\n ),\n child: InkWell(\n onTap: () {\n textEditingController.text =\n listRandomQuestions[2].split(\":\")[1];\n _isFirstQuestionNotAskedYet = false;\n BlocProvider.of(context)\n .generateNewSuggestions(\n scenarioNumber, textEditingController.text,\n isACannedQuestion: false);\n print(\n \"Main: makeSuggestions() : textEditingController.text = = ${textEditingController.text}\");\n },\n child: Card(\n elevation: 5,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.circular(10),\n ),\n child: Padding(\n padding: const EdgeInsets.all(5.0),\n child: Row(\n children: [\n Image.asset(\n \"assets/images/suggestions.png\",\n height: 20,\n width: 20,\n fit: BoxFit.cover,\n ),\n Flexible(\n child: Text(\n //' Can you add CM sold margin to above list? ',\n listRandomQuestions[2].split(\":\")[1],\n style: TextStyle(\n decoration: TextDecoration.underline,\n fontSize: 15,\n ),\n softWrap: true,\n overflow: TextOverflow.ellipsis,\n maxLines: 2),\n ),\n ],\n ),\n ),\n ),\n ),\n ),\n ),\n ])),\n );\n }\n\n Widget createCardsSuggestion(\n {required String? title,\n required String text,\n required int scenarioNumber,\n required bool isHovered,\n required String imagePath,\n required String timeStamp,\n required String userGrouping}) {\n print('Main : createCardsSuggestion() : START');\n print('Main : createCardsSuggestion() : START : text = $text');\n print('Main : createCardsSuggestion() : START : title = $title');\n print('Main : createCardsSuggestion() : START : userGrouping = $userGrouping');\n return Expanded(\n child: InkWell(\n onTap: () {\n print(\n 'Main : createCardsSuggestion() : TextButton : onPressed() : START : text = $text');\n\n textEditingController.text = text;\n _isFirstQuestionNotAskedYet = false;\n //Use the user_grouping associated to the question\n //selectedValueNotifier.value = title;\n selectedValueNotifier.value = userGrouping;\n\n //Setting the current user_grouping value\n TextToDocParameter.currentUserGrouping = userGrouping;\n\n BlocProvider.of(context).generateNewSuggestions(\n scenarioNumber, text,\n isACannedQuestion: false,\n userGrouping: userGrouping);\n },\n onHover: (value) {\n\n },\n child: Container(\n width: screenSize!.width / 2.5,\n height: screenSize!.height / 6,\n child: Card(\n color: Colors.white,\n elevation: 0,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.circular(10),\n ),\n // Define how the card's content should be clipped\n clipBehavior: Clip.antiAliasWithSaveLayer,\n // Define the child widget of the card\n child: Padding(\n padding: const EdgeInsets.all(10),\n child: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n mainAxisSize: MainAxisSize.max,\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n Image.asset(\n imagePath!,\n height: 50,\n width: 50,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Flexible(\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.start,\n //mainAxisSize: MainAxisSize.max,\n children: [\n // Add some spacing between the top of the card and the title\n Container(height: 5),\n // Add a title widget\n Flexible(\n //flex: 1,\n child: Container(\n color: isHovered ? Colors.blueGrey : null,\n child: Text(\n title!,\n style: TextStyle(\n fontSize: 18,\n fontWeight: FontWeight.bold,\n color: Color(0xFF37474F)),\n ),\n ),\n ),\n // Add some spacing between the title and the subtitle\n Container(height: 5),\n // Add some spacing between the subtitle and the text\n // Add a text widget to display some text\n Flexible(\n //flex: 6,\n child: Container(\n color: isHovered ? Colors.blueGrey : null,\n child: Text(\n text,\n style: Theme.of(context)\n .textTheme\n .titleMedium!\n .copyWith(\n color: Colors.grey[700],\n ),\n maxLines: 6,\n softWrap: true,\n overflow: TextOverflow.visible,\n ),\n ),\n ),\n Container(height: 10),\n Flexible(\n //flex: 6,\n child: Container(\n padding: EdgeInsets.only(top: 5),\n color: isHovered ? Colors.blueGrey : null,\n child: Text(\n 'Question typed on the ' + timeStamp,\n style: Theme.of(context)\n .textTheme\n .titleSmall!\n .copyWith(\n color: Colors.blueAccent,\n ),\n maxLines: 6,\n softWrap: true,\n overflow: TextOverflow.visible,\n ),\n ),\n ),\n ],\n ),\n ),\n ],\n ),\n ),\n ),\n ),\n ),\n );\n }\n\n Widget _createDebugInfoCard(Widget leading, String title, String subtitle,\n String imageTrailingPath, Widget info, String infoText) {\n final ButtonStyle flatButtonStyle = TextButton.styleFrom(\n shape: const RoundedRectangleBorder(\n borderRadius: BorderRadius.all(Radius.circular(4.0)),\n ),\n );\n return Padding(\n padding: const EdgeInsets.symmetric(horizontal: 12.0),\n child: ExpansionTileCard(\n //key: cardA,\n initiallyExpanded: true,\n leading: leading,\n title: Row(\n mainAxisAlignment: MainAxisAlignment.spaceBetween,\n mainAxisSize: MainAxisSize.max,\n children: [\n Flexible(\n flex: 5,\n child: Text(title, style: TextStyle(fontWeight: FontWeight.bold)),\n ),\n Flexible(\n flex: 1,\n child: Image.asset(width: 75, height: 75, imageTrailingPath))\n ],\n ),\n subtitle: Text(subtitle, style: TextStyle(fontSize: 12)),\n children: [\n const Divider(\n thickness: 1.0,\n height: 1.0,\n ),\n Align(\n alignment: Alignment.centerLeft,\n child: Padding(\n padding: const EdgeInsets.symmetric(\n horizontal: 16.0,\n vertical: 8.0,\n ),\n child: info,\n ),\n ),\n ButtonBar(\n alignment: MainAxisAlignment.spaceBetween,\n buttonHeight: 52.0,\n buttonMinWidth: 90.0,\n children: [\n TextButton(\n style: flatButtonStyle,\n onPressed: () async {\n await Clipboard.setData(ClipboardData(text: infoText));\n },\n child: const Column(\n children: [\n Icon(Icons.copy),\n Padding(\n padding: EdgeInsets.symmetric(vertical: 2.0),\n ),\n Text('Copy'),\n ],\n ),\n ),\n ],\n ),\n ],\n ),\n );\n }\n\n Container _buildStepper() {\n final canCancel = currentStep > 0;\n final canContinue = currentStep < 3;\n StepState? stateStepperUploaded;\n bool? isActiveUploaded;\n StepState? stateStepperExtracted;\n bool? isActiveExtracted;\n StepState? stateStepperCompared;\n bool? isActiveCompared;\n StepState? stateStepperCommitted;\n bool? isActiveCommitted;\n\n StepState? stateStepperEnterQuestion;\n bool? isActiveEnterQuestion;\n String? messageStepperEnterQuestion = \"\";\n StepState? stateStepperGenerateSQL;\n bool? isActiveGenerateSQL;\n String? messageStepperGenerateSQL = \"\";\n StepState? stateStepperRunQuery;\n bool? isActiveRunQuery;\n String? messageStepperRunQuery = \"\";\n StepState? stateStepperGetGraphDescription;\n bool? isActiveGetGraphDescription;\n String? messageStepperGraphDescription = \"\";\n StepState? stateStepperGetTextSummary;\n bool? isActiveGetTextSummary;\n String? messageStepperGetTextSummary = \"\";\n\n String? message = \"\";\n\n void setStepsStates(UpdateStepperState state) {\n print('Main : _buildStepper() : setStepsStates() : Start');\n print(\n 'Main : _buildStepper() : setStepsStates() : status = ${state.status}');\n print('Main : _buildStepper() : currentStep : $currentStep');\n switch (state.status) {\n case StepperStatus.initial:\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.initial');\n currentStep = 0;\n stateStepperEnterQuestion = StepState.disabled;\n isActiveEnterQuestion = false;\n stateStepperGenerateSQL = StepState.disabled;\n isActiveGenerateSQL = false;\n stateStepperRunQuery = StepState.disabled;\n isActiveRunQuery = false;\n stateStepperGetGraphDescription = StepState.disabled;\n isActiveGetGraphDescription = false;\n stateStepperGetTextSummary = StepState.disabled;\n isActiveGetTextSummary = false;\n message = state.message;\n break;\n case StepperStatus.enter_question:\n stepperExpertInfoList.clear();\n currentStep = 0;\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.enter_question');\n stateStepperEnterQuestion = StepState.complete;\n isActiveEnterQuestion = true;\n stateStepperGenerateSQL = StepState.disabled;\n isActiveGenerateSQL = false;\n stateStepperRunQuery = StepState.disabled;\n isActiveRunQuery = false;\n stateStepperGetGraphDescription = StepState.disabled;\n isActiveGetGraphDescription = false;\n stateStepperGetTextSummary = StepState.disabled;\n isActiveGetTextSummary = false;\n messageStepperEnterQuestion = \"Question entered in 0 s\";\n overallProcessingTime = 0;\n stepperExpertInfoList.add(StepperExpertInfo());\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.enter_question : stepperExpertInfoList.length = ${stepperExpertInfoList.length}');\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.enter_question : stepperExpertInfoList = ${stepperExpertInfoList}');\n break;\n case StepperStatus.generate_sql:\n currentStep = 1;\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.generate_sql');\n stateStepperEnterQuestion = StepState.complete;\n isActiveEnterQuestion = true;\n stateStepperGenerateSQL = StepState.complete;\n isActiveGenerateSQL = true;\n stateStepperRunQuery = StepState.disabled;\n isActiveRunQuery = false;\n stateStepperGetGraphDescription = StepState.disabled;\n isActiveGetGraphDescription = false;\n stateStepperGetTextSummary = StepState.disabled;\n isActiveGetTextSummary = false;\n messageStepperGenerateSQL = state.message! +\n \" \" +\n ((state.debugInfo.stepDuration!.toDouble()) / 1000).toString() +\n \" s\";\n overallProcessingTime = overallProcessingTime +\n (state.debugInfo.stepDuration!.toDouble()) / 1000;\n stepperExpertInfoList.add(state.debugInfo);\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.generate_sql : stepperExpertInfoList.length = ${stepperExpertInfoList.length}');\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.generate_sql : stepperExpertInfoList = ${stepperExpertInfoList}');\n break;\n case StepperStatus.run_query:\n currentStep = 2;\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.run_query');\n stateStepperEnterQuestion = StepState.complete;\n isActiveEnterQuestion = true;\n stateStepperGenerateSQL = StepState.complete;\n isActiveGenerateSQL = true;\n stateStepperRunQuery = StepState.complete;\n isActiveRunQuery = true;\n stateStepperGetGraphDescription = StepState.disabled;\n isActiveGetGraphDescription = false;\n stateStepperGetTextSummary = StepState.disabled;\n isActiveGetTextSummary = false;\n messageStepperRunQuery = state.message! +\n \" \" +\n ((state.debugInfo.stepDuration!.toDouble()) / 1000).toString() +\n \" s\";\n overallProcessingTime = overallProcessingTime +\n (state.debugInfo.stepDuration!.toDouble()) / 1000;\n stepperExpertInfoList.add(state.debugInfo);\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.run_query : stepperExpertInfoList.length = ${stepperExpertInfoList.length}');\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.run_query : stepperExpertInfoList = ${stepperExpertInfoList}');\n break;\n case StepperStatus.get_graph_description:\n currentStep = 3;\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.get_graph_description');\n stateStepperEnterQuestion = StepState.complete;\n isActiveEnterQuestion = true;\n stateStepperGenerateSQL = StepState.complete;\n isActiveGenerateSQL = true;\n stateStepperRunQuery = StepState.complete;\n isActiveRunQuery = true;\n stateStepperGetGraphDescription = StepState.complete;\n isActiveGetGraphDescription = true;\n stateStepperGetTextSummary = StepState.disabled;\n isActiveGetTextSummary = false;\n messageStepperGraphDescription = state.message! +\n \" \" +\n ((state.debugInfo.stepDuration!.toDouble()) / 1000).toString() +\n \" s\";\n overallProcessingTime = overallProcessingTime +\n (state.debugInfo.stepDuration!.toDouble()) / 1000;\n stepperExpertInfoList.add(state.debugInfo);\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.get_graph_description : stepperExpertInfoList.length = ${stepperExpertInfoList.length}');\n print(\n 'Main : _buildStepper() : setStepsStates() : Switch: StepperStatus.get_graph_description : stepperExpertInfoList = ${stepperExpertInfoList}');\n break;\n default:\n currentStep = 0;\n stateStepperUploaded = StepState.disabled;\n isActiveUploaded = false;\n stateStepperExtracted = StepState.disabled;\n isActiveExtracted = false;\n stateStepperCompared = StepState.disabled;\n isActiveCompared = false;\n stateStepperCommitted = StepState.disabled;\n isActiveCommitted = false;\n message = state.message;\n }\n }\n\n Future _dialogRequestGenerated(\n BuildContext context, StepperExpertInfo debugInfo, String title) {\n print(\n \"Main: _dialogRequestGenerated() : title = $title : debugInfo.response = ${debugInfo.response}\");\n return showDialog(\n context: context,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Text(title),\n content: SingleChildScrollView(\n child: Container(\n width: screenSize!.width / 2,\n child: Column(\n children: [\n _createDebugInfoCard(\n Icon(Icons.link),\n 'URI POST',\n 'URI used in this step.',\n 'assets/images/https.png',\n Text(\n debugInfo.uri!,\n style: Theme.of(context)\n .textTheme\n .bodyMedium!\n .copyWith(fontSize: 16),\n ),\n debugInfo.uri!),\n _createDebugInfoCard(\n //FaIcon(icon),\n Icon(Icons.https),\n 'HTTPS Body',\n 'Json body of the request',\n 'assets/images/json.png',\n Container(\n child: JsonViewer(jsonDecode(debugInfo.body!))),\n debugInfo.body!),\n _createDebugInfoCard(\n //FaIcon(icon),\n Icon(Icons.https),\n 'HTTPS Headers',\n 'Headers sent in the HTTPS request.',\n 'assets/images/https_header.png',\n Container(\n child: JsonViewer(jsonDecode(debugInfo.header!))),\n debugInfo.header!),\n _createDebugInfoCard(\n FaIcon(FontAwesomeIcons.reply),\n 'HTTPS Response',\n 'HTTPS body of the response',\n 'assets/images/json.png',\n (!debugInfo.response!.contains('syntax error') && !debugInfo.response!.contains('SyntaxError') ) ?\n Container(\n child:\n JsonViewer(jsonDecode(debugInfo.response!..replaceAll('\"', '\\\\\"')))) : Container(child: Text(debugInfo.response!)),\n debugInfo.response!),\n _createDebugInfoCard(\n FaIcon(FontAwesomeIcons.code),\n 'Status Code',\n 'HTTPS answer status code',\n 'assets/images/status_code.png',\n Text(\n debugInfo.statusCode!.toString(),\n style: Theme.of(context)\n .textTheme\n .bodyMedium!\n .copyWith(fontSize: 16),\n ),\n debugInfo.statusCode!.toString()),\n _createDebugInfoCard(\n Icon(Icons.timer),\n 'Processing Time',\n 'Step duration in seconds',\n 'assets/images/elapsed_time.png',\n Text(\n (debugInfo.stepDuration!.toDouble() / 1000)\n .toString() +\n \" s\",\n style: Theme.of(context)\n .textTheme\n .bodyMedium!\n .copyWith(fontSize: 16),\n ),\n (debugInfo.stepDuration!.toDouble() / 1000)\n .toString() +\n \" s\"),\n //Text(\"SQL request\\n: ${debugInfo.generatedSQLText}\"),\n ],\n )),\n ), //Text(DicInfoExtractedMap.toString(),),\n actions: [\n TextButton(\n style: TextButton.styleFrom(\n textStyle: Theme.of(context).textTheme.labelLarge,\n ),\n child: const Text('Ok'),\n onPressed: () {\n Navigator.of(context).pop();\n },\n ),\n ],\n );\n },\n );\n }\n\n return Container(\n child: BlocBuilder(\n builder: (context, state) {\n setStepsStates(state);\n return Stepper(\n controlsBuilder: (BuildContext context, ControlsDetails controls) {\n return Container(); //to remove the continue and cancel buttons\n },\n type: StepperType.horizontal,\n currentStep: currentStep,\n onStepTapped: (int index) {\n String title = \"\";\n print(\n 'HomePage() : build() : Stepper : onStepTapped() : index = $index');\n switch (index) {\n case 0:\n title = \"\";\n break;\n case 1:\n title = \"SQL Request Generation\";\n break;\n case 2:\n title = \"Data Retrieval\";\n break;\n case 3:\n title = \"Graph Description Retrieval\";\n break;\n default:\n title = \"\";\n break;\n }\n\n print(\n 'HomePage() : build() : END : Stepper : onStepTapped() : index = $index');\n print(\n 'HomePage() : build() : END : Stepper : onStepTapped() : TextToDocParameter.isTextTodocGlobal = ${TextToDocParameter.isTextTodocGlobal}');\n print(\n 'HomePage() : build() : END : Stepper : onStepTapped() : stepperExpertInfoList.length = ${stepperExpertInfoList.length}');\n\n //temporary knob to dispaly info about TextToDoc request/answer\n if (TextToDocParameter.isTextTodocGlobal &&\n index == 3 &&\n stepperExpertInfoList.length == 2) {\n print(\n 'HomePage() : build() : END : Stepper : onStepTapped() : if(TextToDocParameter.isTextTodocGlobal && index == 4 && stepperExpertInfoList.length == 2 ) = $index');\n stepperExpertInfoList.insert(1, StepperExpertInfo());\n stepperExpertInfoList.insert(1, StepperExpertInfo());\n }\n if (TextToDocParameter.isTextTodocGlobal && index < 3) {\n return;\n }\n //end of temporary knob\n\n _dialogRequestGenerated(\n context, stepperExpertInfoList[index], title);\n },\n onStepContinue: () {},\n onStepCancel: () {},\n steps: [\n Step(\n title: Text(\"Question Typed\",\n style:\n TextStyle(fontWeight: FontWeight.bold, fontSize: 18.0)),\n subtitle: Text(messageStepperEnterQuestion!),\n state: stateStepperEnterQuestion!, //stateStepperUploaded!,\n isActive: isActiveEnterQuestion!, //isActiveUploaded!,\n content: LimitedBox(\n maxWidth: screenSize!.width, //100,\n maxHeight: 40,\n child: Container(\n color: Colors.black12, //CupertinoColors.systemGrey,\n child: Center(\n child: Padding(\n padding: const EdgeInsets.only(right: 8, left: 8),\n child: Text(message!,\n style: TextStyle(\n fontWeight: FontWeight.normal,\n fontSize: 18.0,\n color: stateStepperEnterQuestion! ==\n StepState.complete\n ? Colors.green\n : Colors.red)),\n ))),\n ),\n ),\n Step(\n title: Text(\"SQL Request Generated\",\n style:\n TextStyle(fontWeight: FontWeight.bold, fontSize: 18.0)),\n subtitle: Text(messageStepperGenerateSQL!),\n state: stateStepperGenerateSQL!, //stateStepperExtracted!,\n isActive: isActiveGenerateSQL!, //isActiveExtracted!,\n content: LimitedBox(\n maxWidth: screenSize!.width, //100,\n maxHeight: 40,\n child: Container(\n color: Colors.black12,\n child: Center(\n child: Padding(\n padding: const EdgeInsets.only(right: 8, left: 8),\n child: Text(message!,\n style: TextStyle(\n fontWeight: FontWeight.normal,\n fontSize: 18.0,\n color: Colors.green)),\n ),\n )),\n ),\n ),\n Step(\n title: Text(\"Data Retrieved\",\n style:\n TextStyle(fontWeight: FontWeight.bold, fontSize: 18.0)),\n subtitle: Text(messageStepperRunQuery!),\n state:\n stateStepperRunQuery!, //StepState.disabled, //stateStepperCompared!,\n isActive: isActiveRunQuery!, //isActiveCompared!,\n content: LimitedBox(\n maxWidth: screenSize!.width, //100,\n maxHeight: 40,\n child: Container(\n color: Colors.black12,\n child: Center(\n child: Padding(\n padding: const EdgeInsets.only(right: 8, left: 8),\n child: Text(message!,\n style: TextStyle(\n fontWeight: FontWeight.normal,\n fontSize: 18.0,\n color: Colors.green)),\n ))),\n ),\n ),\n Step(\n title: Text(\"Graph & Table Generated\",\n style:\n TextStyle(fontWeight: FontWeight.bold, fontSize: 18.0)),\n subtitle: Text(messageStepperGraphDescription!),\n state:\n stateStepperGetGraphDescription!, //StepState.disabled, //stateStepperCompared!,\n isActive: isActiveGetGraphDescription!, //isActiveCompared!,\n content: LimitedBox(\n maxWidth: screenSize!.width, //100,\n maxHeight: 40,\n child: Container(\n color: Colors.black12,\n child: Center(\n child: Padding(\n padding: const EdgeInsets.only(right: 8, left: 8),\n child: Text(\n \"The overall time to generate the natural language answer is ${double.parse(overallProcessingTime.toStringAsFixed(2))} s.\",\n style: TextStyle(\n fontWeight: FontWeight.normal,\n fontSize: 18.0,\n color: Colors.green)),\n ))),\n ),\n ),\n ],\n );\n },\n ),\n );\n }\n\n Future loadImageAsset(String assetPath) async {\n final ByteData data = await rootBundle.load(assetPath);\n final ui.Codec codec =\n await ui.instantiateImageCodec(data.buffer.asUint8List());\n final ui.FrameInfo frameInfo = await codec.getNextFrame();\n return frameInfo.image;\n }\n\n Future> convertImageToListInt(String assetPath) async {\n ui.Image image = await loadImageAsset(assetPath);\n ByteData? byteData = await image.toByteData(format: ui.ImageByteFormat.png);\n return byteData!.buffer.asUint8List();\n }\n\n Future _loadFile(String uri) async {\n final byteData = await rootBundle.load(uri);\n final buffer = byteData.buffer;\n Uint8List bytes =\n buffer.asUint8List(byteData.offsetInBytes, byteData.lengthInBytes);\n return bytes;\n }\n\n Future>> importQuestions() async {\n print('Main: ttmd : importQuestions() : START');\n List>? rowsAsListOfValues;\n final filePickerResult = await FilePicker.platform.pickFiles(\n allowMultiple: false,\n allowedExtensions: ['csv'],\n type: FileType.custom,\n dialogTitle: \"Import questions\",\n );\n\n if (filePickerResult != null) {\n print(\n 'Main: ttmd : importQuestions() : fileName = ${filePickerResult.files.single.name}');\n print(\n 'Main: ttmd : importQuestions() : size = ${filePickerResult.files.single.size}');\n //print('Main: ttmd : importQuestions() : path = ${filePickerResult.files.single.path}');\n Uint8List fileBytes = filePickerResult.files.single.bytes!;\n\n String fileContent = utf8.decode(fileBytes);\n List lines = fileContent.split('\\n');\n print('Main: ttmd : importQuestions() : lines.length = ${lines.length}');\n\n rowsAsListOfValues = const CsvToListConverter(\n fieldDelimiter: ',', textDelimiter: '\"', textEndDelimiter: '\"')\n .convert(fileContent);\n print(\n 'Main: ttmd : importQuestions() : rowsAsListOfValues = ${rowsAsListOfValues}');\n\n if (rowsAsListOfValues!.length < 2 ||\n rowsAsListOfValues[0].length > 4 ||\n (rowsAsListOfValues[0].length <= 4 &&\n (rowsAsListOfValues[0][0] != \"user_grouping\" ||\n rowsAsListOfValues[0][1] != \"scenario\" ||\n rowsAsListOfValues[0][2] != \"question\"))) {\n print(\n \"Main: importQuestions() : WRONG FORMAT : rowsAsListOfValues[0] = ${rowsAsListOfValues[0]}\");\n checkImportedCSVFile();\n return [[]];\n }\n\n for (var entry in rowsAsListOfValues) {\n if (rowsAsListOfValues[1].length == 4) {\n print(\n \"Main: importQuestions() : rowsAsListOfValues = ${entry[0]}, ${entry[2]}, ${entry[3]}\");\n if (rowsAsListOfValues[1].length != 4) checkImportedCSVFile();\n }\n if (rowsAsListOfValues[1].length == 3) {\n print(\n \"Main: importQuestions() : rowsAsListOfValues = ${entry[0]}, ${entry[2]}\");\n if (rowsAsListOfValues[1].length != 3) checkImportedCSVFile();\n }\n }\n }\n\n return rowsAsListOfValues!;\n }\n\n void checkImportedCSVFile() {\n showDialog(\n context: context,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Text('Alert'),\n content: Text(\n \"The imported CVS file does not have the right format or any entry.\\n\" +\n \"Please follow the format below (CVS comma separated):\\n\" +\n \"grouping,\\tscenario,\\tquestion\\n\" +\n \"grouping1,\\tscenario1,\\tquestion1\\n\\n\" +\n \"For more info, please look at:\\n\" +\n \"https://github.com/GoogleCloudPlatform/Open_Data_QnA/blob/main/scripts/known_good_sql.csv\",\n softWrap: true,\n overflow: TextOverflow.visible), //SwitchExample(),\n actions: [\n TextButton(\n onPressed: () {\n Navigator.of(context).pop(); // Close the dialog\n },\n child: Text('OK'),\n ),\n ],\n );\n },\n );\n }\n\n void noCfgStoredinFirestore() {\n showDialog(\n context: context, //navigatorKey.currentContext!,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Row(\n // Use a Row to align the icon and title\n children: [\n Icon(Icons.warning, color: Colors.red),\n SizedBox(width: 8), // Add some spacing\n Text('Alert'),\n ],\n ),\n content: SelectableText.rich(\n TextSpan(\n children: [\n TextSpan(\n text: \"You must first create and load a configuration file\\n\" +\n \"using the Settings -> Upload frontend config file option.\\n\" +\n 'For that, copy and paste the content below in a file\\n' +\n \"that you'll name config_frontend.json:\\n\\n\"),\n TextSpan(\n text: \"{\\n\",\n style: TextStyle(color: Colors.black),\n ),\n TextSpan(\n text: '\"endpoint_opendataqnq\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"\",\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '\"firestore_database_id\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"opendataqna-session-logs\",\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '\"firestore_history_collection\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"session_logs\",\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '\"firestore_cfg_collection\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"front_end_flutter_cfg\",\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '\"expert_mode\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: ',\\n',\n style: TextStyle(color: Colors.red),\n ),\n TextSpan(\n text: '\"anonymized_data\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: ',\\n',\n style: TextStyle(color: Colors.red),\n ),\n TextSpan(\n text: '\"firebase_app_name\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"opendataqna\"\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '\"imported_questions\": ',\n style: TextStyle(color: Colors.blueAccent),\n ),\n TextSpan(\n text: '\"imported_questions\"\\n',\n style: TextStyle(color: Colors.green),\n ),\n TextSpan(\n text: '}',\n style: TextStyle(color: Colors.black),\n ),\n ],\n ),\n ),\n actions: [\n TextButton(\n onPressed: () {\n Navigator.of(context).pop(); // Close the dialog\n },\n child: Text('OK'),\n ),\n ],\n );\n },\n );\n }\n\n List> createQuestionList(List> questionList) {\n List> finalNodeList = [];\n List>? nodeList = [];\n List>? nodeEmbeddedList;\n String scenario_nameCurrent = \"\";\n int count = 1;\n int countQuestions = 0;\n bool isNewScenario = true;\n String nodeTmp = \"\";\n\n print(\"Main: createQuestionList() : START\");\n print(\n \"Main: createQuestionList() : questionList.length = ${questionList.length}\");\n print(\n \"Main: createQuestionList() : questionList[0].length = ${questionList[0].length}\");\n\n for (var entry in questionList)\n print(\n \"Main: createQuestionList() : questionList = ${entry[1]}, ${entry[2]}\");\n\n scenario_nameCurrent = (questionList[0][1] as String).trim().toLowerCase();\n nodeTmp = \"\";\n\n for (int i = 0; i < questionList.length; i++) {\n print(\n \"Main: createQuestionList() : START LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent\");\n\n if (i < questionList.length - 1) {\n print(\n \"Main: createQuestionList() : START LOOP : i = $i : $i < ${questionList.length - 1}\");\n //We're on the same scenario\n if (scenario_nameCurrent ==\n (questionList[i][1] as String).trim().toLowerCase()) {\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = questionList[i][1] as String).trim().toLowerCase() = $scenario_nameCurrent\");\n\n //processQuestionRow(i);\n if (isNewScenario) {\n print(\n \"Main: createQuestionList() : LOOP : i = $i : This is a new sceanrio : isNewScenario = $isNewScenario\");\n nodeList = [];\n nodeEmbeddedList = [];\n\n finalNodeList.add(TreeNode(\n 'Scenario $count - ${questionList[i][1]} - ${getQuestionCount(questionList, scenario_nameCurrent)}',\n subNodes: nodeList!));\n isNewScenario = false;\n }\n nodeTmp = \"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0];\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : nodeTmp = $nodeTmp\");\n\n if ((questionList[i][3] as String).trim().toLowerCase() == \"y\" &&\n (questionList[i + 1][3] as String).trim().toLowerCase() == \"y\" &&\n (questionList[i + 1][1] as String).trim().toLowerCase() ==\n scenario_nameCurrent) {\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()} : main_question${i + 1} = ${(questionList[i + 1][3] as String).trim().toLowerCase()}\");\n\n nodeList!\n .add(TreeNode(nodeTmp, subNodes: nodeEmbeddedList!));\n } else if ((questionList[i][3] as String).trim().toLowerCase() ==\n \"y\" &&\n (questionList[i + 1][1] as String).trim().toLowerCase() !=\n scenario_nameCurrent) {\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()} : main_question${i + 1} = ${(questionList[i + 1][3] as String).trim().toLowerCase()} : questionList[i+1][1] = ${questionList[i + 1][1]} != scenario_nameCurrent = $scenario_nameCurrent\");\n\n nodeList!.add(TreeNode(nodeTmp));\n } else if ((questionList[i][3] as String).trim().toLowerCase() ==\n \"y\" &&\n (questionList[i + 1][3] as String).trim().toLowerCase() == \"n\") {\n //this is a new main question\n nodeEmbeddedList = [];\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()} != main_question${i + 1} = ${(questionList[i + 1][3] as String).trim().toLowerCase()}\");\n /*nodeEmbeddedList!.add(TreeNode(\"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0]));\n nodeList!.add(TreeNode(nodeTmp, subNodes: nodeEmbeddedList!));*/\n nodeList!\n .add(TreeNode(nodeTmp, subNodes: nodeEmbeddedList!));\n } else if ((questionList[i][3] as String).trim().toLowerCase() ==\n \"n\" &&\n (questionList[i + 1][3] as String).trim().toLowerCase() == \"n\") {\n //this is a follow-up question\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()} == main_question${i + 1} = ${(questionList[i + 1][3] as String).trim().toLowerCase()}\");\n nodeEmbeddedList!.add(TreeNode(\"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0]));\n } else if ((questionList[i][3] as String).trim().toLowerCase() ==\n \"n\" &&\n (questionList[i + 1][3] as String).trim().toLowerCase() == \"y\") {\n //this is a new main question\n print(\n \"Main: createQuestionList() : LOOP : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()} == main_question${i + 1} = ${(questionList[i + 1][3] as String).trim().toLowerCase()}\");\n nodeEmbeddedList!.add(TreeNode(\"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0]));\n }\n } else {\n print(\n \"Main: createQuestionList() : i = $i : $scenario_nameCurrent != ${questionList[i][1]} : countQuestions = $countQuestions\");\n isNewScenario = true;\n count++;\n\n //processQuestionRow(i);\n\n nodeList = [];\n nodeEmbeddedList = [];\n\n finalNodeList.add(TreeNode(\n 'Scenario $count - ${questionList[i][1]} - ${getQuestionCount(questionList, questionList[i][1])}',\n subNodes: nodeList!));\n isNewScenario = false;\n\n nodeTmp = \"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0];\n print(\n \"Main: createQuestionList() : i = $i : $scenario_nameCurrent != ${questionList[i][1]} : nodeTmp = $nodeTmp\");\n nodeList!.add(TreeNode(nodeTmp, subNodes: nodeEmbeddedList!));\n scenario_nameCurrent =\n (questionList[i][1] as String).trim().toLowerCase();\n print(\n \"Main: createQuestionList() : i = $i : new scenario_nameCurrent = $scenario_nameCurrent \");\n }\n } //end of if i < questionList.length - 1\n\n else if (i == questionList.length - 1) {\n print(\n \"Main: createQuestionList() : LOOP : Last row : i = $i : i = ${questionList.length - 1}\");\n if ((questionList[i][3] as String).trim().toLowerCase() == \"y\") {\n print(\n \"Main: createQuestionList() : LOOP : Last row : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()}\");\n\n //nodeList!.add(TreeNode(nodeTmp, subNodes: nodeEmbeddedList!));\n nodeList!.add(TreeNode(\"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0]));\n } else {\n //this is a new main question\n //nodeEmbeddedList = [];\n print(\n \"Main: createQuestionList() : LOOP : Last row : i = $i : scenario_nameCurrent = $scenario_nameCurrent : main_question$i = ${(questionList[i][3] as String).trim().toLowerCase()}\");\n nodeEmbeddedList!.add(TreeNode(\"$count:\" +\n questionList[i][2] +\n \":\" +\n questionList[i][1] +\n \":\" +\n questionList[i][0]));\n }\n }\n } //end of for\n\n print(\"Main: createQuestionList() : END : count = $count\");\n return finalNodeList;\n }\n\n int getQuestionCount(List> questionList, String scenario) {\n int scenarioCount = 0;\n bool hit = false;\n\n print(\"Main: getQuestionCount() : START \");\n\n for (int i = 0; i < questionList.length; i++) {\n if (scenario.toLowerCase() ==\n (questionList[i][1] as String).trim().toLowerCase()) {\n hit = true;\n scenarioCount++;\n } else {\n if (hit) break;\n }\n }\n\n print(\"Main: getQuestionCount() : scenarioCount = $scenarioCount\");\n return scenarioCount;\n }\n\n Future>> _getLastQuestions() async {\n List> resp = [];\n\n print(\"Main : _getLastQuestions() : START\");\n print(\n 'Main : _getLastQuestions() : TextToDocParameter.userID = ${TextToDocParameter.userID}');\n\n print('Main : _getLastQuestions() : db = ${db}');\n print('Main : _getLastQuestions() : db.app = ${db.app}');\n print('Main : _getLastQuestions() : db.databaseId = ${db.databaseId}');\n\n var querySnapshot = await db\n .collection(\"session_logs\")\n .where(\"user_id\", isEqualTo: TextToDocParameter.userID)\n .limit(4)\n .orderBy('timestamp', descending: true)\n .get();\n\n print(\n \"Main : _getLastQuestions() : querySnapshot.docs.length = ${querySnapshot.docs.length}\");\n print(\"Main : _getLastQuestions() : querySnapshot = ${querySnapshot}\");\n for (var docSnapshot in querySnapshot.docs) {\n print(\n 'Main : _getLastQuestions() : ${docSnapshot.id} => ${docSnapshot.data()}');\n resp.add({\n \"user_question\": \"${docSnapshot.data()['user_question']}\",\n \"timestamp\":\n \"${DateTime.fromMillisecondsSinceEpoch(docSnapshot.data()['timestamp'].seconds * 1000)}\",\n \"user_grouping\":\n \"${docSnapshot.data().containsKey('user_grouping') ? docSnapshot.data()['user_grouping'] : 'no data'}\",\n \"scenario_name\":\n \"${docSnapshot.data().containsKey('scenario_name') ? docSnapshot.data()['scenario_name'] : 'no data'}\"\n });\n }\n return resp;\n }\n\n Future> _getLastQuestionsOld(String userGrouping) async {\n List respLLMQuestion = [];\n List respCannedQuestions = [];\n List resp = [];\n List tmpQuestions = [];\n String body = \"\";\n Uri url;\n String question1 = \"\";\n String question2 = \"\";\n String question3 = \"\";\n String question4 = \"\";\n String timeString = \"\";\n String originalQuestion = \"\";\n\n print(\n 'NewSuggestionCubit : getLastQuestions() : generateNewSuggestions : START');\n print(\n 'NewSuggestionCubit : getLastQuestions() : userGrouping = $userGrouping');\n\n timeString = displayDateTime();\n\n //Create the header\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n //Create the body\n body = '''{\n \"user_grouping\": \"$userGrouping\"\n }''';\n\n try {\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/get_known_sql',\n method: \"POST\",\n sendData: body);\n\n print('NewSuggestionCubit : getLastQuestions() : response = ' +\n response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('NewSuggestionCubit: getLastQuestions() : jsonData = $jsonData');\n\n //KnownSQL = [{\"example_user_question\": \"question1\", \"example_generated_sql\": \"sql1\"},\n // {\"example_user_question\": \"question2\", \"example_generated_sql\": \"sql2\"},\n // ...]\n\n var knownSql =\n jsonData[\"KnownSQL\"].replaceAll(RegExp(r'((\\\\n)|(\\\\r))'), '');\n\n var knownSqlMap = jsonDecode(knownSql);\n\n print(\n 'NewSuggestionCubit: getLastQuestions() : knownSqlMap = ${knownSqlMap}');\n\n print(\n 'NewSuggestionCubit: getLastQuestions() : knownSqlMap[0] = ${knownSqlMap[0].toString()}');\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"example_user_question\")\n tmpQuestions.add(entry.value);\n }\n }\n\n print('Main: getLastQuestions() : tmpQuestions = ${tmpQuestions}');\n\n question1 = tmpQuestions[0] ?? \"No suggestion for question1\";\n question2 = tmpQuestions[1] ?? \"No suggestion for question2\";\n question3 = tmpQuestions[2] ?? \"No suggestion for question3\";\n question4 = tmpQuestions[3] ?? \"No suggestion for question4\";\n\n //Adding scenarioNumber + \":\" to have same format as for Business KPI questions\n question1 = \"2:\" + question1;\n question2 = \"2:\" + question2;\n question3 = \"2:\" + question3;\n question4 = \"2:\" + question4;\n\n print('Main: getLastQuestions() : question1 = ${question1}');\n print('Main: getLastQuestions() : question2 = ${question2}');\n print('Main: getLastQuestions() : question3 = ${question3}');\n print('Main: getLastQuestions() : question4 = ${question4}');\n }\n } catch (e) {\n print(\n 'Main: getLastQuestions() : Not a canned question : EXCEPTION = $e');\n throw Exception('Failed to get earning calls question suggestions: $e');\n } finally {\n resp.add(question1);\n resp.add(question2);\n resp.add(question3);\n resp.add(question4);\n\n print('Main: getLastQuestions() : resp = $resp');\n return resp;\n }\n }\n\n void SaveImportedQuestionsToFirestore(\n List> questionList) async {\n int count = 0;\n\n print('Main: SaveImportedQuestionsToFirestore() : START');\n print(\n 'Main: SaveImportedQuestionsToFirestore() : questionList = $questionList');\n\n if (questionList.length > 0) {\n try {\n //Delete all former questions for that user\n var querySnapshot = await db!\n .collection(\"${TextToDocParameter.imported_questions}\")\n .where(\"user_id\", isEqualTo: TextToDocParameter.userID)\n .get();\n\n for (var docSnapshot in querySnapshot.docs) {\n db.collection(\"imported_questions\").doc('${docSnapshot.id}').delete();\n }\n\n //create new questions to be stored on Firestore\n for (int i = 0; i < questionList.length; i++) {\n List list = questionList[i];\n\n print('Main: SaveImportedQuestionsToFirestore() : List = $List');\n\n Map questionMap = {};\n\n questionMap['user_grouping'] = list[0];\n questionMap['scenario'] = list[1];\n questionMap['question'] = list[2];\n questionMap['user_id'] = TextToDocParameter.userID;\n if (list.length == 4)\n questionMap['main_question'] = list[3];\n else\n questionMap['main_question'] = \"Y\";\n\n questionMap['order'] = count++;\n\n db\n .collection(\"imported_questions\")\n .doc(\"question$i\")\n .set(questionMap);\n }\n } catch (e) {\n print('Main: SaveImportedQuestionsToFirestore() : EXCEPTION : e = $e');\n }\n }\n }\n}\n\nclass Config {\n static bool isUseFeedback = false;\n static bool isUseColorMode = false;\n static bool isUseDashboards = false;\n static bool isUseReports = false;\n static bool isExpert = false;\n static bool isUseLog = false;\n static bool isAnonymizedMode = false;\n}\n" }, { "path": "frontend/frontend-flutter/lib/screens/bot.dart", "content": "import 'dart:typed_data';\nimport 'package:cloud_firestore/cloud_firestore.dart';\nimport 'package:flutter/cupertino.dart';\nimport 'package:flutter/material.dart';\nimport 'package:flutter/services.dart';\nimport 'package:flutter_bloc/flutter_bloc.dart';\nimport 'package:flutter_inappwebview/flutter_inappwebview.dart';\nimport 'package:intl/intl.dart';\nimport 'package:flutter_chat_ui/flutter_chat_ui.dart';\nimport 'package:flutter_chat_types/flutter_chat_types.dart' as types;\nimport 'dart:convert';\nimport 'dart:math';\nimport 'package:file_picker/file_picker.dart';\nimport 'package:bubble/bubble.dart';\nimport 'package:http/http.dart' as http;\nimport 'package:deep_pick/deep_pick.dart';\nimport 'package:ttmd/utils/stepper_expert_info.dart';\nimport '../services/first_question/first_question_cubit.dart';\nimport '../services/load_question/load_question_cubit.dart';\nimport '../services/new_suggestions/new_suggestion_cubit.dart';\nimport '../services/update_stepper/update_stepper_cubit.dart';\nimport '../services/update_stepper/update_stepper_state.dart';\nimport '../utils/TextToDocParameter.dart';\nimport '../utils/custom_input_field.dart' as cif;\nimport '../utils/custom_input_field.dart';\nimport 'dart:html' as html;\nimport 'package:screenshot/screenshot.dart';\nimport '../utils/tabbed_container.dart';\nimport 'package:simple_http_api/simple_http_api.dart';\nimport \"dart:js\" as js;\n\n// For the testing purposes, you should probably use https://pub.dev/packages/uuid.\nString randomString() {\n final random = Random.secure();\n final values = List.generate(16, (i) => random.nextInt(255));\n return base64UrlEncode(values);\n}\n\nclass Bot extends StatefulWidget {\n final TextEditingController? textEditingController;\n final FirebaseFirestore? db;\n const Bot({Key? key, this.textEditingController, this.db}) : super(key: key);\n\n @override\n State createState() => BotState();\n}\n\nclass BotState extends State with SingleTickerProviderStateMixin {\n final List _messages = [];\n final Map _graphsImagesMap = {};\n final _user = types.User(\n id: '82091008-a484-4a89-ae75-a22bf8d6f3ac',\n firstName: '${TextToDocParameter.firstName}',\n lastName: '${TextToDocParameter.lastName}'\n );\n final _userAvatar = types.User(\n id: '82091010-a484-4a89-ae75-a22bf8d6f3ab',\n firstName: '${TextToDocParameter.firstName}',\n lastName: '${TextToDocParameter.lastName}'\n );\n final _user1 = const types.User(\n id: '82091009-a485-4a90-ae76-a22bf8d6f3ad',\n firstName: 'Open Data QnA',\n lastName: 'Assistant');\n String textMLKit = \"\";\n String textDocAi = \"\";\n String responsePalMBody = \"\";\n String streamingText = \"\";\n String requestPalMBody = \"\";\n String responseDLPMBody = \"\";\n String requestDLPBody = \"\";\n List sourceList = [];\n Map> mapSource = Map();\n String colorBubble = \"user\";\n Chat? chat;\n Size? screenSize;\n bool isFirstQuestion = true;\n bool isProcessing = false;\n ScreenshotController screenshotController = ScreenshotController();\n ScreenshotController screenshotController1 = ScreenshotController();\n List> tableKeyList = [];\n Map tableKeyMap = {};\n String? imageId;\n bool isGraphKeyAdded = false;\n bool isTableKeyAdded = false;\n Map mainQuestionsFollowUpQuestions = {};\n late TabController _tabController;\n bool _isThumbsUpHovered = false;\n bool _isThumbsDownHovered = false;\n bool _isCopyHovered = false;\n static bool isTextToDoc = false;\n int _selectedIndex = 0;\n InAppWebViewController? webViewController;\n Map mapAnonymisationGraph = {};\n Container? graphContainer;\n\n @override\n void initState() {\n setup();\n super.initState();\n }\n\n @override\n void dispose() {\n _tabController!.dispose();\n webViewController!.dispose();\n super.dispose();\n }\n\n Future setup() async {\n _tabController = TabController(length: 2, vsync: this);\n }\n\n @override\n Widget build(BuildContext context) {\n\n screenSize = MediaQuery.of(context).size;\n\n chat = Chat(\n emptyState: Text(\"\"),\n avatarBuilder: avatarBuilder,\n customBottomWidget: CustomInputField(\n onAttachmentPressed: _handleAttachmentPressed,\n onSendPressed: _handleSendPressed,\n db: widget.db,\n options: cif.InputOptions(\n textEditingController: widget.textEditingController)),\n customMessageBuilder: customMessageBuilder,\n //inputOptions: InputOptions(textEditingController: widget.textEditingController),\n messages: _messages,\n onSendPressed: _handleSendPressed,\n //onAttachmentPressed: _handleImageSelection,\n //onAttachmentPressed: _handleFileSelection\n onAttachmentPressed: _handleAttachmentPressed,\n onMessageTap: _handleMessageTap,\n onPreviewDataFetched: _handlePreviewDataFetched,\n showUserAvatars: true,\n showUserNames: true,\n bubbleBuilder: _bubbleBuilder,\n user: _user,\n messageWidthRatio: 0.9,\n theme: DefaultChatTheme(\n seenIcon: Text(\n 'read',\n style: TextStyle(\n fontSize: 10.0,\n ),\n ),\n backgroundColor: Color(\n 0xFFF0F2F6), //Color.fromRGBO(242, 242, 242, 1.0),//Colors.black12,\n messageMaxWidth: screenSize!.width,\n ),\n );\n\n print(\n \" bot: build() : TextToDocParameter.isTextTodocGlobal = ${TextToDocParameter.isTextTodocGlobal}\");\n return Flexible(\n fit: FlexFit.loose,\n flex: 9,\n child: Stack(children: [\n Screenshot(child: chat!, controller: screenshotController1),\n isProcessing\n ? Positioned(\n left: 650,\n top: 300,\n child: SizedBox(\n width: 100,\n height: 100,\n child: CircularProgressIndicator(\n strokeWidth: 6,\n ),\n ),\n )\n : dummyFunction(),\n ]),\n );\n }\n\n Text dummyFunction() {\n print(\"Bot: dummyFunction() : START\");\n Future.delayed(const Duration(seconds: 2), () {\n generateSnapshot();\n });\n\n return Text(\"\");\n }\n\n Future generateSnapshot() async {\n print(\"Bot: generateSnapshot() : START\");\n //Does not work for now because the flutter_inappwebview is not supported by the creenshot package\n //it return a blank image. InAppWebViewController.takeScreenshot() is not implemented for the web platform.\n //So commenting out the code below :\n\n /*if (isGraphKeyAdded == true &&\n !TextToDocParameter.isTextTodocGlobal) {\n print(\n \"Bot: generateSnapshot() : isGraphKeyAdded = $isGraphKeyAdded\");\n print(\"Bot: generateSnapshot() : imageId = ${imageId}\");\n Uint8List img = await _generateImage(graphContainer!);\n _graphsImagesMap[imageId!] = img;\n isGraphKeyAdded = false;\n print(\"Bot: generateSnapshot() : isGraphKeyAdded = $isGraphKeyAdded\");\n print(\"Bot: generateSnapshot() : _graphsImagesMap.length = ${_graphsImagesMap.length}\");\n } */\n if (tableKeyList.isNotEmpty &&\n isTableKeyAdded == true &&\n !TextToDocParameter.isTextTodocGlobal) {\n print(\n \"Bot: generateSnapshot() : tableKeyList.length = ${tableKeyList.length}\");\n print(\n \"Bot: generateSnapshot() : tableKeyList.isNotEmpty = ${tableKeyList.isNotEmpty} && isTableKeyAdded = $isTableKeyAdded\");\n print(\n \"Bot: generateSnapshot() : tableKeyList.last = ${tableKeyList.last.toString()}\");\n /* Commenting out this code until syncfusion_flutter_pdfviewer package is replaced\n PaginatedDataTable pdfGrid = await tableKeyList.last.currentState!.widget;\n print(\"Bot: generateSnapshot() : pdfGrid = $pdfGrid\");\n tableKeyMap[tableKeyList.last.toString()] = pdfGrid!;\n print(\n \"Bot: generateSnapshot() : tableKeyMap.length = ${tableKeyMap.length}\");\n print(\"Bot: generateSnapshot() : isTableKeyAdded = $isTableKeyAdded\"); */\n isTableKeyAdded = false;\n }\n //tableKeyMap\n }\n\n void _addMessage(types.Message message) {\n print('Bot : _addMessage() message.text = ' + message.toJson().toString());\n setState(() {\n _messages.insert(0, message);\n });\n\n print('Bot : _addMessage() : _messages.length = ' +\n _messages.length.toString());\n print('Bot : _addMessage() : _messages = ' + _messages.toString());\n }\n\n void _handleSendPressed(types.PartialText message) {\n final textMessage = types.TextMessage(\n author: _userAvatar,\n createdAt: DateTime.now().millisecondsSinceEpoch,\n id: randomString(),\n text: message.text,\n type: types.MessageType.text,\n metadata: {\"dataSource\": \"user\"});\n bool isACannedQuestion = false;\n\n print('Bot : _handleSendPressed() textMessage.text = ' + textMessage.text);\n print('Bot : _handleSendPressed() textMessage.author.firstName = ' +\n textMessage.author.firstName!);\n\n _addMessage(textMessage);\n\n //Remove the welcome message after the first question asked\n if (isFirstQuestion) {\n print('Bot : _handleSendPressed() : isFirstQuestion = ' +\n isFirstQuestion.toString());\n isFirstQuestion = false;\n BlocProvider.of(context).removeWelcomeMessage();\n }\n\n print(\n \"main: initState() : After BlocProvider.of(context).updateStepperStatusUploaded() : TextToDocParameter.lastScenarioNumber = ${TextToDocParameter.lastScenarioNumber}\");\n\n BlocProvider.of(context).generateNewSuggestions(\n TextToDocParameter.lastScenarioNumber, message.text,\n lastCannedQuestion: TextToDocParameter.lastCannedQuestion,\n isACannedQuestion: isACannedQuestion);\n\n //Update the question history on the side menu\n //BlocProvider.of(context).loadQuestionToChat(question: message.text, time: displayDateTime());\n BlocProvider.of(context)\n .loadQuestionToChat(question: message.text, time: \"rr\");\n\n //Update stepper state\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.enter_question,\n message: \"Question entered.\",\n stateStepper: StepState.complete,\n isActiveStepper: true);\n print(\n \"main: initState() : After BlocProvider.of(context).updateStepperStatusUploaded() : stepper initialized\");\n\n _handleReceivedResponse(textMessage.text, \"text\");\n\n setState(() {\n isProcessing = true;\n });\n }\n\n void _handleReceivedResponse(String msg, String type) async {\n String mime = \"\";\n String id = randomString();\n bool isText = true;\n dynamic dataViz;\n GlobalKey tableKey = GlobalKey();\n imageId = randomString();\n PaginatedDataTable? tableGrid;\n\n //TextToDocParameter.isTextTodocGlobal = true;\n print('Bot : _handleReceivedResponse(): START ');\n print('Bot : _handleReceivedResponse(): isTextToDoc = ' +\n isTextToDoc.toString());\n print(\n 'Bot : _handleReceivedResponse(): TextToDocParameter.isTextTodocGlobal = ${TextToDocParameter.isTextTodocGlobal}');\n\n switch (type) {\n case \"text\":\n mime = \"application/json\";\n break;\n case \"pdf\":\n mime = \"application/pdf\";\n break;\n case \"png\":\n //case \"image\":\n mime = \"image/png\";\n break;\n case \"jpeg\":\n //case \"image\":\n mime = \"image/jpeg\";\n break;\n default:\n print('Error, out of range : index = $type ');\n }\n\n print('Bot : _handleReceivedResponse(): type = ' +\n type +\n ' : mime = ' +\n mime);\n\n print('Bot : _handleReceivedResponse(): msg = ' + msg);\n\n if (!TextToDocParameter.isTextTodocGlobal) {\n //NL2SQL request\n print('Bot : _handleReceivedResponse(): USING NL2SQL');\n\n //Get generated reponse\n var rep = await getChatResponseNew(msg, mime, id, user: _user1);\n\n print(\n 'Bot : _handleReceivedResponse(): back from getChatResponseNew() : rep = $rep');\n\n //graphConfig = rep![2] as GraphConfig;\n dataViz = rep![2];\n\n if ((rep![0] as String).length == 0) {\n //knownDB\n rep[0] =\n '[{\"response\": \"The request did not return meaningful information. It could be because the question has not been formulated properly or some context is missing.\"}]';\n } else {\n //The request has been successful and an entry has been created on ${TextToDocParameter.firestore_database_id}\n //Adding the user_grouping and scenario_name to the entry because as of now it does not contain this data.\n //If in the future user_grouping is added, _updateUserGroupingInSessionLogs() below can be removed\n _updateUserGroupingInSessionLogs();\n\n tableGrid = createPaginatedTable(rep[0] as String);\n\n tableKeyList.add(tableKey);\n isTableKeyAdded = true;\n print(\n 'Bot : _handleReceivedResponse(): tableKey = $tableKey : isTableKeyAdded = $isTableKeyAdded');\n }\n isText = rep[1] as bool;\n\n print(\n 'Bot : _handleReceivedResponse(): repFinal = ' + (rep[0] as String));\n print(\n 'Bot : _handleReceivedResponse(): isText = ' + (rep[1].toString()));\n\n print(\n 'Bot : _handleReceivedResponseNew() : CustomMessage : isText = ${isText}');\n\n if (isText) {\n print(\n 'Bot : _handleReceivedResponseNew() : isText = $isText : dataViz = ${dataViz}');\n imageId = \"no_image\";\n } else {\n print(\n 'Bot : _handleReceivedResponseNew() : CustomMessage : imageId = $imageId');\n }\n\n final customMessage = types.CustomMessage(\n author: _user1,\n createdAt: DateTime.now().millisecondsSinceEpoch,\n id: randomString(),\n type: types.MessageType.custom,\n metadata: {\n \"graph\": dataViz,\n \"textSummary\": rep[3] as String,\n \"imageId\": imageId,\n \"dataSource\": tableGrid,\n \"tableKey\": tableKey.toString(),\n });\n\n _addMessage(customMessage);\n } else {\n print('Bot : _handleReceivedResponse(): USING TEXT2DOC');\n\n var rep =\n await getChatResponseTextToDoCStream(msg, mime, id, user: _user1);\n\n imageId = \"no_image\";\n\n final customMessage = types.CustomMessage(\n author: _user1,\n createdAt: DateTime.now().millisecondsSinceEpoch,\n id: randomString(),\n type: types.MessageType.custom,\n metadata: {\n \"graph\": dataViz,\n \"textSummary\": rep![3] as String,\n \"imageId\": imageId,\n \"dataSource\": null,\n \"tableKey\": tableKey.toString(),\n \"stream\": rep![4] as Stream> ?? null,\n \"stopWatch\": rep![5] as Stopwatch ?? null\n });\n\n _addMessage(customMessage);\n }\n\n setState(() {\n isProcessing = false;\n });\n }\n\n Future?> getChatResponseTextToDoCStream(\n String msg, String mime, String id,\n {types.User? user}) async {\n List? reqResp = [];\n List RespList = [];\n Uri url;\n String userQuestion = \"\";\n String body = \"\";\n\n print('Bot : getChatResponseTextToDoCStream() : START');\n url = Uri.parse(\n 'https://colab-cloudrun-template-ra1-uz6w7mqrka-uc.a.run.app/generate_streaming');\n\n print('Bot : getChatResponseTextToDoCStream() : url = ' +\n url.host +\n url.path);\n\n Map? headers = {\n \"Content-Type\": \"$mime\",\n };\n\n print('Bot : getChatResponseTextToDoCStream() : headers = $headers');\n\n userQuestion = msg;\n print(\n 'Bot : getChatResponseTextToDoCStream() : userQuestion = $userQuestion');\n\n body = '''{\n \"query\": \"${userQuestion}\"\n }''';\n print('Bot : getChatResponseTextToDoCStream() : request_body = $body');\n\n var eventSource = EventSource(url, ApiMethod.post);\n eventSource.setHeaders(headers);\n\n final cancelToken = TimingToken(Duration(seconds: 5));\n\n final stopwatchtextToDoc = Stopwatch()..start();\n //final stream = eventSource.send(body: body, cancelToken: cancelToken);\n var stream = eventSource.send(body: body).asBroadcastStream();\n\n print('Bot : getChatResponseTextToDoCStream() : stream = $stream');\n\n RespList.add(\"test\"); // 0 : body of the answer\n RespList.add(true); //1 : is text\n RespList.add(\"dump\"); // 2 : Graph config\n //RespList.add(\"Your request can not be answered right now. Please try again.\");\n RespList.add(\"firstChunck\"); // 3: text\n RespList.add(stream); //4: stream\n RespList.add(stopwatchtextToDoc); //5: stopwatchtextToDoc\n\n print('Bot : getChatResponseTextToDoCStream() : END');\n return RespList;\n }\n\n Future waitForFirstSSEData(\n Stream> stream, EventSource eventSource) async {\n int count = 0;\n print('Bot : waitForFirstSSEData() : START');\n\n stream.listen(\n (event) {\n if (eventSource.isWeb) {\n print('Bot : waitForFirstSSEData(): eventSource.isWeb');\n print('Bot : waitForFirstSSEData(): count = $count');\n print('Bot : waitForFirstSSEData(): event.chunk = ${event.chunk}');\n\n responsePalMBody = responsePalMBody + event.chunk.toString();\n var answerPlainTextChunck =\n pickFromJson(event.chunk.toString(), 'response').asStringOrNull();\n print(\n 'Bot : waitForFirstSSEData(): answerPlainTextChunck = ${answerPlainTextChunck}');\n\n setState(() {\n streamingText = streamingText + answerPlainTextChunck!;\n });\n\n if (count == 0) {\n final textMessage = types.TextMessage(\n author: _user1,\n createdAt: DateTime.now().millisecondsSinceEpoch,\n id: randomString(),\n text: streamingText,\n type: types.MessageType.text,\n metadata: {\"dataSource\": null});\n\n _addMessage(textMessage);\n }\n count++;\n } else {\n print('Bot : waitForFirstSSEData(): eventSource.isNotWeb');\n final encoding = event.getEncoding();\n\n print(\n 'Bot : waitForFirstSSEData(): eventSource.isNotWeb : encoding.decode(event.chunk as List) = ${encoding.decode(event.chunk as List)}');\n }\n },\n onError: (err) => print('Bot : waitForFirstSSEData(): err = $err'),\n onDone: () {\n eventSource.close;\n streamingText = \"\";\n },\n );\n\n return \"\";\n }\n\n Future?> getChatResponseTextToDoC(\n String msg, String mime, String id,\n {types.User? user}) async {\n List? reqResp = [];\n List RespList = [];\n Uri url;\n String userQuestion = \"\";\n String body = \"\";\n String finalAnswer = \"\";\n\n print('Bot : getChatResponseTextToDoC() : START');\n url = Uri.parse(\n 'https://multi-modal-rag-dgujjntxuq-uc.a.run.app/generate-answer');\n\n print('Bot : getChatResponseTextToDoC() : url = ' + url.host + url.path);\n\n print('Bot : getChatResponseTextToDoC() : mime = ' + mime);\n\n Map? _headers = {\n \"Content-Type\": \"$mime\",\n };\n\n userQuestion = msg;\n print('Bot : getChatResponseTextToDoC() : userQuestion = ' + userQuestion);\n\n body = '''{\n \"query\": \"${userQuestion}\"\n }''';\n\n print('Bot : getChatResponseTextToDoC() : request_body = ' + body);\n\n final stopwatchtextToDoc = Stopwatch()..start();\n final _response = await http.post(url, headers: _headers, body: body);\n stopwatchtextToDoc.stop();\n\n if (_response.statusCode == 200) {\n print('Bot : getChatResponseTextToDoC() : Status code 200 ');\n responsePalMBody = _response.body\n .replaceAll(RegExp(r'(\\\\u003cb|\\\\u003e|\\\\u003c|\\\\u003e|(\\/n))'), '');\n\n var answerPlainText =\n pickFromJson(responsePalMBody, 'response').asStringOrNull();\n\n if (answerPlainText != null) {\n finalAnswer = answerPlainText;\n print(\n 'Bot : getChatResponseTextToDoC() : answerPlainText != null : finalAnswer = ' +\n finalAnswer);\n } else {\n finalAnswer =\n \"Your request can not be answered right now. Please try again.\";\n print(\n 'Bot : getChatResponseTextToDoC() : answerPlainText == null : finalAnswer = ' +\n finalAnswer);\n }\n\n reqResp.add(body);\n reqResp.add(responsePalMBody);\n\n print(\n 'Bot : getChatResponseTextToDoC() : /generate_answer : reqResp[0] = ' +\n reqResp[0]);\n print(\n 'Bot : getChatResponseTextToDoC() : /generate_answer : reqResp[1] = ' +\n reqResp[1]);\n\n RespList.add(responsePalMBody); // 0 : body of the answer\n RespList.add(true); //1 : is text\n RespList.add(\"dump\"); // 2 : Grah config\n RespList.add(finalAnswer); //3 : answer in plain text\n\n //Update stepper state to get_text_summary\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.get_text_summary,\n message: \"NL answer received in\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri: url!.host + url!.path,\n body: body,\n header: jsonEncode(_headers!),\n response: responsePalMBody,\n summary: finalAnswer,\n statusCode: _response.statusCode,\n stepDuration: stopwatchtextToDoc.elapsed.inMilliseconds,\n ));\n } else {\n print(\n 'Bot : getChatResponseTextToDoC() : _response.statusCode = ${_response.statusCode}');\n RespList.add(\"test\"); // 0 : body of the answer\n RespList.add(true); //1 : is text\n RespList.add(\"dump\"); // 2 : Grah config\n RespList.add(\n \"Your request can not be answered right now. Please try again.\"); //3 : answer in plain text\n }\n\n print('Bot : getChatResponseTextToDoC() : END');\n return RespList;\n }\n\n Future ShowCapturedWidget(\n BuildContext context, Uint8List capturedImage) {\n print(\"Bot: ShowCapturedWidget() : START\");\n return showDialog(\n useSafeArea: false,\n context: context,\n builder: (context) => Scaffold(\n appBar: AppBar(\n title: Text(\"Captured widget screenshot\"),\n ),\n body: Center(child: Image.memory(capturedImage)),\n ),\n );\n }\n\n List transfromDynamicListToStringList(List list) {\n List dataList = [];\n\n print(\n 'Bot: transfromDynamicListToStringList() : list.length = ${list.length}');\n print('Bot: transfromDynamicListToStringList() : list = $list');\n\n for (int i = 0; i <= list.length - 1; i++) {\n dataList.add(list.elementAt(i) as String);\n }\n\n if (dataList is List)\n print(\n 'BarGraph: transfromDynamicListToStringList() : dataList is of type List');\n\n print(\n 'BarGraph: transfromDynamicListToStringList() : dataList = $dataList');\n return dataList;\n }\n\n void _handleFileSelection() async {\n final result = await FilePicker.platform.pickFiles(\n type: FileType.custom,\n allowedExtensions: ['pdf'],\n );\n\n if (result != null && result.files.single.path != null) {\n String res = \"\";\n final message = types.FileMessage(\n author: _user,\n createdAt: DateTime.now().millisecondsSinceEpoch,\n id: randomString(),\n name: result.files.single.name,\n size: result.files.single.size,\n uri: result.files.single.path!,\n );\n\n _addMessage(message);\n\n print('Bot : _handleReceivedResponse() uri = ' + message.uri);\n textDocAi = \"not implemented yet\"; //await extractTextMLKit(message.uri);\n res = textDocAi.replaceAll('\"', \" \");\n\n print('Bot : _handleReceivedResponse() : textDocAi = ' + res);\n\n _handleReceivedResponse(\n 'Ecris en français un résumé du texte ci-dessous en mois de 50 mots:\\n' +\n res,\n \"png\");\n }\n }\n\n void _handleMessageTap(BuildContext _, types.Message message) async {\n print('Bot : _handleMessageTap() : DEBUT = ');\n if (message is types.FileMessage) {\n print('Bot : _handleMessageTap() : types.FileMessage : message.uri = ' +\n message.uri);\n }\n }\n\n void _handlePreviewDataFetched(\n types.TextMessage message,\n types.PreviewData previewData,\n ) {\n final index = _messages.indexWhere((element) => element.id == message.id);\n final updatedMessage = (_messages[index] as types.TextMessage).copyWith(\n previewData: previewData,\n );\n\n setState(() {\n _messages[index] = updatedMessage;\n });\n }\n\n void _handleAttachmentPressed() {\n showModalBottomSheet(\n context: context,\n builder: (BuildContext context) => SafeArea(\n child: SizedBox(\n height: 144,\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.stretch,\n children: [\n TextButton(\n onPressed: () {\n Navigator.pop(context);\n },\n child: const Align(\n alignment: AlignmentDirectional.centerStart,\n child: Text('Photo'),\n ),\n ),\n TextButton(\n onPressed: () {\n Navigator.pop(context);\n _handleFileSelection();\n },\n child: const Align(\n alignment: AlignmentDirectional.centerStart,\n child: Text('File'),\n ),\n ),\n TextButton(\n onPressed: () => Navigator.pop(context),\n child: const Align(\n alignment: AlignmentDirectional.centerStart,\n child: Text('Cancel'),\n ),\n ),\n ],\n ),\n ),\n ),\n );\n }\n\n Widget _bubbleBuilder(\n Widget child, {\n required message,\n required nextMessageInGroup,\n }) {\n String colorString = colorBubble;\n\n return Container(\n width: screenSize!.width,\n child: Bubble(\n child: child,\n color: message.author.id !=\n '82091010-a484-4a89-ae75-a22bf8d6f3ab'\n ? const Color(0xfff5f5f7) //Color(0xfff5f5f7)\n : const Color(0xffffffff), //Color(0xfffaf9de),\n margin: null,\n nip: BubbleNip.no,\n ),\n );\n }\n\n Widget customMessageBuilder(types.CustomMessage customMessage,\n {required int messageWidth}) {\n print('Bot : customMessageBuilder(): START');\n String inputString = \"\";\n\n return Container(\n //width: 500,\n child: Column(\n mainAxisAlignment: MainAxisAlignment.start,\n crossAxisAlignment: CrossAxisAlignment.center,\n mainAxisSize: MainAxisSize.min,\n children: [\n //Display the firstname and lastname\n Container(\n padding: const EdgeInsets.only(top: 20, left: 20),\n width: screenSize!.width,\n child: Text(_user1.firstName! + \" \" + _user1.lastName! + \"\\n\",\n style: TextStyle(\n fontSize: 12,\n color: Colors.green,\n fontWeight: FontWeight.bold),\n textAlign: TextAlign.left)),\n //Display answer\n !customMessage.metadata!.containsKey('stream') ||\n customMessage.metadata!['textSummary'] != \"firstChunck\"\n ? Container(\n padding: const EdgeInsets.only(top: 20, left: 20),\n width: screenSize!.width,\n child: Text(customMessage.metadata!['textSummary'],\n textAlign: TextAlign.start, style: TextStyle(fontSize: 16)),\n )\n : StreamBuilder>(\n stream: customMessage.metadata!['stream'],\n builder: (context, snapshot) {\n if (snapshot.connectionState == ConnectionState.waiting) {\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : snapshot.connectionState : WAITING');\n streamingText = \"\";\n return SizedBox(\n width: 100,\n height: 100,\n child: CircularProgressIndicator(\n strokeWidth: 6,\n ),\n ); // Display a loading indicator when waiting for data.\n } else if (snapshot.hasError) {\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : snapshot.hasError : ERROR');\n return Text(\n 'Error: ${snapshot.error}'); // Display an error message if an error occurs.\n } else if (!snapshot.hasData) {\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : !snapshot.hasData : No data available');\n return Text(\n 'No data available'); // Display a message when no data is available.\n } else {\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : snapshot.hasData');\n var answerPlainTextChunck = pickFromJson(\n snapshot.data!.chunk.toString(), 'response')\n .asStringOrNull();\n\n if (answerPlainTextChunck != \"end_stream\") {\n //streamingText = streamingText + answerPlainTextChunck!;\n inputString = inputString + answerPlainTextChunck!;\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : snapshot.hasData : answerPlainTextChunck != end_stream');\n //customMessage.metadata!['eventSource'].close();\n } else {\n customMessage.metadata!['textSummary'] = inputString;\n print(\n 'Bot : customMessageBuilder(): StreamBuilder : snapshot.hasData : answerPlainTextChunck == end_stream');\n Stopwatch stopwatchStreaming =\n customMessage.metadata!['stopWatch'] as Stopwatch;\n stopwatchStreaming.stop();\n\n //Update stepper state to get_text_summary\n BlocProvider.of(context)\n .updateStepperStatusUploaded(\n status: StepperStatus.get_text_summary,\n message: \"NL answer received in\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri:\n \"https://colab-cloudrun-template-ra1-uz6w7mqrka-uc.a.run.app/generate_streaming\",\n body: '{\"N/A\": \"N/A\"}',\n header: '{\"Content-Type\": \"app/json}',\n response: responsePalMBody,\n summary: inputString,\n statusCode: 200,\n stepDuration: stopwatchStreaming.elapsed\n .inMilliseconds, //stopwatchtextToDoc.elapsed.inMilliseconds,\n ));\n }\n return Container(\n padding: const EdgeInsets.only(top: 20, left: 20),\n width: screenSize!.width,\n child: Text(inputString, //streamingText,\n textAlign: TextAlign.start,\n style: TextStyle(fontSize: 16)),\n );\n }\n }),\n if (customMessage.metadata!['dataSource'] != null &&\n !customMessage.metadata!['textSummary']\n .contains(\"The request did not return meaningful information\"))\n Container(\n child: TabbedContainer(\n initialIndex: customMessage.metadata!['graph'] != null ? 1 : 0,\n controller: _tabController,\n tabs: const [\n Tab(text: \"Table\", icon: Icon(Icons.table_rows_sharp)),\n Tab(text: \"Graph\", icon: Icon(Icons.bar_chart)),\n ],\n tabViews: [\n Center(\n child: SingleChildScrollView(child: Container(width: screenSize!.width ,child: customMessage.metadata!['dataSource'] ?? Text('No Data')))),\n Center(\n child: getGoogleGraph(customMessage.metadata!['graph']) ?? Text('No Graph')),\n ],\n ),\n ),\n SizedBox(height: 40),\n //Add feedback\n Container(\n width: screenSize!.width / 10,\n decoration: BoxDecoration(\n borderRadius: BorderRadius.circular(10.0),\n color: Colors.black12, // Adjust the radius as needed\n ),\n child: Padding(\n padding: const EdgeInsets.all(8),\n child: Row(\n mainAxisAlignment: MainAxisAlignment.spaceBetween,\n children: [\n InkWell(\n onTap: () {\n showFeedbackDialog();\n },\n onHover: (value) {\n setState(() {\n _isThumbsUpHovered = value;\n });\n },\n child: Tooltip(\n message: \"Provide your feedback on the generated answer\",\n child: new Image(\n image: new AssetImage(\"assets/images/thumbs_up1.png\"),\n width: 20,\n height: 20,\n color: _isThumbsUpHovered\n ? Colors.grey.withOpacity(0.5)\n : null,\n colorBlendMode:\n _isThumbsUpHovered ? BlendMode.modulate : null,\n fit: BoxFit.scaleDown,\n alignment: Alignment.center,\n ),\n ),\n ),\n InkWell(\n onTap: () {\n showFeedbackDialog();\n },\n onHover: (value) {\n setState(() {\n _isThumbsDownHovered = value;\n });\n },\n child: Tooltip(\n message: \"Provide your feedback on the generated answer\",\n child: new Image(\n image: new AssetImage(\"assets/images/thumbs_down1.png\"),\n width: 20,\n height: 20,\n color: _isThumbsDownHovered\n ? Colors.grey.withOpacity(0.5)\n : null,\n colorBlendMode:\n _isThumbsDownHovered ? BlendMode.modulate : null,\n fit: BoxFit.scaleDown,\n alignment: Alignment.center,\n ),\n ),\n ),\n InkWell(\n onTap: () {\n print(\"Bot: copy : onTap : START\");\n print('Bot : customMessageBuilder() : copy : onTap: () : START');\n copyGraphToClipBoard(customMessage.metadata!['imageId'],\n customMessage.metadata!['textSummary']);\n },\n onHover: (value) {\n setState(() {\n _isCopyHovered = value;\n });\n },\n child: Tooltip(\n message: \"Copy the answer\",\n child: new Image(\n image: new AssetImage(\"assets/images/copy1.png\"),\n width: 20,\n height: 20,\n color:\n _isCopyHovered ? Colors.grey.withOpacity(0.5) : null,\n colorBlendMode:\n _isCopyHovered ? BlendMode.modulate : null,\n fit: BoxFit.scaleDown,\n alignment: Alignment.center,\n ),\n ),\n ),\n ],\n ),\n ),\n )\n //customMessage.metadata!['graph'],\n ],\n ));\n }\n\n Widget getGoogleGraph(dynamic dataViz) {\n print(\" bot: getGoogleGraph() : START\");\n Container container;\n\n if (dataViz![\"chart_div\"]! != null) {\n print(\n 'bot: getGoogleGraph() : dataViz![\"chart_div\"]! = ${dataViz![\"chart_div\"]!}');\n\n String htmlPre = \"\"\"\n \n \n \n \n \n \n \n \n
\n \n \"\"\";\n\n container = Container(\n child: InAppWebView(\n initialData: InAppWebViewInitialData(\n data: htmlPre + \"\\n\" + dataViz![\"chart_div\"]!.replaceAll('width: 600,','width: 1200,').replaceAll('height: 300,', 'height: 600,') + htmlPost + \"\\n\"),\n onWebViewCreated: (controller) {\n webViewController = controller;\n },\n ),\n );\n\n isGraphKeyAdded = true;\n graphContainer = container;\n\n return container;\n } else {\n return Text(\"No Chart\",\n style: TextStyle(\n fontSize: 16,\n color: Colors.indigoAccent,\n fontWeight: FontWeight.bold));\n }\n }\n\n Widget getGoogleTable(dynamic dataViz) {\n print(\"bot: getGoogleTable() : START\");\n if (dataViz![\"chart_div_1\"]! != null) {\n print(\n 'bot: getGoogleTable() : dataViz![\"chart_div_1\"]! = ${dataViz![\"chart_div_1\"]!}');\n String htmlPre = \"\"\"\n \n \n \n \n \n \n \n \n
\n \n \"\"\";\n\n return Container(\n child: InAppWebView(\n initialData: InAppWebViewInitialData(\n data:\n htmlPre + \"\\n\" + dataViz![\"chart_div_1\"]!.replaceAll('width: 600,','width: 1200,').replaceAll('height: 300,', 'height: 600,') + htmlPost + \"\\n\"),\n onWebViewCreated: (controller) {\n webViewController = controller;\n },\n ),\n );\n } else\n return Text(\"No data\",\n style: TextStyle(\n fontSize: 16,\n color: Colors.indigoAccent,\n fontWeight: FontWeight.bold));\n }\n\n Future _generateImage(Widget widget) async {\n print('Bot : _generateImage() : START');\n\n //Uint8List capturedImage = await screenshotController.captureFromWidget(widget); => not supported in Flutter Web for now\n Uint8List? capturedImage = await webViewController!.takeScreenshot();\n\n print('Bot : _generateImage() : capturedImage != null)');\n _graphsImagesMap[imageId!] = capturedImage!;\n //ShowCapturedWidget(context, capturedImage!);\n\n return capturedImage;\n }\n\n\n void showFeedbackDialog() {\n showDialog(\n context: context,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Text(\"Please rate this answer\"),\n content: ConstrainedBox(\n constraints: BoxConstraints(maxHeight: 260.0),\n child: Container(\n //width: screenSize!.width / 2,\n child: Padding(\n padding: const EdgeInsets.all(8.0),\n child: Column(children: [\n Row(\n mainAxisAlignment: MainAxisAlignment.spaceBetween,\n children: [\n ElevatedButton(\n child: Text(\"Good answer\",\n style: TextStyle(color: Colors.white)),\n style: ElevatedButton.styleFrom(\n backgroundColor: Colors.green,\n elevation: 0,\n ),\n onPressed: () {},\n ),\n SizedBox(width: 30),\n ElevatedButton(\n child: Text(\"Partial answer\",\n style: TextStyle(color: Colors.white)),\n style: ElevatedButton.styleFrom(\n backgroundColor: Colors.orange,\n elevation: 0,\n ),\n onPressed: () {},\n ),\n SizedBox(width: 30),\n ElevatedButton(\n child: Text(\"Incorrect answer\",\n style: TextStyle(color: Colors.white)),\n style: ElevatedButton.styleFrom(\n backgroundColor: Colors.red,\n elevation: 0,\n ),\n onPressed: () {},\n ),\n ],\n ),\n SizedBox(height: 50),\n TextField(\n decoration: InputDecoration(\n border: OutlineInputBorder(),\n labelText:\n 'Please provide additionnal feedback (optional)',\n ),\n maxLines: null,\n minLines: 5,\n ),\n ]),\n ),\n ),\n ),\n //Text(DicInfoExtractedMap.toString(),),\n actions: [\n TextButton(\n style: TextButton.styleFrom(\n textStyle: Theme.of(context).textTheme.labelLarge,\n ),\n child: const Text('Submit'),\n onPressed: () {\n Navigator.of(context).pop();\n },\n ),\n ],\n );\n },\n );\n }\n\n Future copyGraphToClipBoard(String imageKey, String summaryText) async {\n print('Bot : copyGraphToClipBoard() : START');\n print('Bot : copyGraphToClipBoard() : imageKey = $imageKey');\n print('Bot : copyGraphToClipBoard() : summaryText = $summaryText');\n\n //For now, the copy button only copies text. Copy of images/widgets is supported, so I'm commenting out the copy of image until\n //the flutter_inappwebview package used to render Google Charts is implementing InAppWebViewController.takeScreenshot() on Flutter Web.\n if(false) {\n //if (imageKey != \"no_image\") {\n print('Bot : copyGraphToClipBoard() : imageKey != \"no_image\"');\n print('Bot : copyGraphToClipBoard() : _graphsImagesMap.length = ${_graphsImagesMap.length}');\n Uint8List imgBytes = _graphsImagesMap[imageKey!]!;\n\n if (imgBytes != null) {\n print('Bot : copyGraphToClipBoard() : imgBytes != null');\n final base64Image = base64Encode(imgBytes);\n\n try {\n js.context.callMethod('copyBase64ImageToClipboard', [base64Image]);\n\n } catch (e) {\n print('Bot : copyGraphToClipBoard() : EXCEPTION : e = $e');\n }\n }\n } else if (summaryText != null || summaryText.length > 0) {\n print('Bot : copyGraphToClipBoard() : summaryText != null || summaryText.length > 0');\n await Clipboard.setData(ClipboardData(text: summaryText));\n } else {\n print('Bot : copyGraphToClipBoard() : else');\n await Clipboard.setData(ClipboardData(text: \"No data available.\"));\n }\n }\n\n Future?> getChatResponseNew(String msg, String mime, String id,\n {types.User? user}) async {\n List? reqResp = [];\n List RespList = [];\n Uri urlGenerateSQL;\n Uri urlRunQuery;\n String generatedSQLText = \"\";\n String userQuestion = \"\";\n int statusCodeRunQuery = 0;\n String jsonResponseRunQuery = \"\";\n\n print('Bot : getChatResponseNew() : START ');\n\n urlGenerateSQL =\n Uri.parse('${TextToDocParameter.endpoint_opendataqnq}/generate_sql');\n\n print('Bot : getChatResponseNew() : urlGenerateSQL = ' +\n urlGenerateSQL.host +\n urlGenerateSQL.path);\n\n print('Bot : getChatResponseNew() : mime = ' + mime);\n\n Map? _headers = {\n \"Content-Type\": \"$mime\",\n };\n\n print(\n 'Bot : getChatResponseNew() : BEFORE prepending main question : msg = ' +\n msg);\n if (mainQuestionsFollowUpQuestions.containsKey(msg)) {\n msg = mainQuestionsFollowUpQuestions[msg];\n }\n\n print(\n 'Bot : getChatResponseNew() : AFTER prepending main question : msg = ' +\n msg);\n\n userQuestion = msg;\n print('Bot : getChatResponseNew() : userQuestion = ' + userQuestion);\n\n String _body1 = '''{\n \"session_id\" :\"${TextToDocParameter.sessionId}\",\n \"user_id\":\"${TextToDocParameter.userID}\",\n \"user_question\":\"${userQuestion}\",\n \"user_grouping\":\"${TextToDocParameter.currentUserGrouping}\"\n }''';\n\n print('Bot : getChatResponseNew() : _body1 = ' + _body1);\n final stopwatchGenerateSQL = Stopwatch()..start();\n\n final _response =\n await http.post(urlGenerateSQL, headers: _headers, body: _body1);\n\n stopwatchGenerateSQL.stop();\n\n responsePalMBody = _response.body.replaceAll(\n RegExp(r'(\\\\u003cb|\\\\u003e|\\\\u003c|\\\\u003e|(\\/n)|(\\\\r))'), '');\n\n print('Bot : getChatResponseNew() : responsePalMBody = $responsePalMBody');\n\n var error = pickFromJson(responsePalMBody!, 'Error').asStringOrNull();\n\n print('Bot : getChatResponseNew() : error = $error');\n print(\n 'Bot : getChatResponseNew() : _response.statusCode = ${_response.statusCode}');\n\n if (_response.statusCode == 200 &&\n responsePalMBody!.toLowerCase().contains(\"select\") &&\n (error!.length == 0 ?? false)) {\n print(\n 'Bot : getChatResponseNew() : _response.statusCode == 200 && (error!.length == 0 ?? false )');\n\n TextToDocParameter.sessionId =\n pickFromJson(responsePalMBody!, 'SessionID').asStringOrNull()!;\n\n print(\n 'Bot : getChatResponseNew() : _response.statusCode == 200 && (error!.length == 0 ?? false ) : TextToDocParameter.sessionId = ${TextToDocParameter.sessionId}');\n\n reqResp.add(_body1);\n reqResp.add(responsePalMBody);\n\n print('Bot : getChatResponseNew() : /generate_sql : reqResp[0] = ' +\n reqResp[0]);\n print('Bot : getChatResponseNew() : /generate_sql : reqResp[1] = ' +\n reqResp[1]);\n\n //get the generated SQL query\n generatedSQLText = extractContentGenerateSQL(responsePalMBody);\n\n //Update stepper state to generate_sql\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.generate_sql,\n message: \"SQL request generated.\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri: urlGenerateSQL.host + urlGenerateSQL.path,\n body: _body1,\n header: jsonEncode(_headers),\n response: responsePalMBody,\n generatedSQLText: generatedSQLText,\n statusCode: _response.statusCode,\n stepDuration: stopwatchGenerateSQL.elapsed.inMilliseconds,\n answerList: [generatedSQLText]));\n print(\n \"main: getChatResponseNew() : After BlocProvider.of(context).updateStepperStatusUploaded() : generate_sql\");\n print(\n \"main: getChatResponseNew() : generate_sql : generatedSQLText = $generatedSQLText\");\n\n urlRunQuery =\n Uri.parse('${TextToDocParameter.endpoint_opendataqnq}/run_query');\n\n String _body2 = '''{\n \"user_question\": \"${userQuestion}\",\n \"user_grouping\": \"${TextToDocParameter.currentUserGrouping}\",\n \"generated_sql\": \"${generatedSQLText}\",\n \"session_id\" : \"${TextToDocParameter.sessionId}\"\n }''';\n\n //send the request to get the results in tabular format\n var stopwatchRunQuery = Stopwatch()..start();\n final _responseTabResults =\n await http.post(urlRunQuery, headers: _headers, body: _body2);\n stopwatchRunQuery.stop();\n\n statusCodeRunQuery = _responseTabResults.statusCode;\n jsonResponseRunQuery = _responseTabResults.body;\n\n print('Bot : getChatResponseNew() : tabular results: urlRunQuery = ' +\n urlRunQuery.toString());\n\n print(\n 'Bot : getChatResponseNew() : tabular results : _body2 = ' + _body2);\n print(\n 'Bot : getChatResponseNew() : tabular results: _responseTabResults.body = ' +\n jsonResponseRunQuery);\n\n if (TextToDocParameter.anonymized_data) {\n print(\n 'Bot : getChatResponseNew() : tabular results: TextToDocParameter.anonymized_data = ${TextToDocParameter.anonymized_data}');\n jsonResponseRunQuery = anonymizedData(jsonResponseRunQuery!);\n }\n\n //return extractContent(_responseTabResults.body, id, user: user!);\n return extractContentResultsOpenDataQnA(\n jsonResponseRunQuery: jsonResponseRunQuery,\n userQuestion: userQuestion,\n generatedSQLText: generatedSQLText,\n urlRunQuery: urlRunQuery,\n bodyRunQuery: _body2,\n headersRunQuery: _headers,\n statusCodeRunQuery: statusCodeRunQuery,\n elapsedTimeRunQuery: stopwatchRunQuery.elapsed.inMilliseconds);\n } else {\n //Update stepper state to generate_sql\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.generate_sql,\n message: \"SQL request generated in\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri: urlGenerateSQL.host + urlGenerateSQL.path,\n body: _body1,\n header: jsonEncode(_headers),\n response: responsePalMBody,\n generatedSQLText:\n \"An error occurred, no SQL request has been generated.\",\n statusCode: _response.statusCode,\n stepDuration: stopwatchGenerateSQL.elapsed.inMilliseconds,\n answerList: [\n \"An error occurred, no SQL request has been generated.\"\n ]));\n print(\n 'Bot : getChatResponseNew() : tabular results : _response.statusCode = ${_response.statusCode} and error attribute is set');\n RespList.add(\"\");\n RespList.add(true);\n RespList.add(\"\");\n RespList.add(\n \"The request did not return meaningful information. It could be because the question has not been formulated properly or some context is missing.\");\n\n return RespList;\n }\n }\n\n String extractContentGenerateSQL(String jsonResponse) {\n print(\"bot() : extractContentGenerateSQL() : START\");\n print(\"bot() : extractContentGenerateSQL() : jsonResponse = $jsonResponse\");\n\n String generatedSQLText = \"\";\n var error = pickFromJson(jsonResponse, 'Error');\n var generatedSQLTmp =\n pickFromJson(jsonResponse, 'GeneratedSQL').asStringOrNull();\n var responseCode = pickFromJson(jsonResponse, 'ResponseCode');\n\n print(\n \"bot() : extractContentGenerateSQL() : generatedSQLTmp = ${generatedSQLTmp}\");\n\n generatedSQLText = generatedSQLTmp!\n .replaceAll('\\n', ' ')\n .replaceAll('\"', '\\\\\"'); //generates an exception if null => fix it\n\n print(\n \"bot() : extractContentGenerateSQL() : generatedSQLText = ${generatedSQLText}; \");\n\n return generatedSQLText!;\n }\n\n Future> extractContentResultsOpenDataQnA(\n {String? jsonResponseRunQuery,\n String? userQuestion,\n String? generatedSQLText,\n Uri? urlRunQuery,\n String? bodyRunQuery,\n Map? headersRunQuery,\n int? statusCodeRunQuery,\n int? elapsedTimeRunQuery}) async {\n //String generatedSQLText = \"\";\n List RespList = [];\n bool isText = true;\n dynamic googleChartVizRes;\n String? naturalResponseText = \"\";\n\n String textSummary =\n \"The request did not return meaningful information. It could be because the question has not been formulated properly or some context is missing.\";\n String? knownDB = \"\";\n String? error = \"\";\n\n error = pickFromJson(jsonResponseRunQuery!, 'Error').asStringOrNull();\n knownDB = pickFromJson(jsonResponseRunQuery!, 'KnownDB').asStringOrNull();\n var responseCode = pickFromJson(jsonResponseRunQuery!, 'ResponseCode');\n naturalResponseText =\n pickFromJson(jsonResponseRunQuery!, 'NaturalResponse').asStringOrNull();\n\n print(\n \"bot() : extractContentResultsOpenDataQnA() : knownDB.length = ${knownDB!.length}; \");\n print(\n \"bot() : extractContentResultsOpenDataQnA() : knownDB = ${knownDB}; \");\n\n print(\n \"bot() : extractContentResultsOpenDataQnA() : bodyRunQuery = ${bodyRunQuery}; \");\n\n //Update stepper state to run_query\n BlocProvider.of(context).updateStepperStatusUploaded(\n status: StepperStatus.run_query,\n message: \"Tabular data retrieved in\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri: urlRunQuery!.host + urlRunQuery!.path,\n body: bodyRunQuery,\n header: jsonEncode(headersRunQuery!),\n response: jsonResponseRunQuery\n .replaceAll(\"\\\"[\", \"[\")\n .replaceAll(\"]\\\"\", \"]\")\n .replaceAll(\"\\\\\\\"\", \"\\\"\"),\n knownDB: knownDB,\n statusCode: statusCodeRunQuery,\n stepDuration: elapsedTimeRunQuery,\n answerList: [knownDB]));\n print(\n \"main: extractContentResultsOpenDataQnA() : After BlocProvider.of(context).updateStepperStatusUploaded() : run_query\");\n\n try {\n if (knownDB != \"[]\" && error!.length == 0 ?? false) {\n print(\"bot() : extractContentResults() : VALID ANSWER\");\n var knowDBJson = jsonDecode(knownDB!);\n //Check if it is worth displaying data. If just one row is returned, no use.\n //if (true) {\n if ((knowDBJson!.length <= 1 && (knowDBJson![0] as Map).length <= 1) ||\n (knowDBJson![0] as Map).length > 2) {\n print(\n \"bot() : extractContentResultsOpenDataQnA() : knowDBJson!.length <= 1 and MAP has at most 1 element\");\n isText = true;\n //Get graph description in case we want to display a graph\n googleChartVizRes = await getDataVisualization(\n userQuestion!, knownDB!, generatedSQLText!);\n\n print(\n \"bot() : extractContentResultsOpenDataQnA() : googleChartVizRes = ${googleChartVizRes}\");\n } else {\n print(\n \"bot() : extractContentResultsOpenDataQnA() : knowDBJson!.length > 1 or MAP has at least 1 element\");\n isText = false;\n //Get graph description in case we want to display a graph\n googleChartVizRes = await getDataVisualization(\n userQuestion!, knownDB!, generatedSQLText!);\n\n print(\n \"bot() : extractContentResultsOpenDataQnA() : googleChartVizRes = ${googleChartVizRes}\");\n }\n\n print(\n \"bot() : extractContentResultsOpenDataQnA() : isText = ${isText}; \");\n //get ML summarize\n //textSummary = await getTextSummary(userQuestion!, knownDB!);\n\n RespList.add(knownDB!);\n RespList.add(isText);\n RespList.add(googleChartVizRes!);\n RespList.add(naturalResponseText!.trim());\n } else {\n print(\"bot() : extractContentResultsOpenDataQnA() : UNVALID ANSWER\");\n RespList.add(\"\");\n RespList.add(true);\n RespList.add({\"chart_div\": \"empty\", \"chart_div_1\": \"empty\"});\n RespList.add(\n \"The request did not return meaningful information. It could be because the question has not been formulated properly or some context is missing.\");\n }\n } catch (e) {\n print(\"bot() : extractContentResultsOpenDataQnA() : EXCEPTION : $e\");\n } finally {\n print(\"bot() : extractContentResultsOpenDataQnA() : FINALLY CLAUSE \");\n RespList.add(knownDB!);\n RespList.add(isText);\n RespList.add(googleChartVizRes!);\n RespList.add(naturalResponseText!);\n return RespList;\n }\n }\n\n Future getDataVisualization(\n String question, String tabularAnswer, String generatedSQLText) async {\n dynamic generatedChartjsMap;\n\n //Create the header\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n };\n\n String tmpReplace = tabularAnswer.replaceAll('\"', '\\\\\"');\n print('Bot : getDataVisualization() : tnpReplace = ' + tmpReplace);\n\n //Create the body\n String _body = '''{\n \"user_question\": \"$question\",\n \"sql_generated\": \"${generatedSQLText}\",\n \"sql_results\": \"${tmpReplace}\",\n \"session_id\" : \"${TextToDocParameter.sessionId}\"\n }''';\n\n print('Bot : getDataVisualization() : _body = ' + _body);\n\n try {\n var stopwatchGetDataVisulization = Stopwatch()..start();\n\n print('Bot: getDataVisualization() : BEFORE HttpRequest');\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/generate_viz',\n method: \"POST\",\n sendData: _body);\n print('Bot: getDataVisualization() : AFTER HttpRequest');\n stopwatchGetDataVisulization.stop();\n\n print('Bot : getDataVisualization() : response = ' + response.toString());\n\n var jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('Bot: getDataVisualization() : jsonData = $jsonData');\n\n generatedChartjsMap = jsonData[\"GeneratedChartjs\"];\n\n print(\n 'Bot: getDataVisualization() : generatedChartjsMap = $generatedChartjsMap');\n\n if(TextToDocParameter.anonymized_data) {\n //update generatedChartjsMap\n mapAnonymisationGraph.forEach((key, value) {\n print(\n 'Bot : getDataVisualization() : update jsonData : key = $key, value = $value');\n generatedChartjsMap['chart_div'] =\n generatedChartjsMap['chart_div'].replaceAll(key, value);\n });\n }\n\n //Update stepper state to get_graph_description\n BlocProvider.of(context)\n .updateStepperStatusUploaded(\n status: StepperStatus.get_graph_description,\n message: \"Graph generated\",\n stateStepper: StepState.complete,\n isActiveStepper: true,\n debugInfo: StepperExpertInfo(\n uri:\n \"${TextToDocParameter.endpoint_opendataqnq}/generate_viz\",\n body: _body\n .replaceAll(\"\\\"[\", \"[\")\n .replaceAll(\"]\\\"\", \"]\")\n .replaceAll(\"\\\\\\\"\", \"\\\"\"),\n header:\n '''{\"Header not accessible\": \"CORS headers are not accessible as they are sent directly by the web browser\"}''',\n response: response\n .replaceAll(\"\\\"[\", \"[\")\n .replaceAll(\"]\\\"\", \"]\")\n .replaceAll(\"\\\\\\\"\", \"\\\"\"),\n statusCode: 0,\n stepDuration:\n stopwatchGetDataVisulization.elapsed.inMilliseconds,\n ));\n\n print(\n \"Bot: getDataVisualization() : After BlocProvider.of(context).updateStepperStatusUploaded() : get Google Charts\");\n } else {\n print('Bot: getDataVisualization() : jsonData is null');\n }\n } catch (e) {\n generatedChartjsMap = {\"chart_div\": \"empty\", \"chart_div_1\": \"empty\"};\n print('Bot: getDataVisualization() : EXCEPTION : error = $e');\n } finally {\n return generatedChartjsMap!;\n }\n }\n\n List setBubbleColor(String text, {types.User? user}) {\n String tmp = \"\";\n String userType = \"\";\n const String datastoreGenerated = \"ap :\";\n const String nMatchLLMGenerated = \"nh :\";\n int startIndex = 0;\n List rep = [];\n\n print('Bot : setBubbleColor() : text = ' + text);\n\n if (user != null) if (_user.id == user!.id) userType = \"user\";\n\n if (text.contains(datastoreGenerated)) {\n startIndex =\n text.indexOf(datastoreGenerated) + datastoreGenerated.length + 1;\n colorBubble = \"datastoreData\";\n } else if (text.contains(nMatchLLMGenerated)) {\n startIndex =\n text.indexOf(nMatchLLMGenerated) + datastoreGenerated.length + 1;\n colorBubble = \"noMatchLLM\";\n } else if (userType == \"user\") {\n colorBubble = \"user\";\n } else {\n colorBubble = \"regularDF\";\n }\n\n tmp = text.substring(startIndex);\n\n print('Bot : setBubbleColor() : startIndex = ' + startIndex.toString());\n print('Bot : setBubbleColor() : tmp = ' + tmp);\n print('Bot : setBubbleColor() : colorBubble = ' + colorBubble);\n\n rep.add(tmp);\n rep.add(colorBubble);\n return rep;\n }\n\n int countOccurences(String mainString, String search) {\n int lInx = 0;\n int count = 0;\n while (lInx != -1) {\n lInx = mainString.indexOf(search, lInx);\n if (lInx != -1) {\n count++;\n lInx += search.length;\n }\n }\n return count;\n }\n\n Future _dialogExtension(BuildContext context, String extension) {\n return showDialog(\n context: context,\n builder: (BuildContext context) {\n return AlertDialog(\n title: const Text(\"Format d'image non supporté\",\n style: TextStyle(\n fontWeight: FontWeight.normal,\n fontSize: 22.0,\n color: Colors.blue)),\n content: Text(\n \"Le format d'image $extension n'est pas supporté.\\n\" +\n \"Choisissez une image du type :\\ngif, tiff, tif, jpg, jpeg, png, bmp, webp\",\n ),\n actions: [\n TextButton(\n style: TextButton.styleFrom(\n textStyle: Theme.of(context).textTheme.labelLarge,\n ),\n child: const Text('Ok'),\n onPressed: () {\n Navigator.of(context).pop();\n },\n ),\n ],\n );\n },\n );\n }\n\n String displayDateTime() {\n String? dateTimeS;\n\n final now = DateTime.now();\n dateTimeS = DateFormat('yyyy-MM-dd HH:mm:ss').format(now);\n return dateTimeS!;\n }\n\n List get messages {\n return _messages;\n }\n\n Map get graphsImages {\n return _graphsImagesMap;\n }\n\n Map get tableKeysMap {\n return tableKeyMap;\n }\n\n Widget avatarBuilder(types.User user) {\n bool isUserAvatar = user.id == '82091010-a484-4a89-ae75-a22bf8d6f3ab';\n\n return CircleAvatar(\n backgroundColor: Colors.green,\n backgroundImage:\n isUserAvatar ? NetworkImage(TextToDocParameter.picture) : null,\n radius: 16,\n child: !isUserAvatar\n ? Text(\n \"TA\",\n )\n : null,\n );\n }\n\n String anonymizedData(String responseRunQuery) {\n String anonymizedData = \"\";\n Random random = new Random();\n mapAnonymisationGraph.clear();\n\n print(\"Bot : anonymizedData() : START\");\n print(\"Bot : anonymizedData() : responseRunQuery = ${responseRunQuery}\");\n\n var responseRunQueryJson = jsonDecode(responseRunQuery);\n print(\n \"Bot : anonymizedData() : responseRunQueryJson = ${responseRunQueryJson}\");\n\n var knownDB = jsonDecode(responseRunQueryJson[\"KnownDB\"]);\n String naturalResponse = responseRunQueryJson[\"NaturalResponse\"];\n\n print(\"Bot : anonymizedData() : knownDB = ${knownDB}\");\n print(\"Bot : anonymizedData() : naturalResponse = ${naturalResponse}\");\n\n for (int i = 0; i < knownDB.length; i++) {\n var entry = knownDB[i];\n print(\"Bot : anonymizedData() : for : i = $i : entry = ${entry}\");\n\n entry.forEach((key, value) {\n print('Bot : anonymizedData() : key = $key : value = $value');\n\n try {\n if (value is double) {\n print(\n 'Bot : anonymizedData() : i = $i : value = $value is of type double');\n double randomNumber = random.nextDouble() * value;\n int truncatedInt = (randomNumber * 100).toInt();\n randomNumber = truncatedInt / 100;\n print(\n 'Bot : anonymizedData() : i = $i : randomNumber = $randomNumber');\n entry[key] = randomNumber;\n\n NumberFormat formatter = NumberFormat(\"#,##0\", \"en_US\"); // Adjust locale if needed\n String formattedNumber = formatter.format(value);\n\n mapAnonymisationGraph[formattedNumber] = formatter.format(randomNumber);//randomNumber.toString();\n }\n if (value is String) {\n print(\n 'Bot : anonymizedData() : i = $i : value = $value is of type String');\n\n String randomString = generateRandomString(value.toString().length);\n print(\n 'Bot : anonymizedData() : i = $i : randomString = $randomString');\n entry[key] = randomString;\n\n mapAnonymisationGraph[value.toString()] = randomString;\n }\n } catch (e) {\n print('Bot : anonymizedData() : PARSING EXCEPTION = $e');\n }\n });\n }\n responseRunQueryJson[\"KnownDB\"] = jsonEncode(knownDB);\n\n //update NaturalResponse\n mapAnonymisationGraph.forEach((key, value) {\n print(\n 'Bot : anonymizedData() : update NaturalResponse : key = $key, value = $value');\n naturalResponse = naturalResponse.replaceAll(key,value);\n });\n\n print(\"Bot : anonymizedData() : END : knownDB = ${knownDB}\");\n print(\"Bot : anonymizedData() : END : naturalResponse = ${naturalResponse}\");\n\n responseRunQueryJson[\"NaturalResponse\"] = naturalResponse;\n\n print(\n \"Bot : anonymizedData() : END : responseRunQueryJson = ${responseRunQueryJson}\");\n return jsonEncode(responseRunQueryJson);\n }\n\n\n String generateRandomString(int length) {\n const _chars = 'AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz1234567890';\n Random _rnd = Random();\n\n return String.fromCharCodes(Iterable.generate(\n length, (_) => _chars.codeUnitAt(_rnd.nextInt(_chars.length))));\n }\n\n\n Future _updateUserGroupingInSessionLogs() async {\n int count = 0;\n print(\n \"Bot: _updateUserGroupingInSessionLogs() : START\");\n\n //get all the documents corresponding to the user_id of the current user\n try {\n var querySnapshot = await widget.db!\n .collection(\"${TextToDocParameter.firestore_history_collection}\")\n .where(\"user_id\", isEqualTo: TextToDocParameter.userID)\n .orderBy('timestamp', descending: true)\n .limit(1)\n .get();\n\n print(\n \"Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : querySnapshot.docs.length = ${querySnapshot.docs.length}\");\n print(\n \"Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : querySnapshot = ${querySnapshot}\");\n\n //update all these documents with the user_grouping scenario_name\n if(TextToDocParameter.currentScenarioName.isEmpty)\n TextToDocParameter.currentScenarioName = \"Scenario\";\n\n for (var docSnapshot in querySnapshot.docs) {\n print(\n 'Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : TextToDocParameter.currentUserGrouping = ${TextToDocParameter.currentUserGrouping}');\n print(\n 'Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : TextToDocParameter.currentScenarioName = ${TextToDocParameter.currentScenarioName}');\n\n print(\n 'Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : ${docSnapshot.id} => ${docSnapshot.data()}');\n widget.db!.collection(\"${TextToDocParameter.firestore_history_collection}\").doc('${docSnapshot.id}').set(\n {\"user_grouping\": \"${TextToDocParameter.currentUserGrouping}\", \"scenario_name\": \"${TextToDocParameter.currentScenarioName}\"},\n SetOptions(merge: true));\n count++;\n }\n } catch (e) {\n print(\n 'Bot: _updateUserGroupingInSessionLogs() : Add user_grouping : EXCEPTION : $e');\n }\n var snackBar = SnackBar(\n content:\n Text('Updated $count questions on ${TextToDocParameter.firestore_database_id} collection'),\n duration: Duration(seconds: 3),\n );\n ScaffoldMessenger.of(context).showSnackBar(snackBar);\n }\n\n PaginatedDataTable? createPaginatedTable(String data) {\n print(\"bot() : createPaginatedTable() : START\");\n print(\"bot() : createPaginatedTable() : data = $data\");\n List dataColumnList = [];\n\n List dataList = jsonDecode(data!);\n\n print(\"bot() : createPaginatedTable() : dataList = $dataList\");\n print(\"bot() : createPaginatedTable() : dataList.length = ${dataList.length}\");\n\n //get headers of columns\n var entry = dataList.first;\n print(\"bot() : createPaginatedTable() : entry = ${entry}\");\n print(\"bot() : createPaginatedTable() : entry.length = ${entry.length}\");\n print(\"bot() : createPaginatedTable() : entry['tconst'] = ${entry['tconst']}\");\n print(\"bot() : createPaginatedTable() : entry['original_title'] = ${entry['original_title']}\");\n print(\"bot() : createPaginatedTable() : entry['average_rating'] = ${entry['average_rating']}\");\n print(\"bot() : createPaginatedTable() : entry['title_type'] = ${entry['title_type']}\");\n\n for (var element in (entry as Map).entries) {\n print('bot() : createPaginatedTable() : Key: ${element.key}, Value: ${element.value}');\n dataColumnList.add(DataColumn(label: Text(element.key.toString(), style: TextStyle(fontWeight: FontWeight.bold, color : Colors.white))));\n }\n\n if (dataList.length != 0) {\n print(\"bot() : createPaginatedTable() : dataList.length != 0}\");\n\n var rowsList = dataList.map((data) {\n List dataCellsList = [];\n\n for (var element in (data as Map).entries) {\n print('bot() : createPaginatedTable() : Key: ${element.key}, Value: ${element.value}');\n DataCell cell = DataCell(Text(element.value.toString()));\n dataCellsList.add(cell);\n }\n return DataRow(cells: dataCellsList);\n }).toList();\n\n return PaginatedDataTable(\n //header: Text('Results'),\n headingRowColor: WidgetStateProperty.all(Colors.blue),\n rowsPerPage: 3, // Customize as needed\n columns: dataColumnList,\n source: OpenDataQnASource(rowsList as List),\n );\n }\n else\n return null;\n }\n\n}\n\nclass OpenDataQnASource extends DataTableSource {\n final List data;\n\n OpenDataQnASource(this.data);\n\n @override\n int get rowCount => data.length;\n\n @override\n DataRow? getRow(int index) {\n\n if (index < data.length) {\n return data[index];\n } else {\n return null;\n }\n }\n\n @override\n bool get isRowCountApproximate => false;\n\n @override\n int get selectedRowCount => 0;\n}\n\n" }, { "path": "frontend/frontend-flutter/lib/screens/bot_chat_view.dart", "content": "import 'package:chatview/chatview.dart';\nimport 'package:flutter/material.dart';\nimport 'package:flutter/services.dart';\n\nclass BotChatView extends StatefulWidget {\n const BotChatView({Key? key}) : super(key: key);\n\n @override\n State createState() => BotChatViewState();\n}\n\nclass BotChatViewState extends State {\n\n ChatUser? currentUser;\n ChatUser? bot;\n ChatController? _chatController;\n\n @override\n void initState() {\n super.initState();\n\n currentUser = ChatUser(\n id: '0',\n name: 'User',\n profilePhoto: \"https://raw.githubusercontent.com/SimformSolutionsPvtLtd/flutter_showcaseview/master/example/assets/simform.png\",\n );\n\n bot = ChatUser(\n id: '1',\n name: 'Bot',\n profilePhoto: \"https://raw.githubusercontent.com/SimformSolutionsPvtLtd/flutter_showcaseview/master/example/assets/simform.png\",\n );\n\n _chatController = ChatController(\n initialMessageList: [\n Message(\n id: '0',\n message: \"Hi Bot!\",\n createdAt: DateTime.now(),\n sendBy: '0', // userId of who sends the message\n ),\n Message(\n id: '1',\n message: \"Hi!\",\n createdAt: DateTime.now(),\n sendBy: '1',\n ),\n ],\n scrollController: ScrollController(),\n chatUsers: [\n currentUser!,\n bot!,\n ],\n );\n\n }\n\n @override\n Widget build(BuildContext context) {\n return Scaffold(\n body: ChatView(\n chatBackgroundConfig: ChatBackgroundConfiguration(\n backgroundImage: \"https://raw.githubusercontent.com/SimformSolutionsPvtLtd/flutter_showcaseview/master/example/assets/simform.png\",//\"assets/images/background.png\",\n messageTimeIconColor: Colors.white,\n messageTimeTextStyle: TextStyle(color: Colors.white),\n defaultGroupSeparatorConfig: DefaultGroupSeparatorConfiguration(\n textStyle: TextStyle(\n color: Colors.white,\n fontSize: 17,\n ),\n ),\n backgroundColor: Color(0xffFCD8DC),\n ),\n currentUser: currentUser!,\n chatController: _chatController!,\n onSendTap: onSendTap,\n chatViewState: ChatViewState.hasMessages\n\n ),\n );\n }\n\n void onSendTap(String message, ReplyMessage replyMessage, MessageType messageType){\n final message = Message(\n id: '2',\n message: \"How are you\",\n createdAt: DateTime.now(),\n sendBy: currentUser!.id,\n replyMessage: replyMessage,\n messageType: messageType,\n );\n _chatController!.addMessage(message);\n }\n\n}" }, { "path": "frontend/frontend-flutter/lib/screens/disclaimer.dart", "content": "import 'package:firebase_auth/firebase_auth.dart';\nimport 'package:firebase_core/firebase_core.dart';\nimport 'package:flutter/material.dart';\nimport '../utils/TextToDocParameter.dart';\n\nlate String firstName;\nlate String lastName;\nlate String userID;\n\nclass Disclaimer extends StatefulWidget {\n late final FirebaseAuth auth;\n\n Disclaimer(this.auth, {super.key});\n\n @override\n State createState() => DisclaimerState();\n}\n\nclass DisclaimerState extends State {\n bool isChecked = false;\n bool isButtonClicked = false;\n\n Color _buttonColor = Colors.blueAccent;\n\n @override\n Widget build(BuildContext context) {\n Size screenSize = MediaQuery.sizeOf(context);\n\n return Material(\n child: !isButtonClicked\n ? Container(\n //height: screenSize.height,\n color: Colors.black,\n child: SingleChildScrollView(\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n Padding(\n padding: const EdgeInsets.only(left: 35.0, right: 35.0, top: 35.0, bottom: 35.0),\n child: Text('Google Cloud | Applied AI Engineering',\n style: TextStyle(fontSize: 20, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(left: 95.0, top: 0),\n child: Container(\n width: screenSize!.width / 2,\n child: Column(\n mainAxisSize: MainAxisSize.min,\n mainAxisAlignment: MainAxisAlignment.start,\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n Text('Open Data QnA',\n style: TextStyle(\n fontSize: 70,\n color: Colors.white,\n fontWeight: FontWeight.bold)),\n Padding(\n padding: const EdgeInsets.only(top: 5),\n child: Text('Demo',\n style: TextStyle(\n fontSize: 40, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: Text(\n 'User Journey | Architecture Diagram | GCP Console Demo',\n style: TextStyle(\n fontSize: 23, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: Divider()),\n Text(\n 'This is a functioning solution demo for CEs/FSRs to present to prospects to demonstrate the capabilities of Google Cloud AI products.\\n\\n'\n 'It is not an officially supported Google Cloud product or service and is provided without any guarantees of performance or maintenance. Because of the tool\\'s limited and experimental nature, we may need to make modifications to, including potentially taking down, the tool on short, or no, notice.\\n\\n'\n 'You may provide feedback and suggestions about this tool to Google, and Google and its Affiliates may use any feedback or suggestions provided without restriction and without obligation to you.\\n\\n'\n 'Your use of the tool must always comply with our Terms of Service (including the Acceptable Use Policy), or the offline variant of the terms that govern your use of the Google Cloud Platform Services.\\n\\n'\n 'During your use of the tool, information is provided for informational purposes only. The data inputed are dummy data.\\n\\n',\n maxLines: null,\n style: TextStyle(\n fontSize: 18,\n color: Colors.white,\n height: 1.2)),\n CheckboxListTile(\n title: Text(\n \"By checking this box, you accept and agree to the above terms and conditions of this tool.\",\n style: TextStyle(\n fontSize: 18, color: Colors.white)),\n value: isChecked,\n onChanged: (bool? newValue) {\n setState(() {\n isChecked = newValue!;\n });\n },\n controlAffinity:\n ListTileControlAffinity.leading,\n checkColor: Colors.white,\n activeColor: Colors.purple,\n tileColor: Colors.white,\n contentPadding: EdgeInsets.zero,\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0, bottom: 15.0),\n child: ElevatedButton(\n onPressed: () {\n setState(() {\n isButtonClicked = true;\n });\n },\n style: ElevatedButton.styleFrom(\n backgroundColor: isChecked\n ? Colors.blue\n : Colors.blueAccent,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius\n .zero, // Ensures sharp corners\n ),\n ),\n child: Text('Accept and Agree',\n style: TextStyle(\n color: isChecked\n ? Colors.white\n : Colors.grey,\n fontSize: 18)),\n ),\n )\n ],\n )),\n ),\n ],\n ),\n ),\n )\n :\n /*Container(\n //height: screenSize.height,\n color: Colors.black,\n child: Column(\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n Padding(\n padding: const EdgeInsets.all(35.0),\n child: Text('Google Cloud | Applied AI Engineering',\n style: TextStyle(fontSize: 20, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(left: 95.0, top: 50),\n child: Container(\n width: screenSize!.width / 2,\n child: SingleChildScrollView(\n child: Column(\n mainAxisSize: MainAxisSize.min,\n mainAxisAlignment: MainAxisAlignment.start,\n crossAxisAlignment: CrossAxisAlignment.start,\n children: [\n Text('Open Data QnA',\n style: TextStyle(\n fontSize: 70,\n color: Colors.white,\n fontWeight: FontWeight.bold)),\n Padding(\n padding: const EdgeInsets.only(top: 5),\n child: Text('Demo',\n style: TextStyle(\n fontSize: 40, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: Text(\n 'User Journey | Architecture Diagram | GCP Console Demo',\n style: TextStyle(\n fontSize: 23, color: Colors.white)),\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: Divider()),\n Text(\n 'This is a functioning solution demo for CEs/FSRs to present to prospects to demonstrate the capabilities of Google Cloud AI products.\\n\\n'\n 'It is not an officially supported Google Cloud product or service and is provided without any guarantees of performance or maintenance. Because of the tool\\'s limited and experimental nature, we may need to make modifications to, including potentially taking down, the tool on short, or no, notice.\\n\\n'\n 'You may provide feedback and suggestions about this tool to Google, and Google and its Affiliates may use any feedback or suggestions provided without restriction and without obligation to you.\\n\\n'\n 'Your use of the tool must always comply with our Terms of Service (including the Acceptable Use Policy), or the offline variant of the terms that govern your use of the Google Cloud Platform Services.\\n\\n'\n 'During your use of the tool, information is provided for informational purposes only. The data inputed are dummy data.\\n\\n',\n maxLines: null,\n style: TextStyle(\n fontSize: 18,\n color: Colors.white,\n height: 1.2)),\n CheckboxListTile(\n title: Text(\n \"By checking this box, you accept and agree to the above terms and conditions of this tool.\",\n style: TextStyle(\n fontSize: 18, color: Colors.white)),\n value: isChecked,\n onChanged: (bool? newValue) {\n setState(() {\n isChecked = newValue!;\n });\n },\n controlAffinity: ListTileControlAffinity.leading,\n checkColor: Colors.white,\n activeColor: Colors.purple,\n tileColor: Colors.white,\n contentPadding: EdgeInsets.zero,\n ),\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: ElevatedButton(\n onPressed: () {\n setState(() {\n isButtonClicked = true;\n });\n },\n style: ElevatedButton.styleFrom(\n backgroundColor: isChecked\n ? Colors.blue\n : Colors.blueAccent,\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius\n .zero, // Ensures sharp corners\n ),\n ),\n child: Text('Accept and Agree',\n style: TextStyle(\n color: isChecked\n ? Colors.white\n : Colors.grey,\n fontSize: 18)),\n ),\n )\n ],\n ),\n )),\n ),\n ],\n ),\n )*/\n Container(\n color: Colors.black,\n child: Center(\n child: Container(\n //width: screenSize.width / 3,\n height: screenSize.height / 6,\n decoration: BoxDecoration(\n // Background color\n color: Colors.white,\n borderRadius:\n BorderRadius.circular(20.0), // 10-pixel rounded corners\n ),\n child: Column(\n mainAxisSize: MainAxisSize.min,\n mainAxisAlignment: MainAxisAlignment.center,\n crossAxisAlignment: CrossAxisAlignment.center,\n children: [\n Padding(\n padding: const EdgeInsets.only(top: 15.0),\n child: ElevatedButton(\n onPressed: () {\n firebaseAuthentication();\n },\n style: ElevatedButton.styleFrom(\n backgroundColor: Color(0xFFeeecec),\n shape: RoundedRectangleBorder(\n borderRadius: BorderRadius.zero,\n ),\n side: BorderSide(\n width: 1.0,\n color: Colors.black,\n ),\n ),\n child: Row(\n // Use a Row to arrange the icon and text\n mainAxisSize: MainAxisSize\n .min, // Make the Row as small as its children\n children: [\n Image.asset(\n 'assets/images/google_icon.png',\n width: 25,\n height: 25,\n fit: BoxFit.scaleDown,\n ), // Replace with your image path\n SizedBox(\n width:\n 8), // Add some spacing between the icon and text\n Text('Sign in with Google',\n style: TextStyle(\n color: Colors.black, fontSize: 14))\n ],\n ),\n ),\n ),\n Padding(\n padding: const EdgeInsets.only(\n right: 40.0, left: 40.0, top: 20, bottom: 30),\n child: Text('To use this demo, you have to sign in',\n style:\n TextStyle(color: Colors.black, fontSize: 16)),\n ),\n ],\n ),\n ),\n ),\n ),\n );\n }\n\n Future signInWithGoogle() async {\n print('Disclaimer: signInWithGoogle() : START');\n // Create a new provider\n GoogleAuthProvider googleProvider = GoogleAuthProvider();\n\n googleProvider\n .addScope('https://www.googleapis.com/auth/contacts.readonly');\n googleProvider.setCustomParameters({'login_hint': 'user@example.com'});\n\n // Once signed in, return the UserCredential\n //return await FirebaseAuth.instance.signInWithPopup(googleProvider);\n return await widget.auth.signInWithPopup(googleProvider);\n }\n\n Future firebaseAuthentication() async {\n print('Disclaimer: firebaseAuthentication() : START');\n print('Disclaimer: firebaseAuthentication() : auth= ${widget.auth}');\n String? res = null;\n try {\n final credential = await signInWithGoogle();\n\n print('Disclaimer: firebaseAuthentication() : credential = $credential');\n final user = credential.user;\n\n print('Disclaimer: firebaseAuthentication() : user.uid = ${user!.uid}.');\n\n print(\n 'Disclaimer: firebaseAuthentication() : credential.additionalUserInfo.profile[\"given_name\"] = ${credential.additionalUserInfo!.profile![\"given_name\"]}');\n print(\n 'Disclaimer: firebaseAuthentication() : credential.additionalUserInfo.profile[\"family_name\"] = ${credential.additionalUserInfo!.profile![\"family_name\"]}');\n\n TextToDocParameter.firstName =\n credential.additionalUserInfo!.profile![\"given_name\"];\n TextToDocParameter.lastName =\n credential.additionalUserInfo!.profile![\"family_name\"];\n TextToDocParameter.email =\n credential.additionalUserInfo!.profile![\"email\"];\n TextToDocParameter.userID = credential.user!.uid;\n TextToDocParameter.picture =\n credential.additionalUserInfo!.profile![\"picture\"];\n\n print(\n 'Disclaimer: firebaseAuthentication() : TextToDocParameter.userID = ${TextToDocParameter.userID}');\n\n TextToDocParameter.isAuthenticated = true;\n print(\n \"Disclaimer: firebaseAuthentication() : TextToDocParameter.isAuthenticated = ${TextToDocParameter.isAuthenticated}\");\n\n print('Disclaimer: firebaseAuthentication() : user?.uid = ${user?.uid}');\n print(\n 'Disclaimer: firebaseAuthentication() : email = ${TextToDocParameter.email}');\n print(\n 'Disclaimer: firebaseAuthentication() : picture = ${TextToDocParameter.picture}');\n\n Navigator.of(context).pushReplacementNamed('/landingPage');\n } on FirebaseAuthException catch (e) {\n if (e.code == 'user-not-found') {\n print('Disclaimer: firebaseAuthentication() : User does not exists');\n res = 'User does not exists';\n return res;\n } else if (e.code == 'wrong-password') {\n print('Disclaimer: firebaseAuthentication() : Password does not match');\n res = 'Password does not match';\n return res;\n } else {\n print('Disclaimer: firebaseAuthentication() : ${e.code}');\n print('Disclaimer: firebaseAuthentication() : ${e.message}');\n res = e.message;\n return res;\n }\n }\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/screens/settings.dart", "content": "import 'dart:convert';\nimport 'package:cloud_firestore/cloud_firestore.dart';\nimport 'package:file_picker/file_picker.dart';\nimport 'package:flutter_bloc/flutter_bloc.dart';\nimport 'package:flutter_settings_ui/flutter_settings_ui.dart';\nimport 'package:flutter/material.dart';\nimport '../services/display_stepper/display_stepper_cubit.dart';\nimport '../utils/TextToDocParameter.dart';\nimport 'package:flutter/services.dart';\n\nclass Settings extends StatefulWidget {\n Settings(FirebaseFirestore this.db, {Key? key}) : super(key: key);\n bool useFeedback = false;\n bool useColorMode = false;\n bool useDashboards = false;\n bool useReports = false;\n bool useExpertMode = false;\n bool useLog = false;\n bool useAnonymizedMode = false;\n static bool isLoadConfig = false;\n static bool isUseFeedback = false;\n static bool isUseColorMode = false;\n static bool isUseDashboards = false;\n static bool isUseReports = false;\n static bool isExpert = false;\n static bool isUseLog = false;\n static bool isAnonymizedMode = false;\n FirebaseFirestore db;\n\n @override\n State createState() => SettingsState();\n}\n\nclass SettingsState extends State {\n @override\n Widget build(BuildContext context) {\n print(\"Settings : build() : START\");\n print(\"Settings : build() : widget.db = ${widget.db}\");\n\n return Scaffold(\n appBar: AppBar(\n title: Text('Open data QnA',\n style:\n TextStyle(fontWeight: FontWeight.bold, color: Colors.purple)),\n centerTitle: false,\n leading: IconButton(\n icon: const Icon(Icons.arrow_back),\n onPressed: () {\n Navigator.pop(context, Settings.isExpert);\n },\n ),\n actions: [\n Padding(\n padding: const EdgeInsets.only(right: 100),\n child: IconButton(\n icon: Image.asset('assets/images/cymbal_logo.png'),\n onPressed: () {\n ;\n },\n ),\n ),\n ],\n ),\n body: SettingsList(\n sections: [\n SettingsSection(\n title: Text('General',\n style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold)),\n tiles: [\n SettingsTile.switchTile(\n initialValue: TextToDocParameter.isLoadConfig,\n onToggle: (value) {\n setState(() {\n //Settings.isLoadConfig = value;\n TextToDocParameter.isLoadConfig = value;\n print(\n \"Settings : build() : TextToDocParameter.isLoadConfig = ${TextToDocParameter.isLoadConfig}\");\n\n importFrontEndCfgFile();\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/config_frontend.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Upload frontend config file'),\n ],\n ),\n description: Text(\"Set required app's parameters\"),\n ),\n SettingsTile.switchTile(\n initialValue: widget.useFeedback,\n onToggle: (value) {\n setState(() {\n //useFeedback = value;\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/feedback.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Feedback (not implemented yet)'),\n ],\n ),\n description: Text('Send feedback on generated answers'),\n ),\n SettingsTile.switchTile(\n initialValue: widget.useColorMode,\n onToggle: (value) {\n setState(() {\n //useColorMode = value;\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/color_mode.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Enable dark mode (not implemented yet)'),\n ],\n )),\n SettingsTile.switchTile(\n initialValue: TextToDocParameter.anonymized_data,//Settings.isAnonymizedMode,\n onToggle: (value) {\n setState(() {\n //Settings.isAnonymizedMode = value;\n TextToDocParameter.anonymized_data = value;\n //print(\"Settings : build() : widget.useExpertMode = ${widget.useExpertMode}\");\n print(\n \"Settings : build() : TextToDocParameter.anonymized_data = ${TextToDocParameter.anonymized_data}\");\n updateFrontEndFlutterCfg(parameter: \"anonymized_data\", value: value);\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/anonymized.jpeg',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Enable anonymization of data'),\n ],\n )),\n ],\n ),\n SettingsSection(\n title: Text('Statistics',\n style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold)),\n tiles: [\n SettingsTile.switchTile(\n initialValue: widget.useDashboards,\n onToggle: (value) {\n setState(() {\n //useDashboards = value;\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/statistics.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Dashboards (not implemented yet)'),\n ],\n ),\n description: Text('Get visibility on activities'),\n ),\n SettingsTile.switchTile(\n initialValue: widget.useReports,\n onToggle: (value) {\n setState(() {\n //useReports = value;\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/reports.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Reports (not implemented yet)'),\n ],\n ),\n description: Text('Export activity reports in pdf'),\n ),\n ],\n ),\n SettingsSection(\n title: Text('Troubleshooting',\n style: TextStyle(fontSize: 18, fontWeight: FontWeight.bold)),\n tiles: [\n SettingsTile.switchTile(\n initialValue: TextToDocParameter.expert_mode,//Settings.isExpert,\n onToggle: (value) {\n setState(() {\n //widget.useExpertMode = value;\n //Settings.isExpert = value;\n TextToDocParameter.expert_mode = value;\n //Config.isExpert = value;\n //print(\"Settings : build() : widget.useExpertMode = ${widget.useExpertMode}\");\n print(\n \"Settings : build() : TextToDocParameter.expert_mode = ${TextToDocParameter.expert_mode}\");\n BlocProvider.of(context).displayStepper(value);\n });\n\n updateFrontEndFlutterCfg(parameter: \"expert_mode\", value: value);\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/troubleshooting.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Expert mode'),\n ],\n ),\n description: Text(\n 'Get workflow details and internal technical informations'),\n ),\n SettingsTile.switchTile(\n initialValue: widget.useLog,\n onToggle: (value) {\n setState(() {\n //useLog = value;\n });\n },\n title: Row(\n mainAxisAlignment: MainAxisAlignment.start,\n children: [\n Image.asset(\n 'assets/images/logs.png',\n height: 70,\n width: 70,\n fit: BoxFit.cover,\n ),\n Container(width: 20),\n Text('Enable logs (not implemented yet)'),\n ],\n ),\n description: Text('Generate logs for troubleshooting'),\n ),\n ],\n ),\n ],\n ),\n );\n }\n\n void importFrontEndCfgFile() async {\n print('Settings: importFrontEndCfgFile() : START');\n List>? rowsAsListOfValues;\n final filePickerResult = await FilePicker.platform.pickFiles(\n allowMultiple: false,\n allowedExtensions: ['json'],\n type: FileType.custom,\n dialogTitle: \"Import Frontend json Config File\",\n );\n\n if (filePickerResult != null) {\n print(\n 'Settings: importFrontEndCfgFile() : fileName = ${filePickerResult.files.single.name}');\n print(\n 'Settings: importFrontEndCfgFile() : size = ${filePickerResult.files.single.size}');\n //print('Settings: importFrontEndCfgFile() : path = ${filePickerResult.files.single.path}');\n Uint8List fileBytes = filePickerResult.files.single.bytes!;\n\n String fileContent = utf8.decode(fileBytes);\n\n print('Settings: importFrontEndCfgFile() : fileContent = ${fileContent}');\n\n var cfg = jsonDecode(fileContent);\n\n print('Settings: importFrontEndCfgFile() : cfg = ${cfg}');\n\n if (cfg != null) {\n setState(() {\n TextToDocParameter.isLoadConfig = false;\n TextToDocParameter.anonymized_data = cfg[\"anonymized_data\"];\n TextToDocParameter.expert_mode = cfg[\"expert_mode\"];\n TextToDocParameter.endpoint_opendataqnq = cfg[\"endpoint_opendataqnq\"];\n TextToDocParameter.firestore_database_id = cfg[\"firestore_database_id\"];\n TextToDocParameter.firebase_app_name = cfg[\"firebase_app_name\"];\n TextToDocParameter.firestore_history_collection = cfg[\"firestore_history_collection\"];\n TextToDocParameter.firestore_cfg_collection = cfg[\"firestore_cfg_collection\"];\n TextToDocParameter.imported_questions = cfg[\"imported_questions\"];\n });\n\n if (TextToDocParameter.anonymized_data != null &&\n TextToDocParameter.expert_mode != null &&\n TextToDocParameter.endpoint_opendataqnq != null &&\n TextToDocParameter.firestore_database_id != null &&\n TextToDocParameter.firebase_app_name != null &&\n TextToDocParameter.firestore_history_collection != null &&\n TextToDocParameter.firestore_cfg_collection != null &&\n TextToDocParameter.imported_questions != null\n ) {\n print('Settings: importFrontEndCfgFile() : Trying to update front_end_flutter_cfg');\n try {\n widget.db\n .collection(\"${TextToDocParameter.firestore_cfg_collection}\")\n .doc('${TextToDocParameter.userID}')\n .set({\n \"endpoint_opendataqnq\":\n \"${TextToDocParameter.endpoint_opendataqnq}\",\n \"firestore_database_id\":\n \"${TextToDocParameter.firestore_database_id}\",\n \"expert_mode\": TextToDocParameter.expert_mode,\n \"anonymized_data\": TextToDocParameter.anonymized_data,\n \"firebase_app_name\": \"${TextToDocParameter.firebase_app_name}\",\n \"firestore_history_collection\": \"${TextToDocParameter.firestore_history_collection}\",\n \"firestore_cfg_collection\": \"${TextToDocParameter.firestore_cfg_collection}\",\n \"imported_questions\": \"${TextToDocParameter.imported_questions}\"\n });\n\n showSuccessfulUploadMsg();\n\n } catch (e) {\n print('Settings: importFrontEndCfgFile() : EXCEPTION : ${e}');\n displayCfgUploadErrorMsg();\n }\n } else {\n print('Settings: importFrontEndCfgFile() : some fields if cfg are null, could not update firestore_cfg_collection');\n displayCfgUploadErrorMsg();\n }\n }\n }\n }\n\n void showSuccessfulUploadMsg() {\n showDialog(\n context: context,//navigatorKey.currentContext!,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Row( // Use a Row to align the icon and title\n children: [\n Icon(Icons.info, color: Colors.blueAccent),\n SizedBox(width: 8), // Add some spacing\n Text('Information'),\n ],\n ),\n content: Text('The json configuration file has been\\nuploaded successfully to Firestore.'),\n actions: [\n TextButton(\n onPressed: () {\n Navigator.of(context).pop(); // Close the dialog\n },\n child: Text('OK'),\n ),\n ],\n );\n },\n );\n }\n\n void displayCfgUploadErrorMsg() {\n showDialog(\n context: context,\n barrierDismissible: true,\n builder: (BuildContext context) {\n return AlertDialog(\n title: Row( // Use a Row to align the icon and title\n children: [\n Icon(Icons.warning, color: Colors.red),\n SizedBox(width: 8), // Add some spacing\n Text('Alert'),\n ],\n ),\n content: SelectableText.rich(\n TextSpan(\n children: [\n TextSpan(text: \"Please check you have uploaded a config_frontend.json\\n\" +\n \"file similar as the one below:\\n\\n\"),\n TextSpan(text: \"{\\n\", style: TextStyle(color: Colors.black),),\n TextSpan(text:'\"endpoint_opendataqnq\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:'\"\",\\n', style: TextStyle(color: Colors.green),),\n TextSpan(text:'\"firestore_database_id\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:'\"opendataqna-session-logs\",\\n', style: TextStyle(color: Colors.green),),\n TextSpan(text:'\"firestore_history_collection\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:'\"session_logs\",\\n', style: TextStyle(color: Colors.green),),\n TextSpan(text:'\"firestore_cfg_collection\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:'\"front_end_flutter_cfg\",\\n', style: TextStyle(color: Colors.green),),\n TextSpan(text:'\"expert_mode\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:',\\n', style: TextStyle(color: Colors.red),),\n TextSpan(text:'\"anonymized_data\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:',\\n', style: TextStyle(color: Colors.red),),\n TextSpan(text:'\"firebase_app_name\": ', style: TextStyle(color: Colors.blueAccent),),\n TextSpan(text:'\"opendataqna\"\\n', style: TextStyle(color: Colors.green),),\n TextSpan(text:'}', style: TextStyle(color: Colors.black),),\n ],\n ),\n ),\n actions: [\n TextButton(\n onPressed: () {\n Navigator.of(context).pop(); // Close the dialog\n },\n child: Text('OK'),\n ),\n ],\n );\n },\n );\n }\n\n void updateFrontEndFlutterCfg({required String parameter, required bool value}) {\n print(\"Settings : updateFrontEndFlutterCfg() : START\");\n print(\"Settings : updateFrontEndFlutterCfg() : widget.db = ${widget.db}\");\n print(\"Settings : updateFrontEndFlutterCfg() : parameter = ${parameter}\");\n print(\"Settings : updateFrontEndFlutterCfg() : value = ${value}\");\n\n print(\n 'Settings: updateFrontEndFlutterCfg() : TextToDocParameter.firestore_cfg_collection = ${TextToDocParameter.firestore_cfg_collection}');\n print(\n 'Settings: updateFrontEndFlutterCfg() : TextToDocParameter.userID = ${TextToDocParameter.userID}');\n\n //update the document in front_end_flutter_cfg collection corresponding to the user_id to refelect the chnage of the TextToDocParameter.expert_mode parameter\n try {\n widget.db!\n .collection(\"${TextToDocParameter.firestore_cfg_collection}\")\n .doc('${TextToDocParameter.userID}')\n .set(\n {\"$parameter\": value},\n SetOptions(merge: true));\n\n print(\n 'Settings: updateFrontEndFlutterCfg() : update firestore_history_collection.expert_mode successfully: parameter = $parameter : value = $value');\n\n } catch (e) {\n print(\n 'Settings: updateFrontEndFlutterCfg() : update firestore_history_collection.expert_mode : EXCEPTION : $e');\n }\n }\n}\n\nclass Config {\n static bool isUseFeedback = false;\n static bool isUseColorMode = false;\n static bool isUseDashboards = false;\n static bool isUseReports = false;\n static bool isExpert = false;\n static bool isUseLog = false;\n static bool isAnonymizedMode = false;\n\n @override\n String toString() {\n return isExpert.toString();\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/display_stepper/display_stepper_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:cloud_firestore/cloud_firestore.dart';\nimport 'package:ttmd/services/display_stepper/display_stepper_state.dart';\n\nimport '../../utils/TextToDocParameter.dart';\n\n\nclass DisplayStepperCubit extends Cubit {\n\n DisplayStepperCubit()\n : super(DisplayStepperState(\n status: displayStepperStatus.remove_stepper));\n\n Future displayStepper(bool isDisplay) async {\n\n print(\n 'DisplayStepperCubit : displayStepper() : DEBUT ');\n\n if(isDisplay)\n emit(state.copyWith(\n status: displayStepperStatus.display_stepper));\n else\n emit(state.copyWith(\n status: displayStepperStatus.remove_stepper));\n }\n}" }, { "path": "frontend/frontend-flutter/lib/services/display_stepper/display_stepper_state.dart", "content": "import 'package:equatable/equatable.dart';\n\n\nenum displayStepperStatus {display_stepper,remove_stepper}\n\nclass DisplayStepperState extends Equatable {\n final displayStepperStatus status;\n\n DisplayStepperState({\n this.status = displayStepperStatus.remove_stepper,\n });\n\n @override\n List get props => [this.status];\n\n DisplayStepperState copyWith({\n displayStepperStatus? status,\n }) {\n print('DisplayStepperState : copyWith() : status = ' + status.toString());\n return DisplayStepperState(\n status: status ?? this.status,\n );\n }\n}" }, { "path": "frontend/frontend-flutter/lib/services/first_question/first_question_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/first_question/first_question_state.dart';\n\n\nclass FirstQuestionCubit extends Cubit {\n\n FirstQuestionCubit()\n : super(FirstQuestionState(\n status: firstQuestionStatus.display_welcome_message , message: \"\"));\n\n Future removeWelcomeMessage({String? message = \"\"}) async {\n print(\n 'FirstQuestionCubit : removeWelcomeMessage() : DEBUT ');\n\n emit(state.copyWith(\n status: firstQuestionStatus.remove_welcome_message , message: message));\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/first_question/first_question_state.dart", "content": "import 'package:equatable/equatable.dart';\n\n\nenum firstQuestionStatus {display_welcome_message,remove_welcome_message}\n\nclass FirstQuestionState extends Equatable {\n final firstQuestionStatus status;\n final String? message;\n\n FirstQuestionState({\n this.status = firstQuestionStatus.display_welcome_message,\n this.message = \"\",\n });\n\n @override\n List get props => [this.status,this.message!];\n\n FirstQuestionState copyWith({\n firstQuestionStatus? status,\n String? message\n }) {\n print('FirstQuestionState : copyWith() : message = ' + message!);\n return FirstQuestionState(\n status: status ?? this.status,\n message: message ?? this.message,\n );\n }\n}\n\n" }, { "path": "frontend/frontend-flutter/lib/services/load_question/load_question_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/load_question/load_question_state.dart';\nimport 'package:http/http.dart' as http;\nimport 'dart:html' as html;\nimport 'dart:convert';\n\nclass LoadQuestionCubit extends Cubit {\n\n LoadQuestionCubit()\n : super(LoadQuestionState(\n status: LoadQuestionStatus.initial, question: \"\"));\n\n Future loadQuestionToChat({String? question, String ? time}) async {\n print(\n 'LoadQuestionCubit : loadQuestionToChat() : DEBUT ');\n\n emit(state.copyWith(\n status: LoadQuestionStatus.loaded, question: question, time: time));\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/load_question/load_question_state.dart", "content": "import 'package:equatable/equatable.dart';\n\nenum LoadQuestionStatus {initial,loaded, error}\n\nclass LoadQuestionState extends Equatable {\n final LoadQuestionStatus status;\n final String? question;\n final String? time;\n\n LoadQuestionState({\n this.status = LoadQuestionStatus.initial,\n this.question = \"\",\n this.time = \"\"\n });\n\n @override\n List get props => [this.status,this.question!, this.time!];\n\n LoadQuestionState copyWith({\n LoadQuestionStatus? status,\n String? question,\n String? time\n }) {\n print('LoadQuestionState : copyWith() : question = $question! : time = $time');\n return LoadQuestionState(\n status: status ?? this.status,\n question: question ?? this.question,\n time: time ?? this.time\n );\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/new_suggestions/new_suggestion_cubit.dart", "content": "import 'dart:convert';\nimport 'dart:math';\n\nimport 'package:bloc/bloc.dart';\nimport 'package:intl/intl.dart';\nimport 'package:ttmd/utils/TextToDocParameter.dart';\nimport 'package:ttmd/services/new_suggestions/new_suggestion_state.dart';\nimport 'dart:html' as html;\n\nclass NewSuggestionCubit extends Cubit {\n NewSuggestionCubit()\n : super(NewSuggestionState(\n status: NewSuggestionStateStatus.initial,\n suggestionList: const [\"x:\", \"x:\", \"x:\"],\n time: \"\",\n scenarioNumber: 0));\n\n Future generateNewSuggestions(int scenarioNumber, String question,\n {String? lastCannedQuestion,\n bool? isACannedQuestion,\n String? userGrouping}) async {\n List respLLMQuestion = [];\n List respCannedQuestions = [];\n List resp = [];\n List tmpQuestions = [];\n String body = \"\";\n Uri url;\n String question1 = \"\";\n String question2 = \"\";\n String question3 = \"\";\n String question4 = \"\";\n String timeString = \"\";\n String originalQuestion = \"\";\n\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : START');\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : userGrouping = $userGrouping');\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : TextToDocParameter.lastCannedQuestion = ${TextToDocParameter.lastCannedQuestion}');\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : lastCannedQuestion= ${lastCannedQuestion}');\n\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : scenarioNumber = $scenarioNumber');\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : question = $question');\n\n timeString = displayDateTime();\n //TextToDocParameter.lastCannedQuestion = question;\n //TextToDocParameter.lastCannedQuestion = lastCannedQuestion?? \"\";\n\n isACannedQuestion = isACannedQuestion ?? false;\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : Not a canned question : isACannedQuestion = $isACannedQuestion');\n originalQuestion = question;\n\n print(\n 'NewSuggestionCubit : NewSuggestionCubit() : generateNewSuggestions : Not a canned question : originalQuestion = $originalQuestion');\n\n //Create the header\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n //Create the body\n body = '''{\n \"user_grouping\": \"$userGrouping\"\n }''';\n\n print(\n 'NewSuggestionCubit : generateNewSuggestions() : Not a canned question : body = ' +\n body);\n\n try {\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/get_known_sql',\n method: \"POST\",\n sendData: body);\n\n print(\n 'NewSuggestionCubit : generateNewSuggestions() : Not a canned question : response = ' +\n response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : jsonData = $jsonData');\n\n //KnownSQL = [{\"example_user_question\": \"question1\", \"example_generated_sql\": \"sql1\"},\n // {\"example_user_question\": \"question2\", \"example_generated_sql\": \"sql2\"},\n // ...]\n\n var knownSql =\n jsonData[\"KnownSQL\"].replaceAll(RegExp(r'((\\\\n)|(\\\\r))'), '');\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSql = $knownSql');\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSql.runtimeType = ${knownSql.runtimeType}');\n\n if (knownSql is Map)\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSql is a Map');\n if (knownSql is List)\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSql is a List');\n if (knownSql is String)\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSql is a String');\n\n var knownSqlMap = jsonDecode(knownSql);\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSqlMap.runtimeType = ${knownSqlMap.runtimeType}');\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSqlMap[0].runtimeType = ${knownSqlMap[0].runtimeType}');\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSqlMap = ${knownSqlMap}');\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : knownSqlMap[0] = ${knownSqlMap[0].toString()}');\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"example_user_question\")\n tmpQuestions.add(entry.value);\n }\n }\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : Before pickUpRandomQuestion() : tmpQuestions = ${tmpQuestions}');\n\n tmpQuestions = pickUpRandomQuestion(question, tmpQuestions);\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : After pickUpRandomQuestion() : tmpQuestions = ${tmpQuestions}');\n\n if (tmpQuestions.length > 0)\n question1 = tmpQuestions[0] ?? \"No suggestion for question1\";\n if (tmpQuestions.length > 1)\n question2 = tmpQuestions[1] ?? \"No suggestion for question2\";\n if (tmpQuestions.length > 2)\n question3 = tmpQuestions[2] ?? \"No suggestion for question3\";\n if (tmpQuestions.length > 3)\n question4 = tmpQuestions[3] ?? \"No suggestion for question4\";\n\n //Adding scenarioNumber + \":\" and \":userGrouping\" to have same format as for Business KPI questions\n question1 = \"$scenarioNumber:\" + question1 + \":\" + userGrouping!;\n question2 = \"$scenarioNumber:\" + question2 + \":\" + userGrouping!;\n question3 = \"$scenarioNumber:\" + question3 + \":\" + userGrouping!;\n question4 = \"$scenarioNumber:\" + question4 + \":\" + userGrouping!;\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : question1 = ${question1}');\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : question2 = ${question2}');\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : question3 = ${question3}');\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : question4 = ${question4}');\n }\n } catch (e) {\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : EXCEPTION = $e');\n throw Exception('Failed to get suggestions: $e');\n } finally {\n respLLMQuestion.add(question1);\n respLLMQuestion.add(question2);\n respLLMQuestion.add(question3);\n respLLMQuestion.add(question4);\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question :scenarioNumber = $scenarioNumber >= 9 : respLLMQuestion = $respLLMQuestion');\n\n //2024-07-14 respCannedQuestions = pickUpNextQuestions(scenarioNumber, question);\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : scenarioNumber = $scenarioNumber >= 9 : respCannedQuestions = $respCannedQuestions');\n\n resp.addAll(respLLMQuestion);\n\n print(\n 'NewSuggestionCubit: generateNewSuggestions() : Not a canned question : scenarioNumber = $scenarioNumber >= 9 : resp = $resp');\n }\n\n emit(state.copyWith(\n status: NewSuggestionStateStatus.loaded,\n suggestionList: resp,\n time: timeString,\n scenarioNumber: scenarioNumber));\n }\n\n Future getAllquestions(String userGrouping) async {\n List resp = [];\n String body = \"\";\n String timeString = \"\";\n\n print('NewSuggestionCubit : getAllquestions() : START');\n\n timeString = displayDateTime();\n\n //Create the header\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n //Create the body\n body = '''{\n \"user_grouping\": \"$userGrouping\"\n }''';\n\n print('NewSuggestionCubit : getAllquestions() : body = ' + body);\n\n try {\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/get_known_sql',\n method: \"POST\",\n sendData: body);\n\n print('NewSuggestionCubit : getAllquestions() : response = ' +\n response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('NewSuggestionCubit: getAllquestions() : jsonData = $jsonData');\n\n //KnownSQL = [{\"example_user_question\": \"question1\", \"example_generated_sql\": \"sql1\"},\n // {\"example_user_question\": \"question2\", \"example_generated_sql\": \"sql2\"},\n // ...]\n\n var knownSql =\n jsonData[\"KnownSQL\"].replaceAll(RegExp(r'((\\\\n)|(\\\\r))'), '');\n\n print('NewSuggestionCubit: getAllquestions() : knownSql = $knownSql');\n\n var knownSqlMap = jsonDecode(knownSql);\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"example_user_question\") resp.add(entry.value);\n }\n }\n }\n } catch (e) {\n print('NewSuggestionCubit: getAllquestions() : EXCEPTION = $e');\n throw Exception('Failed to get questions: $e');\n } finally {\n print('NewSuggestionCubit: getAllquestions() : resp = ${resp}');\n }\n\n emit(state.copyWith(\n status: NewSuggestionStateStatus.all_questions_loaded,\n suggestionList: resp,\n time: timeString,\n scenarioNumber: 0,\n userGrouping: userGrouping));\n }\n\n String displayDateTime() {\n String? dateTimeS;\n\n final now = DateTime.now();\n dateTimeS = DateFormat('yyyy-MM-dd HH:mm:ss').format(now);\n return dateTimeS!;\n }\n\n List pickUpNextQuestions(int scenarioNumber, String question) {\n List list = [];\n return list;\n }\n\n int getIndexOfQuestion(\n int scenarioNumber, String question, List listQuestionsScenario) {\n print('NewSuggestionCubit: getIndexOfQuestion() : START');\n print(\n 'NewSuggestionCubit: getIndexOfQuestion() : question = $scenarioNumber:$question');\n for (String q in listQuestionsScenario) print('q = $q');\n\n int index = 0;\n index = listQuestionsScenario.indexOf(\"$scenarioNumber:$question\");\n\n print('NewSuggestionCubit: getIndexOfQuestion() : index = $index');\n\n return index;\n }\n\n List pickUpRandomQuestion(\n String question, List questionList) {\n List list = [];\n int lengthScenario = 0;\n Random random = new Random();\n List listQuestionsScenario = [];\n int randomNumber = 0;\n String candidateQuestion = \"\";\n Map map = {};\n\n print('NewSuggestionCubit: pickUpRandomQuestion() : START');\n print('NewSuggestionCubit: pickUpRandomQuestion() : question = $question');\n\n for (int i = 0; list.length <= 3; i++) {\n randomNumber = random.nextInt(questionList.length);\n candidateQuestion = questionList[randomNumber];\n print(\n 'NewSuggestionCubit: pickUpRandomQuestion() : candidateQuestion = $candidateQuestion');\n if (question != candidateQuestion) {\n print(\n 'NewSuggestionCubit: pickUpRandomQuestion() : if(question != candidateQuestion) : candidateQuestion = $candidateQuestion');\n if (!map.containsKey(candidateQuestion)) {\n print(\n 'NewSuggestionCubit: pickUpRandomQuestion() : !map.containsKey(candidateQuestion : adding candidateQuestion');\n map[candidateQuestion] = candidateQuestion;\n list.add(candidateQuestion);\n }\n }\n //listQuestionsScenario may contain less than 3 questions (including the asked question\n //so we check that and break the loop if we retreived all the candidate questions\n if (list.length == questionList.length - 1) {\n break;\n }\n }\n\n print('NewSuggestionCubit: pickUpRandomQuestion() : list = $list');\n\n return list;\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/new_suggestions/new_suggestion_state.dart", "content": "import 'package:equatable/equatable.dart';\n\n\nenum NewSuggestionStateStatus {initial,loading,loaded, all_questions_loaded}\n\nclass NewSuggestionState extends Equatable {\n final NewSuggestionStateStatus status;\n final List? suggestionList;\n final String? time;\n final int? scenarioNumber;\n final String? userGrouping;\n\n NewSuggestionState({\n this.status = NewSuggestionStateStatus.initial,\n this.suggestionList = const [\"x:\",\"x:\", \"x:\"],\n this.time = \"\",\n this.scenarioNumber = 0,\n this.userGrouping = \"\"\n });\n\n @override\n List get props => [this.status,this.suggestionList!, this.time!, this.scenarioNumber!, this.userGrouping!];\n\n NewSuggestionState copyWith({\n NewSuggestionStateStatus? status,\n List? suggestionList,\n String? time,\n int? scenarioNumber,\n String? userGrouping\n }) {\n print('NewSuggestionState : copyWith() : suggestion = $suggestionList');\n return NewSuggestionState(\n status: status ?? this.status,\n suggestionList: suggestionList ?? this.suggestionList,\n time: time ?? this.time,\n scenarioNumber: scenarioNumber ?? this.scenarioNumber,\n userGrouping: userGrouping ?? this.userGrouping\n );\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/text_to_doc_question/text_to_doc_question_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/text_to_doc_question/text_to_doc_question_state.dart';\n\n\nclass TextToDocQuestionCubit extends Cubit {\n\n TextToDocQuestionCubit()\n : super(TextToDocQuestionState(\n status: textToDocStatus.not_text_to_doc , message: \"\"));\n\n Future switchToTextToDoc({required bool isTextToDoc, String? message = \"\"}) async {\n print(\n 'TextToDocQuestionCubit : switchToTextToDoc() : DEBUT ');\n print(\n 'TextToDocQuestionCubit : switchToTextToDoc() : isTextToDoc = $isTextToDoc ');\n\n if(isTextToDoc) {\n emit(state.copyWith(\n status: textToDocStatus.text_to_doc, message: message));\n } else {\n emit(state.copyWith(\n status: textToDocStatus.not_text_to_doc, message: message));\n }\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/text_to_doc_question/text_to_doc_question_state.dart", "content": "import 'package:equatable/equatable.dart';\n\nenum textToDocStatus {not_text_to_doc,text_to_doc}\n\nclass TextToDocQuestionState extends Equatable {\n textToDocStatus status;\n String? message;\n\n TextToDocQuestionState({\n this.status = textToDocStatus.not_text_to_doc,\n this.message = \"\",\n });\n\n @override\n List get props => [this.status,this.message!];\n\n TextToDocQuestionState copyWith({\n textToDocStatus? status,\n String? message\n }) {\n print('TextToDocQuestionState : copyWith() : status = $status : message = $message');\n this.status = status!;\n this.message = message;\n return TextToDocQuestionState(\n status: status ?? this.status,\n message: message ?? this.message,\n );\n }\n}\n\n" }, { "path": "frontend/frontend-flutter/lib/services/update_expert_mode/update_expert_mode_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/update_expert_mode/update_expert_mode_state.dart';\n\nimport '../../utils/TextToDocParameter.dart';\n\n\nclass UpdateExpertModeCubit extends Cubit {\n\n UpdateExpertModeCubit()\n : super(UpdateExpertModeState(\n status: updateExpertModeStatus.expert_mode_off));\n\n Future updateExpertMode(bool isDisplay) async {\n\n print(\n 'UpdateExpertModeCubit : updateExpertMode() : DEBUT ');\n\n if(isDisplay)\n emit(state.copyWith(\n status: updateExpertModeStatus.expert_mode_on));\n else\n emit(state.copyWith(\n status: updateExpertModeStatus.expert_mode_off));\n }\n}" }, { "path": "frontend/frontend-flutter/lib/services/update_expert_mode/update_expert_mode_state.dart", "content": "import 'package:equatable/equatable.dart';\n\n\nenum updateExpertModeStatus {expert_mode_on,expert_mode_off}\n\nclass UpdateExpertModeState extends Equatable {\n final updateExpertModeStatus status;\n\n UpdateExpertModeState({\n this.status = updateExpertModeStatus.expert_mode_off,\n });\n\n @override\n List get props => [this.status];\n\n UpdateExpertModeState copyWith({\n updateExpertModeStatus? status,\n }) {\n print('UpdateExpertModeState : copyWith() : status = ' + status.toString());\n return UpdateExpertModeState(\n status: status ?? this.status,\n );\n }\n}" }, { "path": "frontend/frontend-flutter/lib/services/update_popular_questions/update_popular_questions_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/update_popular_questions/update_popular_questions_state.dart';\n\nimport '../../utils/most_popular_questions.dart';\n\nclass UpdatePopularQuestionsCubit extends Cubit {\n UpdatePopularQuestionsCubit()\n : super(UpdatePopularQuestionsState(\n status: UpdateMostPopularQuestionStatus.initial,\n mostPopularQuestionsList: [\n MostPopularQ(\"\", 0, \"\"),\n MostPopularQ(\"\", 0, \"\"),\n MostPopularQ(\"\", 0, \"\")\n ],\n time: \"\"));\n\n Future updateMostPopularQuestions(\n {List? mostPopularQuestionsList, String? time}) async {\n print(\n 'UpdatePopularQuestionsCubit : UpdatePopularQuestionsCubit() : DEBUT ');\n\n emit(state.copyWith(\n status: UpdateMostPopularQuestionStatus.loaded,\n mostPopularQuestionsList: mostPopularQuestionsList,\n time: time));\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/update_popular_questions/update_popular_questions_state.dart", "content": "import 'package:equatable/equatable.dart';\n\nimport '../../utils/most_popular_questions.dart';\n\n\nenum UpdateMostPopularQuestionStatus {initial,loaded}\n\nclass UpdatePopularQuestionsState extends Equatable {\n final UpdateMostPopularQuestionStatus status;\n final List? mostPopularQuestionsList;\n final String? time;\n\n UpdatePopularQuestionsState({\n this.status = UpdateMostPopularQuestionStatus.initial,\n this.mostPopularQuestionsList,\n this.time = \"\"\n });\n\n @override\n List get props => [this.status,this.mostPopularQuestionsList!, this.time!];\n\n UpdatePopularQuestionsState copyWith({\n UpdateMostPopularQuestionStatus? status,\n List? mostPopularQuestionsList,\n String? time\n }) {\n print('UpdatePopularQuestionsState : copyWith() : List? mostPopularQuestionsList = ' + mostPopularQuestionsList!.toString());\n return UpdatePopularQuestionsState(\n status: status ?? this.status,\n mostPopularQuestionsList: mostPopularQuestionsList ?? this.mostPopularQuestionsList,\n time: time ?? this.time\n );\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/update_stepper/update_stepper_cubit.dart", "content": "import 'package:bloc/bloc.dart';\nimport 'package:ttmd/services/update_stepper/update_stepper_state.dart';\nimport 'package:flutter/material.dart';\nimport 'dart:html' as html;\nimport 'dart:convert';\n\nimport 'package:ttmd/utils/stepper_expert_info.dart';\n\nclass UpdateStepperCubit extends Cubit {\n //List? dicRows;\n //String? message;\n //StepState? stateStepper;\n //bool? isActiveStepper;\n\n UpdateStepperCubit()\n : super(UpdateStepperState(\n status: StepperStatus.initial,\n message: \"Initial state\",\n stateStepper: StepState.disabled,\n isActiveStepper: false,\n debugInfo: StepperExpertInfo()));\n\n Future updateStepperStatusUploaded(\n {required StepperStatus status,\n required String message,\n required StepState stateStepper,\n required bool isActiveStepper,\n StepperExpertInfo? debugInfo}) async {\n print('UpdateStepperCubit : updateStepperStatusUploaded() : DEBUT ');\n print('UpdateStepperCubit : updateStepperStatusUploaded() : status = $status');\n print('UpdateStepperCubit : updateStepperStatusUploaded() : message = $message');\n print(\n 'UpdateStepperCubit : updateStepperStatusUploaded() : stateStepper = $stateStepper');\n print(\n 'UpdateStepperCubit : updateStepperStatusUploaded() : isActiveStepper = $isActiveStepper');\n print(\n 'UpdateStepperCubit : updateStepperStatusUploaded() : debugInfo = $debugInfo');\n\n try {\n emit(state.copyWith(\n status: status,\n message: message,\n stateStepper: stateStepper,\n isActiveStepper: isActiveStepper,\n debugInfo: debugInfo\n ));\n } catch (e) {\n print(\n 'UpdateStepperCubit : updateStepperStatusUploaded() : EXCEPTION : ' +\n e.toString());\n emit(state.copyWith(\n status: StepperStatus.error,\n message: \"Une erreur s'est produite.\",\n stateStepper: StepState.error,\n isActiveStepper: false,\n debugInfo: StepperExpertInfo()\n ));\n }\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/services/update_stepper/update_stepper_state.dart", "content": "import 'package:equatable/equatable.dart';\nimport 'package:flutter/material.dart';\n\nimport '../../utils/stepper_expert_info.dart';\n\nenum StepperStatus {\n initial,\n uploaded,\n extracted,\n compared,\n committed,\n error,\n enter_question,\n generate_sql,\n run_query,\n get_graph_description,\n get_text_summary\n}\n\nclass UpdateStepperState extends Equatable {\n final StepperStatus status;\n String? message;\n StepState? stateStepper;\n bool? isActiveStepper;\n final StepperExpertInfo debugInfo;\n\n UpdateStepperState(\n {this.status = StepperStatus.initial,\n this.message = \"\",\n this.stateStepper = StepState.disabled,\n this.isActiveStepper = false,\n this.debugInfo = const StepperExpertInfo()});\n\n @override\n List get props => [\n this.status,\n this.message!,\n this.stateStepper!,\n this.isActiveStepper!,\n this.debugInfo!\n ];\n\n UpdateStepperState copyWith(\n {StepperStatus? status,\n String? message,\n StepState? stateStepper,\n bool? isActiveStepper,\n StepperExpertInfo? debugInfo}) {\n print(\n 'UpdateStepperState : copyWith() : status = $status : message = $message : stateStepper = $stateStepper : isActiveStepper = $isActiveStepper : debugInfo = $debugInfo');\n\n return UpdateStepperState(\n status: status ?? this.status,\n message: message ?? this.message,\n stateStepper: stateStepper ?? this.stateStepper,\n isActiveStepper: isActiveStepper ?? this.isActiveStepper,\n debugInfo: debugInfo ?? this.debugInfo);\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/utils/Input_custom.dart", "content": "import 'package:flutter/material.dart';\nimport 'package:flutter_chat_ui/flutter_chat_ui.dart';\nimport 'package:flutter_chat_types/flutter_chat_types.dart' as types;\n\nclass InputCustom extends Input {\n final bool? isAttachmentUploading;\n final VoidCallback? onAttachmentPressed;\n final void Function(types.PartialText) onSendPressed;\n final InputOptions options;\n //TextEditingController? _textController;\n\n InputCustom({\n super.key,\n this.isAttachmentUploading,\n this.onAttachmentPressed,\n required this.onSendPressed,\n this.options = const InputOptions(),\n }): super(\n isAttachmentUploading: isAttachmentUploading,\n onAttachmentPressed: onAttachmentPressed,\n onSendPressed: onSendPressed,\n options: options,\n );\n\n}" }, { "path": "frontend/frontend-flutter/lib/utils/TextToDocParameter.dart", "content": "class TextToDocParameter {\n static bool isTextTodocGlobal = false;\n static bool isAuthenticated = false;\n static bool anonymized_data = false;\n\n static int lastScenarioNumber = 0;\n static String lastCannedQuestion = \"\";\n static String sessionId = \"\";\n static String userID = \"\";\n static String currentUserGrouping = \"\";\n static String currentScenarioName = \"\";\n static String email = \"\";\n static String firstName = \"\";\n static String lastName = \"\";\n static String picture = \"\";\n static bool isLoadConfig = false;\n static bool expert_mode = false;\n static String endpoint_opendataqnq = \"\";\n static String firebase_app_name = \"\";\n static String firestore_database_id = \"\";\n static String firestore_history_collection = \"\";\n static String firestore_cfg_collection = \"\";\n static String imported_questions = \"\";\n static int questionCount = 1;\n static List suggestionsList = [];\n static List userGroupingList = [];\n\n}\n" }, { "path": "frontend/frontend-flutter/lib/utils/custom_input_field.dart", "content": "import 'dart:convert';\n\nimport 'package:cloud_firestore/cloud_firestore.dart';\nimport 'package:flutter/foundation.dart';\nimport 'package:flutter/material.dart';\nimport 'package:flutter/services.dart';\nimport 'package:flutter_bloc/flutter_bloc.dart';\nimport 'package:flutter_chat_types/flutter_chat_types.dart' as types;\nimport 'package:flutter_chat_ui/src/models/input_clear_mode.dart';\nimport 'package:flutter_chat_ui/src/models/send_button_visibility_mode.dart';\nimport 'package:flutter_chat_ui/src/util.dart';\nimport 'package:flutter_chat_ui/src/widgets/state/inherited_chat_theme.dart';\nimport 'package:flutter_chat_ui/src/widgets/state/inherited_l10n.dart';\nimport 'package:flutter_chat_ui/src/widgets/input/attachment_button.dart';\nimport 'package:flutter_chat_ui/src/widgets/input/input_text_field_controller.dart';\nimport 'package:flutter_chat_ui/src/widgets/input/send_button.dart';\nimport 'package:flutter_typeahead/flutter_typeahead.dart';\nimport 'dart:html' as html;\nimport '../services/new_suggestions/new_suggestion_cubit.dart';\nimport '../services/new_suggestions/new_suggestion_state.dart';\nimport 'TextToDocParameter.dart';\n\n/// A class that represents bottom bar widget with a text field, attachment and\n/// send buttons inside. By default hides send button when text field is empty.\nclass CustomInputField extends StatefulWidget {\n /// Creates [Input] widget.\n const CustomInputField({\n super.key,\n this.isAttachmentUploading,\n this.onAttachmentPressed,\n required this.onSendPressed,\n this.db,\n this.options = const InputOptions(),\n });\n\n /// Whether attachment is uploading. Will replace attachment button with a\n /// [CircularProgressIndicator]. Since we don't have libraries for\n /// managing media in dependencies we have no way of knowing if\n /// something is uploading so you need to set this manually.\n final bool? isAttachmentUploading;\n final FirebaseFirestore? db;\n\n /// See [AttachmentButton.onPressed].\n final VoidCallback? onAttachmentPressed;\n\n /// Will be called on [SendButton] tap. Has [types.PartialText] which can\n /// be transformed to [types.TextMessage] and added to the messages list.\n final void Function(types.PartialText) onSendPressed;\n\n /// Customisation options for the [Input].\n final InputOptions options;\n\n @override\n State createState() => _InputState();\n}\n\n/// [Input] widget state.\nclass _InputState extends State {\n List suggestionsList = [];\n late final _inputFocusNode = FocusNode(\n onKeyEvent: (node, event) {\n if (event.physicalKey == PhysicalKeyboardKey.enter &&\n !HardwareKeyboard.instance.physicalKeysPressed.any(\n (el) => {\n PhysicalKeyboardKey.shiftLeft,\n PhysicalKeyboardKey.shiftRight,\n }.contains(el),\n )) {\n if (kIsWeb && _textController.value.isComposingRangeValid) {\n return KeyEventResult.ignored;\n }\n if (event is KeyDownEvent) {\n _handleSendPressed();\n }\n return KeyEventResult.handled;\n } else {\n return KeyEventResult.ignored;\n }\n },\n );\n\n bool _sendButtonVisible = false;\n late TextEditingController _textController;\n\n @override\n void initState() {\n super.initState();\n\n _textController =\n widget.options.textEditingController ?? InputTextFieldController();\n _handleSendButtonVisibilityModeChange();\n }\n\n void _handleSendButtonVisibilityModeChange() {\n _textController.removeListener(_handleTextControllerChange);\n if (widget.options.sendButtonVisibilityMode ==\n SendButtonVisibilityMode.hidden) {\n _sendButtonVisible = false;\n } else if (widget.options.sendButtonVisibilityMode ==\n SendButtonVisibilityMode.editing) {\n _sendButtonVisible = _textController.text.trim() != '';\n _textController.addListener(_handleTextControllerChange);\n } else {\n _sendButtonVisible = true;\n }\n }\n\n void _handleSendPressed() {\n print(\n \"CustomInputField: build() : _inputBuilder() : TypeAheadField : _handleSendPressed()\");\n final trimmedText = _textController.text.trim();\n if (trimmedText != '') {\n final partialText = types.PartialText(text: trimmedText);\n widget.onSendPressed(partialText);\n\n if (widget.options.inputClearMode == InputClearMode.always) {\n _textController.clear();\n }\n }\n }\n\n void _handleTextControllerChange() {\n if (_textController.value.isComposingRangeValid) {\n return;\n }\n setState(() {\n _sendButtonVisible = _textController.text.trim() != '';\n });\n }\n\n Widget _inputBuilder() {\n final query = MediaQuery.of(context);\n final buttonPadding = InheritedChatTheme.of(context)\n .theme\n .inputPadding\n .copyWith(left: 16, right: 16);\n final safeAreaInsets = isMobile\n ? EdgeInsets.fromLTRB(\n query.padding.left,\n 0,\n query.padding.right,\n query.viewInsets.bottom + query.padding.bottom,\n )\n : EdgeInsets.zero;\n final textPadding = InheritedChatTheme.of(context)\n .theme\n .inputPadding\n .copyWith(left: 0, right: 0)\n .add(\n EdgeInsets.fromLTRB(\n widget.onAttachmentPressed != null ? 0 : 24,\n 0,\n _sendButtonVisible ? 0 : 24,\n 0,\n ),\n );\n\n return Focus(\n autofocus: !widget.options.autofocus,\n child: Padding(\n padding: InheritedChatTheme.of(context).theme.inputMargin,\n child: Material(\n borderRadius: InheritedChatTheme.of(context).theme.inputBorderRadius,\n color: InheritedChatTheme.of(context).theme.inputBackgroundColor,\n surfaceTintColor:\n InheritedChatTheme.of(context).theme.inputSurfaceTintColor,\n elevation: InheritedChatTheme.of(context).theme.inputElevation,\n child: Container(\n decoration: BoxDecoration(\n color: Color(0xFFF0F2F6), // Background color\n ),\n //InheritedChatTheme.of(context).theme.inputContainerDecoration,\n padding: safeAreaInsets,\n child: Row(\n textDirection: TextDirection.ltr,\n children: [\n /*if (widget.onAttachmentPressed != null)\n AttachmentButton(\n isLoading: widget.isAttachmentUploading ?? false,\n onPressed: widget.onAttachmentPressed,\n padding: buttonPadding,\n ),*/\n Container(width: 30),\n Expanded(\n child: Padding(\n padding: textPadding,\n child: FutureBuilder(\n future: getAllquestions(),\n builder: (context, snapshot) {\n if(snapshot.hasData) {\n suggestionsList = snapshot.data!;\n print(\n \"CustomInputField: build() : _inputBuilder() : suggestionList.length = ${suggestionsList.length}\");\n }\n return TypeAheadField(\n hideOnEmpty: true,\n controller: _textController,\n direction: VerticalDirection.up,\n loadingBuilder: (context) =>\n const Text('Loading...'),\n onSelected: (entry) {\n\n _textController.text = entry.suggestion!;\n entry.scenarioNumber;\n BlocProvider.of(context)\n .generateNewSuggestions(\n entry.scenarioNumber!, entry.suggestion!,\n isACannedQuestion: false,\n userGrouping: entry.userGrouping\n );\n },\n itemBuilder: (context, entry) {\n print(\n \"CustomInputField: build() : TypeAheadField : _inputBuilder(): itemBuilder: entry = $entry\");\n return ListTile(\n title: Text(entry.suggestion!),\n );\n },\n suggestionsCallback:\n _textController.text.length <= 1\n ? suggestionsEmptyCallback\n : suggestionsCallback,\n builder:\n (context, _textController, inputFocusNode) {\n print(\n \"CustomInputField: build() : TypeAheadField : _inputBuilder(): builder: focusNode = $inputFocusNode\");\n return TextField(\n enabled: widget.options.enabled,\n autocorrect: widget.options.autocorrect,\n autofocus: widget.options.autofocus,\n enableSuggestions:\n widget.options.enableSuggestions,\n controller: _textController,\n cursorColor: InheritedChatTheme.of(context)\n .theme\n .inputTextCursorColor,\n decoration: InputDecoration(\n border: OutlineInputBorder(\n borderRadius: BorderRadius.circular(\n 50.0), // Adjust the radius as needed\n ),\n //labelText: 'Password',\n filled: true,\n fillColor: Colors.white,\n suffixIcon: Visibility(\n visible: _sendButtonVisible,\n child: IconButton(\n onPressed: () {\n print(\n \"CustomInputField: build() : _inputBuilder() : TypeAheadField : IconButton : onPressed: ()\");\n _handleSendPressed();\n },\n icon: Icon(Icons.send),\n ),\n ),\n ),\n focusNode: inputFocusNode,\n keyboardType: widget.options.keyboardType,\n maxLines: 5,\n minLines: 1,\n onChanged: widget.options.onTextChanged,\n onTap: widget.options.onTextFieldTap,\n style: TextStyle(color: Colors.black),\n textCapitalization:\n TextCapitalization.sentences,\n );\n },\n );\n },\n )),\n ),\n Container(width: 30),\n ],\n ),\n ),\n ),\n ),\n );\n }\n\n Future loadCfgFromFirestore() async {\n /*db = await FirebaseFirestore.instanceFor(\n app: app, databaseId: 'opendataqna-session-logs');*/\n\n print(\"CustomInputField: loadCfgFromFirestore() : db = $widget.db\");\n\n if (TextToDocParameter.userID.isEmpty) {\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.userID is empty = ${TextToDocParameter.userID}\");\n return;\n }\n\n try {\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.userID = ${TextToDocParameter.userID}\");\n\n DocumentSnapshot doc = await widget.db!\n .collection(\"front_end_flutter_cfg\")\n .doc('${TextToDocParameter.userID}')\n .get();\n\n if (doc != null) {\n final data = doc.data() as Map;\n\n TextToDocParameter.anonymized_data = data[\"anonymized_data\"];\n TextToDocParameter.expert_mode = data[\"expert_mode\"];\n TextToDocParameter.endpoint_opendataqnq = data[\"endpoint_opendataqnq\"];\n TextToDocParameter.firestore_database_id =\n data[\"firestore_database_id\"];\n TextToDocParameter.firebase_app_name = data[\"firebase_app_name\"];\n TextToDocParameter.firestore_history_collection =\n data[\"firestore_history_collection\"];\n TextToDocParameter.firestore_cfg_collection =\n data[\"firestore_cfg_collection\"];\n TextToDocParameter.imported_questions = data[\"imported_questions\"];\n\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.anonymized_data = ${TextToDocParameter.anonymized_data}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.expert_mode = ${TextToDocParameter.expert_mode}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.firestore_database_id = ${TextToDocParameter.firestore_database_id}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.endpoint_opendataqnq = ${TextToDocParameter.endpoint_opendataqnq}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.firebase_app_name = ${TextToDocParameter.firebase_app_name}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.firestore_history_collection = ${TextToDocParameter.firestore_history_collection}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.firestore_cfg_collection = ${TextToDocParameter.firestore_cfg_collection}\");\n print(\n \"CustomInputField: loadCfgFromFirestore() : TextToDocParameter.imported_questions = ${TextToDocParameter.imported_questions}\");\n\n } else {\n print(\"CustomInputField: loadCfgFromFirestore() : doc == null\");\n }\n } catch (e) {\n print(\"CustomInputField: loadCfgFromFirestore() : EXCEPTION ON FIRESTORE : e = $e\");\n //https://www.acodeblog.com/post/2022/5/29/flutter-showdialog-without-context-using-the-navigatorkey\n }\n }\n\n Future> getAllquestions() async {\n List resp = [];\n List userGroupingList = [];\n\n print('CustomInputField: getAllquestions() : START');\n\n await loadCfgFromFirestore();\n userGroupingList = await _getUserGrouping();\n print('CustomInputField: getAllquestions() : userGroupingList = ${userGroupingList}');\n\n for (String userGrouping in userGroupingList) {\n var list = await getAllquestionsFromUserGroup(userGrouping);\n resp.addAll((list as List)\n .map((question) =>\n Suggestion(\n suggestion: question, userGrouping: userGrouping!))\n .toList());\n\n print('CustomInputField: getAllquestions() : userGrouping = $userGrouping : resp.length = ${resp.length}');\n }\n\n print('CustomInputField: getAllquestions() : END : resp.length = ${resp.length}');\n return resp;\n }\n\n Future> getAllquestionsFromUserGroup(String userGrouping) async {\n List resp = [];\n String body = \"\";\n\n print('CustomInputField : getAllquestionsFromUserGroup() : START');\n\n //Create the header\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n //Create the body\n body = '''{\n \"user_grouping\": \"$userGrouping\"\n }''';\n\n print('CustomInputField : getAllquestionsFromUserGroup() : body = ' + body);\n\n try {\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/get_known_sql',\n method: \"POST\",\n sendData: body);\n\n print('CustomInputField : getAllquestionsFromUserGroup() : response = ' +\n response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('CustomInputField: getAllquestionsFromUserGroup() : jsonData = $jsonData');\n\n //KnownSQL = [{\"example_user_question\": \"question1\", \"example_generated_sql\": \"sql1\"},\n // {\"example_user_question\": \"question2\", \"example_generated_sql\": \"sql2\"},\n // ...]\n\n var knownSql =\n jsonData[\"KnownSQL\"].replaceAll(RegExp(r'((\\\\n)|(\\\\r))'), '');\n\n print('CustomInputField: getAllquestionsFromUserGroup() : knownSql = $knownSql');\n\n var knownSqlMap = jsonDecode(knownSql);\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"example_user_question\") resp.add(entry.value);\n }\n }\n }\n } catch (e) {\n print('CustomInputField: getAllquestionsFromUserGroup() : EXCEPTION = $e');\n throw Exception('Failed to get questions: $e');\n } finally {\n print('CustomInputField: getAllquestionsFromUserGroup() : END : resp = ${resp}');\n return resp;\n }\n }\n\n Future> _getUserGrouping() async {\n print('CustomInputField : _getUserGrouping() : START');\n List resp = [];\n\n\n Map? _headers = {\n \"Content-Type\": \"application/json\",\n //\"Authorization\": \" Bearer ${client!.credentials.accessToken.toString()}\",\n };\n\n try {\n print(\n 'CustomInputField : _getUserGrouping() : url = ${TextToDocParameter.endpoint_opendataqnq}/available_databases');\n\n var response = await html.HttpRequest.requestCrossOrigin(\n '${TextToDocParameter.endpoint_opendataqnq}/available_databases',\n method: \"GET\");\n\n print('CustomInputField : _getUserGrouping() : response = ' + response.toString());\n\n final jsonData = jsonDecode(response);\n\n if (jsonData != null) {\n print('CustomInputField : _getUserGrouping() : jsonData = $jsonData');\n\n /* Expected response :\n {\n \"Error\": \"\",\n \"KnownDB\": \"[{\\\"table_schema\\\":\\\"imdb-postgres\\\"},{\\\"table_schema\\\":\\\"retail-postgres\\\"}]\",\n \"ResponseCode\": 200\n }*/\n\n var knownSqlMap = jsonDecode(jsonData['KnownDB']);\n\n print('CustomInputField : _getUserGrouping() : knownSqlMap = ${knownSqlMap}');\n\n print(\n 'CustomInputField : _getUserGrouping() : knownSqlMap[0] = ${knownSqlMap[0].toString()}');\n\n for (int i = 0; i < knownSqlMap.length; i++) {\n for (var entry in knownSqlMap[i].entries) {\n print('${entry.key} : ${entry.value}');\n if (entry.key == \"table_schema\") resp.add(entry.value);\n }\n }\n } else {\n resp.add(\"\");\n }\n\n } catch (e) {\n print('CustomInputField : _getUserGrouping() : EXCEPTION = $e');\n throw Exception('Failed to get earning calls question suggestions: $e');\n } finally {\n print('CustomInputField : _getUserGrouping() : resp = $resp');\n return resp;\n }\n }\n\n @override\n void didUpdateWidget(covariant CustomInputField oldWidget) {\n super.didUpdateWidget(oldWidget);\n if (widget.options.sendButtonVisibilityMode !=\n oldWidget.options.sendButtonVisibilityMode) {\n _handleSendButtonVisibilityModeChange();\n }\n }\n\n @override\n void dispose() {\n _inputFocusNode.dispose();\n _textController.dispose();\n super.dispose();\n }\n\n @override\n Widget build(BuildContext context) {\n print(\"CustomInputField: build(): START\");\n return GestureDetector(\n onTap: () => _inputFocusNode.requestFocus(),\n child: _inputBuilder(),\n );\n }\n\n Future> suggestionsCallback(String pattern) async =>\n Future>.delayed(\n Duration(milliseconds: 300),\n () => suggestionsList.where((entry) {\n final nameLower = entry.suggestion!.toLowerCase();\n final patternLower = pattern.toLowerCase();//pattern.toLowerCase().split(' ').join('');\n return nameLower.contains(patternLower);\n }).toList(),\n );\n\n Future> suggestionsEmptyCallback(String pattern) async =>\n Future>.delayed(Duration(milliseconds: 300), () {\n return [];\n /*return [\n Suggestion(\n suggestion: \"looking for suggestions ...\", scenarioNumber: 0)\n ].where((entry) {\n final nameLower = entry.suggestion!.toLowerCase();\n final patternLower = pattern.toLowerCase().split(' ').join('');\n return nameLower.contains(patternLower);\n }).toList()*/\n ;\n });\n}\n\n@immutable\nclass InputOptions {\n const InputOptions({\n this.inputClearMode = InputClearMode.always,\n this.keyboardType = TextInputType.multiline,\n this.onTextChanged,\n this.onTextFieldTap,\n this.sendButtonVisibilityMode = SendButtonVisibilityMode.editing,\n this.textEditingController,\n this.autocorrect = true,\n this.autofocus = false,\n this.enableSuggestions = true,\n this.enabled = true,\n });\n\n /// Controls the [Input] clear behavior. Defaults to [InputClearMode.always].\n final InputClearMode inputClearMode;\n\n /// Controls the [Input] keyboard type. Defaults to [TextInputType.multiline].\n final TextInputType keyboardType;\n\n /// Will be called whenever the text inside [TextField] changes.\n final void Function(String)? onTextChanged;\n\n /// Will be called on [TextField] tap.\n final VoidCallback? onTextFieldTap;\n\n /// Controls the visibility behavior of the [SendButton] based on the\n /// [TextField] state inside the [Input] widget.\n /// Defaults to [SendButtonVisibilityMode.editing].\n final SendButtonVisibilityMode sendButtonVisibilityMode;\n\n /// Custom [TextEditingController]. If not provided, defaults to the\n /// [InputTextFieldController], which extends [TextEditingController] and has\n /// additional fatures like markdown support. If you want to keep additional\n /// features but still need some methods from the default [TextEditingController],\n /// you can create your own [InputTextFieldController] (imported from this lib)\n /// and pass it here.\n final TextEditingController? textEditingController;\n\n /// Controls the [TextInput] autocorrect behavior. Defaults to [true].\n final bool autocorrect;\n\n /// Whether [TextInput] should have focus. Defaults to [false].\n final bool autofocus;\n\n /// Controls the [TextInput] enableSuggestions behavior. Defaults to [true].\n final bool enableSuggestions;\n\n /// Controls the [TextInput] enabled behavior. Defaults to [true].\n final bool enabled;\n}\n\nclass Suggestion {\n final String suggestion;\n final String userGrouping;\n final int? scenarioNumber;\n\n Suggestion({\n required this.suggestion,\n required this.userGrouping,\n this.scenarioNumber = 0,\n });\n}\n" }, { "path": "frontend/frontend-flutter/lib/utils/most_popular_questions.dart", "content": "class MostPopularQ {\n int count;\n String time;\n String question;\n MostPopularQ(this.question,this.count, this.time);\n\n @override\n String toString() {\n return \"{question = $question : count = $count : time = $time}\";\n }\n}" }, { "path": "frontend/frontend-flutter/lib/utils/pdf_viewer.dart", "content": "import 'package:flutter/material.dart';\nimport 'package:flutter/services.dart';\n\n\n//Not a usable class to display PDF files.\n//I keep it though as a stub for future development\n\nclass PdfViewer extends StatefulWidget {\n final List bytes;\n PdfViewer({super.key, required this.bytes});\n @override\n _PdfViewerState createState() => _PdfViewerState();\n}\n\nclass _PdfViewerState extends State {\n\n @override\n initState() {\n print('_PdfViewerState : initState() : START');\n super.initState();\n }\n\n @override\n Widget build(BuildContext context) {\n print('_PdfViewerState : build() : START');\n\n return MaterialApp(\n debugShowCheckedModeBanner: false,\n title: 'PDFViewer',\n theme: ThemeData(\n appBarTheme: AppBarTheme(\n backgroundColor: Colors.white,\n ),\n ),\n home: Scaffold(\n appBar: AppBar(\n leading: IconButton(\n icon: Icon(Icons.arrow_back),\n onPressed: () {\n Navigator.pop(context);\n },\n ),\n title: Text('mySummaryDoc.pdf'),\n actions: [\n IconButton(\n icon: Icon(\n Icons.search,\n color: Colors.black,\n ),\n onPressed: () {\n\n },\n ),\n ],\n ),\n body: Center(\n child: Container(\n child: Text(\"Needs to be impemented\")),\n ),\n ),\n );\n }\n}\n" }, { "path": "frontend/frontend-flutter/lib/utils/stepper_expert_info.dart", "content": "class StepperExpertInfo {\n final String? uri;\n final String? body;\n final String? header;\n final String? response;\n final String? generatedSQLText;\n final List ? answerList;\n final String? knownDB;\n final String? finalNLAnswer;\n final int? statusCode;\n final int? stepDuration;\n final String? graphTitle;\n final String? xAxisTitle;\n final String? yAxisTitle;\n final String? summary;\n\n const StepperExpertInfo({\n this.uri = \"\",\n this.body = '{\"message\" : \"No data\"}',\n this.header = '{\"message\" : \"No data\"}',\n this.response = '{\"message\" : \"No data\"}',\n this.generatedSQLText = \"\",\n this.knownDB = \"\",\n this.finalNLAnswer = \"\",\n this.statusCode = 0,\n this.stepDuration = 0,\n this.graphTitle = \"\",\n this.xAxisTitle = \"\",\n this.yAxisTitle = \"\",\n this.summary = \"\",\n this.answerList = const [\"\"]\n });\n}\n" }, { "path": "frontend/frontend-flutter/lib/utils/tabbed_container.dart", "content": "\nimport 'package:flutter/material.dart';\n\nclass TabbedContainer extends StatefulWidget {\n final List tabs;\n final List tabViews;\n final TabController controller;\n final int initialIndex;\n\n const TabbedContainer({\n super.key,\n required this.tabs,\n required this.tabViews,\n required this.controller,\n required this.initialIndex\n });\n\n @override\n _TabbedContainerState createState() => _TabbedContainerState();\n}\n\nclass _TabbedContainerState extends State\n with SingleTickerProviderStateMixin {\n int _currentIndex = 0;\n\n @override\n void initState() {\n super.initState();\n _currentIndex = widget.initialIndex;\n widget.controller.index = _currentIndex; // Initialize\n /*widget.controller.addListener(() {\n setState(() {\n _currentIndex = widget.controller.index;\n });\n });*/\n }\n\n @override\n Widget build(BuildContext context) {\n return Container(\n height : 500,\n child: Column(\n children: [\n TabBar(\n controller: widget.controller,\n indicatorSize: TabBarIndicatorSize.tab,\n tabs: widget.tabs),\n Flexible(\n fit: FlexFit.loose,\n child: TabBarView(controller: widget.controller, children: widget.tabViews)\n ),\n ],\n ),\n );\n }\n}" }, { "path": "frontend/frontend-flutter/nl2sql_oss.iml", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "frontend/frontend-flutter/pubspec.yaml", "content": "name: ttmd\ndescription: \"A new Flutter project.\"\n# The following line prevents the package from being accidentally published to\n# pub.dev using `flutter pub publish`. This is preferred for private packages.\npublish_to: 'none' # Remove this line if you wish to publish to pub.dev\n\n# The following defines the version and build number for your application.\n# A version number is three numbers separated by dots, like 1.2.43\n# followed by an optional build number separated by a +.\n# Both the version and the builder number may be overridden in flutter\n# build by specifying --build-name and --build-number, respectively.\n# In Android, build-name is used as versionName while build-number used as versionCode.\n# Read more about Android versioning at https://developer.android.com/studio/publish/versioning\n# In iOS, build-name is used as CFBundleShortVersionString while build-number is used as CFBundleVersion.\n# Read more about iOS versioning at\n# https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html\n# In Windows, build-name is used as the major, minor, and patch parts\n# of the product and file versions while build-number is used as the build suffix.\nversion: 1.0.0+1\n\nenvironment:\n sdk: '>=3.2.3 <4.0.0'\n\n# Dependencies specify other packages that your package needs in order to work.\n# To automatically upgrade your package dependencies to the latest versions\n# consider running `flutter pub upgrade --major-versions`. Alternatively,\n# dependencies can be manually updated by changing the version numbers below to\n# the latest version available on pub.dev. To see which dependencies have newer\n# versions available, run `flutter pub outdated`.\ndependencies:\n flutter:\n sdk: flutter\n\n\n # The following adds the Cupertino Icons font to your application.\n # Use with the CupertinoIcons class for iOS style icons.\n cupertino_icons: ^1.0.2\n googleapis: ^13.1.0\n googleapis_auth: ^1.5.0\n path_provider: ^2.1.2\n font_awesome_flutter: ^10.7.0\n url_launcher: ^6.2.5\n deep_pick: ^1.0.0\n flutter_chat_ui: ^1.6.12\n flutter_chat_types: ^3.6.2\n image_picker: ^1.0.7\n file_picker: ^8.0.5\n bubble: ^1.2.1\n chatview: ^1.3.1\n simple_gradient_text: ^1.3.0\n flutter_bloc: ^8.1.4\n equatable: ^2.0.5\n easy_sidemenu: ^0.6.0\n expandable_tree_menu: ^0.1.0-dev.5\n intl: ^0.18.1\n badges: ^3.1.2\n expansion_tile_card: ^3.0.0\n flutter_json_viewer: ^1.0.1\n flutter_login: ^5.0.0\n screenshot: ^3.0.0\n flutter_settings_ui: ^3.0.1\n flutter_typeahead: ^5.2.0\n simple_http_api: ^1.1.1\n firebase_auth: ^5.1.3\n firebase_core: ^3.3.0\n js: ^0.7.1\n csv: ^6.0.0\n image_picker_web: ^4.0.0\n flutter_inappwebview: ^6.0.0\n cloud_firestore: ^5.4.0\n\ndependency_overrides:\n libphonenumber_plugin: ^0.3.3\n libphonenumber_web: ^0.3.2\n js: ^0.7.1\n\ndev_dependencies:\n flutter_test:\n sdk: flutter\n\n # The \"flutter_lints\" package below contains a set of recommended lints to\n # encourage good coding practices. The lint set provided by the package is\n # activated in the `analysis_options.yaml` file located at the root of your\n # package. See that file for information about deactivating specific lint\n # rules and activating additional ones.\n flutter_lints: ^4.0.0\n\n# For information on the generic Dart part of this file, see the\n# following page: https://dart.dev/tools/pub/pubspec\n\n# The following section is specific to Flutter packages.\nflutter:\n\n # The following line ensures that the Material Icons font is\n # included with your application, so that you can use the icons in\n # the material Icons class.\n uses-material-design: true\n\n # To add assets to your application, add an assets section, like this:\n assets:\n - assets/images/\n # - images/a_dot_burr.jpeg\n # - images/a_dot_ham.jpeg\n\n # An image asset can refer to one or more resolution-specific \"variants\", see\n # https://flutter.dev/assets-and-images/#resolution-aware\n\n # For details regarding adding assets from package dependencies, see\n # https://flutter.dev/assets-and-images/#from-packages\n\n # To add custom fonts to your application, add a fonts section here,\n # in this \"flutter\" section. Each entry in this list should have a\n # \"family\" key with the font family name, and a \"fonts\" key with a\n # list giving the asset and other descriptors for the font. For\n # example:\n # fonts:\n # - family: Schyler\n # fonts:\n # - asset: fonts/Schyler-Regular.ttf\n # - asset: fonts/Schyler-Italic.ttf\n # style: italic\n # - family: Trajan Pro\n # fonts:\n # - asset: fonts/TrajanPro.ttf\n # - asset: fonts/TrajanPro_Bold.ttf\n # weight: 700\n #\n # For details regarding fonts from package dependencies,\n # see https://flutter.dev/custom-fonts/#from-packages\n" }, { "path": "frontend/frontend-flutter/test/widget_test.dart", "content": "// This is a basic Flutter widget test.\n//\n// To perform an interaction with a widget in your test, use the WidgetTester\n// utility in the flutter_test package. For example, you can send tap and scroll\n// gestures. You can also use WidgetTester to find child widgets in the widget\n// tree, read text, and verify that the values of widget properties are correct.\n\nimport 'package:flutter/material.dart';\nimport 'package:flutter_test/flutter_test.dart';\n\nimport 'package:ttmd/main.dart';\n\nvoid main() {\n testWidgets('Counter increments smoke test', (WidgetTester tester) async {\n // Build our app and trigger a frame.\n await tester.pumpWidget(const MyApp());\n\n // Verify that our counter starts at 0.\n expect(find.text('0'), findsOneWidget);\n expect(find.text('1'), findsNothing);\n\n // Tap the '+' icon and trigger a frame.\n await tester.tap(find.byIcon(Icons.add));\n await tester.pump();\n\n // Verify that our counter has incremented.\n expect(find.text('0'), findsNothing);\n expect(find.text('1'), findsOneWidget);\n });\n}\n" }, { "path": "frontend/frontend-flutter/web/index 01.49.28.html", "content": "\n\n\n \n \n\n \n \n \n\n \n \n \n \n \n\n \n \n\n ttmd\n \n\n \n \n \n \n\n \n \n\n\n\n\n\n\n\n" }, { "path": "frontend/frontend-flutter/web/index.html", "content": "\n\n\n \n \n\n \n \n \n\n \n \n \n \n \n\n \n \n\n frontend_flutter\n \n \n\n \n\n\n \n\n\n" }, { "path": "frontend/frontend-flutter/web/manifest.json", "content": "{\n \"name\": \"frontend_flutter\",\n \"short_name\": \"frontend_flutter\",\n \"start_url\": \".\",\n \"display\": \"standalone\",\n \"background_color\": \"#0175C2\",\n \"theme_color\": \"#0175C2\",\n \"description\": \"A new Flutter project.\",\n \"orientation\": \"portrait-primary\",\n \"prefer_related_applications\": false,\n \"icons\": [\n {\n \"src\": \"icons/Icon-192.png\",\n \"sizes\": \"192x192\",\n \"type\": \"image/png\"\n },\n {\n \"src\": \"icons/Icon-512.png\",\n \"sizes\": \"512x512\",\n \"type\": \"image/png\"\n },\n {\n \"src\": \"icons/Icon-maskable-192.png\",\n \"sizes\": \"192x192\",\n \"type\": \"image/png\",\n \"purpose\": \"maskable\"\n },\n {\n \"src\": \"icons/Icon-maskable-512.png\",\n \"sizes\": \"512x512\",\n \"type\": \"image/png\",\n \"purpose\": \"maskable\"\n }\n ]\n}\n" }, { "path": "frontend/frontend.yaml", "content": "steps:\n - name: 'node:21'\n args:\n - install\n - '--legacy-peer-deps'\n dir: frontend\n entrypoint: npm\n - name: 'node:21'\n args:\n - install\n - '--legacy-peer-deps'\n - '@popperjs/core'\n dir: frontend\n entrypoint: npm\n - name: 'node:21'\n dir: frontend\n script: |\n #!/usr/bin/env bash\n npm run build || true\n - name: gcr.io/google.com/cloudsdktool/cloud-sdk\n dir: frontend\n entrypoint: gsutil\n - name: 'gcr.io/${_FIREBASE_PROJECT_ID}/firebase'\n args:\n - use\n - '${_FIREBASE_PROJECT_ID}'\n dir: frontend\n - name: 'gcr.io/${_FIREBASE_PROJECT_ID}/firebase'\n args:\n - deploy\n - '--only'\n - hosting\n dir: frontend\n - name: 'gcr.io/${_FIREBASE_PROJECT_ID}/firebase'\n args:\n - deploy\n - '--only'\n - firestore\n dir: frontend\ntimeout: 1200s\n" }, { "path": "frontend/package.json", "content": "{\n \"name\": \"frontend\",\n \"version\": \"0.0.0\",\n \"scripts\": {\n \"ng\": \"ng\",\n \"start\": \"ng serve\",\n \"build\": \"ng build\",\n \"watch\": \"ng build --watch --configuration development\",\n \"test\": \"ng test\",\n \"serve:ssr:frontend\": \"node dist/frontend/server/server.mjs\"\n },\n \"private\": true,\n \"dependencies\": {\n \"@angular/animations\": \"^17.0.0\",\n \"@angular/cdk\": \"^17.0.5\",\n \"@angular/common\": \"^17.0.0\",\n \"@angular/compiler\": \"^17.0.0\",\n \"@angular/core\": \"^17.0.0\",\n \"@angular/fire\": \"^17.0.1\",\n \"@angular/forms\": \"^17.0.0\",\n \"@angular/material\": \"^17.0.5\",\n \"@angular/platform-browser\": \"^17.0.0\",\n \"@angular/platform-browser-dynamic\": \"^17.0.0\",\n \"@angular/platform-server\": \"^17.0.0\",\n \"@angular/router\": \"^17.0.0\",\n \"@angular/ssr\": \"^17.0.9\",\n \"angular-google-charts\": \"^2.2.3\",\n \"bootstrap\": \"^5.3.2\",\n \"chart.js\": \"^4.4.1\",\n \"deps\": \"^1.0.0\",\n \"express\": \"^4.21.1\",\n \"firebase\": \"^10.12.2\",\n \"ng2-charts\": \"^5.0.4\",\n \"prismjs\": \"^1.29.0\",\n \"rxjs\": \"~7.8.0\",\n \"sql-formatter\": \"^15.2.0\",\n \"tslib\": \"^2.3.0\",\n \"zone.js\": \"~0.14.2\"\n },\n \"devDependencies\": {\n \"@angular-devkit/build-angular\": \"^17.0.10\",\n \"@angular/cli\": \"^17.0.9\",\n \"@angular/compiler-cli\": \"^17.0.0\",\n \"@types/express\": \"^4.17.17\",\n \"@types/jasmine\": \"~5.1.0\",\n \"@types/node\": \"^18.18.0\",\n \"jasmine-core\": \"~5.1.0\",\n \"karma\": \"~6.4.0\",\n \"karma-chrome-launcher\": \"~3.2.0\",\n \"karma-coverage\": \"~2.2.0\",\n \"karma-jasmine\": \"~5.1.0\",\n \"karma-jasmine-html-reporter\": \"~2.1.0\",\n \"typescript\": \"~5.2.2\"\n }\n}\n" }, { "path": "frontend/server.ts", "content": "import { APP_BASE_HREF } from '@angular/common';\nimport { CommonEngine } from '@angular/ssr';\nimport express from 'express';\nimport { fileURLToPath } from 'node:url';\nimport { dirname, join, resolve } from 'node:path';\nimport bootstrap from './src/main.server';\n\n// The Express app is exported so that it can be used by serverless Functions.\nexport function app(): express.Express {\n const server = express();\n const serverDistFolder = dirname(fileURLToPath(import.meta.url));\n const browserDistFolder = resolve(serverDistFolder, '../browser');\n const indexHtml = join(serverDistFolder, 'index.server.html');\n\n const commonEngine = new CommonEngine();\n\n server.set('view engine', 'html');\n server.set('views', browserDistFolder);\n\n // Example Express Rest API endpoints\n // server.get('/api/**', (req, res) => { });\n // Serve static files from /browser\n server.get('*.*', express.static(browserDistFolder, {\n maxAge: '1y'\n }));\n\n // All regular routes use the Angular engine\n server.get('*', (req, res, next) => {\n const { protocol, originalUrl, baseUrl, headers } = req;\n\n commonEngine\n .render({\n bootstrap,\n documentFilePath: indexHtml,\n url: `${protocol}://${headers.host}${originalUrl}`,\n publicPath: browserDistFolder,\n providers: [{ provide: APP_BASE_HREF, useValue: baseUrl }],\n })\n .then((html) => res.send(html))\n .catch((err) => next(err));\n });\n\n return server;\n}\n\nfunction run(): void {\n const port = process.env['PORT'] || 4000;\n\n // Start up the Node server\n const server = app();\n server.listen(port, () => {\n console.log(`Node Express server listening on http://localhost:${port}`);\n });\n}\n\nrun();\n" }, { "path": "frontend/src/app/agent-chat/agent-chat.component.html", "content": "\n \n \n adjust\n \n \n \n
Question Sent
\n \n \n \n \n
Converted to SQL
\n
{{msg?.generate_sql?.responseTime}}
\n \n \n \n \n
Data retrieved
\n
{{msg.run_query?.responseTime}}
\n \n \n \n \n
Visualization generated
\n
{{msg.visualize?.responseTime}}
\n \n \n \n
\n
\n {{msg.generate_sql.Error}}\n
\n
\n \n \n \n
\n
\n \n \n
\n
\n \n \n
\n \n
\n {{msg.run_query?.NaturalResponse}}\n
\n
\n \n \n Table View\n \n
\n \n\n \n \n \n \n\n \n \n
{{col | uppercase}} {{element[col]}}
\n
\n \n\n
\n\n
\n
\n
\n
\n
\n
\n
\n \n
\n \n
\n
\n
{{emptyMsg}}\n
\n
\n \n \n
\n
\n
\n

Source

\n {{dataSetName}}\n
\n\n
\n

Owner

\n OpenData QnA\n
\n
\n

Project ID

\n Demo Project\n
\n
\n

Grouping

\n {{dataSet}}\n
\n
\n
\n \n \n \n\n\n
\n
\n

Is this what you were looking for?

\n
\n\n
\n \"up\"\n\n \n
\n \n \n
\n \n Submit\n
\n
\n \n\n \"down\"\n\n
\n
\n
\n
\n" }, { "path": "frontend/src/app/agent-chat/agent-chat.component.scss", "content": " input {\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n }\n\n mat-icon {\n color: blue;\n }\n\n table {\n width: 100%;\n }\n\n @media (max-width: 768px) {\n\n /* I need the code here */\n .summarize-results {\n flex: 0 50%;\n }\n }\n\n .btnSave {\n border-color: transparent;\n min-width: 146px;\n height: 41px;\n background: #1A73E8;\n border-radius: 100px;\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n display: flex;\n align-items: center;\n text-align: center;\n justify-content: center;\n letter-spacing: 0.25px;\n color: #FFFFFF;\n }\n\n .example-container mat-form-field+mat-form-field {\n margin-left: 8px;\n }\n\n .img-align {\n display: flex;\n width: 100%;\n flex-wrap: wrap;\n justify-content: center;\n }\n\n mat-table {\n border: 1px solid #D5D5D5;\n border-radius: 10px;\n }\n\n mat-cell,\n #dataSources mat-header-cell {\n border: none !important;\n }\n\n .query-ref {\n padding: 20px;\n font-size: 14px;\n font-family: Google Sans;\n color: #575757;\n }\n\n .mat-tab-body-content {\n overflow: hidden !important\n }\n\n .spinner-loading-result-table {\n margin: 10px;\n }\n\n ::ng-deep .custom-style {\n background-color: brown;\n color: white;\n border-radius: 10px;\n }\n\n .cursor-ref {\n cursor: pointer;\n }\n\n .h1-ref {\n font-size: 35px !important;\n font-family: Google Sans;\n font-weight: 600;\n text-align: center;\n margin-bottom: 50px;\n }\n\n .container-fluid {\n display: flex;\n flex-direction: column;\n }\n\n .wrap-background {\n background-color: #F3F6FC;\n border-radius: 25px;\n }\n\n .chart_prep {\n align-items: center;\n justify-content: center;\n display: flex;\n flex-direction: column;\n gap: 15px\n }\n\n .feedback-label {\n color: #525252;\n font-family: \"Google Sans\";\n font-size: 16px;\n font-style: normal;\n font-weight: 500;\n line-height: 20px;\n letter-spacing: -0.154px;\n }\n\n .datasource-prep {\n width: 425px;\n height: 250px;\n border-radius: 20px;\n border: 1px solid #cdc8c8;\n display: flex;\n flex-direction: row;\n padding: 20px;\n flex-wrap: wrap;\n background-color: #FFF;\n }\n\n .right-text {\n width: 50%;\n font-family: Google Sans;\n font-size: 14px;\n font-style: normal;\n font-weight: 400;\n letter-spacing: 0.10000000149011612px;\n text-align: left;\n }\n\n .left-text {\n width: 50%;\n font-family: Google Sans;\n font-size: 14px;\n font-weight: 700;\n font-style: normal;\n letter-spacing: 0.5px;\n text-align: left;\n }\n\n .btns {\n display: flex;\n flex-direction: row;\n }\n\n @media (max-width: 768px) {\n .btns {\n flex-direction: column;\n }\n\n .btns .share {\n width: 20%;\n }\n\n .datasource-prep {\n width: auto;\n }\n\n .result-content {\n flex-direction: column;\n }\n }\n\n .copyBtn {\n border: none;\n float: right;\n background: transparent;\n }\n\n .table-responsive {\n display: block;\n width: 100%;\n overflow-x: auto;\n }\n\n .mat-table {\n width: 100%;\n max-width: 100%;\n margin-bottom: 1rem;\n display: table;\n border-collapse: collapse;\n margin: 0px;\n }\n\n .mat-row,\n .mat-header-row {\n display: table-row;\n }\n\n .mat-cell,\n .mat-header-cell {\n word-wrap: initial;\n display: table-cell;\n padding: 0px 5px;\n line-break: unset;\n width: auto;\n white-space: nowrap;\n overflow: hidden;\n vertical-align: middle;\n }\n\n table {\n border-radius: 25px;\n }\n\n table td,\n table th {\n padding: 2px 10px;\n font-size: 14px;\n font-family: Google Sans;\n }\n\n .result-content {\n display: flex;\n flex-direction: row;\n align-items: center;\n gap: 20px;\n overflow: auto;\n }\n\n .table-prep {\n border-radius: 20px;\n border: 1px solid #cdc8c8;\n padding-bottom: 20px;\n padding-top: 20px;\n margin-bottom: 15px;\n }\n\n .example-list {\n width: 100px;\n border: solid 1px #cdc8c8;\n border-radius: 5px;\n background: #fff;\n text-align: center;\n padding: 10px;\n margin: 0;\n }\n\n .feedback-ref {\n color: #8AB4F8;\n font-size: 14px;\n font-family: Google Sans;\n font-weight: 400;\n background-color: #FFF;\n border: 1px solid #ccc;\n padding: 3px;\n border-radius: 10px;\n margin: 2px;\n cursor: pointer;\n }\n\n .feedback-ref-clicked {\n color: #FFF;\n font-size: 14px;\n font-family: Google Sans;\n font-weight: 400;\n background-color: #1A73E8;\n border: 1px solid #ccc;\n padding: 3px;\n border-radius: 10px;\n margin: 2px;\n cursor: pointer;\n }\n\n .feedback-ref a {\n text-decoration: none;\n }\n\n .feedback-grad {\n background-image: radial-gradient(#AACAFF, #E7E7E7);\n border: 1px solid #ccc;\n }\n\n .feedback-button {\n text-decoration: none;\n font-family: Google Sans;\n font-size: 14px;\n border-radius: 12px;\n background: #5D93FF;\n color: #fff;\n width: 78px;\n height: 28px;\n margin-top: 10px;\n display: flex;\n justify-content: center;\n align-items: center;\n }\n\n .cdk-overlay-pane:has(.mat-mdc-select-panel) {\n width: auto !important;\n }\n\n .g-container {\n position: relative;\n }\n\n //-----chat------\\\\\n\n .msg {\n border-radius: 10px;\n }\n\n .msg.right {\n background-color: #CCC;\n float: right;\n }\n\n .msg.left {\n background-color: #555;\n color: white;\n }\n\n .flex-expand {\n flex: 1 1 auto;\n }\n\n #messages {\n display: flex;\n flex-direction: column;\n overflow-y: auto;\n overflow-x: hidden;\n margin-top: 0;\n margin-bottom: 0;\n padding-left: 0 !important;\n list-style: none;\n }\n\n #messages li {\n list-style: none;\n margin-top: 1rem;\n }\n\n #messages img {\n width: 43px;\n height: auto;\n }\n\n .received {\n display: block;\n text-align: right;\n position: relative;\n }\n\n .user-feedback {\n width: 257px !important;\n height: 93px !important;\n border-radius: 16px;\n background: #FFF;\n margin-bottom: 0 !important;\n margin-left: 15px !important;\n }\n\n .user-feedback-title {\n color: #1F1F1F !important;\n font-family: \"Google Sans\";\n font-size: 14px !important;\n font-style: normal;\n font-weight: 400 !important;\n line-height: 20px;\n margin-top: 20px !important;\n }\n\n mat-stepper {\n width: 824px;\n height: 128px;\n margin: 5px 20px 20px 15px;\n border-radius: 20px;\n }\n\n .sqlError {\n border-radius: 26px;\n background: #FFF;\n margin-bottom: 10px;\n width: 830px;\n color: #f78b8b !important;\n font-family: \"Google Sans\";\n font-size: 14px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n margin-left: 20px;\n padding: 15px;\n min-height: 100px;\n display: flex;\n align-items: center;\n }\n\n .agentTab {\n margin-left: 15px;\n width: 830px;\n min-height: 300px;\n max-height: 600px;\n background: rgb(255, 255, 255);\n border-radius: 25px;\n align-items: center;\n overflow: auto;\n --mat-tab-animation-duration: 500ms;\n }\n\n mat-expansion-panel {\n box-shadow: none !important;\n }" }, { "path": "frontend/src/app/agent-chat/agent-chat.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { AgentChatComponent } from './agent-chat.component';\n\ndescribe('AgentChatComponent', () => {\n let component: AgentChatComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [AgentChatComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(AgentChatComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/agent-chat/agent-chat.component.ts", "content": "import { AfterViewInit, ChangeDetectorRef, Component, ElementRef, Input, ViewChild, signal } from '@angular/core';\nimport { HomeService } from '../shared/services/home.service';\nimport { format } from 'sql-formatter';\nimport { MatSnackBar } from '@angular/material/snack-bar';\nimport { FormBuilder, FormGroup } from '@angular/forms';\nimport { Subject, takeUntil } from 'rxjs';\nimport { ChatService } from '../shared/services/chat.service';\n\n@Component({\n selector: 'app-agent-chat',\n templateUrl: './agent-chat.component.html',\n styleUrl: './agent-chat.component.scss'\n})\nexport class AgentChatComponent implements AfterViewInit {\n msg: any;\n @Input('example_user_question') example_user_question: any;\n @Input('ind') ind: any;\n emptyMsg: any = '';\n @Input('suggestionList') suggestionList: any;\n showResult: boolean = false;\n private _destroy$ = new Subject();\n showChart: boolean = false;\n showLoader: boolean = false;\n isOpen: boolean = false;\n selectedFeedbackOption: any;\n result: any;\n dataSet!: string;\n dataSetName!: string;\n displayedColumns: string[] = [];\n dataSource: any[] = [];\n feedbackForm: FormGroup = this.formBuilder.group({\n feedbackCtrl: [''],\n });\n resultLoader: boolean = false;\n @ViewChild(\"feedback\")\n feedbackElement!: ElementRef;\n readonly panelOpenState = signal(false);\n\n constructor(public homeService: HomeService, private snackBar: MatSnackBar, private formBuilder: FormBuilder, public chatService: ChatService, private cdref: ChangeDetectorRef) { }\n ngOnInit() {\n // added this comment for test PR from frontend_dev\n this.homeService.knownSqlObservable?.pipe(takeUntil(this._destroy$)).subscribe((response: any) => {\n if (response && response != null) {\n this.suggestionList = JSON.parse(response);\n }\n })\n this.dataSet = this.homeService.getSelectedDbGrouping();\n this.dataSetName = this.homeService.getselectedDbName();\n this.chatService.chatSessionObservable.pipe(takeUntil(this._destroy$)).subscribe((res) => {\n this.homeService.updateChatMsgs(res.chatMsgs)\n // this.msg = res.chatMsgs.at(this.ind)\n })\n // this.msg = this.homeService.getChatMsgs().at(this.ind);\n this.showResult = true;\n\n }\n ngAfterContentChecked() {\n this.cdref.detectChanges();\n }\n\n ngAfterViewInit() {\n this.msg = this.homeService.getChatMsgs().at(this.ind);\n this.feedbackElement?.nativeElement.scrollIntoView({ behavior: \"smooth\", block: \"start\" });\n this.cdref.detectChanges();\n }\n getResultforSql() {\n this.resultLoader = true;\n // Subscribe to the response data observable\n this.homeService.runQuery(this.msg?.generate_sql.GeneratedSQL, this.homeService.getSelectedDbGrouping(), this.msg?.user_question, this.homeService.getSessionId())\n .pipe(takeUntil(this._destroy$)).subscribe((res: any) => {\n const data = JSON.parse(res.KnownDB);\n if (res && res.ResponseCode === 200) {\n if (data.length === 0) {\n this.emptyMsg = 'No data found';\n } else {\n this.emptyMsg = '';\n for (var obj in data) {\n if (data.hasOwnProperty(obj)) {\n for (var prop in data[obj]) {\n if (data[obj].hasOwnProperty(prop)) {\n if (this.displayedColumns.indexOf(prop) === -1) {\n this.displayedColumns.push(prop);\n }\n }\n }\n }\n }\n // console.log(this.ind)\n this.updateLocalMessage(this.ind, res, data);\n this.dataSource = data;\n }\n } else {\n this.updateLocalMessage(this.ind, res, data);\n this.emptyMsg = res?.Error;\n }\n this.resultLoader = false;\n });\n }\n\n async tabClick(event: any, displayedColumns: any) {\n const tab = event?.tab?.textLabel;\n switch (tab) {\n case \"Generated SQL\": break;\n case \"Result\": if (!displayedColumns) { this.getResultforSql() }; break;\n case \"Data Sources\": break;\n }\n }\n visualizeBtn(msg: any, ind: any) {\n this.showLoader = true;\n let sql = format(msg.generate_sql.GeneratedSQL, { language: 'mysql' })\n this.homeService.generateViz(this.msg.user_question, sql, msg.dataSource, this.homeService.getSessionId()).subscribe((res: any) => {\n const object = res.GeneratedChartjs;\n this.msg = {\n ...this.msg, \"visualize\": res\n }\n for (const [key, value] of Object.entries(object)) {\n let newvalue: string = (value as string).replace('chart_div', ind + '-chart_div');\n this.onChange(newvalue, ind, key);\n }\n })\n }\n\n onChange(value: any, ind: any, key: any) {\n this.showChart = true;\n this.showLoader = false;\n this.result = eval(value)\n }\n\n thumbsUp(sql: any, ind: any) {\n let chats = this.homeService.getChatMsgs();\n const sqlExist = this.suggestionList.some((res: { example_user_question: any; example_generated_sql: any; }) => res.example_user_question === chats[ind]?.user_question && res.example_generated_sql === sql);\n if (!sqlExist) {\n // let concatedUserQuestionsList = chats.map((msg: any) => {\n // if (msg.author == 'user') {\n // return msg.user_question\n // }\n // })\n // const concatenatedStr = concatedUserQuestionsList.filter((word) => word).reduce((accumulator, currentValue) => accumulator + ' , ' + currentValue);\n this.homeService.thumbsUp(sql, chats[ind]?.user_question, this.homeService.getSelectedDbGrouping(), this.homeService.getSessionId()).subscribe((res: any) => {\n if (res && res.ResponseCode === 201) {\n this.updateLocalMessage(ind, res, \"\");\n this.showSnackbarCssStyles(res?.Message, 'Close', '10000')\n this.isOpen = true;\n } else {\n this.updateLocalMessage(ind, res, \"\");\n this.showSnackbarCssStyles(res?.Error, 'Close', '10000')\n }\n })\n } else {\n this.showSnackbarCssStyles('Data is present in the suggestion list', 'Close', '4000')\n }\n }\n\n updateLocalMessage(ind: any, res: any, data: any) {\n\n let localChatMsgs = this.msg;\n // Update message in case of result tab\n if (data) {\n localChatMsgs =\n {\n ...localChatMsgs, 'dataSource': data,\n 'displayedColumns': this.displayedColumns,\n 'dataSet': this.dataSet,\n 'run_query': res,\n 'ind': ind\n };\n }\n // Update message in case of thumbsup\n else {\n localChatMsgs = {\n ...localChatMsgs,\n \"embed_sql\": res,\n 'ind': ind\n };\n }\n this.msg = localChatMsgs;\n //this.homeService.updateChatMsgsAtIndex(localChatMsgs, this.ind);\n }\n\n showSnackbarCssStyles(content: any, action: any, duration: any) {\n let sb = this.snackBar.open(content, action, {\n duration: duration,\n panelClass: [\"custom-style\"]\n });\n sb.onAction().subscribe(() => {\n sb.dismiss();\n });\n }\n\n showContentCopiedMsg() {\n this.showSnackbarCssStyles(\"Content Copied\", 'Close', '4000')\n }\n\n closeFeedback() {\n this.isOpen = false;\n }\n\n thumbsDown() {\n this.isOpen = true;\n }\n\n submitFeedback(ind: any, comment: any) {\n this.isOpen = false;\n\n this.msg = {\n ...this.msg, \"feedback\": {\n \"option\": this.selectedFeedbackOption,\n \"comment\": comment\n }\n }\n }\n\n feedbackOption(val: any) {\n if (val == 0) {\n this.selectedFeedbackOption = 'Correct answer';\n } else if (val == 1) {\n this.selectedFeedbackOption = 'Easy to understand'\n } else {\n this.selectedFeedbackOption = 'Quick results'\n }\n }\n\n ngOnDestroy() {\n this._destroy$.next();\n }\n}\n" }, { "path": "frontend/src/app/app-routing.module.ts", "content": "import { NgModule } from '@angular/core';\nimport { RouterModule, Routes } from '@angular/router';\nimport { LoginComponent } from './login/login.component';\nimport { UserJourneyComponent } from './user-journey/user-journey.component';\nimport { HomeComponent } from './home/home.component';\nimport { BusinessUserComponent } from './business-user/business-user.component';\n\nconst routes: Routes = [\n { path: '', component: LoginComponent },\n { path: 'user-journey', component: UserJourneyComponent },\n { path: 'home-page', component: HomeComponent },\n { path: 'business-mode', component: BusinessUserComponent },\n]\n\n@NgModule({\n imports: [RouterModule.forRoot(routes)],\n exports: [RouterModule]\n})\nexport class AppRoutingModule { }\n" }, { "path": "frontend/src/app/app.component.html", "content": "\n" }, { "path": "frontend/src/app/app.component.scss", "content": "" }, { "path": "frontend/src/app/app.component.spec.ts", "content": "import { TestBed } from '@angular/core/testing';\nimport { AppComponent } from './app.component';\n\ndescribe('AppComponent', () => {\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [AppComponent],\n }).compileComponents();\n });\n\n it('should create the app', () => {\n const fixture = TestBed.createComponent(AppComponent);\n const app = fixture.componentInstance;\n expect(app).toBeTruthy();\n });\n\n it(`should have the 'genai-sa-ui-app' title`, () => {\n const fixture = TestBed.createComponent(AppComponent);\n const app = fixture.componentInstance;\n expect(app.title).toEqual('genai-sa-ui-app');\n });\n\n it('should render title', () => {\n const fixture = TestBed.createComponent(AppComponent);\n fixture.detectChanges();\n const compiled = fixture.nativeElement as HTMLElement;\n expect(compiled.querySelector('h1')?.textContent).toContain('Hello, genai-sa-ui-app');\n });\n});\n" }, { "path": "frontend/src/app/app.component.ts", "content": "import { Component } from '@angular/core';\n\n@Component({\n selector: 'app-root',\n templateUrl: './app.component.html',\n styleUrl: './app.component.scss'\n})\nexport class AppComponent {\n title = 'data-engineering-ui-app';\n config: any;\n constructor() {}\n ngOnInit() {}\n}\n" }, { "path": "frontend/src/app/app.module.server.ts", "content": "import { NgModule } from '@angular/core';\nimport { ServerModule } from '@angular/platform-server';\nimport { AppComponent } from './app.component';\nimport { AppModule } from './app.module';\n\n@NgModule({\n imports: [\n AppModule,\n ServerModule,\n ],\n bootstrap: [AppComponent],\n})\nexport class AppServerModule {}\n" }, { "path": "frontend/src/app/app.module.ts", "content": "import { CUSTOM_ELEMENTS_SCHEMA, NgModule, importProvidersFrom } from \"@angular/core\";\nimport { AppComponent } from \"./app.component\";\nimport { LoginComponent } from \"./login/login.component\";\nimport { LoginButtonComponent } from \"./login-button/login-button.component\";\nimport { LoginService } from \"./shared/services/login.service\";\nimport { SharedService } from \"./shared/services/shared.service\";\nimport { provideFirestore, initializeFirestore } from \"@angular/fire/firestore\";\nimport { getApp, initializeApp, provideFirebaseApp } from \"@angular/fire/app\";\nimport { BrowserModule } from \"@angular/platform-browser\";\nimport { UserJourneyComponent } from \"./user-journey/user-journey.component\";\nimport { AppRoutingModule } from \"./app-routing.module\";\nimport { HomeComponent } from \"./home/home.component\";\nimport { HeaderComponent } from \"./header/header.component\";\nimport { MatToolbarModule } from \"@angular/material/toolbar\";\nimport { MatIconModule } from \"@angular/material/icon\";\nimport { MatButtonModule } from \"@angular/material/button\";\nimport { RouterLink } from \"@angular/router\";\nimport { MatTabsModule } from \"@angular/material/tabs\";\nimport { MatDividerModule } from \"@angular/material/divider\";\nimport { MatSelectModule } from \"@angular/material/select\";\nimport { MatInputModule } from \"@angular/material/input\";\nimport { MatFormFieldModule } from \"@angular/material/form-field\";\nimport { MatAutocompleteModule } from \"@angular/material/autocomplete\";\nimport { CommonModule, NgFor, NgIf } from \"@angular/common\";\nimport { BrowserAnimationsModule } from \"@angular/platform-browser/animations\";\nimport { MatListModule } from '@angular/material/list';\nimport { MatSidenavModule } from '@angular/material/sidenav';\nimport { MatExpansionModule } from '@angular/material/expansion';\nimport { MenuComponent } from \"./menu/menu.component\";\nimport { FormsModule, ReactiveFormsModule } from \"@angular/forms\";\nimport { BusinessUserComponent } from \"./business-user/business-user.component\";\nimport { MatTableModule } from '@angular/material/table';\nimport { MatCardModule } from '@angular/material/card';\nimport { HTTP_INTERCEPTORS, HttpClientModule, provideHttpClient, withFetch } from '@angular/common/http';\nimport { HomeService } from './shared/services/home.service';\nimport { provideAuth, getAuth } from '@angular/fire/auth';\nimport { MatProgressSpinnerModule } from '@angular/material/progress-spinner';\nimport { MatSnackBarModule } from '@angular/material/snack-bar';\nimport { ClipboardModule } from '@angular/cdk/clipboard';\nimport { MatSlideToggleModule } from '@angular/material/slide-toggle';\nimport { NgChartsModule } from 'ng2-charts';\nimport { AngularFireModule } from '@angular/fire/compat';\nimport { AngularFirestoreModule } from '@angular/fire/compat/firestore';\nimport { AngularFireAuth, AngularFireAuthModule } from '@angular/fire/compat/auth';\nimport { UserPhotoComponent } from \"./user-photo/user-photo.component\";\nimport { PrismComponent } from \"./prism/prism.component\";\nimport 'prismjs/components/prism-sql';\nimport { MatPaginatorModule } from \"@angular/material/paginator\";\nimport { OverlayModule } from \"@angular/cdk/overlay\";\nimport { GoogleChartsModule } from \"angular-google-charts\";\nimport { MatRadioModule } from '@angular/material/radio';\nimport { MatStepperModule } from '@angular/material/stepper';\nimport { STEPPER_GLOBAL_OPTIONS } from \"@angular/cdk/stepper\";\nimport { AgentChatComponent } from \"./agent-chat/agent-chat.component\";\nimport { AppHttpInterceptor } from \"./http.interceptor\";\nimport { firebaseConfig, FIRESTORE_DATABASE_ID } from \"../assets/constants\";\nimport { MatTreeModule } from \"@angular/material/tree\";\nimport { ScenarioListComponent } from \"./scenario-list/scenario-list.component\";\n\n@NgModule({\n declarations: [\n AppComponent,\n LoginComponent,\n LoginButtonComponent,\n UserJourneyComponent,\n UserPhotoComponent,\n HomeComponent,\n HeaderComponent,\n MenuComponent,\n BusinessUserComponent,\n PrismComponent,\n AgentChatComponent,\n ScenarioListComponent\n ],\n imports: [\n CommonModule,\n BrowserModule,\n ReactiveFormsModule,\n FormsModule,\n BrowserAnimationsModule,\n AppRoutingModule,\n MatToolbarModule,\n MatIconModule,\n MatButtonModule,\n RouterLink,\n MatTabsModule,\n MatDividerModule,\n NgIf,\n NgFor,\n MatSelectModule,\n MatInputModule,\n MatFormFieldModule,\n MatAutocompleteModule,\n MatListModule,\n MatSidenavModule,\n MatTableModule,\n MatExpansionModule,\n MatCardModule,\n HttpClientModule,\n MatProgressSpinnerModule,\n MatSnackBarModule,\n ClipboardModule,\n MatSlideToggleModule,\n NgChartsModule,\n AngularFireModule.initializeApp(firebaseConfig),\n AngularFirestoreModule,\n AngularFireAuthModule,\n MatPaginatorModule,\n OverlayModule,\n GoogleChartsModule,\n MatRadioModule,\n MatStepperModule,\n MatExpansionModule,\n MatTreeModule\n ],\n providers: [\n {\n provide: STEPPER_GLOBAL_OPTIONS,\n useValue: { displayDefaultIndicatorType: false }\n },\n {\n provide: HTTP_INTERCEPTORS,\n useClass: AppHttpInterceptor,\n multi: true\n },\n provideHttpClient(withFetch()),\n importProvidersFrom([\n provideFirebaseApp(() => initializeApp(firebaseConfig)),\n provideFirestore(() => {\n const app = getApp();\n const providedFirestore = initializeFirestore(app, {}, FIRESTORE_DATABASE_ID);\n return providedFirestore;\n }),\n\n provideAuth(() => getAuth()),\n LoginService,\n SharedService,\n HomeService,\n AngularFireAuth\n ]),\n ],\n bootstrap: [AppComponent],\n schemas: [CUSTOM_ELEMENTS_SCHEMA]\n})\nexport class AppModule { }" }, { "path": "frontend/src/app/business-user/business-user.component.html", "content": "
\n
\n
Access restricted: you dont have sufficient\n permissions to access this data source
\n
\n\n
\n
\n
    \n
  • \n
    \n
    \n

    \n Welcome to Open Data QnA\n

    \n

    {{msg.user_question}}

    \n
    \n Prompt Suggestion\n
    \n

    \n \n {{res.example_user_question}}\n

    \n
    \n
    \n \n \n \n \n\n
    \n \n \n \n \n\n \n \n\n \n \n\n \n \n \n \n\n \n
    \n
    \n \n
    \n
    \n \n
    \n
    \n \n
    \n
    \n
    \n\n
    • \n
\n
\n\n
\n
\n \n \"follow\n
\n
\n
" }, { "path": "frontend/src/app/business-user/business-user.component.scss", "content": "input {\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n}\n\n.name {\n border-radius: 25px;\n border: 1px solid #E9E9E9;\n background: #FFF;\n box-shadow: 0px 0px 3px 0px rgba(0, 0, 0, 0.15) inset;\n width: 925px;\n height: 48px;\n flex-shrink: 0;\n font-family: Google Sans;\n font-size: 14px;\n font-style: normal;\n font-weight: 500;\n line-height: normal;\n flex: 1 0 8em;\n margin: 5px;\n /* 8em * 16px = 128px */\n}\n\n.name-err {\n border-radius: 25px;\n border: 1px solid red;\n background: #FFF;\n box-shadow: 0px 0px 3px 0px rgba(0, 0, 0, 0.15) inset;\n width: 925px;\n height: 48px;\n flex-shrink: 0;\n font-family: Google Sans;\n font-size: 14px;\n font-style: normal;\n font-weight: 500;\n line-height: normal;\n margin: 5px;\n}\n\n.input-flex {\n display: flex;\n flex-wrap: wrap;\n padding: 15px 10px\n}\n\n.suggestion-ref {\n border-radius: 11.5px;\n background: #D3E3FD;\n}\n\n.summarize-results {\n margin-top: 10px;\n}\n\n@media (max-width: 768px) {\n\n /* I need the code here */\n .summarize-results {\n flex: 0 50%;\n }\n}\n\n::ng-deep .custom-style {\n background-color: brown;\n color: white;\n border-radius: 10px;\n}\n\n.followUp {\n cursor: pointer;\n flex: 0 1 8em;\n}\n\n.container-fluid {\n display: flex;\n flex-direction: column;\n height: 100%;\n}\n\n.insight-results-err {\n font-family: Google Sans;\n font-size: 14px;\n}\n\n.cdk-overlay-pane:has(.mat-mdc-select-panel) {\n width: auto !important;\n}\n\n/*-----chat messages------*/\n.chat-body {\n overflow-y: auto;\n position: relative;\n flex: 1 1 auto;\n border: 0;\n margin: 5px;\n vertical-align: baseline;\n background: #F3F6FC;\n border-radius: 25px;\n display: flex;\n flex-direction: column;\n justify-content: space-between;\n}\n\n.msg {\n border-radius: 10px;\n}\n\n.msg.right {\n background-color: #CCC;\n float: right;\n}\n\n.msg.left {\n background-color: #555;\n color: white;\n\n}\n\n.flex-expand {\n flex: 1 1 auto;\n}\n\n#messages {\n display: flex;\n flex-direction: column;\n overflow-y: auto;\n overflow-x: hidden;\n margin-top: 0;\n margin-bottom: 0;\n padding-left: 0 !important;\n list-style: none;\n}\n\n#messages li {\n list-style: none;\n margin-top: 1rem;\n}\n\n#messages img {\n width: 43px;\n height: auto;\n}\n\n.received {\n display: block;\n text-align: right;\n position: relative;\n}\n\nli.received p {\n padding: 5px 10px;\n display: inline-block;\n border-radius: 20px;\n margin-bottom: 0;\n\n color: #000;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: \"Google Sans\";\n font-size: 14px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n letter-spacing: -0.154px;\n border-radius: 11.5px;\n border: 1px solid #B6CEFF;\n background: #DBE7FF;\n max-width: 500px;\n text-align: left;\n}\n\nli.sent {\n display: block;\n text-align: left;\n position: relative;\n color: #000;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: \"Google Sans\";\n font-size: 14px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n letter-spacing: -0.154px;\n background: #F3F6FC;\n}\n\nli.sent div {\n border-radius: 26px;\n background: #FFF;\n margin-bottom: 10px;\n width: 520px;\n}\n\nli.sent p,\na,\nimg {\n color: #000;\n font-size: 14px;\n line-height: 1.5;\n font-weight: 400;\n padding: 5px 10px;\n display: inline-block;\n border-radius: 20px;\n}\n\nli.sent:after {\n display: block;\n content: '';\n clear: both;\n}\n\n.caption_span {\n color: var(--grey, #575757);\n text-align: right;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: \"Google Sans\";\n font-size: 10px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n letter-spacing: -0.154px;\n padding-left: 10px;\n}\n\nli.sent .caption_span {\n margin-right: 15px;\n color: var(--grey, #575757);\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: \"Google Sans\";\n font-size: 10px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n letter-spacing: -0.154px;\n}\n/*-----chat messages------*/\n\n.welcomeTitle {\n color: #000;\n font-family: 'Google Sans';\n font-size: 25px !important;\n font-style: normal;\n font-weight: 700 !important;\n line-height: 20px;\n letter-spacing: 0.25px;\n padding: 10px;\n width: 500px\n}\n\n.userMsg {\n color: #000;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: 'Google Sans';\n font-size: 16px;\n font-style: normal;\n font-weight: 400;\n line-height: 22px;\n}\n\n.promptTitle {\n color: #525252;\n font-feature-settings: \"clig\" off, \"liga\" off;\n font-family: \"Google Sans\";\n font-size: 16px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n padding-left: 10px;\n}\n/* Progress spinner css */\n.meter {\n height: 18px;\n overflow: hidden;\n margin-bottom: 12px;\n border-radius: 10px;\n display: block;\n background: transparent;\n}\n\n.meter span {\n display: block;\n height: 100%;\n background: transparent;\n}\n\n.progress1 {\n transition: width 2s;\n border-radius: 10.5px;\n animation: progressBar1 7s linear 5s alternate;\n animation-delay: 0.1s;\n background: linear-gradient(80deg, rgba(66, 133, 244, 0.50)0%, rgba(198, 214, 253, 0.60) 41.37%, rgba(230, 222, 255, 0.70) 81.92%);\n animation-fill-mode: forwards;\n}\n\n.progress2 {\n transition: width 2s;\n border-radius: 10.5px;\n animation: progressBar1 10s linear 5s alternate;\n animation-delay: 1.5s;\n background: linear-gradient(100deg, rgba(66, 133, 244, 0.50)0%, rgba(198, 214, 253, 0.60) 41.37%, rgba(230, 222, 255, 0.70) 81.92%);\n animation-fill-mode: forwards;\n}\n\n.progress3 {\n transition: width 2s;\n border-radius: 10.5px;\n animation: progressBar1 5s linear 5s infinite alternate;\n animation-delay: 2.5s;\n animation-timing-function: ease-in-out;\n background: linear-gradient(100deg, rgba(66, 133, 244, 0.50)0%, rgba(198, 214, 253, 0.60) 41.37%, rgba(230, 222, 255, 0.70) 81.92%);\n animation-fill-mode: forwards;\n}\n\n@keyframes progressBar1 {\n from {\n background: linear-gradient(100deg, rgba(230, 222, 255, 0.70) 0%, rgba(198, 214, 253, 0.60) 41.37%, rgba(66, 133, 244, 0.50) 81.92%);\n border-radius: 10.5px;\n opacity: 0.2\n }\n\n to {\n background: linear-gradient(100deg, rgba(66, 133, 244, 0.50)0%, rgba(198, 214, 253, 0.60) 41.37%, rgba(230, 222, 255, 0.70) 81.92%);\n border-radius: 10.5px;\n }\n}\n\n\n/***************************************************\n * Generated by SVG Artista on 5/27/2024, 4:09:34 PM\n * MIT license (https://opensource.org/licenses/MIT)\n * W. https://svgartista.net\n **************************************************/\n\n@-webkit-keyframes animate-svg-stroke-1 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@keyframes animate-svg-stroke-1 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@-webkit-keyframes animate-svg-fill-1 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n@keyframes animate-svg-fill-1 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n.svg-elem-1 {\n -webkit-animation: animate-svg-stroke-1 2s ease 0.8s both, animate-svg-fill-1 2s ease 0.8s both;\n animation: animate-svg-stroke-1 2s ease 0.8s both, animate-svg-fill-1 2s ease 0.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-2 {\n 0% {\n stroke-dashoffset: 24.627429962158203px;\n stroke-dasharray: 24.627429962158203px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 24.627429962158203px;\n }\n}\n\n@keyframes animate-svg-stroke-2 {\n 0% {\n stroke-dashoffset: 24.627429962158203px;\n stroke-dasharray: 24.627429962158203px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 24.627429962158203px;\n }\n}\n\n.svg-elem-2 {\n -webkit-animation: animate-svg-stroke-2 2s ease 1.8s both, animate-svg-fill-2 2s ease 1.8s both;\n animation: animate-svg-stroke-2 2s ease 1.8s both, animate-svg-fill-2 2s ease 1.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-3 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@keyframes animate-svg-stroke-3 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@-webkit-keyframes animate-svg-fill-3 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n@keyframes animate-svg-fill-3 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n.svg-elem-1 {\n -webkit-animation: animate-svg-stroke-1 2s ease 0.8s both, animate-svg-fill-1 2s ease 0.8s both;\n animation: animate-svg-stroke-1 2s ease 0.8s both, animate-svg-fill-1 2s ease 0.8s both;\n}\n\n.svg-elem-3 {\n -webkit-animation: animate-svg-stroke-3 2s ease 2.8s both, animate-svg-fill-3 2s ease 2.8s both;\n animation: animate-svg-stroke-3 2s ease 2.8s both, animate-svg-fill-3 2s ease 2.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-4 {\n 0% {\n stroke-dashoffset: 17.556342124938965px;\n stroke-dasharray: 17.556342124938965px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 17.556342124938965px;\n }\n}\n\n@keyframes animate-svg-stroke-4 {\n 0% {\n stroke-dashoffset: 17.556342124938965px;\n stroke-dasharray: 17.556342124938965px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 17.556342124938965px;\n }\n}\n\n.svg-elem-4 {\n -webkit-animation: animate-svg-stroke-4 2s ease 3.8s both, animate-svg-fill-4 2s ease 3.8s both;\n animation: animate-svg-stroke-4 2s ease 3.8s both, animate-svg-fill-4 2s ease 3.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-5 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@keyframes animate-svg-stroke-5 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@-webkit-keyframes animate-svg-fill-5 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n@keyframes animate-svg-fill-5 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n.svg-elem-5 {\n -webkit-animation: animate-svg-stroke-5 2s ease 4.8s both, animate-svg-fill-5 2s ease 4.8s both;\n animation: animate-svg-stroke-5 2s ease 4.8s both, animate-svg-fill-5 2s ease 4.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-6 {\n 0% {\n stroke-dashoffset: 17.55635643005371px;\n stroke-dasharray: 17.55635643005371px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 17.55635643005371px;\n }\n}\n\n@keyframes animate-svg-stroke-6 {\n 0% {\n stroke-dashoffset: 17.55635643005371px;\n stroke-dasharray: 17.55635643005371px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 17.55635643005371px;\n }\n}\n\n.svg-elem-6 {\n -webkit-animation: animate-svg-stroke-6 2s ease 5.8s both, animate-svg-fill-6 2s ease 5.8s both;\n animation: animate-svg-stroke-6 2s ease 5.8s both, animate-svg-fill-6 2s ease 5.8s both;\n}\n\n@-webkit-keyframes animate-svg-stroke-7 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@keyframes animate-svg-stroke-7 {\n 0% {\n stroke-dashoffset: 30.274333882308138px;\n stroke-dasharray: 30.274333882308138px;\n }\n\n 100% {\n stroke-dashoffset: 0;\n stroke-dasharray: 30.274333882308138px;\n }\n}\n\n@-webkit-keyframes animate-svg-fill-7 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n@keyframes animate-svg-fill-7 {\n 0% {\n fill: transparent;\n }\n\n 100% {\n fill: rgb(66, 133, 244);\n }\n}\n\n.svg-elem-7 {\n -webkit-animation: animate-svg-stroke-7 2s ease 6.8s both, animate-svg-fill-7 2s ease 6.8s both;\n animation: animate-svg-stroke-7 2s ease 6.8s both, animate-svg-fill-7 2s ease 6.8s both;\n}\n/* Progress spinner css */" }, { "path": "frontend/src/app/business-user/business-user.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { BusinessUserComponent } from './business-user.component';\n\ndescribe('BusinessUserComponent', () => {\n let component: BusinessUserComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [BusinessUserComponent]\n })\n .compileComponents();\n\n fixture = TestBed.createComponent(BusinessUserComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/business-user/business-user.component.ts", "content": "import { Component, ChangeDetectorRef, Output, EventEmitter, Input, SimpleChanges, inject } from '@angular/core';\nimport { LoginService } from '../shared/services/login.service';\nimport { FormControl, FormGroup } from '@angular/forms';\nimport { HomeService } from '../shared/services/home.service';\nimport { MatSnackBar } from '@angular/material/snack-bar';\nimport { Subject, Subscription, takeUntil } from 'rxjs';\nimport { Router } from '@angular/router';\nimport { GroupingModalComponent } from '../grouping-modal/grouping-modal.component';\nimport { MatDialog } from '@angular/material/dialog';\nimport { ChatService } from '../shared/services/chat.service';\n\nexport interface Tabledata {\n city_id: string;\n}\n\n@Component({\n selector: 'app-business-user',\n templateUrl: './business-user.component.html',\n styleUrl: './business-user.component.scss'\n})\nexport class BusinessUserComponent {\n @Input('checkSideNav') checkSideNav: any\n @Input(\"selectedHistory\")\n selectedHistory!: any;\n @Input('selectedGrouping') selectedGrouping: any\n @Input('userSessions') userSessions!: string;\n currentScenario: any;\n chatMsgs: any[] = [];\n userLoggedIn: boolean = false;\n photoURL: any;\n resultLoader: boolean = false;\n isSuggestions: boolean = true;\n suggestionList: any;\n showResult: boolean = false;\n generatedSql: any = {\n example_user_question: '',\n example_generated_sql: '',\n unfilteredSql: ''\n };\n @Output() updateStyleEvent = new EventEmitter();\n setErrorCSS: boolean = false;\n readonly dialog = inject(MatDialog);\n isOpen: boolean = false;\n dataSet: string | undefined;\n dataSetName!: string;\n userId: any;\n private _destroy$ = new Subject();\n sessionId !: string;\n subscription!: Subscription;\n sub!: Subscription;\n //ind: number = 0;\n\n constructor(public loginService: LoginService, public homeService: HomeService, public chatService: ChatService,\n private snackBar: MatSnackBar, private change: ChangeDetectorRef, public router: Router) {\n this.loginService.getUserDetails().pipe(takeUntil(this._destroy$)).subscribe((res: any) => {\n this.userId = res.uid;\n this.userLoggedIn = true;\n this.photoURL = res?.photoURL\n });\n }\n sqlSearchForm = new FormGroup({\n name: new FormControl(),\n });\n\n ngOnChanges(changes: SimpleChanges) {\n for (const propName in changes) {\n if (changes.hasOwnProperty(propName)) {\n switch (propName) {\n case 'selectedHistory': {\n if (this.selectedHistory) {\n this.showResult = false;\n this.sessionId = this.homeService.getSessionId()\n }\n }\n break;\n }\n }\n }\n }\n ngOnInit() {\n this.sessionId = '';\n this.loadInitialChat();\n }\n reloadComponent(self: boolean, urlToNavigateTo?: string) {\n //skipLocationChange:true means dont update the url to / when navigating\n console.log(\"Current route I am on:\", this.router.url);\n const url = self ? this.router.url : urlToNavigateTo;\n this.router.navigateByUrl('/', { skipLocationChange: true }).then(() => {\n this.router.navigate([`/${url}`]).then(() => {\n console.log(`After navigation I am on:${this.router.url}`)\n })\n })\n }\n\n loadInitialChat() {\n this.chatService.chatSessionObservable.pipe(takeUntil(this._destroy$)).subscribe((res) => {\n this.chatMsgs = res.chatMsgs\n this.dataSet = this.homeService.getSelectedDbGrouping();\n this.dataSetName = this.homeService.getselectedDbName();\n this.sub = this.chatService.agentResponseLoader$.pipe(takeUntil(this._destroy$)).subscribe((res) => {\n this.resultLoader = res\n })\n this.sessionId = this.homeService.getSessionId()\n })\n }\n followUp(query: any, event?: any) {\n if (this.dataSet) {\n event?.preventDefault();\n if (this.sqlSearchForm.controls.name?.value !== null) {\n this.showResult = false;\n this.chatService.addQuestion(query, this.userId, \"followup\")\n this.sqlSearchForm.controls['name'].setValue(\"\");\n this.chatService.agentResponseLoader.next(true)\n this.resultLoader = true;\n // this.generate_sql(query);\n } else {\n this.setErrorCSS = true;\n }\n } else {\n let dialogRef = this.dialog.open(GroupingModalComponent, {\n disableClose: true,\n width: '450px',\n });\n\n dialogRef.afterClosed().subscribe(result => {\n \n });\n\n }\n }\n\n suggestionResult(selectedsql: any) {\n this.showResult = true;\n this.chatService.addQuestion(selectedsql.example_user_question, this.userId, \"followup\")\n this.resultLoader = true;\n this.generatedSql.example_user_question = selectedsql.example_user_question;\n this.sqlSearchForm.controls['name'].setValue(\"\");\n this.updateStyleEvent.emit(this.showResult);\n //this.change.markForCheck();\n }\n\n showSnackbarCssStyles(content: any, action: any, duration: any) {\n let sb = this.snackBar.open(content, action, {\n duration: duration,\n panelClass: [\"custom-style\"]\n });\n sb.onAction().subscribe(() => {\n sb.dismiss();\n });\n }\n updateStyleItem(value: boolean) {\n this.updateStyleEvent.emit(value);\n }\n showContentCopiedMsg() {\n this.showSnackbarCssStyles(\"Content Copied\", 'Close', '4000')\n }\n\n ngOnDestroy() {\n this.chatMsgs = [];\n //this.subscription.unsubscribe();\n this._destroy$.next();\n this.sessionId = \"\";\n this.sub.unsubscribe()\n }\n}\n" }, { "path": "frontend/src/app/grouping-modal/grouping-modal.component.html", "content": "
\n
\n Please select grouping before proceeding any query.\n
\n \n
" }, { "path": "frontend/src/app/grouping-modal/grouping-modal.component.scss", "content": ".grouping-msg {\n font-family: \"Google Sans\";\n font-size: \"16px\";\n color : #d93035\n}\n\n.closeBtn {\n font-family: \"Google Sans\";\n font-size: \"14px\";\n border: transparent;\n width: 100px;\n border-radius: 10px;\n\n}\n\n::ng-deep .mat-dialog-container {\n border-radius: 40px !important;\n}\n.popup{\n height: 110px;\n display: flex;\n justify-content: space-evenly;\n align-items: center;\n flex-direction: column;\n}" }, { "path": "frontend/src/app/grouping-modal/grouping-modal.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { GroupingModalComponent } from './grouping-modal.component';\n\ndescribe('GroupingModalComponent', () => {\n let component: GroupingModalComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [GroupingModalComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(GroupingModalComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/grouping-modal/grouping-modal.component.ts", "content": "import { Component } from '@angular/core';\nimport { MatDialogRef } from '@angular/material/dialog';\n\n@Component({\n selector: 'app-grouping-modal',\n standalone: true,\n imports: [],\n templateUrl: './grouping-modal.component.html',\n styleUrl: './grouping-modal.component.scss'\n})\nexport class GroupingModalComponent {\n constructor(public dialogRef: MatDialogRef) {\n\n }\n closeDialog() {\n this.dialogRef.close();\n }\n}\n" }, { "path": "frontend/src/app/header/header.component.html", "content": "Open Data QnA\n\n" }, { "path": "frontend/src/app/header/header.component.scss", "content": ".title {\n color: #000;\n font-family: Google Sans;\n font-size: 20px;\n font-style: normal;\n font-weight: 400;\n line-height: normal;\n}\n" }, { "path": "frontend/src/app/header/header.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { HeaderComponent } from './header.component';\n\ndescribe('HeaderComponent', () => {\n let component: HeaderComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [HeaderComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(HeaderComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/header/header.component.ts", "content": "import { Component } from '@angular/core';\n\n@Component({\n selector: 'app-header',\n templateUrl: './header.component.html',\n styleUrl: './header.component.scss'\n})\nexport class HeaderComponent {\n constructor() { }\n\n ngOnInit() {\n\n }\n}\n" }, { "path": "frontend/src/app/home/home.component.html", "content": "
\n \n \n\n \"Cymbal\n \"Cymbal\n \n \n\n
\n \n
\n\n \n \n\n
\n
\n \n \n \n \n \n \n \n
\n \n
\n \n \n \n
\n
\n
" }, { "path": "frontend/src/app/home/home.component.scss", "content": "h1 {\n padding: 0 1rem;\n}\n\nh2 {\n padding: 1rem;\n}\n\nmat-toolbar {\n position: fixed;\n top: 0;\n z-index: 2;\n}\n\n// Move the content down so that it won't be hidden by the toolbar\n.content {\n padding-top: 3.5rem;\n\n @media screen and (min-width: 600px) {\n padding-top: 4rem;\n }\n}\n\n.flex-expand {\n flex: 1 1 auto;\n}\n\n/* Styles for tab labels */\n::ng-deep .mat-mdc-tab.mdc-tab--active .mdc-tab__text-label {\n color: #4D88FF !important;\n text-align: center;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: Google Sans;\n font-size: 16px;\n font-style: normal;\n font-weight: 500;\n line-height: 20px;\n}\n\n/* Styles for the active tab label */\n::ng-deep .mat-mdc-tab .mdc-tab__text-label {\n text-align: center;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: Google Sans;\n font-size: 16px;\n font-style: normal;\n font-weight: 500;\n line-height: 20px;\n letter-spacing: -0.154px;\n}\n\n/* Styles for the ink bar */\n::ng-deep .mat-mdc-tab.mdc-tab--active .mdc-tab-indicator__content--underline {\n stroke-width: 3px;\n stroke: #4D88FF !important;\n border-color: #4D88FF !important\n}\n\n::ng-deep .mat-form-field.mat-focused .mat-form-field-underline {\n display: none;\n}\n\n.mat-form-field-appearance-legacy .mat-form-field-underline {\n background-color: blue;\n}\n\n.mat-mdc-form-field-focus-overlay {\n background-color: red;\n}\n\n.mdc-text-field--filled:not(.mdc-text-field--disabled) {\n background-color: red\n}\n\n::ng-deep .mat-form-field-underline {\n display: none;\n}\n\nmat-form-field mat-select {\n border: none;\n}\n\n.mdc-text-field--outlined {\n border-color: red;\n}\n\noptgroup {\n font-size: 40px;\n}\n\n.mr-10 {\n margin-right: 7rem !important;\n}\n\nmat-sidenav-container {\n height: 100%;\n position: absolute;\n}\n\n// Move the content down so that it won't be hidden by the toolbar\nmat-sidenav {\n padding-top: 3.5rem;\n\n @media screen and (min-width: 600px) {\n padding-top: 4rem;\n }\n\n .entry {\n display: flex;\n align-items: center;\n gap: 1rem;\n padding: 0.75rem;\n }\n}\n\n// Move the content down so that it won't be hidden by the toolbar\nmat-sidenav-content {\n padding-top: 3.5rem;\n\n @media screen and (min-width: 600px) {\n padding-top: 4rem;\n }\n}\n\n.no-border {\n border: none;\n}\n\n::ng-deep mat-expansion-panel-body {\n padding: 0 !important\n}\n\n::ng-deep mat-expansion-panel-header {\n padding: 0 1px 0 0px !important\n}\n\n.toolbar {\n border-bottom: 1px solid #ebe6e6;\n background: #FFF;\n border-radius: 24px\n}" }, { "path": "frontend/src/app/home/home.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { HomeComponent } from './home.component';\n\ndescribe('HomeComponent', () => {\n let component: HomeComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [HomeComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(HomeComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/home/home.component.ts", "content": "import { Component, ViewChild } from '@angular/core';\nimport { FormControl } from '@angular/forms';\nimport { HomeService } from '../shared/services/home.service';\nimport { ThemePalette } from '@angular/material/core';\nimport { MatSidenav } from '@angular/material/sidenav';\nimport { BreakpointObserver } from '@angular/cdk/layout';\nimport { LoginService } from '../shared/services/login.service';\nimport { Router } from '@angular/router';\nimport { Subject, Subscription, take, takeUntil } from 'rxjs';\n\n@Component({\n selector: 'app-home',\n templateUrl: './home.component.html',\n styleUrl: './home.component.scss',\n})\nexport class HomeComponent {\n title = 'material-responsive-sidenav';\n isCollapsed = true;\n groupingsListCtrl = new FormControl('');\n private _destroy$ = new Subject();\n groupingsList: any;\n groupingString: any;\n checkStyle: boolean | undefined;\n userType: String | undefined;\n checkSideNav: string = 'New Query';\n @ViewChild(MatSidenav)\n sidenav!: MatSidenav;\n isMobile = true;\n selectedGrouping: any;\n photoURL: any;\n reloadComp: boolean = false;\n userId: any;\n userSessions: any = [];\n userHistory: any = [];\n Subscription!: Subscription\n selectedHistory: any;\n selectedScenario: any;\n\n constructor(private homeService: HomeService, private observer: BreakpointObserver, private _router: Router, private loginService: LoginService) {\n this.loginService.getUserDetails().subscribe(message => {\n this.userId = message.uid;\n this.photoURL = message?.photoURL\n });\n }\n\n async ngOnInit() {\n if (!this.photoURL) {\n this._router.navigate(['']);\n }\n this.observer.observe(['(max-width: 800px)']).subscribe((screenSize) => {\n if (screenSize.matches) {\n this.isMobile = true;\n } else {\n this.isMobile = false;\n }\n });\n if (this.userId) {\n this.Subscription = this.homeService.getUserSessions(this.userId)\n .pipe(takeUntil(this._destroy$))\n .subscribe({\n next: (res: any) => {\n this.userSessions = res;\n },\n error: (error: any) => {\n throw error;\n },\n complete: () => {\n //console.log(\"complete\")\n }\n })\n }\n this.groupingValAndKnownSql()\n }\n groupingValAndKnownSql() {\n this.homeService.setSelectedDbGrouping(\"\");\n this.groupingString = this.homeService.getAvailableDBList();\n if (this.groupingString !== null && this.groupingString !== undefined) {\n this.groupingsList = JSON.parse(this.groupingString);\n this.homeService.currentSelectedGrouping.next('');\n this.homeService.currentSelectedGroupingObservable.subscribe((res) => {\n this.groupingsListCtrl.setValue(res);\n if (!res) {\n this.homeService.sqlSuggestionList(\"\", \"\").subscribe((data: any) => {\n if (data && data.ResponseCode === 200) {\n this.homeService.knownSqlFromDb.next(data.KnownSQL);\n }\n })\n }\n })\n } else {\n this.homeService.getAvailableDatabases().subscribe((res: any) => {\n if (res && res.ResponseCode === 200) {\n this.groupingsList = JSON.parse(res.KnownDB);\n }\n });\n }\n }\n changeDb(dbtype: any) {\n let selectedDbtype = dbtype.target.value.split(\"-\");\n this.homeService.setSelectedDbGrouping(dbtype.target.value);\n this.homeService.setSessionId('');\n this.homeService.setselectedDbName(selectedDbtype[1])\n this.homeService.currentSelectedGrouping.next(dbtype.target.value)\n this.homeService.sqlSuggestionList(dbtype.target.value, selectedDbtype[1]).subscribe((data: any) => {\n if (data && data.ResponseCode === 200) {\n this.homeService.knownSqlFromDb.next(data.KnownSQL);\n }\n })\n }\n\n updateBackgroundStyle(data: boolean) {\n this.checkStyle = data;\n }\n checkSideNavTAb(data: any) {\n if (data == 'New Query') {\n this.reloadComp = true;\n }\n this.checkSideNav = data;\n }\n\n sendHistory(data: any) {\n this.selectedHistory = data\n }\n\n toggleMenu() {\n if (this.isMobile) {\n this.sidenav.toggle();\n } else {\n // do nothing for now\n }\n }\n\n ngOnDestroy() {\n this.userHistory = [];\n this.Subscription?.unsubscribe();\n this._destroy$.next()\n }\n}\n" }, { "path": "frontend/src/app/http.interceptor.ts", "content": "import { Injectable } from \"@angular/core\";\nimport { HttpEvent, HttpHandler, HttpInterceptor, HttpRequest, HttpResponse } from \"@angular/common/http\";\nimport { Observable, tap } from \"rxjs\";\nimport { LoginService } from \"./shared/services/login.service\";\n\n@Injectable()\nexport class AppHttpInterceptor implements HttpInterceptor {\n\n userDetails: any;\n idToken!: string;\n\n constructor(public loginService: LoginService) {\n this.loginService.getUserDetails().subscribe((res: any) => { this.userDetails = res });\n this.loginService.getIdToken().subscribe((res: string) => {\n this.idToken = res\n });\n }\n\n intercept(req: HttpRequest, next: HttpHandler): Observable> {\n req = req.clone({ headers: req.headers.append('Content-Type', 'application/x-www-form-urlencoded') });\n req = req.clone({ headers: req.headers.append('Access-Control-Allow-Origin', '*') });\n req = req.clone({ headers: req.headers.append('Authorization', `Bearer ${this.userDetails?.accessToken || this.idToken}`) });\n const started = Date.now();\n return next.handle(req).pipe(tap((event: any) => {\n const elapsed = Date.now() - started;\n //console.log(`Request for ${req.urlWithParams} took ${elapsed} ms.`);\n if (event instanceof HttpResponse) {\n let responseTime = elapsed / 1000\n event.body.responseTime = `${responseTime} s`\n return event.body\n };\n })\n )\n }\n\n}" }, { "path": "frontend/src/app/login/login.component.html", "content": "
\n
\n \n \n \n \n \n \n\n \n \n \n
\n
\n

\n Open Data QnA\n

\n
\n
\n

\n Demo\n

\n
\n
\n

\n User Journey | Architecture Diagram | GCP Console Demo\n

\n
\n
\n
\n

\n This is a functioning solution demo is for CEs/FSRs to present to prospects to demonstrate the\n capabilities of Google Cloud AI products.

\n\n It is not an officially supported Google Cloud product or service and is provided without any guarantees\n of performance or maintenance. Because of the tool's limited and experimental nature, we may need to\n make modifications to, including potentially taking down, the tool on short, or no, notice.

\n\n You may provide feedback and suggestions about this tool to Google, and Google and its Affiliates may\n use any feedback or suggestions provided without restriction and without obligation to you.

\n\n Your use of the tool must always comply with our Terms of Service (including the Acceptable Use Policy),\n or the offline variant of the terms that govern your use of the Google Cloud Platform Services.

\n\n During your use of the tool, information is provided for informational purposes only. The data inputed\n are dummy data.\n

\n
\n
\n

\n By checking this box, you\n accept and agree to the above\n terms and conditions of this tool.\n

\n
\n
\n \n
\n
\n" }, { "path": "frontend/src/app/login/login.component.scss", "content": ".search {\n color: #FFFFFF;\n}\n\n.login-user {\n float: right;\n padding-right: 20px;\n}\n\n.login-container .mat-mdc-dialog-container .mdc-dialog__surface {\n border-radius: 44px !important;\n}\n\n.login-btn {\n position: absolute;\n left: 1400px;\n top: 30px;\n background: #F7F2FA;\n box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3), 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n border-radius: 100px;\n display: flex;\n flex-direction: row;\n justify-content: center;\n align-items: center;\n padding: 10px 24px;\n gap: 8px;\n flex: none;\n order: 0;\n align-self: stretch;\n flex-grow: 1;\n width: 39px;\n height: 20px;\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n letter-spacing: 0.1px;\n color: #6750A4;\n}\n\n\n.login-page-background {\n background-image: radial-gradient(grey, black) !important;\n min-width: 100%;\n min-height: 100%\n}\n\n.title {\n /* Customer Service Modernization */\n padding-left: 75px;\n color: #FFFFFF;\n width: 1328px;\n height: 115px;\n left: 159px;\n top: 280px;\n\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 700;\n font-size: 61px;\n line-height: 100px;\n\n /* or 102% */\n display: flex;\n align-items: center;\n letter-spacing: 0.5px;\n margin-top: 77px;\n}\n\n.demo {\n padding-left: 77px;\n width: 176px;\n height: 40px;\n left: 159px;\n margin-top: -25px;\n\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 700;\n font-size: 40px;\n line-height: 40px;\n /* identical to box height, or 62% */\n display: flex;\n align-items: center;\n letter-spacing: 0.5px;\n color: #FFFFFF;\n}\n\n.userJourney {\n padding-left: 75px;\n height: 40px;\n left: 159px;\n margin-top: 22px;\n\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 400;\n font-size: 24px;\n line-height: 40px;\n display: flex;\n align-items: center;\n letter-spacing: 0.5px;\n color: #FFFFFF;\n}\n\n.desc {\n padding-left: 75px;\n color: #FFFFFF;\n width: 959px;\n left: 164px;\n margin-top: 23px;\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 400;\n font-size: 18px;\n line-height: 20px;\n}\n\n.check {\n padding-left: 75px;\n left: 194px;\n top: 1051px;\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 700;\n font-size: 18px;\n line-height: 20px;\n color: #FFFFFF;\n\n}\n\n.checkboxCss {\n left: 164px;\n top: 1049px;\n\n padding-left: 75px;\n left: 0%;\n right: 0%;\n top: 0%;\n bottom: 0%;\n\n font-family: 'Google Symbols';\n font-style: normal;\n font-weight: 400;\n font-size: 24px;\n line-height: 50px;\n align-items: center;\n text-align: center;\n color: #FFFFFF;\n}\n\n.acceptButton {\n gap: 10px;\n margin-left: 70px;\n left: 164px;\n top: 1114px;\n background: #1A73E8;\n border-radius: 4px;\n width: 200px;\n height: 40px;\n font-family: \"Google Sans\";\n font-style: normal;\n font-weight: 500;\n font-size: 18px;\n line-height: 20px;\n display: flex;\n align-items: center;\n text-align: center;\n justify-content: center;\n letter-spacing: 0.25px;\n flex: none;\n order: 0;\n flex-grow: 0;\n color: #FFFFFF;\n border: none !important;\n}\n\n//sign in background gray\n::ng-deep .cdk-overlay-backdrop.cdk-overlay-backdrop-showing {\n opacity: 70 !important;\n background: rgba(0, 0, 0, 0.70)\n}" }, { "path": "frontend/src/app/login/login.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { LoginComponent } from './login.component';\n\ndescribe('LoginComponent', () => {\n let component: LoginComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [LoginComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(LoginComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/login/login.component.ts", "content": "import { Component, ElementRef } from '@angular/core';\nimport { LoginButtonComponent } from '../login-button/login-button.component';\nimport { Subscription } from 'rxjs';\nimport { Router } from '@angular/router';\nimport { Dialog } from '@angular/cdk/dialog';\nimport { LoginService } from '../shared/services/login.service';\nimport { MatSnackBar } from '@angular/material/snack-bar';\n\n@Component({\n // standalone: true,\n selector: 'app-login',\n templateUrl: './login.component.html',\n styleUrl: './login.component.scss'\n})\nexport class LoginComponent {\n photoURL: string | undefined;\n subscription: Subscription | undefined;\n acceptAndAgreeButton: boolean = true;\n loginError = false;\n loginErrorMessage: any\n constructor(private _router: Router, public loginService: LoginService, public dialog: Dialog, public snackbar: MatSnackBar) {\n this.loginService.getLoginError().subscribe((res: any) => {\n this.loginErrorMessage = res\n this.loginError = true;\n });\n this.subscription = this.loginService.getUserDetails().subscribe(res => {\n this.userLoggedIn = true;\n this.photoURL = res?.photoURL\n });\n }\n\n ngAfterViewInit() {\n if (!this.photoURL) {\n this.showLogIn()\n }\n }\n\n userLoggedIn: boolean = false;\n navigateToUserJourney() {\n if (!this.loginError) {\n this.userLoggedIn = true;\n this._router.navigate(['user-journey']);\n }\n else {\n this.showSnackbarCssStyles(this.loginErrorMessage, 'Close', 10000);\n }\n }\n\n\n showLogIn(): void {\n const dialogRef = this.dialog.open(LoginButtonComponent, {\n disableClose: true,\n width: '350px',\n panelClass: 'login-container'\n });\n }\n\n checkboxChecked(event: any) {\n if (event.target.checked) {\n this.acceptAndAgreeButton = false\n } else {\n this.acceptAndAgreeButton = true\n }\n }\n\n showSnackbarCssStyles(content: any, action: any, duration: any) {\n let sb = this.snackbar.open(content, action, {\n duration: duration,\n panelClass: [\"custom-style\"]\n });\n sb.onAction().subscribe(() => {\n sb.dismiss();\n });\n }\n}\n" }, { "path": "frontend/src/app/login-button/login-button.component.html", "content": "\n
\n \n
\n \n
\n
\n To use this demo, you have to sign in\n
\n
\n" }, { "path": "frontend/src/app/login-button/login-button.component.scss", "content": ".login-btn {\n align-items: center;\n margin-left: 75px;\n margin-top: 110px;\n\n /* label-text */\n width: 90px;\n height: 36px;\n /* Auto layout */\n display: flex;\n flex-direction: column;\n justify-content: center;\n align-items: center;\n padding: 0px;\n gap: 8px;\n\n position: relative;\n width: 87px;\n height: 40px;\n\n /* M3/sys/light/surface-container-low */\n background: #F7F2FA;\n /* M3/Elevation Light/1 */\n box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.3), 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n border-radius: 100px;\n\n /* state-layer */\n\n /* Auto layout */\n display: flex;\n flex-direction: row;\n justify-content: center;\n align-items: center;\n padding: 10px 24px;\n gap: 8px;\n\n width: 87px;\n height: 40px;\n\n /* Inside auto layout */\n flex: none;\n order: 0;\n align-self: stretch;\n flex-grow: 1;\n\n /* label-text */\n\n /* M3/label/large */\n font-family: 'Google Sans';\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n /* identical to box height, or 143% */\n display: flex;\n align-items: center;\n text-align: center;\n letter-spacing: 0.1px;\n\n /* M3/sys/light/primary */\n color: #6750A4;\n\n /* Inside auto layout */\n flex: none;\n order: 0;\n flex-grow: 0;\n}\n\n.popup {\n /* Rectangle 6452 */\n height: 110px;\n display: flex;\n justify-content: space-evenly;\n align-items: center;\n flex-direction: column;\n}\n\n.signin-msg {\n font-family: \"Google Sans\";\n font-size: \"16px\";\n}\n\n.gsi-material-button {\n -moz-user-select: none;\n -webkit-user-select: none;\n -ms-user-select: none;\n -webkit-appearance: none;\n background-color: WHITE;\n background-image: none;\n border: 1px solid #747775;\n -webkit-border-radius: 4px;\n border-radius: 4px;\n -webkit-box-sizing: border-box;\n box-sizing: border-box;\n color: #1f1f1f;\n cursor: pointer;\n font-family: \"Google Sans\";\n font-size: 14px;\n height: 40px;\n letter-spacing: 0.25px;\n outline: none;\n overflow: hidden;\n padding: 0 12px;\n position: relative;\n text-align: center;\n -webkit-transition: background-color .218s, border-color .218s, box-shadow .218s;\n transition: background-color .218s, border-color .218s, box-shadow .218s;\n vertical-align: middle;\n white-space: nowrap;\n width: auto;\n max-width: 400px;\n min-width: min-content;\n}\n\n.gsi-material-button .gsi-material-button-icon {\n height: 20px;\n margin-right: 12px;\n min-width: 20px;\n width: 20px;\n}\n\n.gsi-material-button .gsi-material-button-content-wrapper {\n -webkit-align-items: center;\n align-items: center;\n display: flex;\n -webkit-flex-direction: row;\n flex-direction: row;\n -webkit-flex-wrap: nowrap;\n flex-wrap: nowrap;\n height: 100%;\n justify-content: space-between;\n position: relative;\n width: 100%;\n}\n\n.gsi-material-button .gsi-material-button-contents {\n -webkit-flex-grow: 1;\n flex-grow: 1;\n font-family: \"Google Sans\";\n font-weight: 500;\n overflow: hidden;\n text-overflow: ellipsis;\n vertical-align: top;\n}\n\n.gsi-material-button .gsi-material-button-state {\n -webkit-transition: opacity .218s;\n transition: opacity .218s;\n bottom: 0;\n left: 0;\n opacity: 0;\n position: absolute;\n right: 0;\n top: 0;\n}\n\n.gsi-material-button:disabled {\n cursor: default;\n background-color: #ffffff61;\n border-color: #1f1f1f1f;\n}\n\n.gsi-material-button:disabled .gsi-material-button-contents {\n opacity: 38%;\n}\n\n.gsi-material-button:disabled .gsi-material-button-icon {\n opacity: 38%;\n}\n\n.gsi-material-button:not(:disabled):active .gsi-material-button-state,\n.gsi-material-button:not(:disabled):focus .gsi-material-button-state {\n background-color: #303030;\n opacity: 12%;\n}\n\n.gsi-material-button:not(:disabled):hover {\n -webkit-box-shadow: 0 1px 2px 0 rgba(60, 64, 67, .30), 0 1px 3px 1px rgba(60, 64, 67, .15);\n box-shadow: 0 1px 2px 0 rgba(60, 64, 67, .30), 0 1px 3px 1px rgba(60, 64, 67, .15);\n}\n\n.gsi-material-button:not(:disabled):hover .gsi-material-button-state {\n background-color: #303030;\n opacity: 8%;\n}\n\n::ng-deep .mat-dialog-container {\n border-radius: 40px !important;\n}\n\n:host {\n display: block;\n background: #fff;\n border-radius: 20px !important;\n padding: 8px 16px 16px;\n}\n" }, { "path": "frontend/src/app/login-button/login-button.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { LoginButtonComponent } from './login-button.component';\n\ndescribe('LoginButtonComponent', () => {\n let component: LoginButtonComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [LoginButtonComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(LoginButtonComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/login-button/login-button.component.ts", "content": "import { Dialog } from '@angular/cdk/dialog';\nimport { Component } from '@angular/core';\nimport { LoginService } from '../shared/services/login.service';\nimport { SharedService } from '../shared/services/shared.service';\n\n@Component({\n selector: 'app-login-button',\n templateUrl: './login-button.component.html',\n styleUrl: './login-button.component.scss'\n})\nexport class LoginButtonComponent {\n photoURL: any;\n userLoggedIn: boolean = false;\n constructor(public fireservice: SharedService, public loginService: LoginService,\n public dialog: Dialog) {\n }\n getLogin() {\n this.fireservice.googleSignin().then((res => {\n this.userLoggedIn = true;\n this.photoURL = res?.photoURL;\n this.updateUserData(res);\n this.dialog.closeAll()\n }))\n }\n\n updateUserData(userDetails: any): void {\n this.loginService.sendUserDetails(userDetails);\n }\n}\n" }, { "path": "frontend/src/app/menu/menu.component.html", "content": "\n \n New Query\n \n \n \n \n History\n \n

\n \n {{history.question}}\n

\n

\n \n {{history.question}}\n

\n \n \n \n 0\"\n [ngClass]=\"(clickedMenuItem === 'Scenarios') ? 'activeTab' : 'inactiveTab'\" (click)=\"onMenuClick('Scenarios')\">\n\n \n \n star_borderScenarios\n \n \n \n \n\n\n
\n
\n \n
\n
\n \n
\n
" }, { "path": "frontend/src/app/menu/menu.component.scss", "content": "mat-icon {\n vertical-align: bottom;\n}\n\n.app-sidenav {\n color: #000;\n font-feature-settings: 'clig' off, 'liga' off;\n font-family: \"Google Sans\";\n font-size: 16px;\n font-style: normal;\n font-weight: 500;\n line-height: 20px;\n letter-spacing: -0.154px;\n width: 350px;\n}\n\n.activeTab {\n background-color: #E8F0FE;\n border-radius: 25px;\n color: #4D88FF;\n cursor: pointer;\n}\n\n.inactiveTab {\n background-color: #FFF;\n border-radius: 25px;\n color: #444746;\n cursor: pointer;\n}\n\n.historyDetails {\n text-wrap: wrap;\n display: block;\n color: #444746;\n font-family: 'Google Sans';\n font-size: 14px;\n margin-bottom: 10px;\n cursor: pointer;\n}\n\n.showMore {\n font-size: .875rem;\n font-weight: 500;\n line-height: 1.25rem;\n font-family: \"Google Sans\", \"Helvetica Neue\", sans-serif;\n letter-spacing: normal;\n border: transparent;\n border-radius: 20px;\n padding: 10px;\n width: 100%;\n margin-bottom: 10px;\n background: #FFF;\n}\n\n::ng-deep.mat-expansion-indicator {\n margin-right: 10px\n}\n\n// .mdc-list-item {\n// cursor: auto !important;\n// }\n\ninput[type=\"file\"] {\n display: none;\n}\n\n.dropLabel {\n text-align: center;\n font-family: Google Sans;\n font-size: 14px;\n font-style: normal;\n font-weight: 400;\n line-height: 20px;\n cursor: pointer;\n display: flex;\n}\n\n.file-wrapper {\n border-color: transparent;\n height: 41px;\n background: #34A853;\n border-radius: 100px;\n font-family: \"Google Sans\";\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n display: flex;\n align-items: center;\n text-align: center;\n justify-content: center;\n letter-spacing: 0.25px;\n color: #FFFFFF;\n cursor: pointer;\n}\n\n::ng-deep .mat-expansion-panel-body {\n padding: 0 24px 0px !important;\n}\n\n.mat-expansion-panel {\n background: transparent !important;\n}\n\n.mat-expansion-panel:not(.mat-expanded) .mat-expansion-panel-header:not([aria-disabled=true]):hover {\n background: transparent !important;\n}\n\nul,\n#myUL {\n list-style-type: none;\n}\n\n#myUL {\n margin: 0;\n padding: 0;\n}\n\n.caret {\n cursor: pointer;\n -webkit-user-select: none;\n /* Safari 3.1+ */\n -moz-user-select: none;\n /* Firefox 2+ */\n -ms-user-select: none;\n /* IE 10+ */\n user-select: none;\n}\n\n.caret::before {\n content: \"\\25B6\";\n color: black;\n display: inline-block;\n margin-right: 6px;\n}\n\n.caret-down::before {\n -ms-transform: rotate(90deg);\n /* IE 9 */\n -webkit-transform: rotate(90deg);\n /* Safari */\n transform: rotate(90deg);\n}\n\n.scenariosInfoBtn {\n cursor: pointer;\n background: transparent;\n border: none;\n height: 24px;\n}\n\n.uploadScenarioBtn{\n position: absolute; bottom:20px; width: 100%;\n}" }, { "path": "frontend/src/app/menu/menu.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { MenuComponent } from './menu.component';\n\ndescribe('MenuComponent', () => {\n let component: MenuComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [MenuComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(MenuComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/menu/menu.component.ts", "content": "import { Component, Input, EventEmitter, Output, signal, SimpleChanges, inject, ViewChild } from '@angular/core';\nimport { ThemePalette } from '@angular/material/core';\nimport { Router } from '@angular/router';\nimport { HomeService } from '../shared/services/home.service';\nimport { GroupingModalComponent } from '../grouping-modal/grouping-modal.component';\nimport { MatDialog } from '@angular/material/dialog';\nimport { ScenarioListComponent } from '../scenario-list/scenario-list.component';\nimport { ChatService } from '../shared/services/chat.service';\nimport { LoginService } from '../shared/services/login.service';\nimport { UploadTemplateComponent } from '../upload-template/upload-template.component';\n\n@Component({\n selector: 'app-menu',\n templateUrl: './menu.component.html',\n styleUrl: './menu.component.scss',\n\n})\nexport class MenuComponent {\n clickedMenuItem: 'Query' | 'New Query' | 'Reports' | 'History' | 'Operations Mode' | 'My workspace' | 'Team workspaces' | 'Recent' | 'Shared with me' | 'Trash' | 'Templates' | 'Scenarios' | undefined;\n color: ThemePalette = 'accent';\n checked = false;\n disabled = true;\n @Input('userHistory') userHistory: any\n @Input('userSessions') userSessions: any;\n @Output() selectedTab = new EventEmitter();\n @Output() selectedHistory = new EventEmitter();\n panelOpenState = signal(false);\n scenarioPanelOpenState = signal(false)\n readonly dialog = inject(MatDialog);\n userType: any;\n recentHistory: any;\n showMoreHistory: any;\n selectedGrouping: string = '';\n csvData: any;\n @ViewChild(ScenarioListComponent)\n childComponent!: ScenarioListComponent;\n userId: any;\n showUploadSection: boolean = false;\n\n constructor(public _router: Router, public homeService: HomeService, public chatService: ChatService, public loginService: LoginService) {\n this.clickedMenuItem = 'New Query';\n }\n ngOnInit() {\n this.loginService.getUserDetails().subscribe((res: any) => {\n this.userId = res.uid;\n });\n this.selectedTab.emit(this.clickedMenuItem);\n }\n\n ngOnChanges(changes: SimpleChanges) {\n for (const propName in changes) {\n if (changes.hasOwnProperty(propName)) {\n switch (propName) {\n case 'userSessions': {\n if (this.userSessions.length > 0) {\n //group user sessions based on session id\n let grouped = this.userSessions?.reduce(\n (result: any, currentValue: any) => {\n (result[currentValue['session_id']] = result[currentValue['session_id']] || []).push(currentValue);\n return result;\n }, {});\n //map the grouped user sessions as a chatThread , sort and display in side nav\n let sessionToDisplayQuery: any = []\n Object.keys(grouped).map(function (sessionId: string) {\n let chatThreadArray: any[] = grouped[sessionId];\n let obj = {\n 'sessionId': sessionId,\n 'question': chatThreadArray[chatThreadArray.length - 1].user_question,\n 'chatThread': chatThreadArray,\n 'timestamp': chatThreadArray[chatThreadArray.length - 1].timestamp.seconds\n }\n sessionToDisplayQuery.push(obj);\n });\n\n sessionToDisplayQuery?.sort((a: any, b: any) => {\n return b.chatThread[0].timestamp - a.chatThread[0].timestamp\n });\n this.userHistory = sessionToDisplayQuery;\n this.recentHistory = this.userHistory.slice(0, 5);\n }\n break;\n }\n }\n }\n }\n }\n showMore() {\n this.showMoreHistory = this.userHistory.slice(5, 10)\n }\n onMenuClick(item: 'Query' | 'New Query' | 'Reports' | 'History' | 'Operations Mode' | 'My workspace' | 'Team workspaces' | 'Recent' | 'Shared with me' | 'Trash' | 'Templates' | 'Scenarios') {\n this.clickedMenuItem = item;\n this.selectedGrouping = this.homeService.getSelectedDbGrouping();\n\n if (this.clickedMenuItem == 'New Query') {\n this.chatService.createNewSession();\n this.homeService.currentSelectedGrouping.next('')\n this.homeService.setSessionId('')\n this.childComponent?.resetSelectedScenario()\n }\n this.selectedTab.emit(this.clickedMenuItem);\n }\n onClickHistory(chatThread: any) {\n this.childComponent?.resetSelectedScenario()\n this.selectedGrouping = this.homeService.getSelectedDbGrouping();\n if (this.selectedGrouping) {\n this.homeService.updateChatMsgs(chatThread)\n this.chatService.createNewSession()\n this.homeService.setSessionId(chatThread[0].session_id)\n this.homeService.updateSelectedHistory(chatThread)\n this.chatService.addQuestion(chatThread[chatThread?.length - 1]?.user_question, this.userId, 'history', chatThread)\n this.selectedHistory.emit(chatThread);\n } else {\n this.openDialog();\n }\n }\n\n openDialog() {\n let dialogRef = this.dialog.open(GroupingModalComponent, {\n disableClose: true,\n width: '450px',\n });\n dialogRef.afterClosed().subscribe(result => {\n });\n }\n\n uploadTemplate() {\n let dialogRef = this.dialog.open(UploadTemplateComponent, {\n disableClose: true,\n width: '450px',\n });\n\n dialogRef.afterClosed().subscribe(result => {\n this.showUploadSection = result\n });\n\n }\n onFileChange(fileInput: any) {\n if (fileInput) {\n const file: File = fileInput.files[0];\n let reader: FileReader = new FileReader();\n reader.readAsText(file);\n reader.onload = (e) => {\n let csv: any = reader.result;\n csv = csv.split('\\n')\n for (let i = 0; i < csv.length; i++) {\n csv[i] = csv[i].replace(/(\\r\\n|\\n|\\r)/gm, \"\");\n csv[i] = csv[i].split(',');\n if (i != 0) { // 0th element has the column header \n csv[i] = this.arrToObject(csv[i], csv[0]);\n }\n }\n this.csvData = csv.slice(1);\n }\n }\n }\n arrToObject(arr: any[], header: any[]) {\n let rv: any = {};\n for (let i = 0; i < arr.length; ++i)\n if (arr[i] !== undefined && arr.length == header.length) {\n rv[header[i]] = arr[i];\n }\n return rv;\n }\n}\n" }, { "path": "frontend/src/app/prism/prism.component.html", "content": "
\n  {{code}}\n
\n" }, { "path": "frontend/src/app/prism/prism.component.scss", "content": "mat-icon {\n vertical-align: bottom;\n\n}\n\n.app-sidenav {\n font-family: \"Google Sans\";\n font-size: 14px;\n width: 200px;\n}\n\n.app-sidenav mat-list-item {\n cursor: pointer;\n}\n\n.active {\n background-color: #E8F0FE;\n border-radius: 25px;\n color: #4D88FF;\n}" }, { "path": "frontend/src/app/prism/prism.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { PrismComponent } from './prism.component';\n\ndescribe('MenuComponent', () => {\n let component: PrismComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [PrismComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(PrismComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/prism/prism.component.ts", "content": "import { Component, Input, EventEmitter, Output, ViewChild, ElementRef } from '@angular/core';\nimport * as Prism from 'prismjs';\n\n@Component({\n selector: 'app-prism',\n\n templateUrl: './prism.component.html',\n styleUrl: './prism.component.scss'\n})\nexport class PrismComponent {\n @ViewChild('codeEle') codeEle!: ElementRef;\n @Input() code?: string;\n @Input() language?: string;\n\n constructor() { }\n\n ngAfterViewInit() {\n Prism.highlightElement(this.codeEle.nativeElement);\n }\n ngOnChanges(changes: any): void {\n if (changes?.code) {\n if (this.codeEle?.nativeElement) {\n this.codeEle.nativeElement.textContent = this.code;\n Prism.highlightElement(this.codeEle.nativeElement);\n }\n }\n }\n}\n" }, { "path": "frontend/src/app/prism/prism.d.ts", "content": "declare module 'prismjs'\n" }, { "path": "frontend/src/app/scenario-list/scenario-list.component.html", "content": "
\n {{sc.name}}\n \n \n \n \n {{node.name}}\n \n \n \n \n \n
\n \n {{node.name}}\n \n
\n\n
\n \n
\n \n \n
" }, { "path": "frontend/src/app/scenario-list/scenario-list.component.scss", "content": ".example-tree-invisible {\n display: none;\n}\n\n.example-tree ul,\n.example-tree li {\n margin-top: 0;\n margin-bottom: 0;\n list-style-type: none;\n}\n\n/*\n * This padding sets alignment of the nested nodes.\n */\n.example-tree .mat-nested-tree-node div[role=group] {\n padding-left: 40px;\n}\n\n/*\n * Padding for leaf nodes.\n * Leaf nodes need to have padding so as to align with other non-leaf nodes\n * under the same parent.\n */\n.example-tree div[role=group]>.mat-tree-node {\n padding-left: 25px;\n}\n\n.user-question {\n text-wrap: wrap;\n display: block;\n color: #444746;\n font-family: \"Google Sans\";\n font-size: 14px;\n cursor: pointer;\n}\n\n.historyDetails {\n text-wrap: wrap;\n display: block;\n color: #444746;\n font-family: 'Google Sans';\n font-size: 14px;\n margin-bottom: 10px;\n cursor: pointer;\n}\n\n.mat-tree {\n background-color: transparent !important;\n}" }, { "path": "frontend/src/app/scenario-list/scenario-list.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { ScenarioListComponent } from './scenario-list.component';\n\ndescribe('ScenarioListComponent', () => {\n let component: ScenarioListComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [ScenarioListComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(ScenarioListComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/scenario-list/scenario-list.component.ts", "content": "import { Component, EventEmitter, Input, Output, SimpleChanges } from '@angular/core';\nimport { NestedTreeControl, TreeControl } from '@angular/cdk/tree';\nimport { MatTreeNestedDataSource } from '@angular/material/tree';\nimport { HomeService } from '../shared/services/home.service';\nimport { ChatService } from '../shared/services/chat.service';\nimport { LoginService } from '../shared/services/login.service';\n\n/**\n * Scenario data with nested structure.\n * Each node has a name and an optional list of children.\n */\ninterface Question {\n name: string;\n child?: Question[];\n}\n\ninterface ScenarioNode {\n dataSource: MatTreeNestedDataSource;\n questions: Question[];\n name: string;\n userGrouping: string\n}\n\n@Component({\n selector: 'app-scenario-list',\n templateUrl: './scenario-list.component.html',\n styleUrl: './scenario-list.component.scss'\n})\nexport class ScenarioListComponent {\n @Input('csvData') csvData: any\n scenarioData: ScenarioNode[] = [];\n treeControl = new NestedTreeControl(node => node.child);\n hasChild = (_: number, node: Question) => !!node.child && node.child.length > 0;\n selectedGrouping: string = '';\n selectedScenario: string = '';\n userId: any;\n constructor(public homeService: HomeService, public chatService: ChatService, public loginService: LoginService) {\n this.loginService.getUserDetails().subscribe((res: any) => {\n this.userId = res.uid;\n });\n }\n\n ngOnChanges(changes: SimpleChanges) {\n for (const propName in changes) {\n if (changes.hasOwnProperty(propName)) {\n switch (propName) {\n case 'csvData': {\n if (this.csvData) {\n this.formatCsvData()\n }\n }\n }\n }\n }\n }\n ngOnInit() {\n this.formatCsvData();\n }\n\n formatCsvData() {\n this.scenarioData = [];\n let result: any = [];\n this.csvData.forEach((current: any) => {\n let desiredObj: any = result?.find((ele: any) => {\n if (ele.user_grouping == current.user_grouping &&\n ele.scenario == current.scenario) {\n return ele\n }\n });\n\n if (desiredObj === undefined) {\n result.push(\n {\n user_grouping: current.user_grouping,\n scenario: current.scenario,\n question: [current.question]\n });\n }\n else {\n desiredObj.question.push(current.question);\n }\n });\n result?.map((ele: any) => {\n if (ele.user_grouping) {\n var nestedQues: any = this.constructNestedTree(ele.question);\n let dataSource = new MatTreeNestedDataSource();\n dataSource.data = [nestedQues];\n this.scenarioData.push({\n dataSource: dataSource,\n questions: [nestedQues],\n name: ele.scenario,\n userGrouping: ele.user_grouping\n })\n }\n })\n this.treeControl = new NestedTreeControl(node => node.child)\n }\n\n onClickScenario(question: any, scenario: any) {\n this.homeService.setSelectedDbGrouping(scenario.userGrouping);\n this.selectedGrouping = this.homeService.getSelectedDbGrouping();\n this.homeService.currentSelectedGrouping.next(scenario.userGrouping)\n if (this.selectedScenario == scenario.name) {\n this.chatService.addQuestion(question.name, this.userId, 'scenario')\n this.chatService.agentResponseLoader.next(true)\n //this.chatService.setAgentResponseLoader(true)\n }\n else {\n this.chatService.createNewSession()\n this.chatService.addQuestion(question.name, this.userId, 'scenario')\n this.chatService.agentResponseLoader.next(true)\n }\n this.homeService.currentSelectedGroupingObservable.subscribe((res) => {\n this.selectedGrouping = res\n })\n this.selectedScenario = scenario.name\n }\n\n resetSelectedScenario() {\n this.selectedScenario = '';\n }\n constructNestedTree(questions: any[]) {\n if (questions.length == 1) {\n var ques: Question = {\n name: questions[0]\n }\n }\n else {\n var ques: Question = {\n name: questions[0],\n child: [this.constructNestedTree(questions.slice(1))]\n }\n }\n return ques;\n }\n}\n" }, { "path": "frontend/src/app/shared/services/chat.service.spec.ts", "content": "import { TestBed } from '@angular/core/testing';\n\nimport { ChatService } from './chat.service';\n\ndescribe('ChatService', () => {\n let service: ChatService;\n\n beforeEach(() => {\n TestBed.configureTestingModule({});\n service = TestBed.inject(ChatService);\n });\n\n it('should be created', () => {\n expect(service).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/shared/services/chat.service.ts", "content": "import { Injectable } from '@angular/core';\nimport { Subject, takeUntil } from 'rxjs';\nimport { BehaviorSubject } from 'rxjs/internal/BehaviorSubject';\nimport { HomeService } from './home.service';\n\nexport interface Message {\n author: string\n message?: any\n user_question: string\n generate_sql?: any\n run_query?: any\n visualize?: any\n dataSource?: any,\n displayedColumns?: any\n dataSet?: any\n ind?: any\n embed_sql?: any\n}\ninterface ChatSession {\n chatMsgs: Message[]\n}\n\n@Injectable({\n providedIn: 'root'\n})\n\nexport class ChatService {\n private chatSess: ChatSession = {\n chatMsgs: []\n }\n public currentActiveSession = new BehaviorSubject(this.chatSess);\n chatSessionObservable = this.currentActiveSession.asObservable();\n public agentResponseLoader = new BehaviorSubject(false);\n agentResponseLoader$ = this.agentResponseLoader.asObservable();\n private _destroy$ = new Subject();\n private currentSessionId: string = '';\n private suggestionList: [] = [];\n selectedGrouping!: string;\n\n constructor(public homeService: HomeService) {\n this.homeService.currentSelectedGroupingObservable.pipe(takeUntil(this._destroy$)).subscribe((res) => {\n this.selectedGrouping = res\n })\n this.homeService.knownSqlObservable?.pipe(takeUntil(this._destroy$)).subscribe((response: any) => {\n if (response && response != null) {\n this.suggestionList = JSON.parse(response);\n this.createNewSession()\n }\n })\n }\n\n createNewSession() {\n const newSession: ChatSession = {\n chatMsgs: [{\n 'author': 'agent',\n 'message': this.suggestionList,\n 'user_question': 'Looking for a specific insight or want to browse through your database? Ask what you are looking for in a natural language and Open Data QnA will translate to SQL and bring back results in natural language.'\n }]\n }\n this.currentSessionId = '';\n this.currentActiveSession.next(newSession);\n }\n\n addToSessionThread(message: Message) {\n const desiredSession = this.currentActiveSession.value;\n if (desiredSession) {\n desiredSession.chatMsgs.push(message);\n if (message.author == 'agent') {\n this.agentResponseLoader.next(false)\n }\n this.currentActiveSession.next(desiredSession);\n }\n }\n\n addQuestion(question: string, userId: string, agentCase: string, selectedHistory?: any) {\n let newChatMsg: Message = {\n author: 'user',\n message: [],\n user_question: question\n }\n if (agentCase != 'history') {\n this.addToSessionThread(newChatMsg);\n }\n // do API call for agent response.\n switch (agentCase) {\n case 'followup':\n this.generate_sql(question, userId);\n break;\n case 'history':\n this.generate_history_thread(selectedHistory)\n break;\n case 'scenario':\n this.generate_sql(question, userId)\n break;\n }\n }\n\n generate_sql(question: string, userId: string) {\n this.currentSessionId = this.homeService.getSessionId()\n this.agentResponseLoader.next(true)\n this.homeService.generateSql(question, this.homeService.getSelectedDbGrouping(), this.currentSessionId, userId).subscribe((response: any) => {\n if (response !== undefined) {\n this.homeService.setSessionId(response.SessionID)\n this.currentSessionId = response.SessionID;\n const newChatMsg: Message = {\n 'author': 'agent',\n 'user_question': question,\n 'generate_sql': response,\n }\n this.addToSessionThread(newChatMsg);\n }\n })\n }\n\n generate_history_thread(selectedHistory: any) {\n for (let i = selectedHistory?.length - 1; i >= 0; i--) {\n const newUserMsg: Message = {\n author: 'user',\n user_question: selectedHistory[i]?.user_question\n }\n this.addToSessionThread(newUserMsg);\n const newAgentMsg: Message = {\n 'author': 'agent',\n 'user_question': selectedHistory[i]?.user_question,\n 'generate_sql': {\n 'GeneratedSQL': selectedHistory[i]?.bot_response,\n 'SessionID': selectedHistory[i]?.session_id,\n \"ResponseCode\": 200\n }\n }\n this.homeService.setSessionId(selectedHistory[i]?.session_id)\n this.addToSessionThread(newAgentMsg);\n }\n }\n}\n" }, { "path": "frontend/src/app/shared/services/home.service.spec.ts", "content": "import { TestBed } from '@angular/core/testing';\n\nimport { HomeService } from './home.service';\n\ndescribe('HomeService', () => {\n let service: HomeService;\n\n beforeEach(() => {\n TestBed.configureTestingModule({});\n service = TestBed.inject(HomeService);\n });\n\n it('should be created', () => {\n expect(service).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/shared/services/home.service.ts", "content": "import { Injectable, inject } from '@angular/core';\nimport { HttpHeaders, HttpClient, HttpErrorResponse } from '@angular/common/http';\nimport { catchError, throwError, BehaviorSubject } from 'rxjs';\nimport { ENDPOINT_OPENDATAQNA } from '../../../assets/constants';\nimport { Firestore, collection, collectionData, doc, docData, orderBy, query, updateDoc, where } from '@angular/fire/firestore';\n\n@Injectable({\n providedIn: 'root'\n})\nexport class HomeService {\n public knownSqlFromDb = new BehaviorSubject(null);\n knownSqlObservable = this.knownSqlFromDb.asObservable();\n\n public currentSelectedGrouping = new BehaviorSubject('');\n currentSelectedGroupingObservable = this.currentSelectedGrouping.asObservable();\n private databaseList: any;\n private selectedGrouping: any;\n public selectedDBType: any;\n selectedDbName: any;\n chatMsgs: any[] = [];\n selectedHistory: any\n private firestore: Firestore = inject(Firestore);\n session_id: any = '';\n constructor(public http: HttpClient) { }\n\n ngOnInit() { }\n\n getUserSessions(userId: string) {\n const sessionCollection = collection(this.firestore, `session_logs`);\n const orderedCollection = query(sessionCollection, orderBy(\"timestamp\", \"desc\"));\n const filter = query(orderedCollection, where(\"user_id\", \"==\", userId))\n return collectionData(filter, { idField: 'id' })\n }\n\n getAvailableDatabases(): any {\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n\n return this.http.get(ENDPOINT_OPENDATAQNA + '/available_databases', requestOptions).pipe(catchError(this.handleError))\n }\n\n sqlSuggestionList(grouping: any, dbtype: any) {\n\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n\n const body =\n {\n \"user_grouping\": grouping\n }\n this.selectedDBType = dbtype;\n\n return this.http.post(ENDPOINT_OPENDATAQNA + '/get_known_sql', body, requestOptions)\n .pipe(catchError(this.handleError));\n\n }\n \n generateSql(userQuestion: any, grouping: any, session_id: any, user_id: any) {\n\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n const body =\n {\n \"user_question\": userQuestion,\n \"user_grouping\": grouping,\n \"session_id\": session_id,\n \"user_id\": user_id\n }\n let endpoint = ENDPOINT_OPENDATAQNA;\n\n return this.http.post(endpoint + \"/generate_sql\", body, requestOptions)\n .pipe(catchError(this.handleError));\n }\n\n private handleError(error: HttpErrorResponse) {\n if (error.error instanceof ErrorEvent) {\n console.error('An error occurred:', error.error);\n } else {\n }\n return throwError(\n 'Something bad happened; please try again later.');\n }\n \n setAvailableDBList(databaseList: string) {\n this.databaseList = databaseList;\n }\n getAvailableDBList(): string {\n return this.databaseList;\n }\n setSelectedDbGrouping(selectedDBGroup: any) {\n this.selectedGrouping = selectedDBGroup;\n }\n getSelectedDbGrouping(): string {\n return this.selectedGrouping;\n }\n\n setselectedDbName(databaseList: any) {\n this.selectedDbName = databaseList;\n }\n getselectedDbName(): string {\n return this.selectedDbName;\n }\n\n setSessionId(session_id: any) {\n this.session_id = session_id;\n }\n getSessionId(): string {\n return this.session_id;\n }\n\n getChatMsgs(): any[] {\n return this.chatMsgs\n }\n\n updateChatMsgs(chatMsgs: any) {\n this.chatMsgs = chatMsgs;\n }\n\n getSelectedHistory() {\n return this.selectedHistory\n }\n\n updateSelectedHistory(selectedHistory: any) {\n this.selectedHistory = selectedHistory\n }\n\n updateChatMsgsAtIndex(chatMsg: any, ind: any) {\n this.chatMsgs[ind] = chatMsg\n }\n\n runQuery(query: any, grouping: any, user_question: any, session_id: any) {\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n const body =\n {\n \"generated_sql\": query,\n \"user_grouping\": grouping,\n \"user_question\": user_question,\n \"session_id\": this.session_id\n }\n let endpoint = ENDPOINT_OPENDATAQNA;\n\n return this.http.post(endpoint + \"/run_query\", body, requestOptions);\n }\n\n thumbsUp(sql: any, user_question: any, selectedGrouping: any, session_id: any) {\n\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n const body =\n {\n user_grouping: selectedGrouping,\n generated_sql: sql,\n user_question: user_question,\n session_id: this.session_id\n }\n let endpoint = ENDPOINT_OPENDATAQNA;\n return this.http.post(endpoint + \"/embed_sql\", body, requestOptions)\n .pipe(catchError(this.handleError));\n }\n\n generateViz(question: any, query: any, result: any, session_id: any) {\n const header = {\n 'Content-Type': 'application/json',\n }\n const requestOptions = {\n headers: new HttpHeaders(header),\n };\n const body =\n {\n \"user_question\": question,\n \"sql_generated\": query,\n \"sql_results\": result,\n \"session_id\": this.session_id\n }\n return this.http.post(ENDPOINT_OPENDATAQNA + \"/generate_viz\", body, requestOptions)\n .pipe(catchError(this.handleError));\n }\n}\n" }, { "path": "frontend/src/app/shared/services/login.service.spec.ts", "content": "import { TestBed } from '@angular/core/testing';\n\nimport { LoginService } from './login.service';\n\ndescribe('LoginService', () => {\n let service: LoginService;\n\n beforeEach(() => {\n TestBed.configureTestingModule({});\n service = TestBed.inject(LoginService);\n });\n\n it('should be created', () => {\n expect(service).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/shared/services/login.service.ts", "content": "import { Injectable } from '@angular/core';\nimport { Observable, ReplaySubject } from 'rxjs';\n\n@Injectable({\n providedIn: 'root'\n})\nexport class LoginService {\n\n constructor() { }\n private userDetails = new ReplaySubject(1);\n userDetails$: Observable = this.userDetails.asObservable();\n loginErrorMsg: any = new ReplaySubject(1);\n private idToken = new ReplaySubject(1);\n idToken$: Observable = this.idToken.asObservable();\n \n getLoginError(): any {\n return this.loginErrorMsg;\n }\n updateLoginError(msg: any) {\n this.loginErrorMsg.next(msg)\n }\n getUserDetails(): Observable {\n return this.userDetails$;\n }\n getIdToken(): any {\n return this.idToken$;\n }\n\n setIdToken(token: any) {\n this.idToken.next(token);\n }\n\n sendUserDetails(message: any) {\n this.userDetails.next(message);\n }\n}\n" }, { "path": "frontend/src/app/shared/services/shared.service.spec.ts", "content": "import { TestBed } from '@angular/core/testing';\n\nimport { SharedService } from './shared.service';\n\ndescribe('SharedService', () => {\n let service: SharedService;\n\n beforeEach(() => {\n TestBed.configureTestingModule({});\n service = TestBed.inject(SharedService);\n });\n\n it('should be created', () => {\n expect(service).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/shared/services/shared.service.ts", "content": "import { Injectable, inject } from '@angular/core';\nimport { LoginService } from './login.service';\nimport { GoogleAuthProvider, signInWithPopup } from '@firebase/auth';\nimport { Auth } from '@angular/fire/auth';\n\n@Injectable({\n providedIn: 'root'\n})\nexport class SharedService {\n userData: any;\n private auth: Auth = inject(Auth);\n\n constructor(public loginservice: LoginService) { }\n\n async googleSignin() {\n const provider = new GoogleAuthProvider();\n\n return await signInWithPopup(this.auth, provider)\n .then(async (result) => {\n const token = await this.auth.currentUser?.getIdToken(); \n this.loginservice.setIdToken(token); \n return result.user;\n }).\n catch((error) => {\n if (error.message.indexOf('Cloud Function') === 15) {\n const jsonStart = error.message.indexOf('{');\n const jsonEnd = error.message.lastIndexOf('}');\n const jsonString = error.message.substring(jsonStart, jsonEnd + 1);\n const errorObject = JSON.parse(jsonString);\n this.loginservice.updateLoginError(errorObject.error.message)\n } else {\n this.loginservice.updateLoginError(error.message)\n }\n });\n }\n}\n\n" }, { "path": "frontend/src/app/upload-template/upload-template.component.html", "content": "
\n
\n Please upload a csv file with the details below.\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
user_groupingscenarioquestion
user_grouping1scenario1question1
user_grouping1scenario1question2
user_grouping1scenario1question3
user_grouping1scenario2question1
user_grouping1scenario2question2
user_grouping1scenario2question3
user_grouping2scenario1question1
user_grouping2scenario1question2
user_grouping2scenario1question3
\n
\n \n
" }, { "path": "frontend/src/app/upload-template/upload-template.component.scss", "content": ".grouping-msg {\n font-family: \"Google Sans\";\n font-size: 16px;\n color: #d93035;\n margin-bottom: 20px;\n}\n\n.closeBtn {\n border-color: transparent;\n min-width: 146px;\n height: 41px;\n background: #1A73E8;\n border-radius: 100px;\n font-family: \"Google Sans\";\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n display: flex;\n align-items: center;\n text-align: center;\n justify-content: center;\n letter-spacing: 0.25px;\n color: #FFFFFF;\n}\n\n::ng-deep .mat-dialog-container {\n border-radius: 40px !important;\n}\n\n.popup {\n display: flex;\n justify-content: space-evenly;\n align-items: center;\n flex-direction: column;\n gap: 20px;\n margin: 20px;\n}\n\n#customers {\n font-family: 'Google Sans';\n border-collapse: collapse;\n width: 100%;\n}\n\n#customers td,\n#customers th {\n border: 1px solid #ddd;\n padding: 8px;\n}\n\n#customers tr:nth-child(even) {\n background-color: #f2f2f2;\n}\n\n#customers tr:hover {\n background-color: #ddd;\n}\n\n#customers th {\n padding-top: 12px;\n padding-bottom: 12px;\n text-align: left;\n background-color: #04AA6D;\n color: white;\n}" }, { "path": "frontend/src/app/upload-template/upload-template.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { UploadTemplateComponent } from './upload-template.component';\n\ndescribe('UploadTemplateComponent', () => {\n let component: UploadTemplateComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [UploadTemplateComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(UploadTemplateComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/upload-template/upload-template.component.ts", "content": "import { Component } from '@angular/core';\nimport { MatDialogRef } from '@angular/material/dialog';\n\n@Component({\n selector: 'app-upload-template',\n standalone: true,\n imports: [],\n templateUrl: './upload-template.component.html',\n styleUrl: './upload-template.component.scss'\n})\nexport class UploadTemplateComponent {\n constructor(public dialogRef: MatDialogRef) {\n\n }\n closeDialog() {\n this.dialogRef.close(true);\n }\n}\n" }, { "path": "frontend/src/app/user-journey/user-journey.component.html", "content": "
\n
\n \n \n \n \n \n \n
\n\n
\n
\n\n
\n\n
\n\n
\n\n

{{user.userTitle}}

\n

{{user.userId}}

\n
    \n
    \n
  • \n {{content}}\n
    • \n
    \n
\n
\n\n \n
\n \n
\n
\n \n
\n\n
\n
\n
\n \n
\n

Watch the UI Walkthrough below

\n \n
\n
" }, { "path": "frontend/src/app/user-journey/user-journey.component.scss", "content": ".list-css {\n background-image: radial-gradient(grey, black);\n min-width: 100%;\n min-height: 100%;\n}\n\n.search {\n color: #FFFFFF;\n}\n\n.userTitle {\n width: 180px;\n height: 20px;\n top: 788px;\n left: 485px;\n font-family: Google Sans;\n font-size: 24px;\n font-weight: 700;\n line-height: 25px;\n letter-spacing: 0.10000000149011612px;\n text-align: left;\n color: #FFFFFF;\n display: inline;\n\n}\n\n.userId {\n top: 498px;\n left: 485px;\n font-family: \"Google Sans\";\n font-size: 18px;\n font-weight: 400;\n line-height: 20px;\n letter-spacing: 0.10000000149011612px;\n text-align: left;\n color: #FFFFFF;\n display: inline;\n float: right;\n}\n\n.content {\n width: 612px;\n height: 162px;\n top: 426px;\n left: 689px;\n font-family: Google Sans;\n font-size: 18px;\n font-style: normal;\n font-weight: 400;\n line-height: 27px;\n letter-spacing: 0.10000000149011612px;\n text-align: left;\n color: #FFFFFF;\n}\n\n.user-text {\n font-family: Google Sans;\n font-size: 20px;\n font-weight: 400;\n font-style: normal;\n line-height: 27px;\n letter-spacing: 0.10000000149011612px;\n text-align: left;\n color: white;\n background: rgba(255, 255, 255, 0.2);\n border-radius: 19px;\n padding: 20px 40px;\n}\n\n.user-journey-btn {\n display: flex;\n align-items: center;\n gap: 10px;\n margin-left: 70px;\n width: 168px;\n height: 36px;\n left: 164px;\n top: 1114px;\n background: #1A73E8;\n border-radius: 4px;\n width: 200px;\n height: 40px;\n font-family: \"Google Sans\";\n font-style: normal;\n font-weight: 500;\n font-size: 14px;\n line-height: 20px;\n display: flex;\n align-items: center;\n text-align: center;\n justify-content: center;\n letter-spacing: 0.25px;\n flex: none;\n order: 0;\n flex-grow: 0;\n color: #FFFFFF;\n border: none !important;\n}\n\n.align {\n align-items: center;\n}\n\n.list-group .list-group-item {\n border: none !important;\n}\n\n.list-group {\n margin-top: 100px;\n}\n\n.demoBtn {\n float: right;\n display: inline-flex;\n}\n\n.flex-container {\n display: flex;\n flex-direction: row;\n flex-wrap: wrap;\n width: 100%;\n}\n\n.flex-child {\n flex: 1;\n border: 2px solid yellow;\n}\n\n.flex-child:first-child {\n margin-right: 20px;\n}\n\n.gap-10 {\n gap: 10px;\n}\n\n.loading-spinner {\n margin-left: 25px;\n margin-top: 5px;\n}\n\n.video-demo {\n font-size: 18px;\n font-family: 'Google Sans';\n display: flex;\n flex-direction: column;\n justify-content: center;\n align-items: center;\n gap : 20px;\n margin : 10px\n}\n\n.video-title {\n font-family: \"Google Sans\";\n text-align: center;\n margin-bottom: 10px; /* Add a margin to separate the title from the video */\n color: white; \n}" }, { "path": "frontend/src/app/user-journey/user-journey.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { UserJourneyComponent } from './user-journey.component';\n\ndescribe('UserJourneyComponent', () => {\n let component: UserJourneyComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [UserJourneyComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(UserJourneyComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/user-journey/user-journey.component.ts", "content": "import { AfterViewInit, Component } from '@angular/core';\nimport { LoginService } from '../shared/services/login.service';\nimport { Subscription } from 'rxjs';\nimport { Router } from '@angular/router';\nimport { HomeService } from '../shared/services/home.service';\n\n@Component({\n selector: 'app-user-journey',\n templateUrl: './user-journey.component.html',\n styleUrl: './user-journey.component.scss'\n})\nexport class UserJourneyComponent implements AfterViewInit {\n photoURL: string | undefined;\n subscription: Subscription | undefined;\n showProgress: boolean = false\n loginError = false;\n loginErrorMessage: any;\n demoVideo = false;\n userJourneyList: any = [{\n userId: \"User journey 1\",\n userTitle: \"Business User\",\n userContent: [\n \"This demo will help you to ask the questions in natural language, view the SQL, get results and visualize data\",\n ]\n }];\n\n constructor(public _router: Router, public loginService: LoginService, public homeService: HomeService) {\n this.subscription = this.loginService.getUserDetails().subscribe(message => {\n this.photoURL = message?.photoURL;\n if (!this.photoURL) {\n this._router.navigate(['']);\n }\n });\n }\n\n onDemoVideoClick(){\n this.demoVideo = true;\n }\n ngOnInit() {\n\n // this.loginService.getLoginError().subscribe((res: any) => {\n // this.loginErrorMessage = res\n // this.loginError = true;\n // if(this.loginError){\n // this._router.navigate(['']);\n // }\n // });\n }\n\n ngAfterViewInit(){\n window.scroll(0, 0);\n }\n\n async navigateToHome(userTitle: String) {\n\n if (userTitle === 'Business User') {\n //this.homeService.checkuserType = 'Business';\n this.showProgress = true;\n this.homeService.getAvailableDatabases().subscribe((res: any) => {\n if (res && res.ResponseCode === 200) {\n this.homeService.setAvailableDBList(res.KnownDB);\n this.showProgress = false;\n this._router.navigate(['home-page']);\n }\n })\n }\n }\n\n ngOnDestroy() {\n this.subscription?.unsubscribe()\n }\n}\n" }, { "path": "frontend/src/app/user-photo/user-photo.component.html", "content": " \"User\n\n" }, { "path": "frontend/src/app/user-photo/user-photo.component.scss", "content": "\n.login-user {\n float: right;\n padding-right: 20px;\n}\n" }, { "path": "frontend/src/app/user-photo/user-photo.component.spec.ts", "content": "import { ComponentFixture, TestBed } from '@angular/core/testing';\n\nimport { UserPhotoComponent } from './user-photo.component';\n\ndescribe('UserPhotoComponent', () => {\n let component: UserPhotoComponent;\n let fixture: ComponentFixture;\n\n beforeEach(async () => {\n await TestBed.configureTestingModule({\n imports: [UserPhotoComponent]\n })\n .compileComponents();\n \n fixture = TestBed.createComponent(UserPhotoComponent);\n component = fixture.componentInstance;\n fixture.detectChanges();\n });\n\n it('should create', () => {\n expect(component).toBeTruthy();\n });\n});\n" }, { "path": "frontend/src/app/user-photo/user-photo.component.ts", "content": "import { Component, inject } from '@angular/core';\nimport { LoginButtonComponent } from '../login-button/login-button.component';\nimport { Subscription } from 'rxjs';\nimport { Auth, User, user } from '@angular/fire/auth';\nimport { Router } from '@angular/router';\nimport { LoginService } from '../shared/services/login.service';\nimport { Dialog } from '@angular/cdk/dialog';\n\n@Component({\n selector: 'app-user-photo',\n templateUrl: './user-photo.component.html',\n styleUrl: './user-photo.component.scss'\n})\nexport class UserPhotoComponent {\n photoURL: string | undefined;\n subscription: Subscription | undefined;\n userLoggedIn: boolean = false;\n private auth: Auth = inject(Auth);\n user$ = user(this.auth);\n userSubscription: Subscription;\n\n constructor(private _router: Router, public dialog: Dialog, public loginService: LoginService) {\n this.dialog.closeAll();\n this.userSubscription = this.user$.subscribe((aUser: User | null) => {\n //handle user state changes here. Note, that user will be null if there is no currently logged in user\n if (aUser) {\n this.dialog.closeAll();\n this.userLoggedIn = true;\n this.loginService.sendUserDetails(aUser)\n if (aUser.photoURL) {\n this.photoURL = aUser.photoURL;\n }\n }\n else {\n this.userLoggedIn = false;\n this.showLogIn()\n }\n })\n }\n\n // ngOnInit() {\n // if (!this.photoURL) {\n // this.showLogIn()\n // }\n // }\n\n showLogIn(): void {\n this.dialog.open(LoginButtonComponent, {\n disableClose: true,\n width: '350px',\n panelClass: 'login-container'\n });\n }\n\n ngOnDestroy() {\n this.dialog.closeAll();\n // when manually subscribing to an observable remember to unsubscribe in ngOnDestroy\n this.userSubscription.unsubscribe();\n }\n}\n" }, { "path": "frontend/src/assets/.gitkeep", "content": "" }, { "path": "frontend/src/assets/constants.ts", "content": "export const firebaseConfig = {\n \"projectId\": \"\",\n \"appId\": \"\",\n \"storageBucket\": \"\",\n \"apiKey\": \"\",\n \"authDomain\": \"\",\n \"messagingSenderId\": \"\"\n};\nexport const ENDPOINT_OPENDATAQNA = 'https://opendataqna-kdr33rftkq-uc.a.run.app'\n\nexport const FIRESTORE_DATABASE_ID = 'opendataqna-session-logs'\n" }, { "path": "frontend/src/index.html", "content": "\n\n\n\n \n Open Data QnA\n \n \n \n \n \n \n \n\n\n\n \n\n\n" }, { "path": "frontend/src/main.server.ts", "content": "export { AppServerModule as default } from './app/app.module.server';\n" }, { "path": "frontend/src/main.ts", "content": "import { AppModule } from './app/app.module';\nimport { platformBrowserDynamic } from '@angular/platform-browser-dynamic';\n\nplatformBrowserDynamic().bootstrapModule(AppModule)\n .catch((err) => console.error(err));\n" }, { "path": "frontend/src/styles/variables.scss", "content": "$bgColor_light: white;\n$bgColor_dark: black;\n\n$textColor_light: black;\n$textColor_dark: white;\n\n$borderColor_light: black;\n$borderColor_dark: white;\n\n// mixin that enables css variables in light mode\n@mixin lighten() {\n --bgColor: #{$bgColor_light};\n --textColor: #{$textColor_light};\n --borderColor: #{$borderColor_light};\n}\n\n// mixin that enables css variables in dark mode\n@mixin darken() {\n --bgColor: #{$bgColor_dark};\n --textColor: #{$textColor_dark};\n --borderColor: #{$borderColor_dark};\n}" }, { "path": "frontend/src/styles.scss", "content": "// Custom Theming for Angular Material\n// For more information: https://material.angular.io/guide/theming\n\n@use '@angular/material' as mat;\n\n@import url('https://fonts.googleapis.com/icon?family=Google+Sans');\n@import \"prismjs/themes/prism.css\";\n\n\n\n// Plus imports for other components in your app.\n\n// Include the common styles for Angular Material. We include this here so that you only\n// have to load a single css file for Angular Material in your app.\n// Be sure that you only ever include this mixin once!\n@include mat.core();\n\n// Define the palettes for your theme using the Material Design palettes available in palette.scss\n// (imported above). For each palette, you can optionally specify a default, lighter, and darker\n// hue. Available color palettes: https://material.io/design/color/\n$genai-csm-primary: mat.define-palette(mat.$indigo-palette);\n$genai-csm-accent: mat.define-palette(mat.$pink-palette, A200, A100, A400);\n\n// The warn palette is optional (defaults to red).\n$genai-csm-warn: mat.define-palette(mat.$red-palette);\n\n// Create the theme object. A theme consists of configurations for individual\n// theming systems such as \"color\" or \"typography\".\n$genai-csm-theme: mat.define-light-theme((color: (primary: $genai-csm-primary,\n accent: $genai-csm-accent,\n warn: $genai-csm-warn,\n )));\n\n// Include theme styles for core and each component used in your app.\n// Alternatively, you can import and @include the theme mixins for each component\n// that you are using.\n@include mat.all-component-themes($genai-csm-theme);\n\n/* You can add global styles to this file, and also import other style files */\n\nhtml,\nbody {\n height: 100%;\n font-family: 'Google Sans';\n}\n\nbody {\n margin: 0;\n}\n\n\n$gap : 4vh;\n$padding : 4vh;\n\n\n$bp : (mobile : 480px,\n tablet : 768px,\n desktop : 1440px,\n);\n\n\n@mixin query($display) {\n @each $key, $value in $bp {\n\n // defining max-width\n @if ($display ==$key) {\n @media (max-width: $value) {\n @content;\n }\n }\n }\n}\n\n\n.container {\n display: flex;\n //to lay .block-* classes in a column\n flex-direction: column;\n //Setting gap between the .block-* classes\n gap: $gap;\n\n // to set some padding & border inside\n padding: $padding;\n}" }, { "path": "frontend/tsconfig.app.json", "content": "/* To learn more about this file see: https://angular.io/config/tsconfig. */\n{\n \"extends\": \"./tsconfig.json\",\n \"compilerOptions\": {\n \"outDir\": \"./out-tsc/app\",\n \"types\": [\n \"node\"\n ]\n },\n \"files\": [\n \"src/main.ts\",\n \"src/main.server.ts\",\n \"server.ts\"\n ],\n \"include\": [\n \"src/**/*.d.ts\"\n ]\n}\n" }, { "path": "frontend/tsconfig.json", "content": "/* To learn more about this file see: https://angular.io/config/tsconfig. */\n{\n \"compileOnSave\": false,\n \"compilerOptions\": {\n \"outDir\": \"./dist/out-tsc\",\n \"forceConsistentCasingInFileNames\": true,\n \"strict\": true,\n \"noImplicitOverride\": true,\n \"noPropertyAccessFromIndexSignature\": true,\n \"noImplicitReturns\": true,\n \"noFallthroughCasesInSwitch\": true,\n \"skipLibCheck\": true,\n \"esModuleInterop\": true,\n \"sourceMap\": true,\n \"declaration\": false,\n \"experimentalDecorators\": true,\n \"moduleResolution\": \"node\",\n \"importHelpers\": true,\n \"target\": \"ES2022\",\n \"module\": \"ES2022\",\n \"resolveJsonModule\": true,\n \"useDefineForClassFields\": false,\n \"lib\": [\n \"ES2022\",\n \"dom\"\n ]\n },\n \"angularCompilerOptions\": {\n \"enableI18nLegacyMessageIdFormat\": false,\n \"strictInjectionParameters\": true,\n \"strictInputAccessModifiers\": true,\n \"strictTemplates\": true\n }\n}\n" }, { "path": "frontend/tsconfig.spec.json", "content": "/* To learn more about this file see: https://angular.io/config/tsconfig. */\n{\n \"extends\": \"./tsconfig.json\",\n \"compilerOptions\": {\n \"outDir\": \"./out-tsc/spec\",\n \"types\": [\n \"jasmine\"\n ]\n },\n \"include\": [\n \"src/**/*.spec.ts\",\n \"src/**/*.d.ts\"\n ]\n}\n" }, { "path": "notebooks/0_CopyDataToBigQuery.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"
\\n\",\n \" \\n\",\n \" \\n\",\n \" \\n\",\n \"
\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"copyright\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Copyright 2024 Google LLC\\n\",\n \"#\\n\",\n \"# Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\",\n \"# you may not use this file except in compliance with the License.\\n\",\n \"# You may obtain a copy of the License at\\n\",\n \"#\\n\",\n \"# https://www.apache.org/licenses/LICENSE-2.0\\n\",\n \"#\\n\",\n \"# Unless required by applicable law or agreed to in writing, software\\n\",\n \"# distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\",\n \"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\",\n \"# See the License for the specific language governing permissions and\\n\",\n \"# limitations under the License.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"DRyGcAepAPJ5\"\n },\n \"source\": [\n \"\\n\",\n \"# **Open Data QnA: Set up BigQuery Source**\\n\",\n \"\\n\",\n \"---\\n\",\n \"\\n\",\n \"This notebook shows how to copy a BigQuery public dataset to your GCP project \\n\",\n \"\\n\",\n \"\\n\",\n \"This is accomplished through the three following steps: \\n\",\n \"> i. Create a BigQuery dataset in your GCP project\\n\",\n \"\\n\",\n \"> ii. Create a table in the above dataset\\n\",\n \"\\n\",\n \"> iii. Copy data from the public dataset to the dataset on your project\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### **Change your Kernel to the created .venv with poetry from README.md**\\n\",\n \"\\n\",\n \"Below is the Kernel how it should look like before you proceed\\n\",\n \"\\n\",\n \"![Kernel](../utilities/imgs/Kernel%20Changed.png)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🔗 **1. Connect Your Google Cloud Project**\\n\",\n \"Time to connect your Google Cloud Project to this notebook. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Updated property [core/project].\\n\",\n \"Project has been set to three-p-o\\n\"\n ]\n }\n ],\n \"source\": [\n \"#@markdown Please fill in the value below with your GCP project ID and then run the cell.\\n\",\n \"PROJECT_ID = input(\\\"Enter the project id (same as your Setup Project) to copy source data in bigquery for this solution\\\")\\n\",\n \"\\n\",\n \"# Quick input validation\\n\",\n \"assert PROJECT_ID, \\\"⚠️ Please provide your Google Cloud Project ID\\\"\\n\",\n \"\\n\",\n \"# Configure gcloud.\\n\",\n \"!gcloud config set project {PROJECT_ID}\\n\",\n \"print(f'Project has been set to {PROJECT_ID}')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"yygMe6rPWxHS\"\n },\n \"source\": [\n \"## 🔐 **2. Authenticate to Google Cloud**\\n\",\n \"Authenticate to Google Cloud as the IAM user logged into this notebook in order to access your Google Cloud Project.\\n\",\n \"\\n\",\n \"You can do this within Google Colab or using the Application Default Credentials in the Google Cloud CLI.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {\n \"id\": \"PTXN1_DSXj2b\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Authentication step\\n\",\n \"\\n\",\n \"\\\"\\\"\\\"Colab Auth\\\"\\\"\\\" \\n\",\n \"# from google.colab import auth\\n\",\n \"# auth.authenticate_user()\\n\",\n \"\\n\",\n \"\\n\",\n \"\\\"\\\"\\\"Jupiter Notebook Auth\\\"\\\"\\\"\\n\",\n \"import google.auth\\n\",\n \"import os\\n\",\n \"\\n\",\n \"credentials, project_id = google.auth.default()\\n\",\n \"\\n\",\n \"os.environ['GOOGLE_CLOUD_QUOTA_PROJECT']=PROJECT_ID\\n\",\n \"os.environ['GOOGLE_CLOUD_PROJECT']=PROJECT_ID\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"#Enable all the required APIs for the COPY\\n\",\n \"\\n\",\n \"!gcloud services enable \\\\\\n\",\n \" cloudapis.googleapis.com \\\\\\n\",\n \" compute.googleapis.com \\\\\\n\",\n \" iam.googleapis.com \\\\\\n\",\n \" bigquery.googleapis.com --project {PROJECT_ID}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## ☁️ **Copy a Public Dataset to your GCP Project**\\n\",\n \"\\n\",\n \"Copy a table from the public dataset to ask questions against. A sample table is chosen below, feel free to choose a different one. \\n\",\n \"\\n\",\n \"Note: BigQuery does not allow tables to be copied across regions in certain cases. Therefore, BQ_DST_REGION is set to be BQ_SRC_REGION. Change this parameter to check if copy to your region of interest is allowed.\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Details of source Dataset\\n\",\n \"BQ_SRC_PROJECT = \\\"bigquery-public-data\\\"\\n\",\n \"BQ_SRC_DATASETS = [\\\"imdb\\\", \\\"imdb\\\"]\\n\",\n \"BQ_SRC_TABLES_LIST = [[\\\"title_principals\\\",\\\"title_crew\\\", \\\"title_basics\\\",\\\"name_basics\\\"], [\\\"reviews\\\", \\\"title_ratings\\\"]] # [] Specify empty list to copy 'all' tables, or a Specific list, eg: [\\\"table1\\\", \\\"table3\\\", \\\"table10\\\"]\\n\",\n \"BQ_SRC_REGIONS = [\\\"us\\\", \\\"us\\\"]\\n\",\n \"\\n\",\n \"# Details of destination Dataset\\n\",\n \"BQ_DST_PROJECT = PROJECT_ID\\n\",\n \"BQ_DST_DATASETS =[ \\\"imdb_people\\\", \\\"imdb_ratings\\\"] # List of destinaion dataset names\\n\",\n \"BQ_DST_REGIONS = BQ_SRC_REGIONS # Change if needed\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def createBQDataset(bq_project_id, dataset_name,dataset_region):\\n\",\n \" from google.cloud import bigquery\\n\",\n \" import google.api_core \\n\",\n \"\\n\",\n \" client=bigquery.Client(project=PROJECT_ID)\\n\",\n \"\\n\",\n \" dataset_ref = f\\\"{bq_project_id}.{dataset_name}\\\"\\n\",\n \" \\n\",\n \"\\n\",\n \" try:\\n\",\n \" client.get_dataset(dataset_ref)\\n\",\n \" print(\\\"Destination Dataset exists\\\")\\n\",\n \" except google.api_core.exceptions.NotFound:\\n\",\n \" print(\\\"Cannot find the dataset hence creating.......\\\")\\n\",\n \" dataset=bigquery.Dataset(dataset_ref)\\n\",\n \" dataset.location=dataset_region\\n\",\n \" client.create_dataset(dataset)\\n\",\n \" \\n\",\n \" return dataset_ref\\n\",\n \"\\n\",\n \"def createBQTable(bq_project_id,dataset_name, table_name):\\n\",\n \" from google.cloud import bigquery\\n\",\n \" import google.api_core \\n\",\n \"\\n\",\n \" client=bigquery.Client(project=PROJECT_ID)\\n\",\n \"\\n\",\n \" table_ref = client.dataset(dataset_name, project=bq_project_id).table(table_name)\\n\",\n \"\\n\",\n \" try:\\n\",\n \" client.get_table(table_ref)\\n\",\n \" print(f\\\"Destination Table {table_ref} exists\\\")\\n\",\n \" \\n\",\n \" except google.api_core.exceptions.NotFound:\\n\",\n \" print(f\\\"Creating the table {table_ref}.......\\\")\\n\",\n \" table = bigquery.Table(table_ref)\\n\",\n \" client.create_table(table)\\n\",\n \"\\n\",\n \" return table_ref\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Cannot find the dataset hence creating.......\\n\",\n \"Creating the table three-p-o.imdb_people.title_principals.......\\n\",\n \"Creating the table three-p-o.imdb_people.title_crew.......\\n\",\n \"Creating the table three-p-o.imdb_people.title_basics.......\\n\",\n \"Creating the table three-p-o.imdb_people.name_basics.......\\n\",\n \"Cannot find the dataset hence creating.......\\n\",\n \"Creating the table three-p-o.imdb_ratings.reviews.......\\n\",\n \"Creating the table three-p-o.imdb_ratings.title_ratings.......\\n\",\n \"Done!\\n\"\n ]\n }\n ],\n \"source\": [\n \"#Create destination table and copy table data\\n\",\n \"from google.cloud import bigquery\\n\",\n \"\\n\",\n \"# Initialize BQ client\\n\",\n \"client=bigquery.Client(project=PROJECT_ID)\\n\",\n \"\\n\",\n \"for BQ_SRC_DATASET, BQ_SRC_TABLES, BQ_SRC_REGION, BQ_DST_DATASET, BQ_DST_REGION, in zip(BQ_SRC_DATASETS, BQ_SRC_TABLES_LIST, BQ_SRC_REGIONS, BQ_DST_DATASETS, BQ_DST_REGIONS):\\n\",\n \" \\n\",\n \" # Create Destination Dataset (If the dataset already exists, delete the dataset (and the tables with in) and create an empty dataset)\\n\",\n \" dst_dataset_ref=createBQDataset(BQ_DST_PROJECT,BQ_DST_DATASET,BQ_DST_REGION)\\n\",\n \"\\n\",\n \" if not BQ_SRC_TABLES:\\n\",\n \" #if tables are not explicitly provided, get the list of tables from bigquery\\n\",\n \" dataset_id = f'{BQ_SRC_PROJECT}.{BQ_SRC_DATASET}'\\n\",\n \" bq_tables_obj = client.list_tables(dataset_id)\\n\",\n \" BQ_SRC_TABLES = [table_obj.table_id for table_obj in bq_tables_obj]\\n\",\n \" \\n\",\n \" for BQ_SRC_TABLE in BQ_SRC_TABLES:\\n\",\n \"\\n\",\n \" dst_table_ref=createBQTable(BQ_DST_PROJECT,BQ_DST_DATASET,BQ_SRC_TABLE)\\n\",\n \" src_table_ref = client.dataset(BQ_SRC_DATASET, project=BQ_SRC_PROJECT).table(BQ_SRC_TABLE)\\n\",\n \"\\n\",\n \" job_config = bigquery.CopyJobConfig(write_disposition=\\\"WRITE_TRUNCATE\\\")\\n\",\n \"\\n\",\n \" copy_job = client.copy_table(src_table_ref, dst_table_ref, job_config=job_config)\\n\",\n \" # Wait for the job to complete and check for errors\\n\",\n \" copy_job.result() \\n\",\n \"\\n\",\n \"print('Done!')\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### If all the above steps are executed suucessfully, the Bigquery Public dataset should be copied to your GCP project\"\n ]\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.12.4\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "notebooks/0_CopyDataToCloudSqlPG.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"
\\n\",\n \" \\n\",\n \" \\n\",\n \" \\n\",\n \"
\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"copyright\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Copyright 2024 Google LLC\\n\",\n \"#\\n\",\n \"# Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\",\n \"# you may not use this file except in compliance with the License.\\n\",\n \"# You may obtain a copy of the License at\\n\",\n \"#\\n\",\n \"# https://www.apache.org/licenses/LICENSE-2.0\\n\",\n \"#\\n\",\n \"# Unless required by applicable law or agreed to in writing, software\\n\",\n \"# distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\",\n \"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\",\n \"# See the License for the specific language governing permissions and\\n\",\n \"# limitations under the License.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"DRyGcAepAPJ5\"\n },\n \"source\": [\n \"\\n\",\n \"# **Open Data QnA: Set up Dataset on CloudSQL for PostgreSQL**\\n\",\n \"\\n\",\n \"---\\n\",\n \"\\n\",\n \"This notebook shows how to copy a BigQuery public dataset to CloudSQL for PostgreSQL\\n\",\n \"\\n\",\n \"\\n\",\n \"This is accomplished through the three following steps: \\n\",\n \"> i. Set up PostgreSQL instance and databae on Google Cloud SQL\\n\",\n \"\\n\",\n \"> ii. Copy the BigQuery table to Cloud Storage Bucket\\n\",\n \"\\n\",\n \"> iii. Create the table in PostgreSQL databae using csv file in Cloud Storage Bucket\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### **Change your Kernel to the created .venv with poetry from README.md**\\n\",\n \"\\n\",\n \"Below is the Kernel how it should look like before you proceed\\n\",\n \"\\n\",\n \"![Kernel](../utilities/imgs/Kernel%20Changed.png)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"p4W6FPnrYEE8\"\n },\n \"source\": [\n \"## 🔗 **1. Connect Your Google Cloud Project**\\n\",\n \"Time to connect your Google Cloud Project to this notebook. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"#@markdown Please fill in the value below with your GCP project ID and then run the cell.\\n\",\n \"PROJECT_ID = input(\\\"Enter the project id (same as your Setup Project) to copy source data in bigquery for this solution\\\")\\n\",\n \"\\n\",\n \"# Quick input validations.\\n\",\n \"assert PROJECT_ID, \\\"⚠️ Please provide your Google Cloud Project ID\\\"\\n\",\n \"\\n\",\n \"# Configure gcloud.\\n\",\n \"!gcloud config set project {PROJECT_ID}\\n\",\n \"print(f'Project has been set to {PROJECT_ID}')\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"yygMe6rPWxHS\"\n },\n \"source\": [\n \"## 🔐 **2. Authenticate to Google Cloud**\\n\",\n \"Authenticate to Google Cloud as the IAM user logged into this notebook in order to access your Google Cloud Project.\\n\",\n \"\\n\",\n \"You can do this within Google Colab or using the Application Default Credentials in the Google Cloud CLI.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"PTXN1_DSXj2b\"\n },\n \"outputs\": [],\n \"source\": [\n \"\\\"\\\"\\\"Colab Auth\\\"\\\"\\\" \\n\",\n \"# from google.colab import auth\\n\",\n \"# auth.authenticate_user()\\n\",\n \"\\n\",\n \"\\n\",\n \"\\\"\\\"\\\"Jupiter Notebook Auth\\\"\\\"\\\"\\n\",\n \"import google.auth\\n\",\n \"import os\\n\",\n \"\\n\",\n \"credentials, project_id = google.auth.default()\\n\",\n \"\\n\",\n \"os.environ['GOOGLE_CLOUD_QUOTA_PROJECT']=PROJECT_ID\\n\",\n \"os.environ['GOOGLE_CLOUD_PROJECT']=PROJECT_ID\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"#Enable all the required APIs for the COPY\\n\",\n \"\\n\",\n \"!gcloud services enable \\\\\\n\",\n \" cloudapis.googleapis.com \\\\\\n\",\n \" compute.googleapis.com \\\\\\n\",\n \" iam.googleapis.com \\\\\\n\",\n \" sqladmin.googleapis.com \\\\\\n\",\n \" bigquery.googleapis.com --project {PROJECT_ID}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"noWgbDQQO7mr\"\n },\n \"source\": [\n \"## ☁️ **3. Set up Cloud SQL PostgreSQL Instance** \\n\",\n \"A **Postgres** Cloud SQL instance is required for the following stages of this notebook.\\n\",\n \"\\n\",\n \"To connect and access our Postgres Cloud SQL database instance(s) we will leverage the [Cloud SQL Python Connector](https://github.com/GoogleCloudPlatform/cloud-sql-python-connector).\\n\",\n \"\\n\",\n \"The Cloud SQL Python Connector is a library that can be used alongside a database driver to allow users to easily connect to a Cloud SQL database without having to manually allowlist IP or manage SSL certificates. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ypjpse8yBRdI\"\n },\n \"source\": [\n \"💽 **Create a Postgres Instance**\\n\",\n \"\\n\",\n \"Running the below cell will verify the existence of a Cloud SQL instance or create a new one if one does not exist.\\n\",\n \"\\n\",\n \"> ⏳ - Creating a Cloud SQL instance may take a few minutes.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"cellView\": \"form\",\n \"id\": \"_vIX7rNtVLhn\"\n },\n \"outputs\": [],\n \"source\": [\n \"#@markdown Please fill in the both the Google Cloud region and name of your Cloud SQL instance. Once filled in, run the cell.\\n\",\n \"\\n\",\n \"# Below are the recommended defaults; These can be changed to values of your choice\\n\",\n \"PG_REGION = \\\"us-central1\\\" #@param {type:\\\"string\\\"}\\n\",\n \"PG_INSTANCE = \\\"pg15-opendataqna\\\"\\n\",\n \"PG_DATABASE = \\\"opendataqna-db\\\"\\n\",\n \"PG_USER = \\\"pguser\\\"\\n\",\n \"PG_PASSWORD = \\\"pg123\\\"\\n\",\n \"\\n\",\n \"# Quick input validations.\\n\",\n \"assert PG_REGION, \\\"us-central1\\\"\\n\",\n \"assert PG_INSTANCE, \\\"pg15-opendataqna\\\"\\n\",\n \"\\n\",\n \"# check if Cloud SQL instance exists in the provided region and create it if it does not exist\\n\",\n \"database_version = !gcloud sql instances describe {PG_INSTANCE} --format=\\\"value(databaseVersion)\\\"\\n\",\n \"if database_version[0].startswith(\\\"POSTGRES\\\"):\\n\",\n \" print(\\\"Found existing Postgres Cloud SQL Instance!\\\")\\n\",\n \"else:\\n\",\n \" print(\\\"Creating new Cloud SQL instance...\\\")\\n\",\n \" !gcloud sql instances create {PG_INSTANCE} --database-version=POSTGRES_15 \\\\\\n\",\n \" --region={PG_REGION} --cpu=1 --memory=4GB --root-password={PG_PASSWORD} \\\\\\n\",\n \" --database-flags=cloudsql.iam_authentication=On\\n\",\n \"\\n\",\n \"# Create a database on the instance and a user with password\\n\",\n \"!gcloud sql databases create {PG_DATABASE} --instance={PG_INSTANCE}\\n\",\n \"!gcloud sql users create {PG_USER} \\\\\\n\",\n \"--instance={PG_INSTANCE} \\\\\\n\",\n \"--password={PG_PASSWORD}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"nzb0dFO6C4h6\"\n },\n \"source\": [\n \"## ➡️ **4. Migrate a public BigQuery database to PostgreSQL instance**\\n\",\n \"Let's migrate a public BigQuery dataset over to the newly created PostgreSQL instance. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### A) Set up a Google Cloud Storage Bucket \\n\",\n \"This bucket will be used to store the exported BigQuery public dataset.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"0q5uFF0sJnWK\"\n },\n \"outputs\": [],\n \"source\": [\n \"from google.cloud import storage\\n\",\n \"from urllib.error import HTTPError\\n\",\n \"\\n\",\n \"# Choose a name for the bucket; You might have to choose a different name if the name below already exists\\n\",\n \"BUCKET_NAME = str(PROJECT_ID+'-opendataqna') #@param {type:\\\"string\\\"} \\n\",\n \"\\n\",\n \"# Creating a bucket\\n\",\n \"storage_client = storage.Client(project=PROJECT_ID)\\n\",\n \"\\n\",\n \"try: \\n\",\n \" bucket = storage_client.bucket(BUCKET_NAME)\\n\",\n \"\\n\",\n \" if bucket.exists(): \\n\",\n \" print(\\\"This bucket already exists.\\\")\\n\",\n \"\\n\",\n \" else:\\n\",\n \" bucket = storage_client.create_bucket(BUCKET_NAME)\\n\",\n \" print(f\\\"Bucket {bucket.name} created\\\")\\n\",\n \"\\n\",\n \"except:\\n\",\n \" print(\\\"⚠️ This bucket already exists in another project. Make sure to give your bucket a unique name.\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### B) Export BigQuery Dataset to the Bucket\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"#@markdown Please choose a BigQuery Public dataset to export. You can leave the default values. Once filled in, run the cell.\\n\",\n \"\\n\",\n \"# Below are the recommended defaults; You may choose a different database to export\\n\",\n \"BQ_PROJECT = \\\"bigquery-public-data\\\"\\n\",\n \"BQ_DATABASE = \\\"census_bureau_international\\\"\\n\",\n \"bq_tables = [] # Specify empty list to copy 'all' tables, or a Specific list, eg: [\\\"table1\\\", \\\"table3\\\", \\\"table10\\\"]\\n\",\n \"\\n\",\n \"\\n\",\n \"# Quick input validations.\\n\",\n \"assert BQ_PROJECT, \\\"⚠️ Please specify the BigQuery Project\\\"\\n\",\n \"assert BQ_DATABASE, \\\"⚠️ Please specify the BigQuery Database\\\"\\n\",\n \"\\n\",\n \"from google.cloud import bigquery\\n\",\n \"client = bigquery.Client(project=PROJECT_ID)\\n\",\n \"dataset_id = f'{BQ_PROJECT}.{BQ_DATABASE}'\\n\",\n \"\\n\",\n \"if not bq_tables:\\n\",\n \" bq_tables_obj = client.list_tables(dataset_id)\\n\",\n \" bq_tables = [table_obj.table_id for table_obj in bq_tables_obj]\\n\",\n \"\\n\",\n \"print(f'List of tables in {dataset_id}: {bq_tables}')\\n\",\n \"destination_uris = []\\n\",\n \"\\n\",\n \"for bq_table in bq_tables:\\n\",\n \" # Export the bigquery data to Google Bucket\\n\",\n \" destination_uri = f\\\"gs://{BUCKET_NAME}/{BQ_DATABASE}/{bq_table}.csv\\\"\\n\",\n \" dataset_ref = bigquery.DatasetReference(BQ_PROJECT, BQ_DATABASE)\\n\",\n \" table_ref = dataset_ref.table(bq_table)\\n\",\n \"\\n\",\n \" destination_uris.append(destination_uri)\\n\",\n \"\\n\",\n \" extract_job = client.extract_table(\\n\",\n \" table_ref,\\n\",\n \" destination_uri,\\n\",\n \" # Location must match that of the source table.\\n\",\n \" location=\\\"US\\\",\\n\",\n \" ) # API request\\n\",\n \" extract_job.result() # Waits for job to complete.\\n\",\n \"\\n\",\n \" print(f\\\"Exported {BQ_PROJECT}:{BQ_DATABASE}.{bq_table} to {destination_uri}\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### C) Retrieve Data Types and Formats \\n\",\n \"To migrate the exported .csv files to PostgreSQL, we need to fetch the Data Types and Format from the exported csv file.\\n\",\n \"This needs to be done as we're setting up the PostgreSQL table and columns first (and need to provide the columns in the setup).\\n\",\n \"We will load the .csv content into the table afterwards. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import pandas as pd\\n\",\n \"\\n\",\n \"field_names_list = []\\n\",\n \"field_types_list = []\\n\",\n \"\\n\",\n \"for destination_uri in destination_uris:\\n\",\n \"\\n\",\n \" df = pd.read_csv(destination_uri)\\n\",\n \" field_names = df.columns\\n\",\n \" field_names_list.append(field_names)\\n\",\n \" print(f'Column Names: {field_names}\\\\n')\\n\",\n \" field_types = df.dtypes\\n\",\n \" field_types_list.append(field_types)\\n\",\n \" print(f'Column Names: {field_types}')\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### D) Build the SQL Query for Table Creation \\n\",\n \"Every database is different. To acommodate for different table structures depending on which BigQuery dataset is being loaded in, we will build the SQL query for creating the required PostgreSQL table dynamically. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"def get_sql(pg_schema, bq_table, field_names, field_types): \\n\",\n \"\\n\",\n \" cols = \\\"\\\" \\n\",\n \"\\n\",\n \" for i in range(len(field_names)): \\n\",\n \" cols += str(field_names[i]) +\\\" \\\"+ str(field_types[i])\\n\",\n \" if i < (len(field_names)-1): \\n\",\n \" cols += \\\", \\\"\\n\",\n \"\\n\",\n \"\\n\",\n \" sql = f\\\"\\\"\\\"CREATE TABLE {pg_schema}.{bq_table}({cols})\\\"\\\"\\\"\\n\",\n \"\\n\",\n \" return sql\\n\",\n \"\\n\",\n \"#Please specify the PGSchema or leave it as default (public)\\n\",\n \"PG_SCHEMA = BQ_DATABASE\\n\",\n \"create_table_sqls = []\\n\",\n \"\\n\",\n \"for bq_table, field_names, field_types in zip(bq_tables, field_names_list, field_types_list):\\n\",\n \"\\n\",\n \" sql = get_sql(PG_SCHEMA, bq_table, field_names, field_types)\\n\",\n \" print(f'\\\\nsql for creating {bq_table} PostgreSQL table: \\\\n{sql} \\\\n')\\n\",\n \" create_table_sqls.append(sql)\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### E) Create the PostgreSQL Table\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import asyncio \\n\",\n \"import asyncpg\\n\",\n \"from google.cloud.sql.connector import Connector\\n\",\n \"\\n\",\n \"async def create_pg_schema(PROJECT_ID,\\n\",\n \" PG_REGION,\\n\",\n \" PG_INSTANCE,\\n\",\n \" PG_PASSWORD, \\n\",\n \" PG_DATABASE,\\n\",\n \" PG_USER,\\n\",\n \" PG_SCHEMA): \\n\",\n \" \\\"\\\"\\\"Delete if PG Schema Exists and create a fresh copy\\\"\\\"\\\"\\n\",\n \" loop = asyncio.get_running_loop()\\n\",\n \" async with Connector(loop=loop) as connector:\\n\",\n \" # Create connection to Cloud SQL database\\n\",\n \" conn: asyncpg.Connection = await connector.connect_async(\\n\",\n \" f\\\"{PROJECT_ID}:{PG_REGION}:{PG_INSTANCE}\\\", # Cloud SQL instance connection name\\n\",\n \" \\\"asyncpg\\\",\\n\",\n \" user=f\\\"{PG_USER}\\\",\\n\",\n \" db=f\\\"{PG_DATABASE}\\\",\\n\",\n \" password=f\\\"{PG_PASSWORD}\\\"\\n\",\n \" )\\n\",\n \"\\n\",\n \" await conn.execute(f\\\"DROP SCHEMA IF EXISTS {PG_SCHEMA} CASCADE\\\") \\n\",\n \"\\n\",\n \" await conn.execute(f\\\"CREATE SCHEMA {PG_SCHEMA}\\\") \\n\",\n \"\\n\",\n \" await conn.close()\\n\",\n \"\\n\",\n \"\\n\",\n \"async def create_pg_table(PROJECT_ID,\\n\",\n \" PG_REGION,\\n\",\n \" PG_INSTANCE,\\n\",\n \" PG_PASSWORD,\\n\",\n \" bq_tables, \\n\",\n \" PG_DATABASE, \\n\",\n \" create_table_sqls,\\n\",\n \" PG_USER): \\n\",\n \" \\\"\\\"\\\"Create PG Table from BQ Schema\\\"\\\"\\\"\\n\",\n \" \\n\",\n \" \\n\",\n \" loop = asyncio.get_running_loop()\\n\",\n \" async with Connector(loop=loop) as connector:\\n\",\n \" # Create connection to Cloud SQL database\\n\",\n \" conn: asyncpg.Connection = await connector.connect_async(\\n\",\n \" f\\\"{PROJECT_ID}:{PG_REGION}:{PG_INSTANCE}\\\", # Cloud SQL instance connection name\\n\",\n \" \\\"asyncpg\\\",\\n\",\n \" user=f\\\"{PG_USER}\\\",\\n\",\n \" db=f\\\"{PG_DATABASE}\\\",\\n\",\n \" password=f\\\"{PG_PASSWORD}\\\"\\n\",\n \" )\\n\",\n \"\\n\",\n \" \\n\",\n \" for bq_table, sql in zip(bq_tables, create_table_sqls):\\n\",\n \" # Replace the Data Types to work with PostgreSQL supported ones \\n\",\n \" sql = sql.replace(\\\"object,\\\", \\\"TEXT,\\\").replace(\\\"int64\\\", \\\"INTEGER\\\").replace(\\\"float64\\\", \\\"DOUBLE PRECISION\\\")\\n\",\n \"\\n\",\n \"\\n\",\n \" await conn.execute(f\\\"DROP TABLE IF EXISTS {bq_table} CASCADE\\\")\\n\",\n \" \\n\",\n \" # Create the table.\\n\",\n \" await conn.execute(sql)\\n\",\n \"\\n\",\n \" await conn.close()\\n\",\n \"\\n\",\n \"# Delete schema if exists and create a fresh copy\\n\",\n \"await(create_pg_schema(PROJECT_ID, PG_REGION, PG_INSTANCE, PG_PASSWORD, PG_DATABASE, PG_USER, PG_SCHEMA))\\n\",\n \"# # Create PG Tables\\n\",\n \"await(create_pg_table(PROJECT_ID, PG_REGION, PG_INSTANCE, PG_PASSWORD, bq_tables, PG_DATABASE, create_table_sqls,PG_USER))\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### F) Import Data to PostgreSQL Table\\n\",\n \"The below cell will iterate through each export file on our Google Cloud Storage Bucket and load it to the PostgreSQL instance. \\n\",\n \"This may take a while, depending on the size of the BigQuery public dataset. You can optionally set the LIMIT parameter to limit how many export files will be loaded in. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"async def import_to_pg(PROJECT_ID,\\n\",\n \" PG_REGION,\\n\",\n \" PG_INSTANCE,\\n\",\n \" PG_USER,\\n\",\n \" PG_PASSWORD,\\n\",\n \" PG_DATABASE,\\n\",\n \" PG_SCHEMA,\\n\",\n \" bq_tables, \\n\",\n \" BUCKET_NAME): \\n\",\n \" from google.cloud import storage\\n\",\n \" import pandas as pd \\n\",\n \" import asyncio\\n\",\n \" import asyncpg\\n\",\n \" from google.cloud.sql.connector import Connector\\n\",\n \"\\n\",\n \" storage_client = storage.Client(project=PROJECT_ID)\\n\",\n \"\\n\",\n \" # bucket = storage_client.get_bucket(BUCKET_NAME)\\n\",\n \" # blobs = bucket.list_blobs()\\n\",\n \"\\n\",\n \" loop = asyncio.get_running_loop()\\n\",\n \" async with Connector(loop=loop) as connector:\\n\",\n \"\\n\",\n \"\\n\",\n \" # Create connection to Cloud SQL database\\n\",\n \" conn: asyncpg.Connection = await connector.connect_async(\\n\",\n \" f\\\"{PROJECT_ID}:{PG_REGION}:{PG_INSTANCE}\\\", # Cloud SQL instance connection name\\n\",\n \" \\\"asyncpg\\\",\\n\",\n \" user=f\\\"{PG_USER}\\\",\\n\",\n \" password=f\\\"{PG_PASSWORD}\\\",\\n\",\n \" db=f\\\"{PG_DATABASE}\\\",\\n\",\n \" )\\n\",\n \" \\n\",\n \" for bq_table in bq_tables:\\n\",\n \" URI = f\\\"gs://{BUCKET_NAME}/{PG_SCHEMA}/{bq_table}.csv\\\"\\n\",\n \" print(f'URI:{URI}')\\n\",\n \" df = pd.read_csv(URI)\\n\",\n \" df = df.dropna()\\n\",\n \" df.info() \\n\",\n \"\\n\",\n \" # Copy the dataframe to the table.\\n\",\n \" tuples = list(df.itertuples(index=False))\\n\",\n \"\\n\",\n \" await conn.copy_records_to_table(\\n\",\n \" bq_table, records=tuples, columns=list(df), schema_name=PG_SCHEMA, timeout=3600\\n\",\n \" )\\n\",\n \" await conn.close()\\n\",\n \"\\n\",\n \"# # Load Data into PG Table \\n\",\n \"await(import_to_pg(PROJECT_ID, PG_REGION, PG_INSTANCE, PG_USER, PG_PASSWORD, PG_DATABASE, PG_SCHEMA,bq_tables, BUCKET_NAME))\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### If all the above steps are executed suucessfully, the Bigquery public dataset should be copied to Cloud SQL for PostgreSQL on your GCP project\"\n ]\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.9\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "notebooks/1_Setup_OpenDataQnA.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"
\\n\",\n \" \\n\",\n \" \\n\",\n \" \\n\",\n \"
\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"copyright\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Copyright 2024 Google LLC\\n\",\n \"#\\n\",\n \"# Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\",\n \"# you may not use this file except in compliance with the License.\\n\",\n \"# You may obtain a copy of the License at\\n\",\n \"#\\n\",\n \"# https://www.apache.org/licenses/LICENSE-2.0\\n\",\n \"#\\n\",\n \"# Unless required by applicable law or agreed to in writing, software\\n\",\n \"# distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\",\n \"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\",\n \"# See the License for the specific language governing permissions and\\n\",\n \"# limitations under the License.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"DRyGcAepAPJ5\"\n },\n \"source\": [\n \"\\n\",\n \"

Open Data QnA - Chat with your SQL Database

\\n\",\n \"\\n\",\n \"---\\n\",\n \"\\n\",\n \"This notebook first walks through the Vector Store Setup needed for running the Open Data QnA application. \\n\",\n \"\\n\",\n \"Currently supported Source DBs are: \\n\",\n \"- PostgreSQL on Google Cloud SQL \\n\",\n \"- BigQuery\\n\",\n \"\\n\",\n \"Furthermore, the following vector stores are supported \\n\",\n \"- pgvector on PostgreSQL \\n\",\n \"- BigQuery vector\\n\",\n \"\\n\",\n \"\\n\",\n \"The setup part covers the following steps: \\n\",\n \"> 1. Configuration: Intial GCP project, IAM permissions, Environment and Databases setup including logging on Bigquery for analytics\\n\",\n \"\\n\",\n \"> 2. Creation of Table, Column and Known Good Query Embeddings in the Vector Store for Retreival Augmented Generation(RAG)\\n\",\n \"\\n\",\n \"> 3. Setting up firestore for persisting the session history for multiturn\\n\",\n \"\\n\",\n \"\\n\",\n \"Afterwards, you will be able to run the Open Data QnA Pipeline to generate SQL queries and answer questions over your data source. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"jsWGZW_fUJjN\"\n },\n \"source\": [\n \"### 📒 Using this interactive notebook\\n\",\n \"\\n\",\n \"If you have not used this IDE with jupyter notebooks it will ask for installing Python + Jupyter extensions. Please go ahead install them\\n\",\n \"\\n\",\n \"Click the **run** icons ▶️ of each cell within this notebook.\\n\",\n \"\\n\",\n \"> 💡 Alternatively, you can run the currently selected cell with `Ctrl + Enter` (or `⌘ + Enter` on a Mac).\\n\",\n \"\\n\",\n \"> ⚠️ **To avoid any errors**, wait for each section to finish in their order before clicking the next “run” icon.\\n\",\n \"\\n\",\n \"This sample must be connected to a **Google Cloud project**, but nothing else is needed other than your Google Cloud project.\\n\",\n \"\\n\",\n \"You can use an existing project. Alternatively, you can create a new Cloud project [with cloud credits for free.](https://cloud.google.com/free/docs/gcp-free-tier)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### **Change your Kernel to the created .venv with poetry from README.md**\\n\",\n \"\\n\",\n \"Path would look like e.g. /home/admin_/opendata/.venv/bin/python or ~cache/user/opendataqna/.venv/bin/python\\n\",\n \"\\n\",\n \"Below is the Kernel how it should look like before you proceed.\\n\",\n \"\\n\",\n \"\\n\",\n \"![Kernel](../utilities/imgs/Kernel%20Changed.png)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Set Python Module Path to Root\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"\\n\",\n \"module_path = os.path.abspath(os.path.join('..'))\\n\",\n \"sys.path.append(module_path)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"p4W6FPnrYEE8\"\n },\n \"source\": [\n \"### 🔗 **Connect Your Google Cloud Project**\\n\",\n \"Time to connect your Google Cloud Project to this notebook. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Updated property [core/project].\\n\",\n \"Project has been set to three-p-o\\n\"\n ]\n }\n ],\n \"source\": [\n \"#@markdown Please fill in the value below with your GCP project ID and then run the cell.\\n\",\n \"PROJECT_ID = \\\"three-p-o\\\"\\n\",\n \"\\n\",\n \"# Quick input validations.\\n\",\n \"assert PROJECT_ID, \\\"⚠️ Please provide your Google Cloud Project ID\\\"\\n\",\n \"\\n\",\n \"# Configure gcloud.\\n\",\n \"!gcloud config set project {PROJECT_ID}\\n\",\n \"print(f'Project has been set to {PROJECT_ID}')\\n\",\n \"\\n\",\n \"os.environ['GOOGLE_CLOUD_QUOTA_PROJECT']=PROJECT_ID\\n\",\n \"os.environ['GOOGLE_CLOUD_PROJECT']=PROJECT_ID\\n\",\n \"\\n\",\n \"#If errors out for authentication restart the kernel and start from the previous cell\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### ⚙️ **Enable Required API Services in the GCP Project**\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Operation \\\"operations/acat.p2-978842762722-9cbec4fc-9e93-41cf-a354-4a2ff66161d4\\\" finished successfully.\\n\"\n ]\n }\n ],\n \"source\": [\n \"#Enable all the required APIs for the Open Data QnA solution\\n\",\n \"\\n\",\n \"!gcloud services enable \\\\\\n\",\n \" cloudapis.googleapis.com \\\\\\n\",\n \" compute.googleapis.com \\\\\\n\",\n \" iam.googleapis.com \\\\\\n\",\n \" run.googleapis.com \\\\\\n\",\n \" sqladmin.googleapis.com \\\\\\n\",\n \" aiplatform.googleapis.com \\\\\\n\",\n \" bigquery.googleapis.com \\\\\\n\",\n \" firestore.googleapis.com --project {PROJECT_ID}\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## **Configure your inputs for the environments**\\n\",\n \"\\n\",\n \"This section assumes that a datasource is already set up in your GCP project. If a datasource has not been set up, use the notebooks below to copy a public data set from BigQuery to Cloud SQL or BigQuery on your GCP project\\n\",\n \"\\n\",\n \"\\n\",\n \"Enabled Data Sources:\\n\",\n \"* PostgreSQL on Google Cloud SQL (Copy Sample Data: [0_CopyDataToCloudSqlPG.ipynb](0_CopyDataToCloudSqlPG.ipynb))\\n\",\n \"* BigQuery (Copy Sample Data: [0_CopyDataToBigQuery.ipynb](0_CopyDataToBigQuery.ipynb))\\n\",\n \"\\n\",\n \"Enabled Vector Stores:\\n\",\n \"* pgvector on PostgreSQL \\n\",\n \"* BigQuery vector\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 🤔 **Choose Data Source and Vector Store**\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"**Sources to connect**\\n\",\n \"- This solution lets you setup multiple data source at the same time.\\n\",\n \"- You can group multiple tables from different datasets or schema into a grouping and provide the details\\n\",\n \"- If your dataset/schema has many tables and you want to run the solution against few you should specifically choose a group for that tables only\\n\",\n \"\\n\",\n \"**Format for data_source_list.csv**\\n\",\n \"\\n\",\n \"**source | user_grouping | schema | table**\\n\",\n \"\\n\",\n \"**source** - Supported Data Sources. #Options: bigquery , cloudsql-pg\\n\",\n \"\\n\",\n \"**user_grouping** - Logical grouping or use case name for tables from same or different schema/dataset. When left black it default to the schema value in the next column\\n\",\n \"\\n\",\n \"**schema** - schema name for postgres or dataset name in bigquery \\n\",\n \"\\n\",\n \"**table** - name of the tables to run the solutions against. Leave this column blank after filling schema/dataset if you want to run solution for whole dataset/schema\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Fill out the parameters and configuration settings below. \\n\",\n \"These are the parameters for setting configurations for the vector store tables to be created. \\n\",\n \"\\n\",\n \"Additionally, you can specify whether you have and want to use known-good-queries for the pipeline run and whether you want to enable logging.\\n\",\n \"\\n\",\n \"**Known good queries:** if you have known working user question <-> SQL query pairs, you can put them into the file `scripts/known_good_sql.csv`. This will be used as a caching layer and for in-context learning: If an exact match of the user question is found in the vector store, the pipeline will skip SQL Generation and output the cached SQL query. If the similarity score is between 90-100%, the known good queries will be used as few-shot examples by the SQL Generator Agent. \\n\",\n \"\\n\",\n \"**Logging:** you can enable logging. If enabled, a dataset is created in Big Query in your project, which will store the logging table and save information from the pipeline run in the logging table. This is especially helpful for debugging.\\n\",\n \"\\n\",\n \"**use_column_samples:** you can enable use column samples flag to let the pipeline select sample values of the columns from the source database. In some specific usecase where we need to understand the format or case sensitivity of the values this flag help LLM to have better understanding. Though this is one time setup, please be aware that turning this on mean getting samples from each column and it can be an expensive operation when there are lot many columns.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"#[CONFIG]\\n\",\n \"embedding_model = 'vertex' # Options: 'vertex' or 'vertex-lang'\\n\",\n \"description_model = 'gemini-1.5-pro' # 'gemini-1.0-pro', 'gemini-1.5-pro', 'text-bison-32k', 'gemini-1.5-flash'\\n\",\n \"vector_store = 'bigquery-vector' # Options: 'bigquery-vector', 'cloudsql-pgvector'\\n\",\n \"logging = True # True or False \\n\",\n \"kgq_examples = True # True or False\\n\",\n \"use_session_history = True # True or False\\n\",\n \"use_column_samples = True #True or False\\n\",\n \"\\n\",\n \"#[GCP]\\n\",\n \"project_id = PROJECT_ID\\n\",\n \"\\n\",\n \"#[PGCLOUDSQL]\\n\",\n \"# Default values for pgvector setup, change only if needed\\n\",\n \"pg_region = 'us-central1'\\n\",\n \"pg_instance = 'pg15-opendataqna'\\n\",\n \"pg_database = 'opendataqna-db'\\n\",\n \"pg_user = 'pguser'\\n\",\n \"pg_password = 'pg123'\\n\",\n \"\\n\",\n \"#[BIGQUERY]\\n\",\n \"# Name for the BQ dataset created for bigquery-vector and/or logging. Change names only if needed.\\n\",\n \"bq_dataset_region = 'us-central1'\\n\",\n \"bq_opendataqna_dataset_name = 'opendataqna'\\n\",\n \"bq_log_table_name = 'audit_log_table'\\n\",\n \"\\n\",\n \"#Details for firestore to store the chat session history\\n\",\n \"firestore_region='us-central1'\\n\",\n \"## firestore_database is named as 'opendataqna-session-logs' (This is designed to not be customizable as the setup includes creation of composite indexes from backend)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Fill out the parameters and configuration settings below.\\n\",\n \"These are the parameters for setting configurations for the vector store tables to be created.\\n\",\n \"\\n\",\n \"Additionally, you can specify whether you have and want to use known-good-queries for the pipeline run and whether you want to enable logging.\\n\",\n \"\\n\",\n \"**Known good queries:** if you have known working user question <-> SQL query pairs, you can put them into the file `scripts/known_good_sql.csv`. This will be used as a caching layer and for in-context learning: If an exact match of the user question is found in the vector store, the pipeline will skip SQL Generation and output the cached SQL query. If the similarity score is between 90-100%, the known good queries will be used as few-shot examples by the SQL Generator Agent.\\n\",\n \"\\n\",\n \"**Logging:** you can enable logging. If enabled, a dataset is created in Big Query in your project, which will store the logging table and save information from the pipeline run in the logging table. This is especially helpful for debugging.\\n\",\n \"\\n\",\n \"**use_column_samples = yes** if you want the solution to collect some samples values from the data source columns to imporve understanding of values. yes or no\\n\",\n \"\\n\",\n \"**use_column_samples:** you can enable use column samples flag to let the pipeline select sample values of the columns from the source database. In some specific usecase where we need to understand the format or case sensitivity of the values this flag help LLM to have better understanding. Though this is one time setup, please be aware that turning this on mean getting samples from each column and it can be an expensive operation when there are lot many columns.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Quick input verifications below:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"\\n\",\n \"# Input verification - Vector Store\\n\",\n \"assert vector_store in {'bigquery-vector', 'cloudsql-pgvector'}, \\\"⚠️ Invalid VECTOR_STORE. Must be 'bigquery-vector' or 'cloudsql-pgvector'\\\"\\n\",\n \"\\n\",\n \"# Input verification - Firestore Region\\n\",\n \"assert firestore_region, \\\"⚠️ Provide firestore region name\\\"\\n\",\n \"\\n\",\n \"if logging: \\n\",\n \" assert bq_log_table_name, \\\"⚠️ Please provide a name for your log table if you want to use logging\\\"\\n\",\n \"\\n\",\n \"if vector_store == 'bigquery':\\n\",\n \" assert bq_dataset_region, \\\"⚠️ Please provide the Data Set Region\\\"\\n\",\n \" assert bq_opendataqna_dataset_name, \\\"⚠️ Please provide the name of the logging/vector store dataset on Bigquery\\\"\\n\",\n \"\\n\",\n \"elif vector_store == 'cloudsql-pg':\\n\",\n \" assert pg_region, \\\"⚠️ Please provide Region of the Cloud SQL Instance\\\"\\n\",\n \" assert pg_instance, \\\"⚠️ Please provide the name of the Cloud SQL Instance\\\"\\n\",\n \" assert pg_database, \\\"⚠️ Please provide the name of the PostgreSQL Database on the Cloud SQL Instance\\\"\\n\",\n \" assert pg_user, \\\"⚠️ Please provide a username for the Cloud SQL Instance\\\"\\n\",\n \" assert pg_password, \\\"⚠️ Please provide the Password for the PG_USER\\\"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 💾 **Save Configuration to File** \\n\",\n \"Save the configurations set in this notebook to `config.ini`. The parameters from this file are used in notebooks and in various modeules in the repo\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from scripts import save_config\\n\",\n \"\\n\",\n \"save_config(embedding_model, description_model, vector_store, logging, kgq_examples, use_column_samples, PROJECT_ID,\\n\",\n \" pg_region, pg_instance, pg_database, pg_user, pg_password, \\n\",\n \" bq_dataset_region, bq_opendataqna_dataset_name, bq_log_table_name,firestore_region)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# **1. Vector Store Setup** (Run once)\\n\",\n \"---\\n\",\n \"\\n\",\n \"This section walks through the Vector Store Setup needed for running the Open Data QnA application. \\n\",\n \"\\n\",\n \"It covers the following steps: \\n\",\n \"> 1. Configuration: Environment and Databases setup including logging on Bigquery for analytics\\n\",\n \"\\n\",\n \"> 2. Creation of Table, Column and Known Good Query Embeddings in the Vector Store for Retreival Augmented Generation(RAG)\\n\",\n \"\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## ⚙️ **1.1 Database Setup for Vector Store**\\n\",\n \"\\n\",\n \"If BigQuery is your vector store, the dataset is created.\\n\",\n \"\\n\",\n \"If 'cloudsql-pgvector' is chosen as vector store, PostgreSQL Instance on CloudSQL (Note that this version of code supports only creating vector store on same instance as source)\\n\",\n \"\\n\",\n \"The cell will also create a dataset to store the log table on Big Query, **if** logging is enabled if its not bigquery vector.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env_setup import create_vector_store\\n\",\n \"# Setup vector store for embeddings\\n\",\n \"create_vector_store() \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## **1.2 Create Embeddings in Vector Store for RAG** \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 🖋️ **Create Table and Column Embeddings**\\n\",\n \"\\n\",\n \"In this step, table and column metadata is retreived from the data source and embeddings are generated for both\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env_setup import get_embeddings\\n\",\n \"\\n\",\n \"# Generate embeddings for tables and columns\\n\",\n \"table_schema_embeddings, col_schema_embeddings = get_embeddings()\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 💾 **Save the Table and Column Embeddings in the Vector Store**\\n\",\n \"The table and column embeddings created in the above step are save to the Vector Store chosen\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env_setup import store_embeddings\\n\",\n \"\\n\",\n \"# Store table/column embeddings (asynchronous)\\n\",\n \"await(store_embeddings(table_schema_embeddings, col_schema_embeddings))\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# **2. Firestore Database Setup**\\n\",\n \"---\\n\",\n \"\\n\",\n \"This section walks through setting up the firestore DB to store the session history of the conversation for multiturn\\n\",\n \"\\n\",\n \"It covers the following steps: \\n\",\n \"> 1. Creation Firestore Database\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env_setup import create_firestore_db\\n\",\n \"\\n\",\n \"create_firestore_db(firestore_region)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🗄️ **3. Load Known Good SQL into Vector Store**\\n\",\n \"Known Good Queries are used to create query cache for Few shot examples. Creating a query cache is highly recommended for best outcomes! \\n\",\n \"\\n\",\n \"The following cell will load the Natural Language Question and Known Good SQL pairs into our Vector Store. There pairs are loaded from `known_good_sql.csv` file inside scripts folder. If you have your own Question-SQL examples, curate them in .csv file before running the cell below. \\n\",\n \"\\n\",\n \"If no Known Good Queries are available at this time to create query cache, you can use [3_LoadKnownGoodSQL.ipynb](3_LoadKnownGoodSQL.ipynb) to load them later. \\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"#### Format of the Known Good SQL File (known_good_sql.csv)\\n\",\n \"\\n\",\n \"prompt | sql | user_grouping [3 columns]\\n\",\n \"\\n\",\n \"prompt ==> User Question \\n\",\n \"\\n\",\n \"sql ==> SQL for the user question (Note that the sql should enclosed in quotes and only in single line. Please remove the line break)\\n\",\n \"\\n\",\n \"user_grouping ==>This name should exactly match the grouping name you mentioned while creating vector store\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from env_setup import create_kgq_sql_table, store_kgq_sql_embeddings\\n\",\n \"\\n\",\n \"# Create table for known good queries (if enabled)\\n\",\n \"await(create_kgq_sql_table()) \\n\",\n \"\\n\",\n \"# Store known good query embeddings (if enabled)\\n\",\n \"await(store_kgq_sql_embeddings()) \\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 🥁 If all the above steps are executed suucessfully, the following should be set up:\\n\",\n \"\\n\",\n \"* GCP project and all the required IAM permissions\\n\",\n \"\\n\",\n \"* Environment to run the solution\\n\",\n \"\\n\",\n \"* Data source and Vector store for the solution\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"__________________________________________________________________________________________________________________\"\n ]\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.9\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "notebooks/2_Run_OpenDataQnA.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"
\\n\",\n \" \\n\",\n \" \\n\",\n \" \\n\",\n \"
\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {\n \"id\": \"copyright\"\n },\n \"outputs\": [],\n \"source\": [\n \"# Copyright 2024 Google LLC\\n\",\n \"#\\n\",\n \"# Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\",\n \"# you may not use this file except in compliance with the License.\\n\",\n \"# You may obtain a copy of the License at\\n\",\n \"#\\n\",\n \"# https://www.apache.org/licenses/LICENSE-2.0\\n\",\n \"#\\n\",\n \"# Unless required by applicable law or agreed to in writing, software\\n\",\n \"# distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\",\n \"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\",\n \"# See the License for the specific language governing permissions and\\n\",\n \"# limitations under the License.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"DRyGcAepAPJ5\"\n },\n \"source\": [\n \"\\n\",\n \"

Open Data QnA - Chat with your SQL Database

\\n\",\n \"\\n\",\n \"\\n\",\n \"The pipeline run covers the following steps: \\n\",\n \"\\n\",\n \"> 1. Take user question and generate sql in the dialect corresponding to data source\\n\",\n \"\\n\",\n \"> 2. Execute the sql query and retreive the data\\n\",\n \"\\n\",\n \"> 3. Generate natural language respose and charts to display\\n\",\n \"\\n\",\n \"> 4. Clean Up resources\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"jsWGZW_fUJjN\"\n },\n \"source\": [\n \"### 📒 Using this interactive notebook\\n\",\n \"\\n\",\n \"If you have not used this IDE with jupyter notebooks it will ask for installing Python + Jupyter extensions. Please go ahead install them\\n\",\n \"\\n\",\n \"Click the **run** icons ▶️ of each cell within this notebook.\\n\",\n \"\\n\",\n \"> 💡 Alternatively, you can run the currently selected cell with `Ctrl + Enter` (or `⌘ + Enter` on a Mac).\\n\",\n \"\\n\",\n \"> ⚠️ **To avoid any errors**, wait for each section to finish in their order before clicking the next “run” icon.\\n\",\n \"\\n\",\n \"This sample must be connected to a **Google Cloud project**, but nothing else is needed other than your Google Cloud project.\\n\",\n \"\\n\",\n \"You can use an existing project. Alternatively, you can create a new Cloud project [with cloud credits for free.](https://cloud.google.com/free/docs/gcp-free-tier)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"RicDCkdI-hmp\"\n },\n \"source\": [\n \"# 🚧 **0. Pre-requisites**\\n\",\n \"\\n\",\n \"Make sure that you have completed the intial setup process using [1_Setup_OpenDataQnA.ipynb](1_Setup_OpenDataQnA.ipynb). If the 1_Setup_OpenDataQnA notebook has been run successfully, the following are set up:\\n\",\n \"* GCP project and all the required IAM permissions\\n\",\n \"\\n\",\n \"* Data source and Vector store for the solution\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### **Change your Kernel to the created .venv with poetry from README.md**\\n\",\n \"\\n\",\n \"Path would look like e.g. /home/admin_/opendata/.venv/bin/python or ~cache/user/opendataqna/.venv/bin/python\\n\",\n \"\\n\",\n \"Below is the Kernel how it should look like before you proceed.\\n\",\n \"\\n\",\n \"\\n\",\n \"![Kernel](../utilities/imgs/Kernel%20Changed.png)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## ⚙️ **1. Retreive Configuration Parameters**\\n\",\n \"The notebook will load all the configuration parameters from the `config.ini` file in the root directory. \\n\",\n \"Most of these parameters were set in the initial notebook `1_Setup_OpenDataQnA.ipynb` and save to the 'config.ini file.\\n\",\n \"Use the below cells to retrieve these values and specify additional ones required for this notebook. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"module_path = os.path.abspath(os.path.join('..'))\\n\",\n \"sys.path.append(module_path)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Read your `PROJECT_ID` from the config.ini file, or set it manually below: \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from utilities import PROJECT_ID\\n\",\n \"\\n\",\n \"#Only set the variable if you don't want the project id from config.ini to use\\n\",\n \"#PROJECT_ID = ''\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🔐 **2. Authenticate and Connect to Google Cloud Project**\\n\",\n \"Authenticate to Google Cloud as the IAM user logged into this notebook in order to access your Google Cloud Project.\\n\",\n \"\\n\",\n \"You can do this within Google Colab or using the Application Default Credentials in the Google Cloud CLI.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"\\\"\\\"\\\"Colab Auth\\\"\\\"\\\" \\n\",\n \"# from google.colab import auth\\n\",\n \"# auth.authenticate_user()\\n\",\n \"\\n\",\n \"\\n\",\n \"\\\"\\\"\\\"Google CLI Auth\\\"\\\"\\\"\\n\",\n \"# !gcloud auth application-default login\\n\",\n \"\\n\",\n \"\\n\",\n \"import google.auth\\n\",\n \"credentials, project_id = google.auth.default()\\n\",\n \"\\n\",\n \"# Configure gcloud.\\n\",\n \"!gcloud config set project {PROJECT_ID}\\n\",\n \"print(f'Project has been set to {PROJECT_ID}')\\n\",\n \"# !gcloud auth application-default set-quota-project {PROJECT_ID}\\n\",\n \"\\n\",\n \"import os\\n\",\n \"os.environ['GOOGLE_CLOUD_QUOTA_PROJECT']=PROJECT_ID\\n\",\n \"os.environ['GOOGLE_CLOUD_PROJECT']=PROJECT_ID\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## ▶️ **3. Run the Open Data QnA Pipeline**\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 🏃 **Run the Pipeline**\\n\",\n \"\\n\",\n \"The next cell executes the pipeline for answering natural language questions over structured data.\\n\",\n \"\\n\",\n \"The pipeline performs the following steps:\\n\",\n \"\\n\",\n \"1. **Agent Loading:** Initializes various agents for embedding questions, building SQL queries, validating SQL, debugging SQL, and generating responses.\\n\",\n \"2. **Data Source and Vector Store Configuration:** Sets the data source (BigQuery or PostgreSQL) and vector store (BigQuery or PostgreSQL) based on provided parameters and input files.\\n\",\n \"3. **Exact Match Search:** Attempts to find an exact match for the user's question in a knowledge graph cache (if enabled). If found, the cached SQL query is used.\\n\",\n \"4. **Similar Match and Schema Retrieval:** If no exact match is found, retrieves similar questions and associated SQL queries from the knowledge graph (if enabled). Also retrieves relevant table and column schemas based on similarity to the question.\\n\",\n \"5. **SQL Generation and Debugging:** Builds an initial SQL query using the retrieved information. If enabled, the debugger iteratively refines the query with potential validation and error feedback.\\n\",\n \"6. **SQL Execution and Response Generation:** Executes the final SQL query (if enabled) against the data source and retrieves results. A response agent then generates a natural language answer based on the results.\\n\",\n \"7. **Auditing:** Records the entire pipeline process, including generated SQL, responses, and potential errors, for later analysis.\\n\",\n \"\\n\",\n \"Args:\\n\",\n \"\\n\",\n \"* **session_id (str)** Session ID to identify the conversation\\n\",\n \"\\n\",\n \"* **user_question (str):** The natural language question to answer.\\n\",\n \"\\n\",\n \"* **user_grouping (str):** Based on what the grouping has been set across the table during setup same will be use to filter or support multiple approaches. Check you data_source_list.csv for the group name. If you didn't specify the group name it will default to `schema_name-source` format\\n\",\n \"\\n\",\n \"* **RUN_DEBUGGER (bool, optional):** Whether to run the SQL debugger. Defaults to True.\\n\",\n \"\\n\",\n \"* **EXECUTE_FINAL_SQL (bool, optional):** Whether to execute the final SQL query. Defaults to True.\\n\",\n \"\\n\",\n \"* **DEBUGGING_ROUNDS (int, optional):** The number of debugging rounds. Defaults to 2.\\n\",\n \"\\n\",\n \"* **LLM_VALIDATION (bool, optional):** Whether to use LLM for SQL validation during debugging. Defaults to True.\\n\",\n \"\\n\",\n \"* **Embedder_model (str, optional):** The name of the embedding model. Defaults to 'vertex'.\\n\",\n \"\\n\",\n \"* **SQLBuilder_model (str, optional):** The name of the SQL building model. Defaults to 'gemini-1.0-pro'.\\n\",\n \"\\n\",\n \"* **SQLChecker_model (str, optional):** The name of the SQL validation model. Defaults to 'gemini-1.0-pro'.\\n\",\n \"\\n\",\n \"* **SQLDebugger_model (str, optional):** The name of the SQL debugging model. Defaults to 'gemini-1.0-pro'.\\n\",\n \"\\n\",\n \"* **Responder_model (str, optional):** The name of the response generation model. Defaults to 'gemini-1.0-pro'.\\n\",\n \"\\n\",\n \"* **num_table_matches (int, optional):** The number of similar tables to retrieve. Defaults to 5.\\n\",\n \"\\n\",\n \"* **num_column_matches (int, optional):** The number of similar columns to retrieve. Defaults to 10.\\n\",\n \"\\n\",\n \"* **table_similarity_threshold (float, optional):** The similarity threshold for tables. Defaults to 0.3.\\n\",\n \"\\n\",\n \"* **column_similarity_threshold (float, optional):** The similarity threshold for columns. Defaults to 0.3.\\n\",\n \"\\n\",\n \"* **example_similarity_threshold (float, optional):** The similarity threshold for example questions. Defaults to 0.3.\\n\",\n \"\\n\",\n \"* **num_sql_matches (int, optional):** The number of similar SQL queries to retrieve. Defaults to 3.\\n\",\n \"\\n\",\n \"\\n\",\n \"Returns:\\n\",\n \"- **final_sql (str):** The final generated SQL query.\\n\",\n \"- **response (pandas.DataFrame or str):** The result of executing the SQL query or an error message.\\n\",\n \"- **_resp (str):** The final response generated by the response agent.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## DO NOT RE-RUN THE CELL UNTIL YOU WANT TO START NEW CONVERSATION\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"\\n\",\n \"\\n\",\n \"import uuid\\n\",\n \"\\n\",\n \"session_id=str(uuid.uuid1())\\n\",\n \"\\n\",\n \"#Keep the session id to test for multi turn, change it for number conversation\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### ❓ **Ask your Natural Language Question**\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### CHANGE THE QUESTION FOR EACH FOLLOW UP. IF NEW CONVERSATION RE-RUN ABOVE CELL AND THEN CHANGE THE QUESTION\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"print(\\\"Session ID ::: \\\" + str(session_id))\\n\",\n \"\\n\",\n \"#MovieExplorer-bigquery\\n\",\n \"user_question = \\\"What are the top 5 most common genre's in the dataset?\\\"\\n\",\n \"\\n\",\n \"#follow up-1\\n\",\n \"# user_question= 'Can you only do this for just movies'\\n\",\n \"\\n\",\n \"print(\\\"User Question:- \\\"+user_question)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from opendataqna import run_pipeline\\n\",\n \"\\n\",\n \"final_sql, response, _resp = await run_pipeline(session_id,\\n\",\n \" user_question,\\n\",\n \" user_grouping = 'MovieExplorer-bigquery',\\n\",\n \" RUN_DEBUGGER=True,\\n\",\n \" EXECUTE_FINAL_SQL=True,\\n\",\n \" DEBUGGING_ROUNDS = 2, \\n\",\n \" LLM_VALIDATION=False,\\n\",\n \" Embedder_model='vertex', # Options: 'vertex' or 'vertex-lang'\\n\",\n \" SQLBuilder_model= 'gemini-1.5-pro',\\n\",\n \" SQLChecker_model= 'gemini-1.5-pro',\\n\",\n \" SQLDebugger_model= 'gemini-1.5-pro',\\n\",\n \" Responder_model= 'gemini-1.5-pro',\\n\",\n \" num_table_matches = 5,\\n\",\n \" num_column_matches = 10,\\n\",\n \" table_similarity_threshold = 0.1,\\n\",\n \" column_similarity_threshold = 0.1, \\n\",\n \" example_similarity_threshold = 0.1, \\n\",\n \" num_sql_matches=3)\\n\",\n \"\\n\",\n \" \\n\",\n \"\\n\",\n \"print(\\\"*\\\"*50 +\\\"\\\\nGenerated SQL\\\\n\\\"+\\\"*\\\"*50+\\\"\\\\n\\\"+final_sql)\\n\",\n \"print(\\\"\\\\n\\\"+\\\"*\\\"*50 +\\\"\\\\nResults\\\\n\\\"+\\\"*\\\"*50)\\n\",\n \"display(response)\\n\",\n \"print(\\\"*\\\"*50 +\\\"\\\\nNatural Response\\\\n\\\"+\\\"*\\\"*50+\\\"\\\\n\\\"+_resp)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### 📊 **Create Charts for the results** (Run only when you have proper results in the above cells)\\n\",\n \"Agent provides two suggestive google charts to display on a UI with element IDs chart_div and chart_div_1\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from opendataqna import visualize\\n\",\n \"\\n\",\n \"chart_js=''\\n\",\n \"chart_js,invalid_response = visualize(session_id,user_question,final_sql,response) #sending \\n\",\n \"# print(chart_js[\\\"chart_div_1\\\"])\\n\",\n \"if not invalid_response:\\n\",\n \" print(\\\"Chart Code Generated\\\")\\n\",\n \"else:\\n\",\n \" print(\\\"Error in visualization code generation: \\\"+ chart_js)\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from IPython.display import HTML\\n\",\n \"\\n\",\n \"html_code = f'''\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"'''\\n\",\n \"\\n\",\n \"HTML(html_code)\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"html_code = f'''\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"'''\\n\",\n \"\\n\",\n \"HTML(html_code)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🗑 **Clean Up Notebook Resources**\\n\",\n \"Make sure to delete your Cloud SQL instance and BigQuery Datasets when your are finished with this notebook to avoid further costs. 💸 💰\\n\",\n \"\\n\",\n \"Uncomment and run the cell below to delete \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# # delete Cloud SQL instance\\n\",\n \"# !gcloud sql instances delete {PG_INSTANCE} -q\\n\",\n \"\\n\",\n \"# #delete BigQuery Dataset using bq utility\\n\",\n \"# !bq rm -r -f -d {BQ_DATASET_NAME}\\n\",\n \"\\n\",\n \"# #delete BigQuery 'Open Data QnA' Vector Store Dataset using bq utility\\n\",\n \"\\n\",\n \"# !bq rm -r -f -d {BQ_OPENDATAQNA_DATASET_NAME}\\n\",\n \"\\n\"\n ]\n }\n ],\n \"metadata\": {\n \"colab\": {\n \"provenance\": []\n },\n \"kernelspec\": {\n \"display_name\": \"Python 3\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.9\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 0\n}\n" }, { "path": "notebooks/3_LoadKnownGoodSQL.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# Copyright 2024 Google LLC\\n\",\n \"#\\n\",\n \"# Licensed under the Apache License, Version 2.0 (the \\\"License\\\");\\n\",\n \"# you may not use this file except in compliance with the License.\\n\",\n \"# You may obtain a copy of the License at\\n\",\n \"#\\n\",\n \"# https://www.apache.org/licenses/LICENSE-2.0\\n\",\n \"#\\n\",\n \"# Unless required by applicable law or agreed to in writing, software\\n\",\n \"# distributed under the License is distributed on an \\\"AS IS\\\" BASIS,\\n\",\n \"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\\n\",\n \"# See the License for the specific language governing permissions and\\n\",\n \"# limitations under the License.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"\\n\",\n \"# **Open Data QnA: Cache Known Good Queries in Vector Store**\\n\",\n \"\\n\",\n \"---\\n\",\n \"\\n\",\n \"This notebook shows how to cahce known good queries in a Vector Store that has already been set up using [1_SetUpVectorStore.ipynb](1_SetUpVectorStore.ipynb). The queries are loaded into the vector store from the csv files (/scripts/known_good_sql.csv)\\n\",\n \"\\n\",\n \"Supported vector stores: \\n\",\n \"- pgvector on PostgreSQL \\n\",\n \"- BigQuery vector\\n\",\n \"\\n\",\n \"\\n\",\n \"The notebook covers the following steps: \\n\",\n \"> 1. Clean an existing embeddings table for known good queries (if loading_mode = 'refresh')\\n\",\n \"\\n\",\n \"> 2. Add known good queries from csv file to the embeddings table in the vector store\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🚧 **0. Pre-requisites**\\n\",\n \"\\n\",\n \"Make sure that you have completed the intial setup process using [1_SetUpVectorStore.ipynb](1_SetUpVectorStore.ipynb). If the 1_SetUpVectorStore notebook has been run successfully, the following are set up:\\n\",\n \"* GCP project and all the required IAM permissions\\n\",\n \"\\n\",\n \"* **Environment to run the solution**\\n\",\n \"\\n\",\n \"* Data source and Vector store for the solution\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## ⚙️ **1. Retrieve Configuration Parameters**\\n\",\n \"The notebook will load all the configuration parameters from the `config.ini` file in the root directory. \\n\",\n \"Most of these parameters were set in the initial notebook `1_SetUpVectorStore.ipynb` and save to the 'config.ini file.\\n\",\n \"Use the below cells to retrieve these values and specify additional ones required for this notebook. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 2,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import os\\n\",\n \"import sys\\n\",\n \"module_path = os.path.abspath(os.path.join('..'))\\n\",\n \"sys.path.append(module_path)\\n\",\n \"\\n\",\n \"import configparser\\n\",\n \"config = configparser.ConfigParser()\\n\",\n \"config.read(module_path+'/config.ini')\\n\",\n \"\\n\",\n \"PROJECT_ID = config['GCP']['PROJECT_ID']\\n\",\n \"VECTOR_STORE = config['CONFIG']['VECTOR_STORE']\\n\",\n \"PG_DATABASE = config['PGCLOUDSQL']['PG_DATABASE']\\n\",\n \"PG_USER = config['PGCLOUDSQL']['PG_USER']\\n\",\n \"PG_REGION = config['PGCLOUDSQL']['PG_REGION'] \\n\",\n \"PG_INSTANCE = config['PGCLOUDSQL']['PG_INSTANCE'] \\n\",\n \"PG_PASSWORD = config['PGCLOUDSQL']['PG_PASSWORD']\\n\",\n \"BQ_OPENDATAQNA_DATASET_NAME = config['BIGQUERY']['BQ_OPENDATAQNA_DATASET_NAME']\\n\",\n \"BQ_LOG_TABLE_NAME = config['BIGQUERY']['BQ_LOG_TABLE_NAME'] \\n\",\n \"BQ_DATASET_REGION = config['BIGQUERY']['BQ_DATASET_REGION']\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 🔐 **2. Authenticate and Connect to Google Cloud Project**\\n\",\n \"Authenticate to Google Cloud as the IAM user logged into this notebook in order to access your Google Cloud Project.\\n\",\n \"\\n\",\n \"You can do this within Google Colab or using the Application Default Credentials in the Google Cloud CLI.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Project has been set to three-p-o\\n\"\n ]\n }\n ],\n \"source\": [\n \"\\\"\\\"\\\"Colab Auth\\\"\\\"\\\" \\n\",\n \"# from google.colab import auth\\n\",\n \"# auth.authenticate_user()\\n\",\n \"\\n\",\n \"\\n\",\n \"\\\"\\\"\\\"Google CLI Auth\\\"\\\"\\\"\\n\",\n \"# !gcloud auth application-default login\\n\",\n \"\\n\",\n \"\\n\",\n \"import google.auth\\n\",\n \"import os\\n\",\n \"\\n\",\n \"credentials, project_id = google.auth.default()\\n\",\n \"\\n\",\n \"os.environ['GOOGLE_CLOUD_QUOTA_PROJECT']=PROJECT_ID\\n\",\n \"os.environ['GOOGLE_CLOUD_PROJECT']=PROJECT_ID\\n\",\n \"\\n\",\n \"# Configure gcloud.\\n\",\n \"print(f'Project has been set to {PROJECT_ID}')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## 💾 **3. Cache Knwon Good Queries**\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Format of the Known Good SQL File (known_good_sql.csv)\\n\",\n \"\\n\",\n \"prompt | sql | user_grouping [3 columns]\\n\",\n \"\\n\",\n \"prompt ==> Natural Language Question corresponding to query\\n\",\n \"\\n\",\n \"sql ==> SQL for the user question (Note that the sql should enclosed in quotes and only in single line. Please remove the line break)\\n\",\n \"\\n\",\n \"user_grouping ==>This name should exactly match the user_grouping for Postgres Source or Big Query as data_source_list.csv\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 4,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Known Good SQL Found at Path :: /home/admin_/Open_Data_QnA/scripts/known_good_sql.csv\\n\",\n \" prompt \\\\\\n\",\n \"0 Which country had the highest population in 2023? \\n\",\n \"1 List 10 countries with highest fertility rate \\n\",\n \"2 What is the life expectancy for men and women ... \\n\",\n \"3 Which countries had the highest and lowest inf... \\n\",\n \"4 What is the population distribution by age and... \\n\",\n \"5 What is the sex ratio at birth for China in 2023? \\n\",\n \"6 What is the world average sex ratio at birth i... \\n\",\n \"7 Which country has the highest boy to girl rati... \\n\",\n \"8 What are the top 10 counties with lowest morta... \\n\",\n \"9 What were the birth, death, and growth rates i... \\n\",\n \"10 In 2023, what was the population for countrie... \\n\",\n \"11 What are the top 3 countries with the highest ... \\n\",\n \"12 What are the top 5 coutries with highest popul... \\n\",\n \"13 Which country has highest male life expectancy? \\n\",\n \"14 What are the birth and death rates of the top ... \\n\",\n \"\\n\",\n \" sql user_grouping \\n\",\n \"0 SELECT country_name, midyear_population\\\\r\\\\nFRO... WorldCensus-cloudsql-pg \\n\",\n \"1 SELECT country_name, total_fertility_rate\\\\r\\\\nF... WorldCensus-cloudsql-pg \\n\",\n \"2 SELECT country_name, life_expectancy_male, lif... WorldCensus-cloudsql-pg \\n\",\n \"3 WITH RankedInfantMortality AS (\\\\r\\\\n SELECT\\\\r\\\\... WorldCensus-cloudsql-pg \\n\",\n \"4 SELECT age, sex, population\\\\r\\\\nFROM census_bur... WorldCensus-cloudsql-pg \\n\",\n \"5 SELECT country_name, sex_ratio_at_birth, year\\\\... WorldCensus-cloudsql-pg \\n\",\n \"6 SELECT AVG(sex_ratio_at_birth) AS world_avg_se... WorldCensus-cloudsql-pg \\n\",\n \"7 SELECT country_name, sex_ratio_at_birth\\\\r\\\\nFRO... WorldCensus-cloudsql-pg \\n\",\n \"8 SELECT country_name, mortality_rate_under5\\\\r\\\\n... WorldCensus-cloudsql-pg \\n\",\n \"9 SELECT year, crude_birth_rate, crude_death_rat... WorldCensus-cloudsql-pg \\n\",\n \"10 SELECT mp.midyear_population as Population, mp... WorldCensus-cloudsql-pg \\n\",\n \"11 SELECT mp.country_name, mp.midyear_population,... WorldCensus-cloudsql-pg \\n\",\n \"12 SELECT mp.country_name, \\\\r\\\\n mp.midyear_... WorldCensus-cloudsql-pg \\n\",\n \"13 SELECT country_name, life_expectancy_male\\\\r\\\\nF... WorldCensus-cloudsql-pg \\n\",\n \"14 SELECT bdg.country_name, bdg.crude_birth_rate,... WorldCensus-cloudsql-pg \\n\",\n \"Known Good SQLs Loaded into a Dataframe\\n\"\n ]\n }\n ],\n \"source\": [\n \"# Find the csv file and load as dataframe\\n\",\n \"import pandas as pd\\n\",\n \"\\n\",\n \"current_dir = os.getcwd()\\n\",\n \"root_dir = os.path.expanduser('~') # Start at the user's home directory\\n\",\n \"\\n\",\n \"while current_dir != root_dir:\\n\",\n \" for dirpath, dirnames, filenames in os.walk(current_dir):\\n\",\n \" config_path = os.path.join(dirpath, 'known_good_sql.csv')\\n\",\n \" if os.path.exists(config_path):\\n\",\n \" file_path = config_path # Update root_dir to the found directory\\n\",\n \" break # Stop outer loop once found\\n\",\n \"\\n\",\n \" current_dir = os.path.dirname(current_dir)\\n\",\n \"\\n\",\n \"print(\\\"Known Good SQL Found at Path :: \\\"+file_path)\\n\",\n \"\\n\",\n \"# Load the file\\n\",\n \"df_kgq = pd.read_csv(file_path)\\n\",\n \"df_kgq = df_kgq.loc[:, [\\\"prompt\\\", \\\"sql\\\", \\\"user_grouping\\\"]]\\n\",\n \"df_kgq = df_kgq.dropna()\\n\",\n \"print(df_kgq)\\n\",\n \"\\n\",\n \"print('Known Good SQLs Loaded into a Dataframe')\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Specify mode for loading the known good sql\\n\",\n \"\\n\",\n \"The known good sql can loaded in two modes:\\n\",\n \"* Append mode: Apended to the existing KGQ in the vector store \\n\",\n \"* Refresh mode: Delete the existing KGQ and create of fresh copy from KGQ in known_good_sql.csv file\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 5,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"loading_mode = 'refresh' # Options 'append' or 'refresh'\\n\",\n \"assert loading_mode in {'append', 'refresh'}, \\\"⚠️ Invalid loading_mode. Must be 'append' and 'refresh'\\\"\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"current dir: /home/admin_/Open_Data_QnA/notebooks\\n\",\n \"root_dir set to: /home/admin_/Open_Data_QnA\\n\"\n ]\n },\n {\n \"name\": \"stderr\",\n \"output_type\": \"stream\",\n \"text\": [\n \"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\\n\",\n \"I0000 00:00:1721735219.442999 2463 config.cc:230] gRPC experiments enabled: call_status_override_on_cancellation, event_engine_dns, event_engine_listener, http2_stats_fix, monitoring_experiment, pick_first_new, trace_record_callops, work_serializer_clears_time_cache\\n\"\n ]\n },\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"Adding Known Good Queries to the Vector store.....\\n\",\n \"Done!!\\n\"\n ]\n }\n ],\n \"source\": [\n \"# If you have Known Good Queries, load them to known_good_sql.csv file; \\n\",\n \"# These will be used as few shot examples for query generation. \\n\",\n \"\\n\",\n \"from embeddings.kgq_embeddings import setup_kgq_table, store_kgq_embeddings\\n\",\n \"\\n\",\n \"if loading_mode == 'refresh':\\n\",\n \" # Delete any old tables and create a new table to KGQ embeddings\\n\",\n \" if VECTOR_STORE=='bigquery-vector':\\n\",\n \" await(setup_kgq_table(project_id=PROJECT_ID,\\n\",\n \" instance_name=None,\\n\",\n \" database_name=None,\\n\",\n \" schema=BQ_OPENDATAQNA_DATASET_NAME,\\n\",\n \" database_user=None,\\n\",\n \" database_password=None,\\n\",\n \" region=BQ_DATASET_REGION,\\n\",\n \" VECTOR_STORE = VECTOR_STORE\\n\",\n \" ))\\n\",\n \"\\n\",\n \" elif VECTOR_STORE=='cloudsql-pgvector':\\n\",\n \" await(setup_kgq_table(project_id=PROJECT_ID,\\n\",\n \" instance_name=PG_INSTANCE,\\n\",\n \" database_name=PG_DATABASE,\\n\",\n \" schema=None,\\n\",\n \" database_user=PG_USER,\\n\",\n \" database_password=PG_PASSWORD,\\n\",\n \" region=PG_REGION,\\n\",\n \" VECTOR_STORE = VECTOR_STORE\\n\",\n \" ))\\n\",\n \"\\n\",\n \"\\n\",\n \"print(\\\"Adding Known Good Queries to the Vector store.....\\\")\\n\",\n \"# Add KGQ to the vector store\\n\",\n \"if VECTOR_STORE=='bigquery-vector':\\n\",\n \" await(store_kgq_embeddings(df_kgq,\\n\",\n \" project_id=PROJECT_ID,\\n\",\n \" instance_name=None,\\n\",\n \" database_name=None,\\n\",\n \" schema=BQ_OPENDATAQNA_DATASET_NAME,\\n\",\n \" database_user=None,\\n\",\n \" database_password=None,\\n\",\n \" region=BQ_DATASET_REGION,\\n\",\n \" VECTOR_STORE = VECTOR_STORE\\n\",\n \" ))\\n\",\n \"\\n\",\n \"elif VECTOR_STORE=='cloudsql-pgvector':\\n\",\n \" await(store_kgq_embeddings(df_kgq,\\n\",\n \" project_id=PROJECT_ID,\\n\",\n \" instance_name=PG_INSTANCE,\\n\",\n \" database_name=PG_DATABASE,\\n\",\n \" schema=None,\\n\",\n \" database_user=PG_USER,\\n\",\n \" database_password=PG_PASSWORD,\\n\",\n \" region=PG_REGION,\\n\",\n \" VECTOR_STORE = VECTOR_STORE\\n\",\n \" ))\\n\",\n \"print('Done!!')\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"talktodata-Fy2pM2BF-py3.9\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.10.12\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "opendataqna.py", "content": "import asyncio\nimport argparse\nimport uuid\n\nfrom agents import EmbedderAgent, BuildSQLAgent, DebugSQLAgent, ValidateSQLAgent, ResponseAgent,VisualizeAgent\nfrom utilities import (PROJECT_ID, PG_REGION, BQ_REGION, EXAMPLES, LOGGING, VECTOR_STORE,\n BQ_OPENDATAQNA_DATASET_NAME, USE_SESSION_HISTORY)\nfrom dbconnectors import bqconnector, pgconnector, firestoreconnector\nfrom embeddings.store_embeddings import add_sql_embedding\n\n\n\n#Based on VECTOR STORE in config.ini initialize vector connector and region\nif VECTOR_STORE=='bigquery-vector':\n region=BQ_REGION\n vector_connector = bqconnector\n call_await = False\n\nelif VECTOR_STORE == 'cloudsql-pgvector':\n region=PG_REGION\n vector_connector = pgconnector\n call_await=True\n\nelse: \n raise ValueError(\"Please specify a valid Data Store. Supported are either 'bigquery-vector' or 'cloudsql-pgvector'\")\n\n\ndef generate_uuid():\n \"\"\"Generates a random UUID (Universally Unique Identifier) Version 4.\n\n Returns:\n str: A string representation of the UUID in the format \n xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.\n \"\"\"\n return str(uuid.uuid4())\n\n\n############################\n#_____GET ALL DATABASES_____#\n############################\ndef get_all_databases():\n \"\"\"Retrieves a list of all distinct databases (with source type) from the vector store.\n\n This function queries the vector store (BigQuery or PostgreSQL) to fetch a list of \n unique databases, including their source type. The source type indicates whether \n the database is a BigQuery dataset or a PostgreSQL schema.\n\n Returns:\n tuple: A tuple containing two elements:\n - result (str or list): A JSON-formatted string containing the list of databases and their source types,\n or an error message if an exception occurs.\n - invalid_response (bool): A flag indicating whether an error occurred during retrieval (True)\n or if the response is valid (False).\n\n Raises:\n Exception: If there is an issue connecting to or querying the vector store.\n The exception message will be included in the returned `result`.\n \"\"\"\n \n try:\n if VECTOR_STORE=='bigquery-vector': \n final_sql=f'''SELECT\n DISTINCT user_grouping AS table_schema\n FROM\n `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.table_details_embeddings`'''\n\n else:\n final_sql=\"\"\"SELECT\n DISTINCT user_grouping AS table_schema\n FROM\n table_details_embeddings\"\"\"\n result = vector_connector.retrieve_df(final_sql)\n result = result.to_json(orient='records')\n invalid_response=False\n\n except Exception as e:\n result=\"Issue was encountered while extracting databases in vector store:: \" + str(e)\n invalid_response=True\n return result,invalid_response\n\n\n############################\n#_____GET SOURCE TYPE_____##\n############################\ndef get_source_type(user_grouping):\n \"\"\"Retrieves the source type of a specified database from the vector store.\n\n This function queries the vector store (BigQuery or PostgreSQL) to determine whether the\n given database is a BigQuery dataset ('bigquery') or a PostgreSQL schema ('postgres').\n\n Args:\n user_grouping (str): The name of the database to look up.\n\n Returns:\n tuple: A tuple containing two elements:\n - result (str): The source type of the database ('bigquery' or 'postgres'), or an error message if not found or an exception occurs.\n - invalid_response (bool): A flag indicating whether an error occurred during retrieval (True) or if the response is valid (False).\n\n Raises:\n Exception: If there is an issue connecting to or querying the vector store. The exception message will be included in the returned `result`.\n \"\"\"\n try: \n if VECTOR_STORE=='bigquery-vector': \n sql=f'''SELECT\n DISTINCT source_type\n FROM\n `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.table_details_embeddings`\n where user_grouping='{user_grouping}' '''\n\n else:\n sql=f'''SELECT\n DISTINCT source_type\n FROM\n table_details_embeddings where user_grouping='{user_grouping}' '''\n \n result = vector_connector.retrieve_df(sql)\n result = (str(result.iloc[0, 0])).lower() \n invalid_response=False\n except Exception as e:\n result=\"Error at finding the datasource :: \"+str(e)\n invalid_response=True\n return result,invalid_response\n\n\n\n############################\n###_____GENERATE SQL_____###\n############################\nasync def generate_sql(session_id,\n user_question,\n user_grouping, \n RUN_DEBUGGER,\n DEBUGGING_ROUNDS, \n LLM_VALIDATION,\n Embedder_model,\n SQLBuilder_model,\n SQLChecker_model,\n SQLDebugger_model,\n num_table_matches,\n num_column_matches,\n table_similarity_threshold,\n column_similarity_threshold,\n example_similarity_threshold,\n num_sql_matches,\n user_id=\"opendataqna-user@google.com\"):\n \"\"\"Generates an SQL query based on a user's question and database.\n\n This asynchronous function orchestrates a pipeline to generate an SQL query from a natural language question.\n It leverages various agents for embedding, SQL building, validation, and debugging.\n\n Args:\n session_id (str): Session ID to identify the chat conversation\n user_question (str): The user's natural language question.\n user_grouping (str): The name of the database to query.\n RUN_DEBUGGER (bool): Whether to run the SQL debugger.\n DEBUGGING_ROUNDS (int): The number of debugging rounds to perform.\n LLM_VALIDATION (bool): Whether to use LLM for validation.\n Embedder_model (str): The name of the embedding model.\n SQLBuilder_model (str): The name of the SQL builder model.\n SQLChecker_model (str): The name of the SQL checker model.\n SQLDebugger_model (str): The name of the SQL debugger model.\n num_table_matches (int): The number of table matches to retrieve.\n num_column_matches (int): The number of column matches to retrieve.\n table_similarity_threshold (float): The similarity threshold for table matching.\n column_similarity_threshold (float): The similarity threshold for column matching.\n example_similarity_threshold (float): The similarity threshold for example matching.\n num_sql_matches (int): The number of similar SQL queries to retrieve.\n\n Returns:\n tuple: A tuple containing:\n - final_sql (str): The final generated SQL query, or an error message if generation failed.\n - invalid_response (bool): True if the response is invalid (e.g., due to an error), False otherwise.\n \"\"\"\n\n try:\n\n if session_id is None or session_id==\"\":\n print(\"This is a new session\")\n session_id=generate_uuid()\n\n ## LOAD AGENTS \n\n print(\"Loading Agents.\")\n embedder = EmbedderAgent(Embedder_model) \n SQLBuilder = BuildSQLAgent(SQLBuilder_model)\n SQLChecker = ValidateSQLAgent(SQLChecker_model)\n SQLDebugger = DebugSQLAgent(SQLDebugger_model)\n\n re_written_qe=user_question\n\n print(\"Getting the history for the session.......\\n\")\n session_history = firestoreconnector.get_chat_logs_for_session(session_id) if USE_SESSION_HISTORY else None\n print(\"Grabbed history for the session:: \"+ str(session_history))\n\n if session_history is None or not session_history:\n print(\"No records for the session. Not rewriting the question\\n\")\n else:\n concated_questions,re_written_qe=SQLBuilder.rewrite_question(user_question,session_history)\n\n\n found_in_vector = 'N' # if an exact query match was found \n final_sql='Not Generated Yet' # final generated SQL \n process_step='Not Started'\n error_msg=''\n corrected_sql = ''\n DATA_SOURCE = 'Yet to determine'\n\n DATA_SOURCE,src_invalid = get_source_type(user_grouping)\n\n if src_invalid:\n raise ValueError(DATA_SOURCE)\n\n #vertexai.init(project=PROJECT_ID, location=region)\n #aiplatform.init(project=PROJECT_ID, location=region)\n\n print(\"Source selected as : \"+ str(DATA_SOURCE) + \"\\nSchema or Dataset Name is : \"+ str(user_grouping))\n print(\"Vector Store selected as : \"+ str(VECTOR_STORE))\n\n # Reset AUDIT_TEXT\n AUDIT_TEXT = 'Creating embedding for given question'\n # Fetch the embedding of the user's input question \n embedded_question = embedder.create(re_written_qe)\n\n \n\n AUDIT_TEXT = AUDIT_TEXT + \"\\nUser Question : \" + str(user_question) + \"\\nUser Database : \" + str(user_grouping)\n process_step = \"\\n\\nGet Exact Match: \"\n\n # Look for exact matches in known questions IF kgq is enabled \n if EXAMPLES: \n exact_sql_history = vector_connector.getExactMatches(user_question) \n\n else: exact_sql_history = None \n\n # If exact user query has been found, retrieve the SQL and skip Generation Pipeline \n if exact_sql_history is not None:\n found_in_vector = 'Y' \n final_sql = exact_sql_history\n invalid_response = False\n AUDIT_TEXT = AUDIT_TEXT + \"\\nExact match has been found! Going to retrieve the SQL query from cache and serve!\"\n\n\n else:\n # No exact match found. Proceed looking for similar entries in db IF kgq is enabled \n if EXAMPLES: \n AUDIT_TEXT = AUDIT_TEXT + process_step + \"\\nNo exact match found in query cache, retrieving relevant schema and known good queries for few shot examples using similarity search....\"\n process_step = \"\\n\\nGet Similar Match: \"\n if call_await:\n similar_sql = await vector_connector.getSimilarMatches('example', user_grouping, embedded_question, num_sql_matches, example_similarity_threshold)\n else:\n similar_sql = vector_connector.getSimilarMatches('example', user_grouping, embedded_question, num_sql_matches, example_similarity_threshold)\n\n else: similar_sql = \"No similar SQLs provided...\"\n\n process_step = \"\\n\\nGet Table and Column Schema: \"\n # Retrieve matching tables and columns\n if call_await: \n table_matches = await vector_connector.getSimilarMatches('table', user_grouping, embedded_question, num_table_matches, table_similarity_threshold)\n column_matches = await vector_connector.getSimilarMatches('column', user_grouping, embedded_question, num_column_matches, column_similarity_threshold)\n else:\n table_matches = vector_connector.getSimilarMatches('table', user_grouping, embedded_question, num_table_matches, table_similarity_threshold)\n column_matches = vector_connector.getSimilarMatches('column', user_grouping, embedded_question, num_column_matches, column_similarity_threshold)\n\n AUDIT_TEXT = AUDIT_TEXT + process_step + \"\\nRetrieved Similar Known Good Queries, Table Schema and Column Schema: \\n\" + '\\nRetrieved Tables: \\n' + str(table_matches) + '\\n\\nRetrieved Columns: \\n' + str(column_matches) + '\\n\\nRetrieved Known Good Queries: \\n' + str(similar_sql)\n \n \n # If similar table and column schemas found: \n if len(table_matches.replace('Schema(values):','').replace(' ','')) > 0 or len(column_matches.replace('Column name(type):','').replace(' ','')) > 0 :\n\n # GENERATE SQL\n process_step = \"\\n\\nBuild SQL: \"\n generated_sql = SQLBuilder.build_sql(DATA_SOURCE,user_grouping,user_question,session_history,table_matches,column_matches,similar_sql)\n final_sql=generated_sql\n AUDIT_TEXT = AUDIT_TEXT + process_step + \"\\nGenerated SQL : \" + str(generated_sql)\n \n if 'unrelated_answer' in generated_sql :\n invalid_response=True\n final_sql=\"This is an unrelated question or you are not asking a valid query\"\n\n # If agent assessment is valid, proceed with checks \n else:\n invalid_response=False\n\n if RUN_DEBUGGER: \n generated_sql, invalid_response, AUDIT_TEXT = SQLDebugger.start_debugger(DATA_SOURCE,user_grouping, generated_sql, user_question, SQLChecker, table_matches, column_matches, AUDIT_TEXT, similar_sql, DEBUGGING_ROUNDS, LLM_VALIDATION) \n # AUDIT_TEXT = AUDIT_TEXT + '\\n Feedback from Debugger: \\n' + feedback_text\n\n final_sql=generated_sql\n AUDIT_TEXT = AUDIT_TEXT + \"\\nFinal SQL after Debugger: \\n\" +str(final_sql)\n\n\n # No matching table found \n else:\n invalid_response=True\n print('No tables found in Vector ...')\n AUDIT_TEXT = AUDIT_TEXT + \"\\nNo tables have been found in the Vector DB. The question cannot be answered with the provide data source!\"\n\n # print(f'\\n\\n AUDIT_TEXT: \\n {AUDIT_TEXT}')\n\n if LOGGING: \n bqconnector.make_audit_entry(DATA_SOURCE, user_grouping, SQLBuilder_model, user_question, final_sql, found_in_vector, \"\", process_step, error_msg,AUDIT_TEXT) \n\n\n except Exception as e:\n error_msg=str(e)\n final_sql=\"Error generating the SQL Please check the logs. \"+str(e)\n invalid_response=True\n AUDIT_TEXT=AUDIT_TEXT+ \"\\nException at SQL generation\"\n print(\"Error :: \"+str(error_msg))\n if LOGGING: \n bqconnector.make_audit_entry(DATA_SOURCE, user_grouping, SQLBuilder_model, user_question, final_sql, found_in_vector, \"\", process_step, error_msg,AUDIT_TEXT) \n\n if USE_SESSION_HISTORY and not invalid_response:\n firestoreconnector.log_chat(session_id,user_question,final_sql,user_id)\n print(\"Session history persisted\") \n\n\n return final_sql,session_id,invalid_response\n\n\n############################\n###_____GET RESULTS_____####\n############################\ndef get_results(user_grouping, final_sql, invalid_response=False, EXECUTE_FINAL_SQL=True):\n \"\"\"Executes the final SQL query (if valid) and retrieves the results.\n\n This function first determines the data source (BigQuery or PostgreSQL) based on the provided database name.\n If the SQL query is valid and execution is enabled, it fetches the results using the appropriate connector.\n\n Args:\n user_grouping (str): The name of the database to query.\n final_sql (str): The final SQL query to execute.\n invalid_response (bool, optional): A flag indicating whether the SQL query is invalid. Defaults to False.\n EXECUTE_FINAL_SQL (bool, optional): Whether to execute the final SQL query. Defaults to True.\n\n Returns:\n tuple: A tuple containing:\n - result_df (pandas.DataFrame or str): The results of the SQL query as a DataFrame, or an error message if the query is invalid or execution failed.\n - invalid_response (bool): True if the response is invalid (e.g., due to an error), False otherwise.\n\n Raises:\n ValueError: If the data source is invalid or not supported.\n Exception: If there's an error executing the SQL query or retrieving the results.\n \"\"\"\n \n try:\n\n DATA_SOURCE,src_invalid = get_source_type(user_grouping)\n \n if not src_invalid:\n ## SET DATA SOURCE \n if DATA_SOURCE=='bigquery':\n src_connector = bqconnector\n else: \n src_connector = pgconnector\n else:\n raise ValueError(DATA_SOURCE)\n\n if not invalid_response:\n try: \n if EXECUTE_FINAL_SQL is True:\n final_exec_result_df=src_connector.retrieve_df(final_sql.replace(\"```sql\",\"\").replace(\"```\",\"\").replace(\"EXPLAIN ANALYZE \",\"\"))\n result_df = final_exec_result_df\n\n else: # Do not execute final SQL\n print(\"Not executing final SQL since EXECUTE_FINAL_SQL variable is False\\n \")\n result_df = \"Please enable the Execution of the final SQL so I can provide an answer\" \n invalid_response = True\n \n except ValueError: \n result_df= \"Error has been encountered :: \" + str(e)\n invalid_response=True\n \n else: # Do not execute final SQL\n result_df = \"Not executing final SQL as it is invalid, please debug!\"\n \n except Exception as e: \n print(f\"An error occured. Aborting... Error Message: {e}\")\n result_df=\"Error has been encountered :: \" + str(e)\n invalid_response=True\n\n return result_df,invalid_response\n\ndef get_response(session_id,user_question,result_df,Responder_model='gemini-1.0-pro'):\n try:\n Responder = ResponseAgent(Responder_model)\n\n if session_id is None or session_id==\"\":\n print(\"This is a new session\")\n else:\n session_history =firestoreconnector.get_chat_logs_for_session(session_id) if USE_SESSION_HISTORY else None\n if session_history is None or not session_history:\n print(\"No records for the session. Not rewriting the question\\n\")\n else:\n concated_questions,re_written_qe=Responder.rewrite_question(user_question,session_history)\n user_question=re_written_qe\n \n _resp=Responder.run(user_question, result_df)\n invalid_response=False\n except Exception as e: \n print(f\"An error occured. Aborting... Error Message: {e}\")\n _resp= \"Error has been encountered :: \" + str(e)\n invalid_response=True\n\n return _resp,invalid_response\n\n############################\n###_____RUN PIPELINE_____###\n############################\nasync def run_pipeline(session_id,\n user_question,\n user_grouping,\n RUN_DEBUGGER=True,\n EXECUTE_FINAL_SQL=True,\n DEBUGGING_ROUNDS = 2, \n LLM_VALIDATION=False,\n Embedder_model='vertex',\n SQLBuilder_model= 'gemini-1.5-pro',\n SQLChecker_model= 'gemini-1.0-pro',\n SQLDebugger_model= 'gemini-1.0-pro',\n Responder_model= 'gemini-1.0-pro',\n num_table_matches = 5,\n num_column_matches = 10,\n table_similarity_threshold = 0.3,\n column_similarity_threshold = 0.3, \n example_similarity_threshold = 0.3, \n num_sql_matches=3): \n \"\"\"Orchestrates the end-to-end SQL generation and response pipeline.\n\n This asynchronous function manages the entire process of generating an SQL query from a user's question,\n executing the query (if valid), and formulating a natural language response based on the results.\n\n Args:\n user_question (str): The user's natural language question.\n user_grouping (str): The name of the user grouping to query.\n RUN_DEBUGGER (bool, optional): Whether to run the SQL debugger. Defaults to True.\n EXECUTE_FINAL_SQL (bool, optional): Whether to execute the final SQL query. Defaults to True.\n DEBUGGING_ROUNDS (int, optional): The number of debugging rounds to perform. Defaults to 2.\n LLM_VALIDATION (bool, optional): Whether to use LLM for validation. Defaults to True.\n Embedder_model (str, optional): The name of the embedding model. Defaults to 'vertex'.\n SQLBuilder_model (str, optional): The name of the SQL builder model. Defaults to 'gemini-1.5-pro'.\n SQLChecker_model (str, optional): The name of the SQL checker model. Defaults to 'gemini-1.0-pro'.\n SQLDebugger_model (str, optional): The name of the SQL debugger model. Defaults to 'gemini-1.0-pro'.\n Responder_model (str, optional): The name of the responder model. Defaults to 'gemini-1.0-pro'.\n num_table_matches (int, optional): The number of table matches to retrieve. Defaults to 5.\n num_column_matches (int, optional): The number of column matches to retrieve. Defaults to 10.\n table_similarity_threshold (float, optional): The similarity threshold for table matching. Defaults to 0.3.\n column_similarity_threshold (float, optional): The similarity threshold for column matching. Defaults to 0.3.\n example_similarity_threshold (float, optional): The similarity threshold for example matching. Defaults to 0.3.\n num_sql_matches (int, optional): The number of similar SQL queries to retrieve. Defaults to 3.\n\n Returns:\n tuple: A tuple containing:\n - final_sql (str): The final generated SQL query, or an error message if generation failed.\n - results_df (pandas.DataFrame or str): The results of the SQL query as a DataFrame, or an error message if the query is invalid or execution failed.\n - _resp (str): The generated natural language response based on the results, or an error message if response generation failed.\n \"\"\"\n\n\n final_sql,session_id, invalid_response = await generate_sql(session_id,\n user_question,\n user_grouping,\n RUN_DEBUGGER,\n DEBUGGING_ROUNDS, \n LLM_VALIDATION,\n Embedder_model,\n SQLBuilder_model,\n SQLChecker_model,\n SQLDebugger_model,\n num_table_matches,\n num_column_matches,\n table_similarity_threshold,\n column_similarity_threshold,\n example_similarity_threshold,\n num_sql_matches)\n\n if not invalid_response:\n \n results_df, invalid_response = get_results(user_grouping, \n final_sql,\n invalid_response=invalid_response,\n EXECUTE_FINAL_SQL=EXECUTE_FINAL_SQL)\n\n if not invalid_response:\n _resp,invalid_response=get_response(session_id,user_question,results_df.to_json(orient='records'),Responder_model=Responder_model)\n else:\n _resp=results_df\n else:\n results_df=final_sql\n _resp=final_sql\n\n return final_sql, results_df, _resp\n\n\n############################\n#####_____GET KGQ_____######\n############################\ndef get_kgq(user_grouping):\n \"\"\"Retrieves known good SQL queries (KGQs) for a specific database from the vector store.\n\n This function queries the vector store (BigQuery or PostgreSQL) to fetch a limited number of\n distinct user questions and their corresponding generated SQL queries that are relevant to the\n specified database. These KGQs can be used as examples or references for generating new SQL queries.\n\n Args:\n user_grouping (str): The name of the user grouping for which to retrieve KGQs.\n\n Returns:\n tuple: A tuple containing two elements:\n - result (str): A JSON-formatted string containing the list of KGQs (user questions and SQL queries),\n or an error message if an exception occurs.\n - invalid_response (bool): A flag indicating whether an error occurred during retrieval (True)\n or if the response is valid (False).\n\n Raises:\n Exception: If there is an issue connecting to or querying the vector store.\n The exception message will be included in the returned `result`.\n \"\"\" \n try:\n if VECTOR_STORE=='bigquery-vector': \n sql=f'''SELECT distinct\n example_user_question,\n example_generated_sql \n FROM\n `{PROJECT_ID}.{BQ_OPENDATAQNA_DATASET_NAME}.example_prompt_sql_embeddings`\n where user_grouping='{user_grouping}' LIMIT 5 '''\n\n else:\n sql=\"\"\"select distinct\n example_user_question,\n example_generated_sql \n from example_prompt_sql_embeddings\n where user_grouping = '{user_grouping}' LIMIT 5\"\"\".format(user_grouping=user_grouping)\n\n result = vector_connector.retrieve_df(sql)\n result = result.to_json(orient='records')\n invalid_response = False\n\n except Exception as e:\n result=\"Issue was encountered while extracting known good sqls in vector store:: \" + str(e)\n invalid_response=True\n return result,invalid_response\n\n\n############################\n####_____EMBED SQL_____#####\n############################\nasync def embed_sql(session_id,user_grouping,user_question,generate_sql):\n \"\"\"Embeds a generated SQL query into the vector store as an example.\n\n This asynchronous function takes a user's question, a generated SQL query, and a database name as input.\n It calls the `add_sql_embedding` function to create an embedding of the SQL query and store it in the vector store,\n potentially for future reference as a known good query (KGQ).\n\n Args:\n user_grouping (str): The name of the grouping associated with the query.\n user_question (str): The user's original question.\n generate_sql (str): The SQL query generated from the user's question.\n\n Returns:\n tuple: A tuple containing two elements:\n - embedded (str or None): The embedded SQL query if successful, or an error message if an exception occurs.\n - invalid_response (bool): A flag indicating whether an error occurred during embedding (True)\n or if the response is valid (False).\n\n Raises:\n Exception: If there is an issue with the embedding process.\n The exception message will be included in the returned `embedded` value.\n \"\"\" \n try:\n Rewriter=ResponseAgent('gemini-1.5-pro')\n\n if session_id is None or session_id==\"\":\n print(\"This is a new session\")\n else:\n session_history =firestoreconnector.get_chat_logs_for_session(session_id) if USE_SESSION_HISTORY else None\n if session_history is None or not session_history:\n print(\"No records for the session. Not rewriting the question\\n\")\n else:\n concated_questions,re_written_qe=Rewriter.rewrite_question(user_question,session_history)\n user_question=re_written_qe\n \n embedded = await add_sql_embedding(user_question, generate_sql,user_grouping)\n invalid_response=False\n\n except Exception as e: \n embedded=\"Issue was encountered while embedding the SQL as example.\" + str(e)\n invalid_response=True\n\n return embedded,invalid_response\n\ndef visualize(session_id,user_question,generated_sql,sql_results):\n try:\n Rewriter=ResponseAgent('gemini-1.5-pro')\n \n if session_id is None or session_id==\"\":\n print(\"This is a new session\")\n else:\n session_history =firestoreconnector.get_chat_logs_for_session(session_id) if USE_SESSION_HISTORY else None\n if session_history is None or not session_history:\n print(\"No records for the session. Not rewriting the question\\n\")\n else:\n concated_questions,re_written_qe=Rewriter.rewrite_question(user_question,session_history)\n user_question=re_written_qe\n \n _viz=VisualizeAgent()\n js_chart = _viz.generate_charts(user_question, generate_sql,sql_results)\n invalid_response=False\n\n except Exception as e: \n js_chart=\"Issue was encountered while Generating Charts ::\" + str(e)\n invalid_response=True\n\n return js_chart,invalid_response\n\n\n\n\n\n############################\n#######_____MAIN_____#######\n############################\nif __name__ == '__main__': \n # user_question = \"How many movies have review ratings above 5?\"\n # user_grouping='MovieExplorer-bigquery'\n\n parser = argparse.ArgumentParser(description=\"Open Data QnA SQL Generation\")\n parser.add_argument(\"--session_id\", type=str, required=True, help=\"Session Id\")\n parser.add_argument(\"--user_question\", type=str, required=True, help=\"The user's question.\")\n parser.add_argument(\"--user_grouping\", type=str, required=True, help=\"The user grouping specificed in the source list CSV file\")\n\n # Optional Arguments for run_pipeline Parameters\n parser.add_argument(\"--run_debugger\", action=\"store_true\", help=\"Enable the debugger (default: False)\")\n parser.add_argument(\"--execute_final_sql\", action=\"store_true\", help=\"Execute the final SQL (default: False)\")\n parser.add_argument(\"--debugging_rounds\", type=int, default=2, help=\"Number of debugging rounds (default: 2)\")\n parser.add_argument(\"--llm_validation\", action=\"store_true\", help=\"Enable LLM validation (default: False)\")\n parser.add_argument(\"--embedder_model\", type=str, default='vertex', help=\"Embedder model name (default: 'vertex')\")\n parser.add_argument(\"--sqlbuilder_model\", type=str, default='gemini-1.5-pro', help=\"SQL builder model name (default: 'gemini-1.0-pro')\")\n parser.add_argument(\"--sqlchecker_model\", type=str, default='gemini-1.5-pro', help=\"SQL checker model name (default: 'gemini-1.0-pro')\")\n parser.add_argument(\"--sqldebugger_model\", type=str, default='gemini-1.5-pro', help=\"SQL debugger model name (default: 'gemini-1.0-pro')\")\n parser.add_argument(\"--responder_model\", type=str, default='gemini-1.5-pro', help=\"Responder model name (default: 'gemini-1.0-pro')\")\n parser.add_argument(\"--num_table_matches\", type=int, default=5, help=\"Number of table matches (default: 5)\")\n parser.add_argument(\"--num_column_matches\", type=int, default=10, help=\"Number of column matches (default: 10)\")\n parser.add_argument(\"--table_similarity_threshold\", type=float, default=0.1, help=\"Threshold for table similarity (default: 0.1)\")\n parser.add_argument(\"--column_similarity_threshold\", type=float, default=0.1, help=\"Threshold for column similarity (default: 0.1)\")\n parser.add_argument(\"--example_similarity_threshold\", type=float, default=0.1, help=\"Threshold for example similarity (default: 0.1)\")\n parser.add_argument(\"--num_sql_matches\", type=int, default=3, help=\"Number of SQL matches (default: 3)\")\n\n args = parser.parse_args()\n\n # Use Argument Values in run_pipeline\n final_sql, response, _resp = asyncio.run(run_pipeline(args.session_id,\n args.user_question,\n args.user_grouping,\n RUN_DEBUGGER=args.run_debugger,\n EXECUTE_FINAL_SQL=args.execute_final_sql,\n DEBUGGING_ROUNDS=args.debugging_rounds,\n LLM_VALIDATION=args.llm_validation,\n Embedder_model=args.embedder_model,\n SQLBuilder_model=args.sqlbuilder_model,\n SQLChecker_model=args.sqlchecker_model,\n SQLDebugger_model=args.sqldebugger_model,\n Responder_model=args.responder_model,\n num_table_matches=args.num_table_matches,\n num_column_matches=args.num_column_matches,\n table_similarity_threshold=args.table_similarity_threshold,\n column_similarity_threshold=args.column_similarity_threshold,\n example_similarity_threshold=args.example_similarity_threshold,\n num_sql_matches=args.num_sql_matches\n ))\n\n # user_question = \"How many +18 movies have a rating above 4?\"\n\n # final_sql, response, _resp = asyncio.run(run_pipeline(user_question,\n # 'imdb', \n # RUN_DEBUGGER=True,\n # EXECUTE_FINAL_SQL=True,\n # DEBUGGING_ROUNDS = 2, \n # LLM_VALIDATION=True,\n # Embedder_model='vertex',\n # SQLBuilder_model= 'gemini-1.0-pro',\n # SQLChecker_model= 'gemini-1.0-pro',\n # SQLDebugger_model= 'gemini-1.0-pro',\n # Responder_model= 'gemini-1.0-pro',\n # num_table_matches = 5,\n # num_column_matches = 10,\n # table_similarity_threshold = 0.1,\n # column_similarity_threshold = 0.1, \n # example_similarity_threshold = 0.1, \n # num_sql_matches=3))\n \n \n print(\"*\"*50 +\"\\nGenerated SQL\\n\"+\"*\"*50+\"\\n\"+final_sql)\n print(\"\\n\"+\"*\"*50 +\"\\nResults\\n\"+\"*\"*50)\n print(response)\n print(\"*\"*50 +\"\\nNatural Response\\n\"+\"*\"*50+\"\\n\"+_resp)" }, { "path": "prompts.yaml", "content": "\n#################################################################################################\n ## USE CASE SPECIFIC PROMPTS ##\n#################################################################################################\n\n# CHOOSE PROMPT VARIABLE NAME STRICLY FOLLOWING THE NAMING CONVENTION BELOW\n# Variable Naming for use case prompt: usecase_{source_type}_{user_grouping} [Grab this values from the data_source_list.csv]\n# In the use case prompt, include any relevant context information to the LLMs including but not limited \n# common acronyms, column and table naming conventions (like prefix-Sufix used), buniess jargon and \n# KPI definitions w.r.t the tables and columns, guidance on how to handle ambiguity w.r.t questions \n\n# Use case prompt: source_type = 'bigquery'; user_grouping = 'MovieExplorer-bigquery'\n\nusecase_bigquery_MovieExplorer-bigquery: |\n The Movie Explorer Dataset is group of tables that contains details of movies and tv shows. We have titles, names of the personas worked in those movies. We also\n have the reviews and ratings of those movies as well.\n\n# Use case prompt: source_type = 'cloudsql-pg'; user_grouping = 'WorldCensus-cloudsql-pg'\n\nusecase_cloudsql-pg_WorldCensus-cloudsql-pg: |\n The World Census Dataset is group of tables \n\n#################################################################################################\n\n\n\n\n#################################################################################################\n ## GENERIC PROMPTS FOR DIFFERENT AGENTS ##\n#################################################################################################\n\n\n# DO NOT CHANGE PROMPT VARIABLE NAME\n# Prompt for building sql query for bigquery\nbuildsql_bigquery: |\n\n You are an Bigquery SQL guru. Your task is to write a Bigquery SQL query that answers the following question while using the provided context.\n\n\n \n - Join as minimal tables as possible.\n - When joining tables ensure all join columns are the same data_type.\n - Analyze the database and the table schema provided as parameters and undestand the relations (column and table relations).\n - Use always SAFE_CAST. If performing a SAFE_CAST, use only Bigquery supported datatypes. (i.e {specific_data_types})\n - Always SAFE_CAST and then use aggregate functions\n - Don't include any comments in code.\n - Remove ```sql and ``` from the output and generate the SQL in single line.\n - Tables should be refered to using a fully qualified name with enclosed in ticks (`) e.g. `project_id.owner.table_name`.\n - Use all the non-aggregated columns from the \"SELECT\" statement while framing \"GROUP BY\" block.\n - Return syntactically and symantically correct SQL for BigQuery with proper relation mapping i.e project_id, owner, table and column relation.\n - Use ONLY the column names (column_name) mentioned in Table Schema. DO NOT USE any other column names outside of this.\n - Associate column_name mentioned in Table Schema only to the table_name specified under Table Schema.\n - Use SQL 'AS' statement to assign a new name temporarily to a table column or even a table wherever needed.\n - Table names are case sensitive. DO NOT uppercase or lowercase the table names.\n - Always enclose subqueries and union queries in brackets.\n - Refer to the examples provided below, if given. \n - When given question is out of context of from this session respond always with dummy SQL statement - {not_related_msg}\n - You always generate SELECT queries ONLY. If asked for other statements for DELETE or MERGE etc respond with dummy SQL statement - {not_related_msg}\n\n \n\n \n {usecase_context}\n \n\n \n {similar_sql}\n \n\n \n {tables_schema}\n
\n\n \n {columns_schema}\n \n\n# DO NOT CHANGE PROMPT VARIABLE NAME\n# Prompt for building sql query for PostgreSQL\nbuildsql_cloudsql-pg: |\n\n You are an PostgreSQL SQL guru. Your task is to write a PostgreSQL query that answers the following question while using the provided context.\n\n\n VERY IMPORTANT:- Use ONLY the PostgreSQL available appropriate datatypes (i.e {specific_data_types}) while casting the column in the SQL.\n IMPORTANT:- In \"FROM\" and \"JOIN\" blocks always refer the table_name as schema.table_name.\n IMPORTANT:- Use ONLY the table name(table_name) and column names (column_name) mentioned in Table Schema (i.e {tables_schema}). DO NOT USE any other column names outside of this.\n IMPORTANT:- Associate column_name mentioned in Table Schema only to the table_name specified under Table Schema.\n NOTE:- Use SQL 'AS' statement to assign a new name temporarily to a table column or even a table wherever needed.\n\n \n - Only answer questions relevant to the tables or columns listed in the table schema If a non-related question comes, answer exactly - {not_related_msg}\n - You always generate SELECT queries ONLY. If asked for other statements for DELETE or MERGE etc answer exactly - {not_related_msg}\n - Join as minimal tables as possible.\n - When joining tables ensure all join columns are the same data_type.\n - Analyse the database and the table schema provided as parameters and understand the relations (column and table relations).\n - Don't include any comments in code.\n - Remove ```sql and ``` from the output and generate the SQL in single line.\n - Tables should be refered to using a fully qualified name including owner and table name.\n - Use table_alias.column_name when referring to columns. Example:- dept_id=hr.dept_id\n - Capitalize the table names on SQL \"where\" condition.\n - Use the columns from the \"SELECT\" statement while framing \"GROUP BY\" block.\n - Always refer the column-name with rightly mapped table-name as seen in the table schema.\n - Return syntactically and symantically correct SQL for Postgres with proper relation mapping i.e owner, table and column relation.\n - Refer to the examples provided i.e. {similar_sql}\n \n\n \n {usecase_context}\n \n\n \n {similar_sql}\n \n\n \n {tables_schema}\n
\n\n \n {columns_schema}\n \n\ndebugsql_bigquery: |\n You are an BigQuery SQL guru. Your task is to troubleshoot a BigQuery SQL query. As the user provides versions of the query and the errors returned by BigQuery,\n return a new alternative SQL query that fixes the errors. It is important that the query still answers the original question.\n\n \n - Join as minimal tables as possible.\n - When joining tables ensure all join columns are the same data_type.\n - Analyze the database and the table schema provided as parameters and undestand the relations (column and table relations).\n - Use always SAFE_CAST. If performing a SAFE_CAST, use only Bigquery supported datatypes.\n - Always SAFE_CAST and then use aggregate functions\n - Don't include any comments in code.\n - Remove ```sql and ``` from the output and generate the SQL in single line.\n - Tables should be refered to using a fully qualified name with enclosed in ticks (`) e.g. `project_id.owner.table_name`.\n - Use all the non-aggregated columns from the \"SELECT\" statement while framing \"GROUP BY\" block.\n - Return syntactically and symantically correct SQL for BigQuery with proper relation mapping i.e project_id, owner, table and column relation.\n - Use ONLY the column names (column_name) mentioned in Table Schema. DO NOT USE any other column names outside of this.\n - Associate column_name mentioned in Table Schema only to the table_name specified under Table Schema.\n - Use SQL 'AS' statement to assign a new name temporarily to a table column or even a table wherever needed.\n - Table names are case sensitive. DO NOT uppercase or lowercase the table names.\n - Always enclose subqueries and union queries in brackets.\n \n\n \n {usecase_context}\n \n\n \n {similar_sql}\n \n\n \n {tables_schema}\n
\n\n \n {columns_schema}\n \n\ndebugsql_cloudsql-pg: |\n You are an Postgres SQL guru. This session is trying to troubleshoot an Postgres SQL query. As the user provides versions of the query and the errors returned by Postgres,\n return a new alternative SQL query that fixes the errors. It is important that the query still answer the original question.\n\n \n - Remove ```sql and ``` from the output and generate the SQL in single line.\n - Rewritten SQL can't be igual to the original one.\n - Write a SQL comformant query for Postgres that answers the following question while using the provided context to correctly refer to Postgres tables and the needed column names.\n - All column_name in the query must exist in the table_name.\n - If a join includes d.country_id and table_alias d is equal to table_name DEPT, then country_id column_name must exist with table_name DEPT in the table column metadata\n - When joining tables ensure all join columns are the same data_type.\n - Analyse the database and the table schema provided as parameters and undestand the relations (column and table relations).\n - Don't include any comments in code.\n - Tables should be refered to using a fully qualified name including owner and table name.\n - Use table_alias.column_name when referring to columns. Example: dept_id=hr.dept_id\n - Capitalize the table names on SQL \"where\" condition.\n - Use the columns from the \"SELECT\" statement while framing \"GROUP BY\" block.\n - Always refer the column-name with rightly mapped table-name as seen in the table schema.\n - Return syntactically and symantically correct SQL for Postgres with proper relation mapping i.e owner, table and column relation.\n - Use only column names listed in the column metadata.\n - Always ensure to refer the table as schema.table_name.\n \n\n \n {usecase_context}\n \n\n \n {similar_sql}\n \n\n \n {tables_schema}\n
\n\n \n {columns_schema}\n \n\n\n# This is the prompt for Response agent that takes the user question and data from SQL Query execution and\n# generates a natural language response. Include additional instructions here if the natural language response\n# needs to be customized.\nnl_reponse: |\n You are a Data Assistant that helps to answer users' questions on their data within their databases.\n \n The user has provided the following question in natural language: \n {user_question}\n\n The system has returned the following result after running the SQL query: \n {sql_result}\n\n Provide a natural sounding response to the user question using only the data provided to you.\n\n\nvalidatesql: |\n\n Classify if the SQL query as valid or invalid\n \n The SQL written here is for SQL dialect for source type : {source_type} \n \n Only validate for this source\n\n \n - Validate the SQL for syntax\n - Check for semantic correctness based on the table and column details\n - Check for the data type compatibility\n \n\n \n {tables_schema}\n
\n\n \n {columns_schema}\n \n\n \n Question:- {user_question}\n SQL query:- {generated_sql}\n\n Respond using a valid JSON format with two elements valid and errors. \n Remove ```json and ``` from the output\n \n See example output below \n {{ \"valid\": true or false, \"errors\":errors }}\n\n\n\n# Prompt to suggest a chart type for a given user question and corresponding SQL query\nvisualize_chart_type: |\n You are expert in generating visualizations.\n\n \n Some commonly used charts and when do use them:-\n - Text or Score card is best for showing single value answer\n - Table is best for Showing data in a tabular format.\n - Bullet Chart is best for Showing individual values across categories.\n - Bar Chart is best for Comparing individual values across categories, especially with many categories or long labels.\n - Column Chart is best for Comparing individual values across categories, best for smaller datasets.\n - Line Chart is best for Showing trends over time or continuous data sets with many data points.\n - Area Chart is best for Emphasizing cumulative totals over time, or the magnitude of change across multiple categories.\n - Pie Chart is best for Show proportions of a whole, but only for a few categories (ideally less than 6).\n - Scatter Plot is best for Investigating relationships or correlations between two variables.\n - Bubble Chart is best for Comparing and showing relationships between three variables.\n - Histogram is best for Displaying the distribution and frequency of continuous data.\n - Map Chart is best for Visualizing data with a geographic dimension (countries, states, regions, etc.).\n - Gantt Chart\tis best for Managing projects, visualizing timelines, and task dependencies.\n - Heatmap is best for\tShowing the density of data points across two dimensions, highlighting areas of concentration.\n \n\n \n -Do not add any explanation to the response. Only stick to format Chart-1, Chart-2\n -Do not enclose the response with js or javascript or ```\n \n\n Below is the Question and corresponding SQL Generated, suggest best two of the chart types\n Question : {user_question}\n Corresponding SQL : {generated_sql}\n\n Respond using a valid JSON format with two elements chart_1 and chart_2 as below\n {{\"chart_1\":suggestion-1,\n \"chart_2\":suggestion-2}}\n\n# Prompt for generation code for google charts. \nvisualize_generate_chart_code: |\n You are expert in generating visualizations.\n \n Guidelines:\n -Do not add any explanation to the response.\n -Do not enclose the response with js or javascript or ```\n\n You are asked to generate a visualization for the following question:\n {user_question}\n\n The SQL generated for the question is:\n {generated_sql}\n\n The results of the sql which should be used to generate the visualization are in json format as follows:\n {sql_results}\n\n Needed chart type is : {chart_type}\n\n Guidelines:\n\n - Generate js code for {chart_type} for the visualization using google charts and its possible data column. You do not need to use all the columns if not possible.\n - The generated js code should be able to be just evaluated as javascript so do not add any extra text to it.\n - ONLY USE the template below and STRICTLY USE ELEMENT ID {chart_div} TO CREATE THE CHART\n\n google.charts.load('current', );\n google.charts.setOnLoadCallback(drawChart);\n drawchart function \n var data = \n with options\n Title=<>\n width=600,\n height=300,\n hAxis.textStyle.fontSize=5\n vAxis.textStyle.fontSize=5\n legend.textStyle.fontSize=10\n\n other necessary options for the chart type\n\n var chart = new google.charts.(document.getElementById('{chart_div}'));\n\n chart.draw()\n\n Example Response: \n\n google.charts.load('current', {{packages: ['corechart']}});\n google.charts.setOnLoadCallback(drawChart);\n function drawChart() \n {{var data = google.visualization.arrayToDataTable([['Product SKU', 'Total Ordered Items'],\n ['GGOEGOAQ012899', 456], ['GGOEGDHC074099', 334], \n ['GGOEGOCB017499', 319], ['GGOEGOCC077999', 290], \n ['GGOEGFYQ016599', 253], ]); \n \n var options =\n {{ title: 'Top 5 Product SKUs Ordered', \n width: 600, height: 300, hAxis: {{ \n textStyle: {{ fontSize: 12 }} }}, \n vAxis: {{ textStyle: {{ fontSize: 12 }} }},\n legend: {{ textStyle: {{ fontSize: 12\\n }} }}, \n bar: {{ groupWidth: '50%' }} }};\n var chart = new google.visualization.BarChart(document.getElementById('{chart_div}')); \n chart.draw(data, options);}}" }, { "path": "pyproject.toml", "content": "[tool.poetry]\nname = \"opendataqna\"\nversion = \"0.1.0\"\ndescription = \"\"\nauthors = [\"msubasioglu\",\"l-sri\",\"orpzs\"]\nlicense = \"Apache 2.0\"\nreadme = \"README.md\"\n\n[tool.poetry.dependencies]\npython = \">=3.9, <3.9.7 || >3.9.7, <4.0\"\ngoogle = \"^3.0.0\"\ngoogle-api-python-client = \"^2.120.0\"\ngoogle-cloud = \"^0.34.0\"\ngoogle-cloud-aiplatform = \"^1.43.0\"\npandas = \"^2.2.1\"\npandas-gbq = \"^0.21.0\"\ncloud-sql-python-connector = {extras = [\"asyncpg\"], version = \"^1.7.0\"}\nsqlalchemy = \"^2.0.27\"\npgvector = \"^0.2.5\"\npg8000 = \"^1.30.5\"\ntabulate = \"^0.9.0\"\nlangchain = \"^0.1.9\"\nfsspec = \"^2024.2.0\"\ngcsfs = \"^2024.2.0\"\nstreamlit = \"^1.32.2\"\ngoogle-cloud-bigquery = \"^3.17.2\"\ngoogle-cloud-bigquery-connection = \"^1.15.3\"\npyarrow = \"^15.0.2\"\nFlask = { version = \"^2.1.0\", extras = [\"async\"]}\ngunicorn = \"^20.1.0\"\nflask-cors = \"^4.0.0\"\nWerkzeug = \"^2.3.7\"\ngoogle-cloud-firestore= \"^2.16.0\"\nfirebase-admin = \"^5.2.0\"\n\n\n\n[tool.poetry.group.dev.dependencies]\npyzmq =\"<26.0.0\"\nipykernel = \"^6.29.3\"\n\n[build-system]\nrequires = [\"poetry-core\"]\nbuild-backend = \"poetry.core.masonry.api\"" }, { "path": "scripts/.~lock.Scenarios Sample.csv#", "content": ",leelapriya,leelapriya.c.googlers.com,23.07.2024 14:37,file:///usr/local/google/home/leelapriya/.config/libreoffice/4;" }, { "path": "scripts/Scenarios Sample.csv", "content": "user_grouping,scenario,question\r\nMovieExplorer-bigquery,Genres,What are the top 5 most common movie genres in the dataset?\r\nMovieExplorer-bigquery,Genres,How many are musicals?\r\nMovieExplorer-bigquery,Genres,Romance?\r\nMovieExplorer-bigquery,Movie,What is the average user rating of the God Father movie?\r\nMovieExplorer-bigquery,Movie,Which year was it released?\r\nMovieExplorer-bigquery,Movie,How long is it?\r\nMovieExplorer-bigquery,Movie,Who is the lead actor?\r\nMovieExplorer-bigquery,Movie,Director\r\nMovieExplorer-bigquery,Movie,Cast\r\nWorldCensus-cloudsql-pg,Life Expectancy,What is the life expectancy for men and women in a United States in 2022?\r\nWorldCensus-cloudsql-pg,Life Expectancy,In India?\r\nWorldCensus-cloudsql-pg,Life Expectancy,Which country has highest male life expectancy?\r\nWorldCensus-cloudsql-pg,Life Expectancy,Female life expectancy?\r\nWorldCensus-cloudsql-pg,Population Density,What are the top 5 coutries with highest population density in 2024? \r\nWorldCensus-cloudsql-pg,Population Density,What are the birth and death rates in these counties?\r\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,What is the sex ratio at birth in China in 2023?\r\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Which country has the highest?\r\nWorldCensus-cloudsql-pg,Sex Ratio at Birth,Whats the world average?" }, { "path": "scripts/__init__.py", "content": "from .save_config import save_config\n\n__all__ = [\"save_config\"]" }, { "path": "scripts/copy_select_table_column_bigquery.py", "content": "import pandas as pd\r\nfrom google.cloud import bigquery\r\n\r\ndef copy_tables(project_id, source_dataset, destination_dataset, df):\r\n \r\n client = bigquery.Client()\r\n unique_table_names = df['TableName'].unique()\r\n\r\n for table_name in unique_table_names:\r\n print(f\"Processing table: {table_name}\")\r\n dest_table_id = f\"{project_id}.{destination_dataset}.{table_name}\"\r\n orig_table_id = f\"{project_id}.{source_dataset}.{table_name}\"\r\n\r\n # Copy the table (all columns initially)\r\n job = client.copy_table(orig_table_id, dest_table_id) \r\n job.result() \r\n\r\n # Get columns to preserve\r\n columns_to_preserve = df[df['TableName'] == table_name]['ColumnName'].tolist()\r\n\r\n # Drop unwanted columns \r\n dest_table = client.get_table(dest_table_id)\r\n all_columns = [c.name for c in dest_table.schema]\r\n for column in all_columns:\r\n print(f'Cheking if column {column} should be preserved')\r\n if column not in columns_to_preserve:\r\n query = f\"ALTER TABLE {dest_table_id} DROP COLUMN IF EXISTS {column}\"\r\n client.query(query).result() \r\n\r\n # Update descriptions\r\n for column_name in columns_to_preserve:\r\n print(f'Updating description for column {column_name}')\r\n description = df[(df['TableName'] == table_name) & (df['ColumnName'] == column_name)]['Description'].iloc[0]\r\n query = f\"\"\"\r\n ALTER TABLE {dest_table_id} \r\n ALTER COLUMN {column_name} SET OPTIONS(description='{description}')\r\n \"\"\"\r\n\r\n try:\r\n client.query(query).result() \r\n except Exception as e:\r\n print(f\"An error occurred while additon desciption to the column {column_name} in table {dest_table_id}: {e}\")\r\n\r\n\r\ndef add_table_description(project_id, dataset, df):\r\n \r\n client = bigquery.Client()\r\n\r\n for _, row in df.iterrows():\r\n table_name = row['TableName']\r\n table_description = row['TableDescription']\r\n table_id = f\"{project_id}.{dataset}.{table_name}\"\r\n\r\n print(f'Updating description for Table - {table_name}')\r\n query = f\"\"\"\r\n ALTER TABLE {table_id} SET OPTIONS(description='{table_description}')\r\n \"\"\"\r\n\r\n try:\r\n client.query(query).result() \r\n except Exception as e:\r\n print(f\"An error occurred while additon desciption to the column {column_name} in table {table_id}: {e}\")\r\n\r\n\r\ndef add_column_description(project_id, dataset, df):\r\n \r\n client = bigquery.Client()\r\n\r\n for _, row in df.iterrows():\r\n table_name = row['TableName']\r\n column_name = row['ColumnName']\r\n column_description = row['ColumnDescription']\r\n table_id = f\"{project_id}.{dataset}.{table_name}\"\r\n\r\n print(f'Updating description for Column - {table_name}.{column_name}')\r\n query = f\"\"\"\r\n ALTER TABLE {table_id} \r\n ALTER COLUMN {column_name} SET OPTIONS(description='{column_description}')\r\n \"\"\"\r\n\r\n try:\r\n client.query(query).result() \r\n except Exception as e:\r\n print(f\"An error occurred while additon desciption to the column {column_name} in table {table_id}: {e}\")\r\n\r\n\r\n\r\nif __name__ == \"__main__\":\r\n\r\n import os\r\n current_dir = os.path.dirname(__file__)\r\n\r\n # --- Read 'TablesAndColumns' sheet from Excel ---\r\n file_path = f\"{current_dir}/tables_columns_descriptions.csv\"\r\n df = pd.read_csv(file_path)\r\n\r\n # --- Drop rows with null values ---\r\n df.dropna(subset=['TableName', 'ColumnName'], inplace=True)\r\n\r\n # --- BigQuery setup ---\r\n client = bigquery.Client()\r\n project_id = \"\" \r\n source_dataset = \"\"\r\n destination_dataset = \"\" \r\n\r\n # # Copy Tables\r\n # copy_tables(project_id, source_dataset, \r\n # destination_dataset, \r\n # df=df)\r\n\r\n\r\n # Add table descriptions\r\n df_table_desc = df[['TableName', 'TableDescription']] # Select required columns\r\n df_table_desc = df_table_desc.dropna() # Handle missing ColumnDescriptions (optional)\r\n df_table_desc = df_table_desc.drop_duplicates(subset=['TableName', 'TableDescription']) # Drop duplicate table-column pairs\r\n\r\n add_table_description(project_id, destination_dataset, df_table_desc)\r\n\r\n # Add column descriptions\r\n df_col_desc = df[['TableName', 'ColumnName', 'ColumnDescription']] # Select required columns\r\n df_col_desc = df_col_desc.dropna() # Handle missing ColumnDescriptions (optional)\r\n df_col_desc = df_col_desc.drop_duplicates(subset=['TableName', 'ColumnName']) # Drop duplicate table-column pairs\r\n\r\n add_column_description(project_id, destination_dataset, df_col_desc)" }, { "path": "scripts/data_source_list.csv", "content": "source,user_grouping,schema,table\r\nbigquery,MovieExplorer-bigquery,imdb_people,name_basics\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_principals\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_crew\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_basics\r\nbigquery,MovieExplorer-bigquery,imdb_ratings,reviews\r\nbigquery,MovieExplorer-bigquery,imdb_ratings,title_ratings\r\ncloudsql-pg,WorldCensus-cloudsql-pg,census_bureau_international," }, { "path": "scripts/data_source_list_sample.csv", "content": "source,user_grouping,schema,table\r\nbigquery,MovieExplorer-bigquery,imdb_people,name_basics\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_principals\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_crew\r\nbigquery,MovieExplorer-bigquery,imdb_people,title_basics\r\nbigquery,MovieExplorer-bigquery,imdb_ratings,reviews\r\nbigquery,MovieExplorer-bigquery,imdb_ratings,title_ratings\r\ncloudsql-pg,WorldCensus-cloudsql-pg,census_bureau_international," }, { "path": "scripts/known_good_sql.csv", "content": "prompt,sql,user_grouping\r\nWhich country had the highest population in 2023?,\"SELECT country_name, midyear_population\r\nFROM census_bureau_international.midyear_population\r\nWHERE year = 2022\r\nORDER BY midyear_population DESC\r\nLIMIT 1\",WorldCensus-cloudsql-pg\r\nList 10 countries with highest fertility rate,\"SELECT country_name, total_fertility_rate\r\nFROM census_bureau_international.age_specific_fertility_rates\r\nWHERE year = 2023\r\nORDER BY total_fertility_rate DESC\r\nLIMIT 10\r\n\",WorldCensus-cloudsql-pg\r\nWhat is the life expectancy for men and women in a United States and 2022?,\"SELECT country_name, life_expectancy_male, life_expectancy_female\r\nFROM census_bureau_international.mortality_life_expectancy\r\nWHERE country_name = 'United States' AND year = 2022\",WorldCensus-cloudsql-pg\r\nWhich countries had the highest and lowest infant mortality rates in 2023?,\"WITH RankedInfantMortality AS (\r\n SELECT\r\n country_name,\r\n infant_mortality,\r\n RANK() OVER (ORDER BY infant_mortality ASC) AS LowestRank,\r\n RANK() OVER (ORDER BY infant_mortality DESC) AS HighestRank\r\n FROM census_bureau_international.mortality_life_expectancy\r\n WHERE year = 2023\r\n)\r\nSELECT country_name, infant_mortality\r\nFROM RankedInfantMortality\r\nWHERE LowestRank = 1 OR HighestRank = 1;\",WorldCensus-cloudsql-pg\r\nWhat is the population distribution by age and sex in India for 2023?,\"SELECT age, sex, population\r\nFROM census_bureau_international.midyear_population_agespecific\r\nWHERE country_name = 'United States' AND year = 2023\",WorldCensus-cloudsql-pg\r\nWhat is the sex ratio at birth for China in 2023?,\"SELECT country_name, sex_ratio_at_birth, year\r\nFROM census_bureau_international.age_specific_fertility_rates\r\nWHERE country_name = 'China' AND year = 2023\",WorldCensus-cloudsql-pg\r\nWhat is the world average sex ratio at birth in 2023?,\"SELECT AVG(sex_ratio_at_birth) AS world_avg_sex_ratio_at_birth\r\nFROM census_bureau_international.age_specific_fertility_rates\r\nWHERE year = 2023 AND sex_ratio_at_birth IS NOT NULL;\",WorldCensus-cloudsql-pg\r\nWhich country has the highest boy to girl ratio at birth in 2023?,\"SELECT country_name, sex_ratio_at_birth\r\nFROM census_bureau_international.age_specific_fertility_rates\r\nWHERE year = 2023\r\n AND sex_ratio_at_birth IS NOT NULL\r\nORDER BY sex_ratio_at_birth DESC\r\nLIMIT 1;\",WorldCensus-cloudsql-pg\r\nWhat are the top 10 counties with lowest mortality rate of children under 5 in 2022?,\"SELECT country_name, mortality_rate_under5\r\nFROM census_bureau_international.mortality_life_expectancy\r\nWHERE year = 2023\r\nORDER BY mortality_rate_under5 ASC\r\nLIMIT 10\",WorldCensus-cloudsql-pg\r\n\"What were the birth, death, and growth rates in SIngapore between 2019-2024?\",\"SELECT year, crude_birth_rate, crude_death_rate, growth_rate\r\nFROM census_bureau_international.birth_death_growth_rates\r\nWHERE country_name = 'Singapore' AND year BETWEEN 2019 AND 2024\",WorldCensus-cloudsql-pg\r\n\"In 2023, what was the population for countries with a population growth rate above 3%?\",\"SELECT mp.midyear_population as Population, mp.country_name, bdg.growth_rate\r\nFROM census_bureau_international.midyear_population mp\r\nJOIN census_bureau_international.birth_death_growth_rates bdg ON mp.country_code = bdg.country_code AND mp.year = bdg.year\r\nWHERE bdg.year = 2023 AND bdg.growth_rate > 3;\",WorldCensus-cloudsql-pg\r\nWhat are the top 3 countries with the highest midyear populations in 2024 and their respective land areas?,\"SELECT mp.country_name, mp.midyear_population, cna.country_area\r\nFROM census_bureau_international.midyear_population mp\r\nJOIN census_bureau_international.country_names_area cna ON mp.country_code = cna.country_code\r\nWHERE mp.year = 2023\r\nORDER BY mp.midyear_population DESC\r\nLIMIT 3;\",WorldCensus-cloudsql-pg\r\nWhat are the top 5 coutries with highest population density in 2024? ,\"SELECT mp.country_name, \r\n mp.midyear_population / cna.country_area AS population_density_persqkm\r\nFROM census_bureau_international.midyear_population mp\r\nJOIN census_bureau_international.country_names_area cna ON mp.country_code = cna.country_code\r\nWHERE mp.year = 2024 AND cna.country_area > 0 -- Exclude countries with zero or null area\r\nORDER BY population_density_persqkm DESC\r\nLIMIT 5;\",WorldCensus-cloudsql-pg\r\nWhich country has highest male life expectancy?,\"SELECT country_name, life_expectancy_male\r\nFROM census_bureau_international.mortality_life_expectancy\r\nWHERE year = 2022 -- Assuming you want the data for the year 2022\r\nORDER BY life_expectancy_male DESC\r\nLIMIT 1;\",WorldCensus-cloudsql-pg\r\nWhat are the birth and death rates of the top 5 densest countries in 2024?,\"SELECT bdg.country_name, bdg.crude_birth_rate, bdg.crude_death_rate, population_density_persqkm\r\nFROM census_bureau_international.birth_death_growth_rates bdg\r\nJOIN (\r\n SELECT mp.country_code, mp.midyear_population / cna.country_area as population_density_persqkm\r\n FROM census_bureau_international.midyear_population mp\r\n JOIN census_bureau_international.country_names_area cna ON mp.country_code = cna.country_code\r\n WHERE mp.year = 2024 AND cna.country_area > 0\r\n ORDER BY mp.midyear_population / cna.country_area DESC\r\n LIMIT 5\r\n) top_countries ON bdg.country_code = top_countries.country_code\r\nWHERE bdg.year = 2024;\",WorldCensus-cloudsql-pg" }, { "path": "scripts/save_config.py", "content": "import os\nimport sys\nimport configparser\n\n\ndef is_root_dir():\n \"\"\"\n Checks if the current working directory is the root directory of a project \n by looking for either the \"/notebooks\" or \"/agents\" folders.\n\n Returns:\n bool: True if either directory exists in the current directory, False otherwise.\n \"\"\"\n\n current_dir = os.getcwd()\n print(\"current dir: \", current_dir)\n notebooks_path = os.path.join(current_dir, \"notebooks\")\n agents_path = os.path.join(current_dir, \"agents\")\n \n return os.path.exists(notebooks_path) or os.path.exists(agents_path)\n\n\n\ndef save_config(embedding_model,\n description_model,\n vector_store,\n logging,\n kgq_examples,\n use_session_history,\n use_column_samples,\n PROJECT_ID,\n pg_region, \n pg_instance, \n pg_database, \n pg_user, \n pg_password,\n bq_dataset_region,\n bq_opendataqna_dataset_name, \n bq_log_table_name,\n firestore_region): \n \n config = configparser.ConfigParser()\n\n if is_root_dir():\n current_dir = os.getcwd()\n config.read(current_dir + '/config.ini')\n root_dir = current_dir\n else:\n root_dir = os.path.abspath(os.path.join(os.getcwd(), '..'))\n config.read(root_dir+'/config.ini')\n\n if not 'root_dir' in locals(): # If not found in any parent dir\n raise FileNotFoundError(\"config.ini not found in current or parent directories.\")\n\n\n config['GCP']['PROJECT_ID'] = PROJECT_ID\n # config['CONFIG']['DATA_SOURCE'] = data_source\n config['CONFIG']['VECTOR_STORE'] = vector_store\n config['CONFIG']['EMBEDDING_MODEL'] = embedding_model\n config['CONFIG']['DESCRIPTION_MODEL'] = description_model\n config['CONFIG']['FIRESTORE_REGION'] = firestore_region\n\n\n # Save the parameters based on Vector Store Choices\n if vector_store == 'cloudsql-pgvector':\n config['PGCLOUDSQL']['PG_INSTANCE'] = pg_instance\n config['PGCLOUDSQL']['PG_DATABASE'] = pg_database\n config['PGCLOUDSQL']['PG_USER'] = pg_user\n config['PGCLOUDSQL']['PG_PASSWORD'] = pg_password\n config['PGCLOUDSQL']['PG_REGION'] = pg_region\n # config['PGCLOUDSQL']['PG_SCHEMA'] = pg_schema\n\n if vector_store := 'bigquery':\n config['BIGQUERY']['BQ_DATASET_REGION'] = bq_dataset_region\n config['BIGQUERY']['BQ_OPENDATAQNA_DATASET_NAME'] = bq_opendataqna_dataset_name\n config['BIGQUERY']['BQ_LOG_TABLE_NAME'] = bq_log_table_name\n\n if logging: \n config['CONFIG']['LOGGING'] = 'yes'\n config['BIGQUERY']['BQ_LOG_TABLE_NAME'] = bq_log_table_name\n\n else: \n config['CONFIG']['LOGGING'] = 'no'\n\n if kgq_examples: \n config['CONFIG']['KGQ_EXAMPLES'] = 'yes'\n\n else: \n config['CONFIG']['KGQ_EXAMPLES'] = 'no'\n\n if use_session_history:\n config['CONFIG']['USE_SESSION_HISTORY'] = 'yes'\n\n else:\n config['CONFIG']['USE_SESSION_HISTORY'] = 'no'\n\n if use_column_samples:\n config['CONFIG']['USE_COLUMN_SAMPLES'] = 'yes'\n\n else:\n config['CONFIG']['USE_COLUMN_SAMPLES'] = 'no'\n\n\n with open(root_dir+'/config.ini', 'w') as configfile: \n config.write(configfile)\n\n print('All configuration paramaters saved to file!')" }, { "path": "scripts/tables_columns_descriptions.csv", "content": "TableName,ColumnName,TableDescription,ColumnDescription\r\nTable_1,Col11,Table_1_DESC,Col11_DESC\r\nTable_1,Col12,,Col12_DESC\r\nTable_1,Col13,,Col13_DESC\r\nTable_1,Col14,,Col14_DESC\r\n,,,\r\nTable_2,Col21,Table_2_DESC,Col21_DESC\r\nTable_2,Col22,,Col22_DESC\r\nTable_2,Col23,,Col31_DESC" }, { "path": "terraform/.gitignore", "content": ".terraform\nterraform.tfstate" }, { "path": "terraform/README.md", "content": "\n# Infrastructure deployment\nThe provided terraform streamlines the setup of this solution and serves as a blueprint for deployment. The script provides a one-click, one-time deployment option. However, it doesn't include CI/CD capabilities and is intended solely for initial setup.\n\n> [!NOTE]\n> Current version of the Terraform Google Cloud provider does not support deployment of a few resources, this soultion uses null_resource to create those resources using Google Cloud SDK.\n\n## Table of Contents\n- [Terraform folder structure](#terraform-folder-structure)\n - [Structure](#structure)\n - [.tf Files Description](#tf-files-description)\n - [Scripts](#scripts)\n - [Template files](#template-files)\n- [Step 0: Prerequisites](#step-0-prerequisites)\n - [Data Sources](#data-sources-set-up)\n - [Enable Firebase](#enable-firebase)\n - [Local Configuration](#optional-local-configuration)\n - [Terraform Deployment](#terraform-deployment)\n- [Step 1a: One-click Deployment](#step-1-one-click-deployment) [choose either 1a or 1b]\n- [Step 1b: Step by step Deployment](#step-1-step-by-step-deployment) [choose either 1a or 1b]\n- [Step 2: Review your environment](#step-2-review-your-environment)\n- [Deployed Resources](#deployed-resources)\n- [Troubleshooting Known Issues](#troubleshooting-known-issues)\n\n## Terraform folder structure\n#### Structure\n```\nOpen_Data_QnA/\n |\n |--->terraform/\n |---> .tf files\n |---> scripts/\n | |--> .sh, .py\n |---> templates/\n |--> .tftpl files\n\n```\n1. .tf files - terraform scripts to spin up resources.\n1. scripts/ - A folder containing all the required shell scripts and python files.\n1. templates/ - the templates in this directory are used to replace all necessary configuration values for the application in config.ini and constants.ts\n\n#### .tf Files Description\n1. versions.tf - Contains provider blocks with appropriate versions that are needed to deploy the resources\n1. locals.tf - Contains the list of APIs to be enabled and BQ tables to be created.\n1. main.tf - Imports data about the mentioned project and also activates required APIs in the project.\n1. iam.tf - Handles implementation of all the necessary IAM roles to various members like service accounts and users.\n1. bq.tf - This script is responsible for creating Bigquery dataset.\n1. pg-vector.tf - This is responsible for spinning up the CloudSQL instance, database, username and password.\n1. embeddings-setup.tf - This script is responsible for the below tasks.\n * Update [config.ini](../config.ini) with values provided in variables.tf / terraform.tfvars.\n * Fetch list of data sources from [data_source_list.csv](../scripts/data_source_list.csv).\n * Create vector embeddings for the metadata associated with the listed tables.\n * Fetch known good sqls from [known_good_sql.csv](../scripts/known_good_sql.csv) (if opted) and create vector embeddings for the same.\n * Store table metadata embeddings and known good sql embeddings to the vector data store.\n1. backend.tf - Creates firstore database and cloud run service with dummy container image.\n1. frontend.tf - Responsible for creating the below resources.\n * Updates firebase.json\n * Creates firebase web app\n * Imports config data about the web app.\n * Updates [constants.ts](../frontend/src/assets/constants.ts) with relevant values from web app's config.\n1. outputs.tf - Contains the important details about the resources that will be deployed. Example: instance name, ip address etc, cloud run url. This will be printed out on the console when the terraform executes successfully.\n1. variables.tf - Contains the list of variables, their default values and descriptions.\n1. terraform.tfvars - Can be used to override default values in variables.tf\n\n#### Scripts\n1. [deploy-all.sh](scripts/deploy-all.sh) - This shell script is used for one-click deployment. this executes the terraform scripts as well as the gcloud commands for deploying latest cloud build images for backend and frontend\n1. [install-dependencies.sh](scripts/install-dependencies.sh) - This script installs poetry module which is used to manage the dependencies required for the application. This shell script is used within the terraform script [embeddings-setup.tf](embeddings-setup.tf)\n1. [create-and-store-embeddings.py](scripts/create-and-store-embeddings.py) - This is a python file that executes the relevant functions to create and store vector embeddings\n1. [execute-python-files.sh](scripts/execute-python-files.sh) - This shell script is used within the terraform script [embeddings-setup.tf](embeddings-setup.tf) to execute the [create-and-store-embeddings.py](scripts/create-and-store-embeddings.py)\n1. [execute-gcloud-cmd.sh](scripts/execute-gcloud-cmd.sh) - This shell script can overwrite an org policy or delete the over-written rule based on the input parameters.\n1. [copy-firebase-json.sh](scripts/copy-firebase-json.sh) - This script is used to overwrite firebase.json file with the information required by the application. This is information is copied from [firebase_setup.json](../frontend/firebase_setup.json)\n1. [backend-deployment.sh](scripts/backend-deployment.sh) - This shell script updates the cloud run service created by terraform, with the latest code.\n1. [frontend-deployment.sh](scripts/frontend-deployment.sh) - This shell script submits the latest build for the frontent code.\n\n#### Template Files\n1. [config.ini.tftpl](./templates/config.ini.tftpl) - This template is used to populate [config.ini](../config.ini) with corresponding values from variables.tf or terraform.tfvars. This file is populated via [embeddings-setup.tf](embeddings-setup.tf).\n1. [constants.ts.tftpl](./templates/constants.ts.tftpl) - This template is used to populate [constants.ts](../frontend/src/assets/constants.ts). This file is populated via [frontend.tf](frontend.tf)\n\n## Step 0: Prerequisites\n\nPrior to executing terraform, ensure that the below mentioned steps have been completed.\n\n#### Data Sources Set Up\n\n1. Source data should already be available. If you do not have readily available source data, use the notebooks [0_CopyDataToBigQuery.ipynb](../notebooks/0_CopyDataToBigQuery.ipynb) or [0_CopyDataToCloudSqlPG.ipynb](../notebooks/0_CopyDataToCloudSqlPG.ipynb) based on the preferred source to populate sample data.\n2. Ensure that the [data_source_list.csv](../scripts/data_source_list.csv) is populated with the list of datasources to be used in this solution. Terraform will take care of creating the embeddings in the destination. Use [data_source_list_sample.csv](../scripts/data_source_list_sample.csv) to fill the [data_source_list.csv](../scripts/data_source_list.csv)\n3. If you want to use known good sqls for few shot prompting, ensure that the [known_good_sql.csv](../scripts/known_good_sql.csv) is populated with the required data. Terraform will take care of creating the embeddings in the destination.\n\n#### Enable Firebase\nFirebase will be used to host the frontend of the application.\n\n1. Go to https://console.firebase.google.com/\n1. Select add project and load your Google Cloud Platform project\n1. Add Firebase to one of your existing Google Cloud projects\n1. Confirm Firebase billing plan\n1. Continue and complete\n\n#### (Optional) Local configuration\nIf you are running this outside Cloud Shell you need to set up your Google Cloud SDK Credentials\n\n```shell\ngcloud config set project \ngcloud auth application-default set-quota-project \n```\n\n#### Terraform deployment\n> [!NOTE] \n> Terraform apply command for this application uses gcloud config to fetch & pass the set project id to the scripts. Please ensure that gcloud config has been set to your intended project id before proceeding.\n\n1. Install [terraform 1.7 or higher](https://developer.hashicorp.com/terraform/install).\n1. [OPTIONAL] Update default values of variables in [variables.tf](variables.tf) according to your preferences. You can find the description for each variable inside the file. This file will be used by terraform to get information about the resources it needs to deploy. If you do not update these, terraform will use the already specified default values in the file.\n1. Move to the terraform directory in the terminal: ```Open_Data_QnA/terraform```.\n1. There are 2 ways to deploy this application: \n * **[One-click deployment](#step-1-one-click-deployment)** : Use this method when you want to do a single click deployment i.e execute terraform and shell scripts for frontend & backend services deployment all at once.\n * **[Step by step deployment](#step-1-step-by-step-deployment)**: Use this method if you want to deploy teraaform resources, frontend and backend services separately after manual validation of configuration files.\n\n> [!IMPORTANT] \n> The Terraform scripts require specific IAM permissions to function correctly. The user needs either the broad `roles/resourcemanager.projectIamAdmin` role or a custom role with tailored permissions to manage IAM policies and roles.\n> Additionally, one script TEMPORARILY disables Domain Restricted Sharing Org Policies to enable the creation of a public endpoint. This requires the user to also have the `roles/orgpolicy.policyAdmin` role.\n\n## Step 1: One-Click Deployment\n### Deployment command and script overview\n```sh\nsh ./scripts/deploy-all.sh\n```\nThis script will perform the following steps:\n1. **Run terraform scripts** - These terraform scripts will generate all the GCP resources and configurations files required for the frontend & backend. It will also generate embeddings and store it in the destination vector db.\n1. **Deploy cloud run backend service with latest backend code** - The terraform in the previous step uses a dummy container image to deploy the initial version of cloud run service. This is the step where the actual backend code gets deployed.\n1. **Deploy frontend app** - All the config files, web app etc required to create the frontend are deployed via terraform. However, the actual UI deployment takes place in this step.\n\n### After deployment\n***Auth Provider***\n\nYou need to enable at least one authentication provider in Firebase, you can enable it using the following steps:\n1. Go to https://console.firebase.google.com/project/your_project_id/authentication/providers (change the `your_project_id` value)\n2. Click on Get Started (if needed)\n3. Select Google and enable it\n4. Set the name for the project and support email for project\n5. Save\n\n## Step 1: Step by step Deployment\n#### Start Terraform deployment\n\n```sh\nterraform init\nterraform apply -var=project_id=$(gcloud config get project)\n```\n\nThis terraform will generate all configurations files required in the frontend and backend_apis. It will also generate embeddings and store it in the destination vector db\n\n#### After Terraform deployment\n***Auth Provider***\n\nYou need to enable at least one authentication provider in Firebase, you can enable it using the following steps:\n1. Go to https://console.firebase.google.com/project/your_project_id/authentication/providers (change the `your_project_id` value)\n2. Click on Get Started (if needed)\n3. Select Google and enable it\n4. Set the name for the project and support email for project\n5. Save\n\n#### Validate the config files\nThis deployment uses the templates in the [templates/](templates/) directory to replace all necessary configuration values for the application. Before deploying the application check that all the values have been populated correctly in the [config.ini](../config.ini) file and [constants.ts](../frontend/src/assets/constants.ts) file.\n\n#### Application deployment\nTo deploy the backend cloud run service for the application, run the following command. You will find all the needed values from terraform output. Expected working directory : ```Open_Data_QnA/terraform``` \n```\nsh scripts/backend-deployment.sh --servicename --project --region --sa \n```\nTo deploy the frontent of the application, you need to execute the below command:\n```\nsh scripts/frontend-deployment.sh --project --region \n```\n\n## Step 2: Review your environment\nOnce deployment is completed the scripts (terraform & shell scripts) will output relevant resource values.\n\nResulting example outputs:\n```sh\ninstance_name = \"pg15-opendataqna-2\"\npostgres_db = \"opendataqna-db-2\"\npublic_ip_address = \"XX.XXX.XXX.XXX\"\nservice_account = \"opendataqna@your-project-id.iam.gserviceaccount.com\"\nservice_name = \"opendataqna-v2\"\nservice_url = \"https://opendataqna-xxxxxxxx.a.run.app\"\n```\nYour url to access the frontend is usually in this format: \"https://your-project-id.web.app\"\n\n## Deployed resources\nThis deployment creates all the resources described in the main [README.md](../README.md) file, the following is a list of the created resources:\n1. Enable Google Cloud Service APIs. List can be found in [locals.tf](locals.tf)\n2. [BiqQuery](https://console.cloud.google.com/bigquery) Dataset and tables:\n - **1 Dataset**: where the log table and embedding tables will be created.\n - **Tables**:\n - **Audit Log table**: This table will store all application logs\n - **Table Metadata Embeddings table**: Created only when Bigquery is chosen as the vector db. This will contain all table meta data and its text embeddings. \n - **Column Embeddings table**: Created only when Bigquery is chosen as the vector db. This will contain all column meta data and its text embeddings.\n - **Example SQL table**: Created only when Bigquery is chosen as the vector db and marked kgq_examples='yes' in tfvars. This contains all the known good sqls and their respective text embeddings. You need to populate the [known_good_sql.csv](../scripts/known_good_sql.csv) before deployment. Terraform will take care of creation of embeddings in the destination.\n3. **CloudSQL instance**: Created if cloudsql-pgvector is chosen as the vector store.\n4. **pg-vector database**: Created if cloudsql-pgvector is chosen as the vector store. This database will store all text embeddings.\n - **Tables**:\n - **Table Metadata Embeddings table**: Created only when cloudsql-pgvector is chosen as the vector db. This will contain all table meta data and its text embeddings.\n - **Column Embeddings table**: Created only when cloudsql-pgvector is chosen as the vector db. This will contain all column meta data and its text embeddings.\n - **Example SQL table**: Created only when cloudsql-pgvector is chosen as the vector db and marked kgq_examples='yes' in tfvars. This contains all the known good sqls and their respective text embeddings.\n> [!NOTE] \n> The above mentioned tables in **CloudSQL & BigQuery** are created via the python script [create-and-store-embeddings.py](scripts/create-and-store-embeddings.py)\n\n\n> [!IMPORTANT] \n> If you have an existing CloudSQL instance that you want to use for storing vector embeddings, then update the variable - **use_existing_cloudsql_instance = \"yes\"**. Terraform will not create any new cloudsql instance, database or user name & password. It will be assumed that these resources already exist.\n\n5. A backend service account for cloud run service with the required permissions\n6. All embedding tables will be populated via terraform.\n7. [Cloud Run](https://console.cloud.google.com/run) for backend APIs\n8. A Firestore database for storing chat history\n9. Firebase web app for hosting frontend.\n\n## Troubleshooting Known Issues\n1. #### `pipx: command not found`\n * This error will be caused by execution of [install-dependencies.sh](./scripts/install-dependencies.sh) file.\n * The error indicates that even though you've installed pipx, your shell isn't aware of its location. This usually means the directory where pipx is installed hasn't been added to your system's PATH environment variable.\n * Resolution:\n 1. We have already tried to ensure that pipx gets added to PATH via execution of the command `export PATH=\"$PATH:$(python3 -c \"import sysconfig; print(sysconfig.get_paths()['scripts'])\")\"` in the [install-dependencies.sh](./scripts/install-dependencies.sh) file. This command is specifically designed to return the path to the global site-packages directory (where system-wide packages are installed) and add to PATH temporarily. However, This command might not be giving you the correct directory where pipx is installed.\n 2. If you encounter this error, you can try replacing the above command with `export PATH=\"$PATH:$(python3 -m site --user-base)/bin\"`. This command is specifically designed to return the path to the user-specific site-packages directory and add to PATH temporarily.\n 3. If the above step also fails, you will need to manually find the PATH where pipx is installed. Once you know the location, just replace `export PATH=\"$PATH:$(python3 -c \"import sysconfig; print(sysconfig.get_paths()['scripts'])\")\"` inside [install-dependencies.sh](./scripts/install-dependencies.sh) with `export PATH=\"$PATH:/path/to/pipx/directory\"`. Replace `/path/to/pipx/directory` with the actual path you found.\n\n1. #### `poetry: command not found`\n * This error will be caused by execution of [install-dependencies.sh](./scripts/install-dependencies.sh) file.\n * The error indicates that even though you've installed poetry, your shell isn't aware of its location. This usually means the directory where pipx is installed hasn't been added to your system's PATH environment variable.\n * Resolution:\n 1. Although we have ensured that poetry will automatically get added to PATH via commands `pipx ensurepath` and `source ~/.bashrc` in the [install-dependencies.sh](./scripts/install-dependencies.sh) file, if the script still fails with this error, you need to manually find the path where poetry is installed.\n 2. Add the new PATH to `~/.bashrc` file and source it before re-running the script again. \n\n1. #### `aiohttp.client_exceptions.ClientConnectorCertificateError`\n * `aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host sqladmin.googleapis.com:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1006)')]`\n * SSL/TLS and Certificates: Secure connections over the internet (HTTPS) rely on SSL/TLS protocols. These protocols use digital certificates to verify the identity of the server you're connecting to. Your system needs a set of trusted root certificates to validate these server certificates.\n * Missing Root Certificates: If your system lacks the necessary root certificates or they are outdated, it can't establish trust with the server, leading to the CERTIFICATE_VERIFY_FAILED error.\n * Resolution:\n 1. Install or Update certifi - certifi is a package that provides a curated collection of Root Certificates for validating the trustworthiness of SSL certificates while making secure network requests.\n Install or update it using pip:\n ```\n pip install --upgrade certifi\n ```\n 2. Set the SSL_CERT_FILE environment variable\n Tell your Python environment to use the certificates provided by certifi:\n ```\n export SSL_CERT_FILE=$(python -c \"import certifi; print(certifi.where())\")\n ```\n This command finds the location of the certificate file provided by certifi and sets the SSL_CERT_FILE environment variable to that path.\n \n 3. Retry your code.\n\n1. #### `Firebase project not found`\n * `Error creating WebApp: googleapi: Error 404: Firebase project XXXXXXXXX not found.`\n * This error is encountered when you fail to add firebase to your project before execution of the scripts.\n * Follow the instructions for [enabling firebase](#enable-firebase) under [pre-requisites](#prerequisites) section of the README\n\n1. #### `Error: Provider produced inconsistent result after apply`\n * Sometimes, when Terraform creates or updates a resource on your cloud provider, there can be a slight delay before that change is fully reflected in the provider's system. This can cause Terraform to see an inconsistency between what it expects and what the provider reports, leading to an error.\n * As a result, you might encounter an error message if Terraform attempts to read information about a newly created or updated resource before the provider's system has fully processed the change.\n * Solutions:\n 1. **Manual Deletion and Retry**: Delete the problematic resource directly on the remote system and re-run terraform apply.\n 2. **Terraform Import**: Use `terraform import` to synchronize the resource's state from the remote system into your Terraform state file, then re-run `terraform apply`. Follow the [How to Import Resources into a Remote State Managed by Terraform Cloud](https://support.hashicorp.com/hc/en-us/articles/360061289934-How-to-Import-Resources-into-a-Remote-State-Managed-by-Terraform-Cloud) for detailed steps.\n\n\n" }, { "path": "terraform/backend.tf", "content": "resource \"google_firestore_database\" \"chathistory_db\" {\n project = var.project_id\n name = \"opendataqna-session-logs\"\n location_id = var.firestore_region\n type = \"FIRESTORE_NATIVE\"\n depends_on = [ module.project_services ]\n}\n\nresource \"google_cloud_run_service\" \"backend\" {\n name = var.cloud_run_service_name\n location = var.deploy_region\n project = var.project_id\n\n template {\n spec {\n containers {\n image = \"us-docker.pkg.dev/cloudrun/container/hello\"\n }\n service_account_name = module.genai_cloudrun_service_account.email\n }\n }\n\n traffic {\n percent = 100\n latest_revision = true\n }\n\n depends_on = [ module.project_services,null_resource.org_policy_temp, module.genai_cloudrun_service_account, local_file.config_ini,]\n}\n" }, { "path": "terraform/bq.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nmodule \"bigquery\" {\n source = \"terraform-google-modules/bigquery/google\"\n version = \"~> 7.0\"\n dataset_id = var.bq_opendataqna_dataset\n dataset_name = var.bq_opendataqna_dataset\n project_id = var.project_id\n location = var.bq_dataset_region\n default_table_expiration_ms = null\n\n depends_on = [module.project_services]\n}" }, { "path": "terraform/embeddings-setup.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nresource \"local_file\" \"config_ini\" {\n content = templatefile(\"${path.module}/templates/config.ini.tftpl\", {\n embedding_model = var.embedding_model,\n description_model = var.description_model,\n vector_store = var.vector_store,\n debugging = var.debugging,\n logging = var.logging,\n kgq_examples = var.kgq_examples,\n firestore_region = var.firestore_region,\n use_column_samples = var.use_column_samples,\n use_session_history = var.use_session_history,\n project_id = var.project_id,\n pg_region = var.pg_region,\n pg_instance = var.pg_instance,\n pg_database = var.pg_database,\n pg_user = var.pg_user\n pg_password = var.pg_password\n bq_dataset_region = var.bq_dataset_region\n bq_opendataqna_dataset = var.bq_opendataqna_dataset\n bq_log_table = var.bq_log_table\n }\n )\n filename = \"../config.ini\"\n}\n\nresource \"null_resource\" \"install_dependencies\" {\n triggers = {\n always_run = \"${timestamp()}\"\n }\n provisioner \"local-exec\" {\n \n working_dir = \"${path.module}\"\n command = \"sh ${path.module}/scripts/install-dependencies.sh\"\n }\n}\n\nresource \"null_resource\" \"create_and_store_embeddings\" {\n depends_on = [local_file.config_ini, null_resource.install_dependencies,module.bigquery, google_sql_database_instance.pg15_opendataqna[0]]\n triggers = {\n always_run = \"${timestamp()}\"\n }\n provisioner \"local-exec\" {\n \n working_dir = \"${path.module}\"\n command = \"sh ${path.module}/scripts/execute-python-files.sh './scripts'\"\n }\n}" }, { "path": "terraform/frontend.tf", "content": "resource \"google_firebase_project\" \"default\" {\n provider = google-beta\n project = var.project_id\n\n provisioner \"local-exec\" {\n \n working_dir = \"${path.module}\"\n command = \"sh ${path.module}/scripts/copy-firebase-json.sh\"\n }\n}\n\nresource \"google_firebase_web_app\" \"app_frontend\" {\n provider = google-beta\n project = var.project_id\n display_name = var.firebase_web_app_name\n}\n\ndata \"google_firebase_web_app_config\" \"app_frontend_config\" {\n provider = google-beta\n web_app_id = google_firebase_web_app.app_frontend.app_id\n project = var.project_id\n}\n\nresource \"local_file\" \"constants_ts\" {\n depends_on = [ google_firebase_web_app.app_frontend, google_cloud_run_service.backend ]\n content = templatefile(\"${path.module}/templates/constants.ts.tftpl\", {\n projectId = var.project_id\n appId = google_firebase_web_app.app_frontend.app_id\n apiKey = data.google_firebase_web_app_config.app_frontend_config.api_key\n authDomain = data.google_firebase_web_app_config.app_frontend_config.auth_domain\n storageBucket = lookup(data.google_firebase_web_app_config.app_frontend_config, \"storage_bucket\", \"\")\n messagingSenderId = lookup(data.google_firebase_web_app_config.app_frontend_config, \"messaging_sender_id\", \"\")\n endpoint_opendataqna = google_cloud_run_service.backend.status[0].url\n }\n )\n filename = \"../frontend/src/assets/constants.ts\"\n}\n\n\n" }, { "path": "terraform/iam.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nresource \"null_resource\" \"org_policy_temp\" {\n depends_on = [module.project_services]\n\n provisioner \"local-exec\" {\n \n working_dir = \"${path.module}\"\n command = \"sh ${path.module}/scripts/execute-gcloud-cmd.sh ${var.project_id} YES\"\n }\n}\n\nresource \"null_resource\" \"delete_org_policy_temp\" {\n provisioner \"local-exec\" {\n working_dir = \"${path.module}\"\n command = \"sh ${path.module}/scripts/execute-gcloud-cmd.sh ${var.project_id} NO\"\n }\n\n depends_on = [module.project_services, null_resource.org_policy_temp, google_cloud_run_service.backend ]\n}\n\nmodule \"genai_cloudrun_service_account\" {\n source = \"terraform-google-modules/service-accounts/google\"\n version = \"~> 4.0\"\n project_id = var.project_id\n names = [var.service_account]\n project_roles = [\n \"${var.project_id}=>roles/cloudsql.client\",\n \"${var.project_id}=>roles/bigquery.admin\",\n \"${var.project_id}=>roles/aiplatform.user\",\n \"${var.project_id}=>roles/datastore.owner\"\n ]\n depends_on = [module.project_services]\n}\n\n\nresource \"google_project_iam_member\" \"default_ce_sa_role\" {\n for_each = toset([\n \"roles/storage.admin\",\n \"roles/artifactregistry.admin\",\n \"roles/firebase.admin\",\n \"roles/cloudbuild.builds.builder\",\n \"roles/logging.logWriter\"\n ])\n role = each.key\n member = \"serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com\"\n project = var.project_id\n depends_on = [module.project_services]\n}\n\nresource \"google_project_iam_member\" \"default_cloudbuild_sa_role\" {\n for_each = toset([\n \"roles/firebase.admin\",\n \"roles/artifactregistry.admin\",\n \"roles/serviceusage.apiKeysAdmin\",\n \"roles/cloudbuild.builds.builder\"\n ])\n role = each.key\n member = \"serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com\"\n project = var.project_id\n depends_on = [module.project_services]\n}\n\nresource \"google_cloud_run_service_iam_member\" \"invoker\" {\n location = google_cloud_run_service.backend.location\n project = google_cloud_run_service.backend.project\n service = google_cloud_run_service.backend.name\n role = \"roles/run.invoker\"\n member = \"allUsers\"\n depends_on = [ google_cloud_run_service.backend ]\n}\n" }, { "path": "terraform/locals.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nlocals {\n services = [\n \"cloudapis.googleapis.com\",\n \"iam.googleapis.com\",\n \"run.googleapis.com\",\n \"sqladmin.googleapis.com\",\n \"aiplatform.googleapis.com\",\n \"bigquery.googleapis.com\",\n \"storage.googleapis.com\",\n \"cloudresourcemanager.googleapis.com\",\n \"serviceusage.googleapis.com\",\n \"firebase.googleapis.com\",\n \"firebasehosting.googleapis.com\",\n \"logging.googleapis.com\",\n \"monitoring.googleapis.com\",\n \"cloudbuild.googleapis.com\",\n \"firestore.googleapis.com\"\n ]\n\n}" }, { "path": "terraform/main.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n\ndata \"google_project\" \"project\" {\n project_id = var.project_id\n}\n\n\nmodule \"project_services\" {\n source = \"terraform-google-modules/project-factory/google//modules/project_services\"\n version = \"~> 14.5\"\n\n project_id = var.project_id\n enable_apis = true\n\n activate_apis = local.services\n disable_services_on_destroy = false\n}\n" }, { "path": "terraform/outputs.tf", "content": "output \"project_number\" {\n value = data.google_project.project.number\n}\n\noutput \"project_id\" {\n value = var.project_id\n}\n\noutput \"instance_name\" {\n value = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\" ? google_sql_database_instance.pg15_opendataqna[0].name : null\n description = \"The instance name for the master instance\"\n}\n\noutput \"public_ip_address\" {\n description = \"The first public (PRIMARY) IPv4 address assigned for the master instance\"\n value = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\" ? google_sql_database_instance.pg15_opendataqna[0].public_ip_address : null\n}\n\noutput \"postgres_db\" {\n description = \"database name\"\n value = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\" ? google_sql_database.pg_db[0].name : null\n}\n\noutput \"sql_user\" {\n description = \"user name generated for the instance\"\n value = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\" ? google_sql_user.pguser[0] : null\n sensitive = true\n}\n\noutput \"sql_user_password\" {\n description = \"user name generated for the instance\"\n value = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\" ? google_sql_user.pguser[0].password : null\n sensitive = true\n}\n\noutput \"service_name\" {\n value = google_cloud_run_service.backend.name\n description = \"Name of the created service\"\n}\n\noutput \"revision\" {\n value = google_cloud_run_service.backend.status[0].latest_ready_revision_name\n description = \"Deployed revision for the service\"\n}\n\noutput \"service_url\" {\n value = google_cloud_run_service.backend.status[0].url\n description = \"The URL on which the deployed service is available\"\n}\n\noutput \"firebase_appId\" {\n value = google_firebase_web_app.app_frontend.app_id\n}\n\noutput \"firebase_apiKey\" {\n value = data.google_firebase_web_app_config.app_frontend_config.api_key\n}\n\noutput \"firebase_authDomain\" {\n value = data.google_firebase_web_app_config.app_frontend_config.auth_domain\n}\n\noutput \"firebase_storageBucket\" {\n value = lookup(data.google_firebase_web_app_config.app_frontend_config, \"storage_bucket\", \"\")\n}\n\noutput \"firebase_messagingSenderId\" {\n value = lookup(data.google_firebase_web_app_config.app_frontend_config, \"messaging_sender_id\", \"\")\n}\n\noutput \"firebase_measurementId\" {\n value = lookup(data.google_firebase_web_app_config.app_frontend_config, \"measurement_id\", \"\")\n}\n\noutput \"hosting_url\" {\n value = google_firebase_web_app.app_frontend.app_urls\n}\n\noutput \"service_account\"{\n value = module.genai_cloudrun_service_account.email\n}\n\noutput \"deploy_region\" {\n value = var.deploy_region\n}" }, { "path": "terraform/pg-vector.tf", "content": "resource \"google_sql_database_instance\" \"pg15_opendataqna\" {\n count = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\"? 1:0\n name = var.pg_instance\n project = var.project_id\n region = var.pg_region\n database_version = \"POSTGRES_15\"\n root_password = \"abcABC123!\"\n settings {\n tier = \"db-custom-2-7680\"\n password_validation_policy {\n min_length = 6\n reuse_interval = 2\n complexity = \"COMPLEXITY_DEFAULT\"\n disallow_username_substring = true\n password_change_interval = \"30s\"\n enable_password_policy = true\n }\n }\n deletion_protection = false\n depends_on = [ module.project_services ]\n}\n\nresource \"google_sql_database\" \"pg_db\" {\n count = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\"? 1:0\n name = var.pg_database\n project = var.project_id\n instance = google_sql_database_instance.pg15_opendataqna[count.index].name\n depends_on = [google_sql_database_instance.pg15_opendataqna]\n}\n\nresource \"google_sql_user\" \"pguser\" {\n count = var.vector_store==\"cloudsql-pgvector\" && var.use_existing_cloudsql_instance==\"no\"? 1:0\n name = var.pg_user\n project = var.project_id\n instance = google_sql_database_instance.pg15_opendataqna[count.index].name\n password = var.pg_password\n depends_on = [\n google_sql_database_instance.pg15_opendataqna,\n ]\n}\n" }, { "path": "terraform/scripts/backend-deployment.sh", "content": "usage() {\n echo \"Usage: $0 --servicename --project --region --sa \"\n exit 1 # Indicate an error\n}\n\n# Function to validate a parameter's value\nvalidate_param() {\n local param_name=\"$1\"\n local param_value=\"$2\"\n\n if [ -z \"$param_value\" ]; then\n echo \"Error: Parameter '$param_name' cannot be empty.\"\n usage # Show usage and exit if the value is empty\n fi\n}\n\n# Check if enough arguments are provided\nif [ $# -lt 8 ]; then \n echo \"Error: Insufficient arguments.\"\n usage\nfi\n\n# Parse and validate named parameters\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n --servicename)\n validate_param \"$1\" \"$2\"\n SERVICE_NAME=$2\n shift 2 # Move to the next parameter\n ;;\n --project)\n validate_param \"$1\" \"$2\"\n PROJECT_ID=$2\n shift 2 # Move to the next parameter\n ;;\n --region)\n validate_param \"$1\" \"$2\"\n DEPLOY_REGION=$2\n shift 2\n ;;\n --sa)\n validate_param \"$1\" \"$2\"\n SERVICE_ACCOUNT=$2\n shift 2\n ;;\n *) \n echo \"Error: Unknown parameter '$1'.\"\n usage\n ;;\n esac\ndone\n\n# Deploys backend to Cloud Run using the provided region and service account\nmain(){\n pwd\n cd ../backend-apis\n\n echo \"Setting orgpolicy to allow all IAM domains\"\n \n gcloud resource-manager org-policies set-policy --project=$PROJECT_ID policy.yaml || exit 1 #This command will create policy that overrides to allow all domain\n \n cd ../\n \n echo \"Deploying cloud run service..\"\n pwd\n gcloud beta run deploy $SERVICE_NAME --region $DEPLOY_REGION --source . --service-account=$SERVICE_ACCOUNT --service-min-instances=1 --allow-unauthenticated --project=$PROJECT_ID || exit 1\n\n echo \"Deleting the previously create dorg policy..\"\n\n gcloud resource-manager org-policies delete iam.allowedPolicyMemberDomains --project=$PROJECT_ID || exit 1\n\n}\n\nmain\n" }, { "path": "terraform/scripts/copy-firebase-json.sh", "content": "cd ../frontend/\nrm firebase.json .firebaserc\ncp firebase_setup.json firebase.json" }, { "path": "terraform/scripts/create-and-store-embeddings.py", "content": "import os\nimport sys\nimport asyncio\nmodule_path = os.path.abspath(os.path.join('..', '..'))\nsys.path.append(module_path)\n\nprint(f\"module_path: {module_path}\")\n\nos.chdir(module_path) \ncurrent_dir = os.getcwd()\n\nprint(f\"current_dir : {current_dir}\")\n\nfrom env_setup import get_embeddings, store_embeddings, store_kgq_sql_embeddings\n\nasync def create_and_store_embeddings():\n# Generate embeddings for tables and columns\n table_schema_embeddings, col_schema_embeddings = get_embeddings()\n\n# Store table/column embeddings (asynchronous)\n await(store_embeddings(table_schema_embeddings, col_schema_embeddings))\n\nasyncio.run(create_and_store_embeddings())\n\n# Store known good query embeddings (if enabled)\nasyncio.run(store_kgq_sql_embeddings())\n" }, { "path": "terraform/scripts/deploy-all.sh", "content": "echo \"#####################################################################################\"\necho \" STARTING DEPLOYMENT \"\necho \"#####################################################################################\"\n\noriginal_dir=$(pwd)\necho \"Current Working Directory: $original_dir\"\necho \"-------------------------------------------------------------------------------------\"\necho \" EXECUTING TERRAFORM \"\necho \"-------------------------------------------------------------------------------------\"\nterraform init && \\\nterraform apply -var=project_id=$(gcloud config get project) -auto-approve || exit 1\necho \"-------------------------------------------------------------------------------------\"\necho \" TERRAFORM EXECUTION SUCCESSFUL \"\necho \"-------------------------------------------------------------------------------------\"\necho \"The below values from terraform output will be used for deployment of frontend and backend services:\" \n\ncloudrun_service_name=$(terraform output -raw service_name)\necho \"cloudrun_service_name: $cloudrun_service_name\"\nproject_id=$(terraform output -raw project_id)\necho \"project_id: $project_id\"\ndeploy_region=$(terraform output -raw deploy_region)\necho \"deploy_region: $deploy_region\"\nservice_account=$(terraform output -raw service_account)\necho \"service_account: $service_account\"\n\necho \"-------------------------------------------------------------------------------------\"\necho \" DEPLOYING BACKEND CLOUD RUN \"\necho \"-------------------------------------------------------------------------------------\"\nsh scripts/backend-deployment.sh --servicename $cloudrun_service_name --project $project_id --region $deploy_region --sa $service_account || exit 1\n\necho \"-------------------------------------------------------------------------------------\"\necho \" BACKEND DEPLOYMENT SUCCESSFUL \"\necho \"-------------------------------------------------------------------------------------\"\n\necho \"Current working directory: $original_dir\"\necho \"-------------------------------------------------------------------------------------\"\necho \" DEPLOYING FRONTEND SERVICE \"\necho \"-------------------------------------------------------------------------------------\"\nsh scripts/frontend-deployment.sh --project $project_id --region $deploy_region || exit 1\necho \"-------------------------------------------------------------------------------------\"\necho \" FRONTEND DEPLOYMENT SUCCESSFUL \"\necho \"-------------------------------------------------------------------------------------\"\n\necho \"#####################################################################################\"\necho \" APPLICATION DEPLOYMENT COMPLETED! PLEASE FOLLOW README FOR NEXT STEPS \"\necho \"#####################################################################################\"\n" }, { "path": "terraform/scripts/execute-gcloud-cmd.sh", "content": "export PROJECT_ID=$1\nexport SETPOLICY=$2\ncd ../backend-apis\n\nif [ $SETPOLICY = \"YES\" ]\nthen\n echo \"Enforcing Org Policy..\"\n gcloud resource-manager org-policies set-policy --project=$PROJECT_ID policy.yaml #This command will create policy that overrides to allow all domain\nelse\n echo \"Deleting Org Policy..\"\n gcloud resource-manager org-policies delete iam.allowedPolicyMemberDomains --project=$PROJECT_ID\nfi\n" }, { "path": "terraform/scripts/execute-python-files.sh", "content": "#!/bin/bash\n\n# Get the directory of the current script\n\nSCRIPT_DIR=$(dirname \"$0\")\n\necho \"$SCRIPT_DIR\"\n\nsource ~/.bashrc\n# Activate the poetry virtual environment\nsource \"$(poetry env info --path)/bin/activate\"\n\n# Navigate to the correct directory (assuming the script is in the 'scripts' folder)\ncd \"$SCRIPT_DIR/\"\npwd\n# Execute the Python file within the virtual environment\npoetry run python create-and-store-embeddings.py" }, { "path": "terraform/scripts/frontend-deployment.sh", "content": "\n\nusage() {\n echo \"Usage: $0 --project --region \"\n exit 1 # Indicate an error\n}\n\n# Function to validate a parameter's value\nvalidate_param() {\n local param_name=\"$1\"\n local param_value=\"$2\"\n\n if [ -z \"$param_value\" ]; then\n echo \"Error: Parameter '$param_name' cannot be empty.\"\n usage # Show usage and exit if the value is empty\n fi\n}\n\n# Check if enough arguments are provided\nif [ $# -lt 4 ]; then \n echo \"Error: Insufficient arguments.\"\n usage\nfi\n\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n --project)\n validate_param \"$1\" \"$2\"\n PROJECT_ID=$2\n shift 2 # Move to the next parameter\n ;;\n --region)\n validate_param \"$1\" \"$2\"\n REGION=$2\n shift 2\n ;;\n *) \n echo \"Error: Unknown parameter '$1'.\"\n usage\n ;;\n esac\ndone\n\nmain(){\n pwd\n cd ../..\n git clone https://github.com/GoogleCloudPlatform/cloud-builders-community.git || exit 1\n cd cloud-builders-community/firebase \n gcloud builds submit --region=$REGION . --project=$PROJECT_ID || exit 1\n cd ../..\n rm -rf cloud-builders-community/ || exit 1\n cd Open_Data_QnA/frontend\n gcloud builds submit . --config frontend.yaml --substitutions _FIREBASE_PROJECT_ID=$PROJECT_ID --project=$PROJECT_ID || exit 1\n}\n\nmain" }, { "path": "terraform/scripts/install-dependencies.sh", "content": "#!/bin/bash\npwd\n\npip3 install pipx\nexport PATH=\"$PATH:$(python3 -c \"import sysconfig; print(sysconfig.get_paths()['scripts'])\")\"\necho \"$PATH\"\npipx install poetry\npipx ensurepath\nsource ~/.bashrc\npoetry --version \npoetry lock\npoetry install\npoetry env info" }, { "path": "terraform/templates/config.ini.tftpl", "content": "[CONFIG]\nembedding_model = ${embedding_model}\ndescription_model = ${description_model}\nvector_store = ${vector_store}\ndebugging = ${debugging}\nlogging = ${logging}\nkgq_examples = ${kgq_examples}\nfirestore_region = ${firestore_region}\nuse_session_history = ${use_session_history}\nuse_column_samples = ${use_column_samples}\n\n[GCP]\nproject_id = ${project_id}\n\n[PGCLOUDSQL]\npg_region = ${pg_region}\npg_instance = ${pg_instance}\npg_database = ${pg_database}\npg_user = ${pg_user}\npg_password = ${pg_password}\n\n[BIGQUERY]\nbq_dataset_region = ${bq_dataset_region}\nbq_opendataqna_dataset_name = ${bq_opendataqna_dataset}\nbq_log_table_name = ${bq_log_table}\n\n\n\n\n" }, { "path": "terraform/templates/constants.ts.tftpl", "content": "export const firebaseConfig = {\n \"projectId\": \"${projectId}\",\n \"appId\": \"${appId}\",\n \"storageBucket\": \"${storageBucket}\",\n \"apiKey\": \"${apiKey}\",\n \"authDomain\": \"${authDomain}\",\n \"messagingSenderId\": \"${messagingSenderId}\"\n};\nexport const ENDPOINT_OPENDATAQNA = '${endpoint_opendataqna}'\n\nexport const FIRESTORE_DATABASE_ID = 'opendataqna-session-logs'\n" }, { "path": "terraform/terraform.tfvars.sample", "content": "project_id = \"projectv1-340909\"\nvector_store = \"cloudsql-pgvector\"\nembedding_model = \"vertex\"\ndescription_model = \"gemini-1.5-pro\"\ndebugging = \"yes\"\nlogging = \"yes\"\nkgq_examples = \"yes\"\nuse_session_history = \"yes\"\nuse_column_samples = \"yes\"\nuse_existing_cloudsql_instance = \"no\"\npg_instance = \"pg15-opendataqna-2\"\npg_region = \"us-central1\"\npg_database = \"opendataqna-db-2\"\npg_user = \"pguser2\"\npg_password = \"Pguser@!12345\"\nbq_opendataqna_dataset = \"opendataqna2\"\nbq_dataset_region = \"us-central1\"\nbq_log_table = \"audit_log_table\"\nfirestore_region = \"us-central1\"\nservice_account = \"opendataqna-v2-sa-3\"\ndeploy_region = \"us-central1\"\ncloud_run_service_name = \"opendataqna-v2-run-3\"\nfirebase_web_app_name = \"opendataqna-v2-chatbot-3\"" }, { "path": "terraform/variables.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nvariable \"project_id\" {\n type = string\n}\n\nvariable \"embedding_model\" {\n type = string\n default = \"vertex\"\n description = \"name of the model that you want to use to create embeddings. Options: 'vertex' or 'vertex-lang'\"\n}\n\nvariable \"description_model\" {\n type = string\n default = \"gemini-1.5-pro\"\n description = \"name of the model that you want to use to generate missing description for tables and columns. Options: 'gemini-1.0-pro', 'gemini-1.5-pro', 'text-bison-32k', 'gemini-1.5-flash'\"\n}\n\nvariable \"vector_store\" {\n type = string\n default = \"bigquery-vector\"\n description = \"name of the datastore you want to use to store text embeddings of your meta data. Options: bigquery-vector, cloudsql-pgvector\"\n}\n\nvariable \"debugging\" {\n type = string\n default = \"yes\"\n description = \"yes, if you want to enable debugging. No, otherwise\"\n}\n\nvariable \"logging\" {\n type = string\n default = \"yes\"\n description = \"yes, if you want to enable application logging. No, otherwise\"\n}\n\nvariable \"kgq_examples\" {\n type = string\n default = \"yes\"\n description = \"yes, if you want to use known good sqls for few shot prompting and creating cache. No, otherwise\"\n}\n\nvariable \"use_session_history\" {\n type = string\n default = \"yes\"\n description = \"yes, if you want to use current session's questions without re-evaluating them. No, otherwise\"\n}\n\nvariable \"use_column_samples\" {\n type = string\n default = \"no\"\n description = \"yes, if you want to add some sample column values to the embeddings to enrich it with more information. No, otherwise\"\n}\n\nvariable \"use_existing_cloudsql_instance\" {\n default = \"no\"\n type = string\n description = \"If you want to use an existing cloudsql instance to store the vector embeddings, then choose 'yes' else choose 'no'. Terraform will create a new cloudsql instance if 'no' is chosen.\"\n}\n\nvariable \"pg_instance\" {\n default = \"pg15-opendataqna\"\n type = string\n description = \"Name of the Cloudsql postgres instance to store vector embeddings. Keep this empty if vector db is bigquery.\"\n}\n\nvariable \"pg_region\" {\n default = \"us-central1\"\n type = string\n description = \"Location of the pg_instance\"\n}\n\n\nvariable \"pg_database\" {\n type = string\n default = \"opendataqna-db\"\n description = \"Name of the Database associated with pg_instance\"\n}\n\nvariable \"pg_user\" {\n type = string\n default = \"pguser\"\n description = \"user name for the database\"\n}\n\nvariable \"pg_password\" {\n type = string\n default = \"pg123\"\n description = \"password for pg_user\"\n}\n\nvariable \"bq_opendataqna_dataset\" {\n type = string\n default = \"opendataqna\"\n description = \"This dataset will be used to store text embeddings and application logs. If pg-vector is chosen as vector db, only application logs will be stored here.\"\n}\n\nvariable \"bq_dataset_region\" {\n type = string\n default = \"us-central1\"\n description = \"Location of bq_opendataqna_dataset.\"\n}\n\nvariable \"bq_log_table\" {\n type = string\n default = \"audit_log_table\"\n description = \"Name of the table where audit logs will be stored. This table will be create under the bq_opendataqna_dataset.\"\n}\n\nvariable \"firestore_region\" {\n type = string\n default = \"us-central1\"\n description = \"Location of the firestore database.\"\n}\n\nvariable service_account {\n type = string\n default = \"opendataqna\"\n description = \"service account used by backend service\"\n}\n\nvariable \"deploy_region\" {\n \n type = string\n default = \"us-central1\"\n description = \"region where cloudrun service will be deployed\"\n}\n\nvariable \"cloud_run_service_name\" {\n type = string\n default = \"opendataqna\"\n description = \"name of the cloud run service where backend apis will be deployed\"\n}\n\nvariable \"firebase_web_app_name\" {\n type = string\n default = \"opendataqna\"\n description = \"name of the firebase web app.\"\n}\n\n" }, { "path": "terraform/versions.tf", "content": "/**\n * Copyright 2024 Google LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nterraform {\n required_version = \">= 1.3\"\n required_providers {\n google = {\n source = \"hashicorp/google\"\n version = \">= 4.64, < 6\"\n }\n google-beta = {\n source = \"hashicorp/google-beta\"\n version = \">= 4.64, < 6\"\n }\n local = {\n source = \"hashicorp/local\"\n }\n }\n}" }, { "path": "utilities/__init__.py", "content": "import configparser\nimport os\nimport sys\nimport yaml\n\nconfig = configparser.ConfigParser()\n\ndef is_root_dir():\n \"\"\"\n Checks if the current working directory is the root directory of a project \n by looking for either the \"/notebooks\" or \"/agents\" folders.\n\n Returns:\n bool: True if either directory exists in the current directory, False otherwise.\n \"\"\"\n\n current_dir = os.getcwd()\n print(\"current dir: \", current_dir)\n notebooks_path = os.path.join(current_dir, \"notebooks\")\n agents_path = os.path.join(current_dir, \"agents\")\n \n return os.path.exists(notebooks_path) or os.path.exists(agents_path)\n\ndef load_yaml(file_path: str) -> dict:\n with open(file_path, \"r\", encoding=\"utf-8\") as f:\n return yaml.safe_load(f)\n\nif is_root_dir():\n current_dir = os.getcwd()\n config.read(current_dir + '/config.ini')\n root_dir = current_dir\nelse:\n root_dir = os.path.abspath(os.path.join(os.getcwd(), '..'))\n config.read(root_dir+'/config.ini')\n\nif not 'root_dir' in locals(): # If not found in any parent dir\n raise FileNotFoundError(\"config.ini not found in current or parent directories.\")\n\nprint(f'root_dir set to: {root_dir}')\n\ndef format_prompt(context_prompt, **kwargs):\n \"\"\"\n Formats a context prompt by replacing placeholders with values from keyword arguments.\n Args:\n context_prompt (str): The prompt string containing placeholders (e.g., {var1}).\n **kwargs: Keyword arguments representing placeholder names and their values.\n Returns:\n str: The formatted prompt with placeholders replaced.\n \"\"\"\n return context_prompt.format(**kwargs)\n\n# [CONFIG]\nEMBEDDING_MODEL = config['CONFIG']['EMBEDDING_MODEL']\nDESCRIPTION_MODEL = config['CONFIG']['DESCRIPTION_MODEL']\n# DATA_SOURCE = config['CONFIG']['DATA_SOURCE'] \nVECTOR_STORE = config['CONFIG']['VECTOR_STORE']\n\n#CACHING = config.getboolean('CONFIG','CACHING')\n#DEBUGGING = config.getboolean('CONFIG','DEBUGGING')\nLOGGING = config.getboolean('CONFIG','LOGGING')\nEXAMPLES = config.getboolean('CONFIG', 'KGQ_EXAMPLES')\nUSE_SESSION_HISTORY = config.getboolean('CONFIG', 'USE_SESSION_HISTORY')\nUSE_COLUMN_SAMPLES = config.getboolean('CONFIG','USE_COLUMN_SAMPLES')\n\n#[GCP]\nPROJECT_ID = config['GCP']['PROJECT_ID']\n\n#[PGCLOUDSQL]\nPG_REGION = config['PGCLOUDSQL']['PG_REGION']\n# PG_SCHEMA = config['PGCLOUDSQL']['PG_SCHEMA'] \nPG_INSTANCE = config['PGCLOUDSQL']['PG_INSTANCE']\nPG_DATABASE = config['PGCLOUDSQL']['PG_DATABASE'] \nPG_USER = config['PGCLOUDSQL']['PG_USER'] \nPG_PASSWORD = config['PGCLOUDSQL']['PG_PASSWORD']\n\n#[BIGQUERY]\nBQ_REGION = config['BIGQUERY']['BQ_DATASET_REGION']\n# BQ_DATASET_NAME = config['BIGQUERY']['BQ_DATASET_NAME']\nBQ_OPENDATAQNA_DATASET_NAME = config['BIGQUERY']['BQ_OPENDATAQNA_DATASET_NAME']\nBQ_LOG_TABLE_NAME = config['BIGQUERY']['BQ_LOG_TABLE_NAME']\n# BQ_TABLE_LIST = config['BIGQUERY']['BQ_TABLE_LIST']\n\n#[FIRESTORE]\nFIRESTORE_REGION = config['CONFIG']['FIRESTORE_REGION']\n\n#[PROMPTS]\nPROMPTS = load_yaml(root_dir + '/prompts.yaml')\n\n__all__ = [\"EMBEDDING_MODEL\",\n \"DESCRIPTION_MODEL\",\n #\"DATA_SOURCE\",\n \"VECTOR_STORE\",\n #\"CACHING\",\n #\"DEBUGGING\",\n \"LOGGING\",\n \"EXAMPLES\", \n \"PROJECT_ID\",\n \"PG_REGION\",\n # \"PG_SCHEMA\",\n \"PG_INSTANCE\",\n \"PG_DATABASE\",\n \"PG_USER\",\n \"PG_PASSWORD\", \n \"BQ_REGION\",\n # \"BQ_DATASET_NAME\",\n \"BQ_OPENDATAQNA_DATASET_NAME\",\n \"BQ_LOG_TABLE_NAME\",\n # \"BQ_TABLE_LIST\",\n \"FIRESTORE_REGION\",\n \"PROMPTS\"\n \"root_dir\",\n \"save_config\"]" } ]