",
]
license = "MIT"
readme = "README.md"
keywords = ["Pathway", "LLM"]
package-mode = false
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Programming Language :: Python :: Implementation :: CPython",
"License :: OSI Approved :: MIT License",
]
[tool.poetry.urls]
"Homepage" = "https://github.com/pathwaycom/llm-app"
"Source Code" = "https://github.com/pathwaycom/llm-app"
[tool.poetry.dependencies]
python = ">=3.10,<3.13"
pathway = "^0.12.0"
[tool.poetry.group.linters]
optional = true
[tool.poetry.group.linters.dependencies]
black = "^24.2.0"
isort = "^5.13.2"
mypy = "~1.10.0"
flake8 = "~7.0.0"
pytest = "^8.0.2"
types-requests = "^2.31.0"
types-PyYAML = "^6.0.0"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.black]
target-version = ["py310", "py311"]
[tool.isort]
profile = "black"
[tool.mypy]
python_version = "3.10"
exclude = ["templates/adaptive_rag", "templates/private_rag", "templates/document_indexing", "templates/multimodal_rag", "templates/question_answering_rag"]
ignore_missing_imports = true
check_untyped_defs = true
warn_redundant_casts = true
warn_unused_ignores = true
strict_equality = true
================================================
FILE: setup.cfg
================================================
[flake8]
exclude =
.git
max-line-length = 119
docstring-convention = google
extend-ignore =
# Black (https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8)
E203, E701
================================================
FILE: templates/adaptive_rag/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
CMD ["python", "app.py"]
================================================
FILE: templates/adaptive_rag/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# End to end Adaptive RAG with Pathway
This is the accompanying code for deploying the `adaptive RAG` technique with Pathway. To understand the technique and learn how it can save tokens without sacrificing accuracy, read [our showcase](https://pathway.com/developers/templates/rag/adaptive-rag).
To learn more about building & deploying RAG applications with Pathway, including containerization, refer to [demo question answering](../question_answering_rag/README.md).
## Introduction
This app relies on modules provided under `pathway.xpacks.llm`.
`BaseRAGQuestionAnswerer` is the base class to build RAG applications with Pathway vector store and Pathway xpack components.
It is meant to get you started with your RAG application right away.
Here, we extend the `BaseRAGQuestionAnswerer` to implement the adaptive retrieval and reply to requests in the endpoint `/v2/answer`.
Since we are interested in changing the behavior and logic of the RAG, we only modify `answer` function that handles all this logic, and then replies to the post request.
`answer` function takes the `pw_ai_queries` table as the input, this table contains the prompt, and other arguments coming from the post request, see the `BaseRAGQuestionAnswerer` class and defined schemas to learn more about getting inputs with post requests.
We use the data in this table to call our adaptive retrieval logic.
To do that, we use `answer_with_geometric_rag_strategy_from_index` implementation provided under the `pathway.xpacks.llm.question_answering`.
This function takes an index, LLM, prompt and adaptive parameters such as the starting number of documents. Then, iteratively asks the question to the LLM with an increasing number of context documents retrieved from the index.
We encourage you to check the implementation of `answer_with_geometric_rag_strategy_from_index`.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/templates/configure-yaml).
In the `app.yaml` file we define:
- input connectors
- LLM
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
You can also check our other templates - [Question Answering RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag),
[Multimodal RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/multimodal_rag) or
[Private RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/private_rag). As all of these only differ
in the YAML configuration file, you can also use them as an inspiration for your custom pipeline.
Here some examples of what can be modified.
### LLM Model
You can choose any of the models offered by Open AI, like GPT-5, GPT-4.1, or GPT-4o.
You can find the whole list on their [models page](https://platform.openai.com/docs/models).
You simply need to change the `model` to the one you want to use, e.g., to use GPT-5:
```yaml
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-5"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache
capacity: 8
```
The default model is `gpt-4.1-mini`.
You can also use different provider, by using different class from [Pathway LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview),
e.g. here is configuration for locally run Mistral model.
```yaml
$llm: !pw.xpacks.llm.llms.LiteLLMChat
model: "ollama/mistral"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DiskCache
temperature: 0
top_p: 1
api_base: "http://localhost:11434"
```
### Webserver
You can configure the host and the port of the webserver.
Here is the default configuration:
```yaml
host: "0.0.0.0"
port: 8000
```
### Cache
You can configure whether you want to enable cache or persistence, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
persistence_mode: !pw.PersistenceMode.UDF_CACHING
persistence_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `data` folder.
#### Local Data Source
The local data source is configured by using map with tag `!pw.io.fs.read`. Then set `path` to denote the path to a folder with files to be indexed.
#### Google Drive Data Source
The Google Drive data source is enabled by using map with tag `!pw.io.gdrive.read`. The map must contain two main parameters:
- `object_id`, containing the ID of the folder that needs to be indexed. It can be found from the URL in the web interface, where it's the last part of the address. For example, the publicly available demo folder in Google Drive has the URL `https://drive.google.com/drive/folders/1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`. Consequently, the last part of this address is `1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`, hence this is the `object_id` you would need to specify.
- `service_user_credentials_file`, containing the path to the credentials files for the Google [service account](https://cloud.google.com/iam/docs/service-account-overview). To get more details on setting up the service account and getting credentials, you can also refer to [this tutorial](https://pathway.com/developers/user-guide/connectors/gdrive-connector#setting-up-google-drive).
Besides, to speed up the indexing process you may want to specify the `refresh_interval` parameter, denoted by an integer number of seconds. It corresponds to the frequency between two sequential folder scans. If unset, it defaults to 30 seconds.
For the full list of the available parameters, please refer to the Google Drive connector [documentation](https://pathway.com/developers/api-docs/pathway-io/gdrive#pathway.io.gdrive.read).
#### SharePoint Data Source
This data source requires Scale or Enterprise [license key](https://pathway.com/pricing) - you can obtain free Scale key on [Pathway website](https://pathway.com/get-license).
To use it, set the map tag to be `!pw.xpacks.connectors.sharepoint.read`, and then provide values of `url`, `tenant`, `client_id`, `cert_path`, `thumbprint` and `root_path`. To read about the meaning of these arguments, check the Sharepoint connector [documentation](https://pathway.com/developers/api-docs/pathway-xpacks-sharepoint#pathway.xpacks.connectors.sharepoint.read).
## Running the app
To run the app, depending on the configuration, you may need to set up environmntal variables with LLM provider keys. By default, this template uses OpenAI API, so to run it you need to set `OPENAI_API_KEY` environmental key or create an `.env` file in this directory with your key: `OPENAI_API_KEY=sk-...`. If you modify the code to use another LLM provider, you may need to set a relevant API key.
### With Docker
In order to let the pipeline get updated with each change in local files, you need to mount the folder onto the docker. The following commands show how to do that.
```bash
# Build the image in this folder
docker build -t adaptiverag .
# Run the image, mount the `data` folder into image
# -e is used to pass value of OPENAI_API_KEY environmental variable
docker run -v ./data:/app/data -e OPENAI_API_KEY -p 8000:8000 adaptiverag
```
### Locally
To run locally you need to install the Pathway app with LLM dependencies using:
```bash
pip install pathway[all]
```
Then change your directory in the terminal to this folder and run the app:
```bash
python app.py
```
## Using the app
Finally, query the application with;
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "What is the start date of the contract?"
}'
```
> `{"response": "December 21, 2015 [6]"}`
================================================
FILE: templates/adaptive_rag/app.py
================================================
import logging
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.question_answering import SummaryQuestionAnswerer
from pathway.xpacks.llm.servers import QASummaryRestServer
from pydantic import BaseModel, ConfigDict, InstanceOf
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
question_answerer: InstanceOf[SummaryQuestionAnswerer]
host: str = "0.0.0.0"
port: int = 8000
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
server = QASummaryRestServer( # noqa: F841
self.host, self.port, self.question_answerer
)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/adaptive_rag/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Adaptive RAG template.
# It defines various components such as data sources, language models, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for generating responses.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: data
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Configures the LLM model settings for generating responses.
# The list of available Pathway LLM wrappers is available here:
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
# You can learn more about those in our documentation:
# https://pathway.com/developers/templates/rag-customization/llm-chats
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-4.1-mini"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
capacity: 8
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
model: "text-embedding-3-small"
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
# Defines the splitter settings for dividing text into smaller chunks.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Configures the question-answering component using the RAG approach.
# The component builds a RAG over an index.
# You can interact with obtained RAG using a REST API.
# You can learn more about the available operations here:
# https://pathway.com/developers/templates/rag-customization/rest-api
question_answerer: !pw.xpacks.llm.question_answering.AdaptiveRAGQuestionAnswerer
llm: $llm
indexer: $document_store
n_starting_documents: 2
factor: 2
max_iterations: 4
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: 8000
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/adaptive_rag/requirements.txt
================================================
python-dotenv~=1.0
================================================
FILE: templates/document_indexing/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
CMD ["python", "app.py"]
================================================
FILE: templates/document_indexing/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# Realtime Document Indexing: Vector Store with always-up-to-date knowledge
This is a basic service for a real-time document indexing pipeline powered by [Pathway](https://github.com/pathwaycom/pathway).
The capabilities of the pipeline include:
- Real-time document indexing from Microsoft 365 SharePoint, Google Drive, or a local directory;
- Similarity search by user query;
- Filtering by the metadata according to the condition given in [JMESPath format](https://jmespath.org/);
- Basic stats on the indexer's health.
## Summary of the Pipeline
This example spawns a lightweight webserver that accepts queries on three possible endpoints:
- `/v1/retrieve` to perform similarity search;
- `/v1/statistics` to get the basic stats about the indexer's health;
- `/v1/inputs` to retrieve the metadata of all files currently processed by the indexer.
Please refer to the Open API doc on Hosted Pipelines [website](https://pathway.com/solutions/ai-pipelines) for the format of the requests to the endpoints.
## How It Works
This pipeline uses several Pathway connectors to read the data from the local drive, Google Drive, and Microsoft SharePoint sources. It allows you to poll the changes with low latency and to do the modifications tracking. So, if something changes in the tracked files, the corresponding change is reflected in the internal collections. The contents are read into a single Pathway Table as binary objects.
After that, those binary objects are parsed with [unstructured](https://unstructured.io/) library and split into chunks. With the usage of OpenAI API, the pipeline embeds the obtained chunks.
Finally, the embeddings are indexed with the capabilities of Pathway's machine-learning library. The user can then query the created index with simple HTTP requests to the endpoints mentioned above.
## Pipeline Organization
This folder contains several objects:
- `app.py`, the pipeline code using Pathway and written in Python;
- `app.yaml`, the file containing configuration of the pipeline, like embedding model, sources, or the server address;
- `requirements.txt`, the textfile denoting the pip dependencies for running this pipeline. It can be passed to `pip install -r ...` to install everything that is needed to launch the pipeline locally;
- `Dockerfile`, the Docker configuration for running the pipeline in the container;
- `docker-compose.yml`, the docker-compose configuration for running the pipeline along with the chat UI;
- `.env`, a short environment variables configuration file where the OpenAI key must be stored;
- `files-for-indexing/`, a folder with exemplary files that can be used for the test runs.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/templates/configure-yaml).
In the `app.yaml` file we define:
- input connectors
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
Here some examples of what can be modified.
### Embedding Model
By default this template uses locally run model `mixedbread-ai/mxbai-embed-large-v1`. If you wish, you can replace this with any other model, by changing
`$embedder` in `app.yaml`. For example, to use OpenAI embedder, set:
```yaml
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
model: "text-embedding-ada-002"
cache_strategy: !pw.udfs.DiskCache
```
If you choose to use a provider, that requires API key, remember to set appropriate environmental values (you can also set them in `.env` file).
### Webserver
You can configure the host and the port of the webserver.
Here is the default configuration:
```yaml
host: "0.0.0.0"
port: 8000
```
### Cache
You can configure whether you want to enable cache, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
with_cache: True
cache_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `files-from-indexing` folder.
#### Local Data Source
The local data source is configured by using map with tag `!pw.io.fs.read`. Then set `path` to denote the path to a folder with files to be indexed.
#### Google Drive Data Source
The Google Drive data source is enabled by using map with tag `!pw.io.gdrive.read`. The map must contain two main parameters:
- `object_id`, containing the ID of the folder that needs to be indexed. It can be found from the URL in the web interface, where it's the last part of the address. For example, the publicly available demo folder in Google Drive has the URL `https://drive.google.com/drive/folders/1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`. Consequently, the last part of this address is `1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`, hence this is the `object_id` you would need to specify.
- `service_user_credentials_file`, containing the path to the credentials files for the Google [service account](https://cloud.google.com/iam/docs/service-account-overview). To get more details on setting up the service account and getting credentials, you can also refer to [this tutorial](https://pathway.com/developers/user-guide/connectors/gdrive-connector#setting-up-google-drive).
Besides, to speed up the indexing process you may want to specify the `refresh_interval` parameter, denoted by an integer number of seconds. It corresponds to the frequency between two sequential folder scans. If unset, it defaults to 30 seconds.
For the full list of the available parameters, please refer to the Google Drive connector [documentation](https://pathway.com/developers/api-docs/pathway-io/gdrive#pathway.io.gdrive.read).
#### SharePoint Data Source
This data source requires Scale or Enterprise [license key](https://pathway.com/pricing) - you can obtain free Scale key on [Pathway website](https://pathway.com/get-license).
To use it, set the map tag to be `!pw.xpacks.connectors.sharepoint.read`, and then provide values of `url`, `tenant`, `client_id`, `cert_path`, `thumbprint` and `root_path`. To read about the meaning of these arguments, check the Sharepoint connector [documentation](https://pathway.com/developers/api-docs/pathway-xpacks-sharepoint#pathway.xpacks.connectors.sharepoint.read).
## Running the Example
### Locally
This example can be run locally by executing `python main.py` in this directory. It has several command-line arguments:
- `--host` denoting the host, where the server will run. The default setting is `0.0.0.0`;
- `--port` denoting the port, where the server will accept requests. The default setting is `8000`;
- `--sources-config` pointing to a data source configuration file, `sources_configuration.yaml` by default. You can customize it to change the folders indexed by the vector store. The free version supports `local` and `gdrive` hosted files, while the commercial one also supports `sharepoint` hosted folders. By default, the `local` option indexes files from the `files-for-indexing/` folder that is prefilled with exemplary documents.
Please note that the local run requires the dependencies to be installed. It can be done with a simple pip command:
```bash
pip install -r requirements.txt
```
### With Docker
First, create or fill the `.env` file in this folder (`/document_indexing_rag`) with your OpenAI key `OPENAI_API_KEY=sk-*******`.
To run jointly the vector indexing pipeline and a simple UI please execute:
```bash
docker compose up --build
```
Then, the UI will run at http://127.0.0.1:8501 by default. You can access it by following this URL in your web browser.
The `docker-compose.yml` file declares a [volume bind mount](https://docs.docker.com/reference/cli/docker/container/run/#volume) that makes changes to files under `files-for-indexing/` made on your host computer visible inside the docker container. If the index does not react to file changes, please check that the bind mount works
by running `docker compose exec pathway_vector_indexer ls -l /app/files-for-indexing/` and verifying that all files are visible.
Alternatively, you can launch just the indexing pipeline as a single Docker container:
```bash
docker build -t vector_indexer .
docker run -v `pwd`/files-for-indexing:/app/files-for-indexing -p 8000:8000 vector_indexer
```
The volume overlay is important - without it, docker will not see changes to files under the `files-for-indexing` folder.
## Querying the Example
Since the pipeline spawns an HTTP server and interacts with the user using the REST protocol, you can test it locally by sending requests with `curl`. To get the target address for the request, you can refer to the routes in the first section: `v1/retrieve`, `v1/statistics`, and `v1/inputs` and the Open API docs available at the Hosted Pipelines webpage.
Let's assume that the server runs with the default settings and therefore uses port 8000. That being said, you can query the similarity search endpoint with the following command:
```bash
curl -X 'GET' \
'http://localhost:8000/v1/retrieve?query=Pathway%20data%20processing%20framework&k=2' \
-H 'accept: */*'
```
Above, there are two CGI parameters to be customized: the `query`, which should contain the [urlencoded](https://en.wikipedia.org/wiki/Percent-encoding) search terms, and an integer `k` denoting the number of documents you want to retrieve.
Apart from the querying part, there is a health-check endpoint that shows the number of documents currently indexed and the most recent indexing time. It can be accessed, in a similar way, with the following command:
```bash
curl -X 'GET' \
'http://localhost:8000/v1/statistics' \
-H 'accept: */*'
```
Finally, there is a way to see the metadata of all documents that are currently indexed. To do that, you need to query the `v1/inputs` endpoint. The request command then looks as follows:
```bash
curl -X 'GET' \
'http://localhost:8000/v1/inputs' \
-H 'accept: */*'
```
Please make sure that you've changed the port in the requests if you did that when launching the main script.
## Adding Files to Index
To test index updates, simply add more files to the `files-for-indexing` folder if the local data source is used.
If you are using Google Drive, simply upload your files in the folder configured in the `sources_configuration.yaml` file.
Then you can use the similarity search and stats endpoints, provided below.
================================================
FILE: templates/document_indexing/app.py
================================================
import logging
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.document_store import DocumentStore
from pathway.xpacks.llm.servers import DocumentStoreServer
from pydantic import BaseModel, ConfigDict, InstanceOf
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
document_store: InstanceOf[DocumentStore]
host: str = "0.0.0.0"
port: int = 8000
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
server = DocumentStoreServer( # noqa: F841
self.host, self.port, self.document_store
)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/document_indexing/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Document indexing RAG template.
# It defines various components such as data sources, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for answering the queries.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: files-for-indexing
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Model used for embedding
$embedding_model: "mixedbread-ai/mxbai-embed-large-v1"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.SentenceTransformerEmbedder
model: $embedding_model
call_kwargs:
show_progress_bar: False
# Defines the splitter settings for dividing text into smaller chunks.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: 8000
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/document_indexing/docker-compose.yml
================================================
version: "3.8"
services:
pathway_vector_indexer:
build:
context: .
ports:
- "8000:8000"
environment:
OPENAI_API_KEY: "${OPENAI_API_KEY}"
volumes:
- "./files-for-indexing:/app/files-for-indexing"
streamlit_ui:
depends_on:
- pathway_vector_indexer
build:
context: https://github.com/pathway-labs/realtime-indexer-qa-chat.git
ports:
- "8501:8501"
environment:
OPENAI_API_KEY: "${OPENAI_API_KEY}"
PATHWAY_HOST: "pathway_vector_indexer"
PATHWAY_PORT: "8000"
================================================
FILE: templates/document_indexing/requirements.txt
================================================
python-dotenv~=1.0
================================================
FILE: templates/document_store_mcp_server/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8068
CMD ["python", "app.py"]
================================================
FILE: templates/document_store_mcp_server/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# MCP Server with Realtime Document Indexing
This is a template for exposing a real-time document indexing pipeline powered by [Pathway](https://github.com/pathwaycom/pathway) as an Model Context Protocol (MCP) server.
The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/docs/getting-started/intro) is designed to standardize the way applications interact with large language models (LLMs). It serves as a bridge, much like a universal connector, enabling seamless integration between AI models and various data sources and tools. This protocol facilitates the creation of sophisticated AI workflows and agents, enhancing the capabilities of LLMs by connecting them with real-world data and functionalities.
The capabilities of the pipeline include:
- Real-time document indexing from Microsoft 365 SharePoint, Google Drive, or a local directory;
- Similarity search by user query;
- Filtering by the metadata according to the condition given in [JMESPath format](https://jmespath.org/);
- The documents are available from a standardized MCP server.
## Summary of the Pipeline
This example spawns an MCP server that has three tools:
- `retrieve_query` to perform similarity search on the indexed documents,
- `statistics_query` to get the basic stats about the indexer's health,
- `inputs_query` to retrieve the metadata of all files currently processed by the indexer.
You can get specification of those tools by querying the `list_tools` on the MCP server.
## How It Works
This pipeline uses several Pathway connectors to read the data from the local drive, Google Drive, or Microsoft SharePoint sources. It allows you to poll the changes with low latency and to do the modifications tracking. So, if something changes in the tracked files, the corresponding change is reflected in the internal collections. The contents are read into a single Pathway Table as binary objects.
After that, those binary objects are parsed with the [Docling](https://www.docling.ai/) library and split into chunks. With the usage of the [SentenceTransformer](https://www.sbert.net/) embedder, the pipeline embeds the obtained chunks.
Finally, the embeddings are indexed with the capabilities of Pathway's machine-learning library. The user can then query the created index by connecting to the MCP server using an MCP client.
## Pipeline Organization
This folder contains several objects:
- `app.py`, the pipeline code using Pathway and written in Python;
- `app.yaml`, the file containing configuration of the pipeline, like embedding model, sources, or the server address;
- `requirements.txt`, the textfile denoting the pip dependencies for running this pipeline. It can be passed to `pip install -r requirements.txt` to install everything that is needed to launch the pipeline locally;
- `Dockerfile`, the Docker configuration for running the pipeline in the container;
- `docker-compose.yml`, the docker-compose configuration for running the pipeline along with the chat UI;
- `files-for-indexing/`, a folder with exemplary files that can be used for the test runs.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/templates/configure-yaml).
In the `app.yaml` file we define:
- input connectors
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
Here some examples of what can be modified.
### Embedding Model
By default this template uses locally run model `mixedbread-ai/mxbai-embed-large-v1`. If you wish, you can replace this with any other model, by changing
`$embedder` in `app.yaml`. For example, to use OpenAI embedder, set:
```yaml
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
model: "text-embedding-3-small"
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
```
If you choose to use a provider, that requires API key, remember to set appropriate environmental values (you can also set them in the `.env` file) - e.g. for using OpenAI embedders, set the `OPENAI_API_KEY` variable.
### Webserver
You can configure the name, the host and the port of the MCP server.
Here is the default configuration:
```yaml
mcp_http: !pw.xpacks.llm.mcp_server.PathwayMcp
name: "Streamable MCP Server"
transport: "streamable-http"
host: "localhost"
port: 8068
serve:
- $document_store
```
### Cache
You can configure whether you want to enable cache or persistence, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
persistence_mode: !pw.PersistenceMode.UDF_CACHING
persistence_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `files-from-indexing` folder.
#### Local Data Source
The local data source is configured by using map with tag `!pw.io.fs.read`. Then set `path` to denote the path to a folder with files to be indexed.
#### Google Drive Data Source
The Google Drive data source is enabled by using map with tag `!pw.io.gdrive.read`. The map must contain two main parameters:
- `object_id`, containing the ID of the folder that needs to be indexed. It can be found from the URL in the web interface, where it's the last part of the address. For example, the publicly available demo folder in Google Drive has the URL `https://drive.google.com/drive/folders/1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`. Consequently, the last part of this address is `1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`, hence this is the `object_id` you would need to specify.
- `service_user_credentials_file`, containing the path to the credentials files for the Google [service account](https://cloud.google.com/iam/docs/service-account-overview). To get more details on setting up the service account and getting credentials, you can also refer to [this tutorial](https://pathway.com/developers/user-guide/connectors/gdrive-connector#setting-up-google-drive).
Besides, to speed up the indexing process you may want to specify the `refresh_interval` parameter, denoted by an integer number of seconds. It corresponds to the frequency between two sequential folder scans. If unset, it defaults to 30 seconds.
For the full list of the available parameters, please refer to the Google Drive connector [documentation](https://pathway.com/developers/api-docs/pathway-io/gdrive#pathway.io.gdrive.read).
#### SharePoint Data Source
This data source requires Scale or Enterprise [license key](https://pathway.com/pricing) - you can obtain free Scale key on [Pathway website](https://pathway.com/get-license).
To use it, set the map tag to be `!pw.xpacks.connectors.sharepoint.read`, and then provide values of `url`, `tenant`, `client_id`, `cert_path`, `thumbprint` and `root_path`. To read about the meaning of these arguments, check the Sharepoint connector [documentation](https://pathway.com/developers/api-docs/pathway-xpacks-sharepoint#pathway.xpacks.connectors.sharepoint.read).
## Running the Template
### Pathway License Key
Pathway MCP Server requires a Pathway license key, so before you run the template, you need to set the license key. This template is available for free via [Pathway Scale](https://pathway.com/features), for which you can get the license key [here](https://pathway.com/user/license). Once you have your license key, create a `.env` file, in which set `PATHWAY_LICENSE_KEY` to your license key - see `.env.example` for an example of `.env` file.
### Locally
This template can be run locally by executing `python app.py` in this directory. Please note that the local run requires the `Pathway` library and other dependencies to be installed. It can be done with a pip command:
```bash
pip install pathway[all]
pip install -r requirements.txt
```
### With Docker`.
To run jointly the MCP server with real-time document indexint, please execute:
```bash
docker compose up --build
```
The `docker-compose.yml` file declares a [volume bind mount](https://docs.docker.com/reference/cli/docker/container/run/#volume) that makes changes to files under `files-for-indexing/` made on your host computer visible inside the docker container. If the index does not react to file changes, please check that the bind mount works
by running `docker compose exec pathway_vector_indexer ls -l /app/files-for-indexing/` and verifying that all files are visible.
## Querying the Template with an MCP client
To test your examples, you need an MCP client which will connect to your MCP server. You can use the fastmcp package to define a client as follows:
```python
import asyncio
from fastmcp import Client
# Change the URL if you change the default values in the app.yaml
PATHWAY_MCP_URL = "http://localhost:8068/mcp/"
client = Client(PATHWAY_MCP_URL)
async def main():
async with client:
tools = await client.list_tools()
print(tools)
async with client:
result = await client.call_tool(
name="retrieve_query",
arguments={"query": "How to create a webserver in Pathway?", "k": 3},
)
print(result)
asyncio.run(main())
```
You can list the different tools available in the MCP server using the `list_tools` of the client. To access a given tool, you can use the method call_tool, with the name and arguments parameters. The arguments should be a dict of the different values: in this case, the `retrieve_query` tool has two required arguments: `query` and `k`.
## Using MCP server in Claude Desktop
To use MCP server created by this template in Claude Desktop, follow the [guide in Pathway's documentation](https://pathway.com/developers/user-guide/llm-xpack/pathway-mcp-claude-desktop).
## Adding Files to Index
To test index updates, simply add more files to the `files-for-indexing` folder if the local data source is used.
If you are using Google Drive, simply upload your files in the folder configured in the `sources_configuration.yaml` file.
Then you can use the similarity search and stats endpoints, provided below.
================================================
FILE: templates/document_store_mcp_server/__init__.py
================================================
================================================
FILE: templates/document_store_mcp_server/app.py
================================================
import logging
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.mcp_server import PathwayMcp
from pydantic import BaseModel, ConfigDict
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
mcp_http: PathwayMcp
host: str = "0.0.0.0"
port: int = 8000
terminate_on_error: bool = False
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
def run(self) -> None:
if self.persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=self.persistence_mode,
)
else:
persistence_config = None
pw.run(
terminate_on_error=self.terminate_on_error,
persistence_config=persistence_config,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
print(config)
app = App(**config)
app.run()
================================================
FILE: templates/document_store_mcp_server/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Document indexing RAG template.
# It defines various components such as data sources, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for answering the queries.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: files-for-indexing
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Model used for embedding
$embedding_model: "mixedbread-ai/mxbai-embed-large-v1"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.SentenceTransformerEmbedder
model: $embedding_model
call_kwargs:
show_progress_bar: False
# Defines the splitter settings for dividing text into smaller chunks.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Streamable MCP server, can be proxied
mcp_http: !pw.xpacks.llm.mcp_server.PathwayMcp
name: "Streamable MCP Server"
transport: "streamable-http"
host: "0.0.0.0"
port: 8068
serve:
- $document_store
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/document_store_mcp_server/docker-compose.yml
================================================
version: "3.8"
services:
pathway_mcp_server:
build:
context: .
ports:
- "8068:8068"
environment:
PATHWAY_LICENSE_KEY: $PATHWAY_LICENSE_KEY
volumes:
- "./files-for-indexing:/app/files-for-indexing"
================================================
FILE: templates/document_store_mcp_server/requirements.txt
================================================
python-dotenv~=1.0
================================================
FILE: templates/drive_alert/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
COPY . .
EXPOSE 8080
CMD ["python", "app.py"]
================================================
FILE: templates/drive_alert/README.md
================================================
# Pathway + LLM + Slack notification: RAG App with real-time alerting when answers change in documents
Microservice for a context-aware alerting ChatGPT assistant.
This demo is very similar to the `alert` example, the only difference is the data source (Google Drive)
For the demo, alerts are sent to Slack (you need to provide `slack_alert_channel_id` and `slack_alert_token`),
you can either put these env variables in .env file,
or create env variables in the terminal (i.e. export in bash).
The program then starts a REST API endpoint serving queries about Google Docs stored in a
Google Drive folder.
We can create notifications by asking from Streamlit or sending query to API stating we want to be notified.
One example would be `Tell me and alert about the start date of the campaign for Magic Cola`
## How Does It Work?
First, Pathway connects to Google Drive, extracts all documents, splits them into chunks, turns them into
vectors using OpenAI embedding service, and store in a nearest neighbor index.
Each query text is first turned into a vector, then relevant document chunks are found
using the nearest neighbor index. A prompt is built from the relevant chunk
and sent to the OpenAI GPT3.5 chat service for processing and answering.
After an initial answer is provided, Pathway monitors changes to documents and selectively
re-triggers potentially affected queries. If the new answer is significantly different from
the previously presented one, a new notification is created.
## How to run the project
Before running the app, you will need to give the app access to the Google Drive folder, we follow the steps below.
In order to access files on your Google Drive from the Pathway app, you will need a Google Cloud project and a service user.
### Create a new project in the Google API console:
- Go to [https://console.cloud.google.com/projectcreate](https://console.cloud.google.com/projectcreate) and create new project
- Enable Google Drive API by going to [https://console.cloud.google.com/apis/library/drive.googleapis.com](https://console.cloud.google.com/apis/library/drive.googleapis.com), make sure the newly created project is selected in the top left corner
- Configure consent screen:
- Go to [https://console.cloud.google.com/apis/credentials/consent](https://console.cloud.google.com/apis/credentials/consent)
- If using a private Gmail, select "External", and go next.
- Fill required parameters: application name, user support, and developer email (your email is fine)
- On the next screen click "Add or remove scopes" search for "drive.readonly" and select this scope
- Save and click through other steps
- Create service user:
- Go to [https://console.cloud.google.com/apis/credentials](https://console.cloud.google.com/apis/credentials)
- Click "+ Create credentials" and create a service account
- Name the service user and click through the next steps
- Generate service user key:
- Once more go to [https://console.cloud.google.com/apis/credentials](https://console.cloud.google.com/apis/credentials) and click on your newly created user (under Service Accounts)
- Go to "Keys", click "Add key" -> "Create new key" -> "JSON"
A JSON file will be saved to your computer.
Rename this JSON file to `secrets.json` and put it under `templates/drive_alert` next to `app.py` so that it is easily reachable from the app.
You can now share desired Google Drive resources with the created user.
Note the email ending with `gserviceaccount.com` we will share the folder with this email.
Once you've done it, you will need an ID of some file or directory. You can obtain it manually by right-clicking on the file -> share -> copy link. It will be part of the URL.
[https://drive.google.com/file/d/[FILE_ID]/view?usp=drive_link](https://drive.google.com/file/d/%5BFILE_ID%5D/view?usp=drive_link)
For folders,
First, right-click on the folder and click share, link will be of the format: [https://drive.google.com/drive/folders/[folder_id]?usp=drive_link](https://drive.google.com/drive/folders/%7Bfolder_id%7D?usp=drive_link)
Copy the folder_id from the URL.
Second, click on share and share the folder with the email ending with `gserviceaccount.com`
### Setup Slack notifications:
For this demo, Slack notifications are optional and notifications will be printed if no Slack API keys are provided. See: [Slack Apps](https://api.slack.com/apps) and [Getting a token](https://api.slack.com/tutorials/tracks/getting-a-token).
Your Slack application will need at least `chat:write.public` scope enabled.
### Setup environment:
Set your env variables in the .env file placed in this directory.
```bash
OPENAI_API_KEY=sk-...
SLACK_ALERT_CHANNEL_ID= # If unset, alerts will be printed to the terminal
SLACK_ALERT_TOKEN=
FILE_OR_DIRECTORY_ID= # file or folder ID that you want to track that we have retrieved earlier
GOOGLE_CREDS=./secrets.json # Default location of Google Drive authorization secrets
PATHWAY_PERSISTENT_STORAGE= # Set this variable if you want to use caching
```
### Run with Docker
To run jointly the Alert pipeline and a simple UI execute:
```bash
docker compose up --build
```
Then, the UI will run at http://0.0.0.0:8501 by default. You can access it by following this URL in your web browser.
The `docker-compose.yml` file declares a [volume bind mount](https://docs.docker.com/reference/cli/docker/container/run/#volume) that makes changes to files under `data/` made on your host computer visible inside the docker container. The files in `data/live` are indexed by the pipeline - you can paste new files there and they will impact the computations.
### Run manually
Alternatively, you can run each service separately.
Make sure you have installed poetry dependencies with `--extras unstructured`.
```bash
poetry install --with examples --extras unstructured
```
Then run:
```bash
poetry run python app.py
```
If all dependencies are managed manually rather than using poetry, you can alternatively use:
```bash
python app.py
```
To run the Streamlit UI, run:
```bash
streamlit run ui/server.py --server.port 8501 --server.address 0.0.0.0
```
### Querying the pipeline
To create alerts, you can call the REST API:
```bash
curl --data '{
"user": "user",
"query": "When does the magic cola campaign start? Alert me if the start date changes."
}' http://localhost:8080/ | jq
```
or access the Streamlit UI at `0.0.0.0:8501`.
================================================
FILE: templates/drive_alert/__init__.py
================================================
from .app import run
__all__ = ["run"]
================================================
FILE: templates/drive_alert/app.py
================================================
"""
Microservice for a context-aware alerting ChatGPT assistant.
This demo is very similar to the `alert` example, the only difference is the data source (Google Drive)
For the demo, alerts are sent to Slack (you need to provide `slack_alert_channel_id` and `slack_alert_token`),
you can either put these env variables in .env file under llm-app directory,
or create env variables in the terminal (i.e. export in bash).
The program then starts a REST API endpoint serving queries about Google Docs stored in a
Google Drive folder.
We can create notifications by asking from Streamlit or sending query to API stating we want to be notified.
One example would be `Tell me and alert about the start date of the campaign for Magic Cola`
How Does It Work?
First, Pathway connects to Google Drive, extracts all documents, splits them into chunks, turns them into
vectors using OpenAI embedding service, and store in a nearest neighbor index.
Each query text is first turned into a vector, then relevant document chunks are found
using the nearest neighbor index. A prompt is built from the relevant chunk
and sent to the OpenAI GPT3.5 chat service for processing and answering.
After an initial answer is provided, Pathway monitors changes to documents and selectively
re-triggers potentially affected queries. If the new answer is significantly different from
the previously presented one, a new notification is created.
Please check the README.md in this directory for how-to-run instructions.
"""
import asyncio
import os
import dotenv
import pathway as pw
from pathway.stdlib.ml.index import KNNIndex
from pathway.xpacks.llm.embedders import OpenAIEmbedder
from pathway.xpacks.llm.llms import OpenAIChat, prompt_chat_single_qa
from pathway.xpacks.llm.parsers import UnstructuredParser
from pathway.xpacks.llm.splitters import TokenCountSplitter
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
dotenv.load_dotenv()
class DocumentInputSchema(pw.Schema):
doc: str
class QueryInputSchema(pw.Schema):
query: str
user: str
# Helper Functions
@pw.udf
def build_prompt(documents, query):
docs_str = "\n".join(
[f"Doc-({idx}) -> {doc}" for idx, doc in enumerate(documents[::-1])]
)
prompt = f"""Given a set of documents, answer user query. If answer is not in docs, say it cant be inferred.
Docs: {docs_str}
Query: '{query}'
Final Response:"""
return prompt
@pw.udf
def build_prompt_check_for_alert_request_and_extract_query(query: str) -> str:
prompt = f"""Evaluate the user's query and identify if there is a request for notifications on answer alterations:
User Query: '{query}'
Respond with 'Yes' if there is a request for alerts, and 'No' if not,
followed by the query without the alerting request part.
Examples:
"Tell me about windows in Pathway" => "No. Tell me about windows in Pathway"
"Tell me and alert about windows in Pathway" => "Yes. Tell me about windows in Pathway"
"""
return prompt
@pw.udf
def split_answer(answer: str) -> tuple[bool, str]:
alert_enabled = "yes" in answer[:3].lower()
true_query = answer[3:].strip(' ."')
return alert_enabled, true_query
def build_prompt_compare_answers(new: str, old: str) -> str:
prompt = f"""
Are the two following responses deviating?
Answer with Yes or No.
First response: "{old}"
Second response: "{new}"
"""
return prompt
def make_query_id(user, query) -> str:
return str(hash(query + user))
@pw.udf
def construct_notification_message(query: str, response: str) -> str:
return f'New response for question "{query}":\n{response}'
@pw.udf
def construct_message(response, alert_flag, metainfo=None):
if alert_flag:
if metainfo:
response += "\n" + str(metainfo)
return response + "\n\n🔔 Activated"
return response
def decision_to_bool(decision: str) -> bool:
return "yes" in decision.lower()
def run(
*,
object_id=os.environ.get("FILE_OR_DIRECTORY_ID", ""),
api_key: str = os.environ.get("OPENAI_API_KEY", ""),
host: str = os.environ.get("PATHWAY_REST_CONNECTOR_HOST", "0.0.0.0"),
port: int = int(os.environ.get("PATHWAY_REST_CONNECTOR_PORT", "8080")),
embedder_locator: str = "text-embedding-ada-002",
embedding_dimension: int = 1536,
model_locator: str = "gpt-3.5-turbo",
max_tokens: int = 400,
temperature: float = 0.0,
slack_alert_channel_id=os.environ.get("SLACK_ALERT_CHANNEL_ID", ""),
slack_alert_token=os.environ.get("SLACK_ALERT_TOKEN", ""),
service_user_credentials_file=os.environ.get(
"GOOGLE_CREDS", "templates/drive_alert/secrets.json"
),
**kwargs,
):
# Part I: Build index
embedder = OpenAIEmbedder(
api_key=api_key,
model=embedder_locator,
retry_strategy=pw.asynchronous.FixedDelayRetryStrategy(),
cache_strategy=pw.asynchronous.DefaultCache(),
)
# We start building the computational graph. Each pathway variable represents a
# dynamically changing table.
# The files table contains contents of documents in Google Drive.
# Pathway automatically tracks changes to files and propagates these changes through
# following computations.
# Other Pathway connectors can be used as well - notably:
# - pw.io.fs.read to load and track changes to the local drive and
# - pw.io.s3.read to use an S3-compatible storage
files = pw.io.gdrive.read(
object_id=object_id,
service_user_credentials_file=service_user_credentials_file,
refresh_interval=30, # interval between fetch operations in seconds, lower this for more responsiveness
)
parser = UnstructuredParser()
documents = files.select(texts=parser(pw.this.data))
documents = documents.flatten(pw.this.texts)
documents = documents.select(texts=pw.this.texts[0])
splitter = TokenCountSplitter()
documents = documents.select(
chunks=splitter(pw.this.texts, min_tokens=40, max_tokens=120)
)
documents = documents.flatten(pw.this.chunks)
documents = documents.select(chunk=pw.this.chunks[0])
enriched_documents = documents + documents.select(data=embedder(pw.this.chunk))
# The index is updated each time a file changes.
index = KNNIndex(
enriched_documents.data, enriched_documents, n_dimensions=embedding_dimension
)
# Part II: receive queries, detect intent and prepare cleaned query
# The rest_connector returns a table of all queries under processing
query, response_writer = pw.io.http.rest_connector(
host=host,
port=port,
schema=QueryInputSchema,
autocommit_duration_ms=50,
delete_completed_queries=False,
)
model = OpenAIChat(
api_key=api_key,
model=model_locator,
temperature=temperature,
max_tokens=max_tokens,
retry_strategy=pw.asynchronous.FixedDelayRetryStrategy(),
cache_strategy=pw.asynchronous.DefaultCache(),
)
# Pre-process the queries:
# - detect alerting intent
# - then embed the query for nearest neighbor retrieval
query += query.select(
prompt=build_prompt_check_for_alert_request_and_extract_query(query.query)
)
query += query.select(
tupled=split_answer(
model(
prompt_chat_single_qa(pw.this.prompt),
max_tokens=100,
)
),
)
query = query.select(
pw.this.user,
alert_enabled=pw.this.tupled[0],
query=pw.this.tupled[1],
)
query += query.select(
data=embedder(pw.this.query),
query_id=pw.apply(make_query_id, pw.this.user, pw.this.query),
)
# Part III: respond to queries
# The context is a dynamic table: Pathway updates it each time:
# - a new query arrives
# - a source document is changed significantly enough to change the set of
# nearest neighbors
query_context = query + index.get_nearest_items(query.data, k=3).select(
documents_list=pw.this.chunk
).with_universe_of(query)
# then we answer the queries using retrieved documents
prompt = query_context.select(
pw.this.query_id,
pw.this.query,
pw.this.alert_enabled,
prompt=build_prompt(pw.this.documents_list, pw.this.query),
)
responses = prompt.select(
pw.this.query_id,
pw.this.query,
pw.this.alert_enabled,
response=model(
prompt_chat_single_qa(pw.this.prompt),
),
)
output = responses.select(
result=construct_message(pw.this.response, pw.this.alert_enabled)
)
# and send the answers back to the asking users
response_writer(output)
# Part IV: send alerts about responses which changed significantly.
# However, for the queries with alerts the processing continues
# whenever the set of documents retrieved for a query changes,
# the table of responses is updated.
responses = responses.filter(pw.this.alert_enabled)
def acceptor(new: str, old: str) -> bool:
if new == old:
return False
# TODO: clean after udfs can be used as common functions
prompt = [dict(role="system", content=build_prompt_compare_answers(new, old))]
decision = asyncio.run(model.__wrapped__(prompt, max_tokens=20))
return decision_to_bool(decision)
# Each update is compared with the previous one for deduplication
deduplicated_responses = pw.stateful.deduplicate(
responses,
col=responses.response,
acceptor=acceptor,
instance=responses.query_id,
)
# Significant alerts are sent to the user
alerts = deduplicated_responses.select(
message=construct_notification_message(pw.this.query, pw.this.response)
)
pw.io.slack.send_alerts(alerts.message, slack_alert_channel_id, slack_alert_token)
# Finally, we execute the computation graph
pw.run(monitoring_level=pw.MonitoringLevel.NONE)
if __name__ == "__main__":
run()
================================================
FILE: templates/drive_alert/docker-compose.yml
================================================
version: "3.8"
services:
pathway:
build:
context: .
ports:
- "8080:8080"
environment:
OPENAI_API_KEY:
PATHWAY_PERSISTENT_STORAGE:
streamlit_ui:
depends_on:
- pathway
build:
context: ./ui
ports:
- "8501:8501"
environment:
PATHWAY_REST_CONNECTOR_HOST: "pathway"
================================================
FILE: templates/drive_alert/ui/Dockerfile
================================================
FROM python:3.11
WORKDIR /app
RUN pip install streamlit python-dotenv
COPY . .
EXPOSE 8501
CMD ["streamlit", "run", "server.py", "--server.port", "8501", "--server.address", "0.0.0.0"]
================================================
FILE: templates/drive_alert/ui/server.py
================================================
import os
import requests
import streamlit as st
from dotenv import load_dotenv
api_host = "localhost"
api_port = 8080
load_dotenv()
api_host = os.environ.get("PATHWAY_REST_CONNECTOR_HOST", "127.0.0.1")
api_port = int(os.environ.get("PATHWAY_REST_CONNECTOR_PORT", 8080))
with st.sidebar:
st.markdown("## How to query your data\n")
st.markdown(
"""Enter your question, optionally
ask to be alerted.\n"""
)
st.markdown(
"Example: 'When does the magic cola campaign start? Alert me if the start date changes'",
)
st.markdown(
"""[View the source code on GitHub](
https://github.com/pathwaycom/llm-app/templates/drive_alert/app.py)"""
)
st.markdown("## Current Alerts:\n")
# Streamlit UI elements
st.title("Google Drive notifications with LLM")
prompt = st.text_input("How can I help you today?")
# prompt = st.chat_input("How can I help you today?")
# Initialize chat history
if "messages" not in st.session_state:
st.session_state.messages = []
# Display chat messages from history on app rerun
for message in st.session_state.messages:
with st.chat_message(message["role"]):
st.markdown(message["content"])
# React to user input
if prompt:
# Display user message in chat message container
with st.chat_message("user"):
st.markdown(prompt)
# Add user message to chat history
st.session_state.messages.append({"role": "user", "content": prompt})
for message in st.session_state.messages:
if message["role"] == "user":
st.sidebar.text(f"📩 {message['content']}")
url = f"http://{api_host}:{api_port}/"
data = {"query": prompt, "user": "user"}
response = requests.post(url, json=data)
if response.status_code == 200:
response = response.json()
with st.chat_message("assistant"):
st.markdown(response)
st.session_state.messages.append({"role": "assistant", "content": response})
else:
st.error(f"Failed to send data. Status code: {response.status_code}")
================================================
FILE: templates/multimodal_rag/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
CMD ["python", "app.py"]
================================================
FILE: templates/multimodal_rag/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# Multimodal RAG with Pathway: Process your Financial Reports and Tables with GPT-4o
## **Overview**
This app template showcases how you can build a multimodal RAG application and launch a document processing pipeline that utilizes a vision language model like the `GPT-4o` for parsing. Pathway processes unstructured financial documents within specified directories, extracting and storing the information in a scalable in-memory index. This index is optimized for dynamic RAG, ensuring that search results are continuously updated as documents are modified or new files are added.
Using this approach, you can make your AI application run in permanent connection with your drive, in sync with your documents which include visually formatted elements: tables, charts, images, etc.
We specifically use `GPT-4o` to improve the table data extraction accuracy and demonstrate how this approach outperforms the industry-standard RAG toolkits.
In this showcase, we focused on the finance domain because financial documents often rely heavily on tables in various forms. This showcase highlights the limitations of traditional RAG setups, which struggle to answer questions based on table data. By contrast, our multimodal RAG approach excels in extracting accurate information from tables.
The following GIF shows a snippet from our experiments:
If you want to skip the explanations, you can directly find the code [here](#Running-the-app).
## Table of contents
This includes the technical details to the steps to create a REST Endpoint to run the dynamic RAG application via Docker and modify it for your use-cases.
- [Overview](#Overview)
- [Architecture](#Architecture)
- [Pipeline Organization](#Pipeline-Organization)
- [Customizing the pipeline](#Customizing-the-pipeline)
- [Running the app](#Running-the-app)
- [Conclusion](#Conclusion)
## Architecture
We use `GPT-4o` in two separate places in the flow of data:
- Extracting and understanding the tables inside the PDF
- Answering questions with the retrieved context
The architecture of this multimodal RAG application involves several key components:
- **Data Ingestion**: Ingests data from various sources like local folders, Google Drive, or SharePoint.
- **Document Parsing and Embedding**: Utilizes `DoclingParser` for parsing documents and `OpenAIEmbedder` for embedding text. This includes handling and processing images within PDFs.
- **Document Store**: The `DocumentStoreServer` indexes parsed documents and retrieves relevant chunks for answering questions.
- **Question Answering**: Uses the `BaseRAGQuestionAnswerer` class to call `GPT-4o` for generating responses based on the retrieved context.
- **Server Setup**: Sets up a REST endpoint to serve the RAG application.
For more advanced RAG options, make sure to check out [rerankers](https://pathway.com/developers/api-docs/pathway-xpacks-llm/rerankers) and the [adaptive rag example](../adaptive_rag/).
## Pipeline Organization
This folder contains several objects:
- `app.py`, the main application code using Pathway and written in Python. It reads configuration from `app.yaml`, and runs a server answering queries to the defined pipeline.
- `app.yaml`, YAML configuration file, that defines components of the pipeline.
- `Dockerfile`, the Docker configuration for running the pipeline in a container. It includes instructions for installing dependencies and setting up the runtime environment.
- `requirements.txt`, the dependencies for the pipeline. This file can be passed to `pip install -r requirements.txt` to install everything needed to launch the pipeline locally.
- `.env`, a short environment variables configuration file where the OpenAI key must be stored. This file ensures secure handling of sensitive information.
- `data/`, a folder with exemplary files that can be used for test runs. It includes sample financial documents to demonstrate the pipeline's capabilities.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/user-guide/llm-xpack/yaml-templates).
In the `app.yaml` file we define:
- input connectors
- LLM
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
You can also check our other templates - [Question Answering RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag),
[Multimodal RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/multimodal_rag) or
[Private RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/private_rag). As all of these only differ
in the YAML configuration file, you can also use them as an inspiration for your custom pipeline.
Here some examples of what can be modified.
### LLM Model
This template by default uses two llm models - GPT-4.1-mini for answering queries and GPT-4o for parsing tables and images.
You can replace either of them with other Open AI models, like GPT-4.1 or GPT-5, but keep in mind that the model used for parsing needs to support image input.
You can find the whole list on their [models page](https://platform.openai.com/docs/models).
To change the model of the answering llm, you simply need to change the `model` in the `$llm` variable to the one you want to use, e.g. to use `GPT-5` set:
```yaml
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-5"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
capacity: 8
```
You can also use different provider, by using different class from [Pathway LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview),
e.g. here is configuration for locally run Mistral model with Ollama.
```yaml
$llm: !pw.xpacks.llm.llms.LiteLLMChat
model: "ollama/mistral"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DiskCache
temperature: 0
top_p: 1
api_base: "http://localhost:11434"
```
You can also change LLM used for parsing in the same way, by changing `!parsing_llm` in `app.yaml`, just keep in mind to use a multimodal model.
### Webserver
You can configure the host and the port of the webserver.
Here is the default configuration:
```yaml
host: "0.0.0.0"
port: 8000
```
### Cache
You can configure whether you want to enable cache or persistence, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
persistence_mode: !pw.PersistenceMode.UDF_CACHING
persistence_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `data` folder.
## Running the app
> Note: Recommended way of running the Pathway on Windows is Docker, refer to [Running with the Docker section](#with-docker).
First, make sure to install the requirements by running:
```bash
pip install -r requirements.txt -U
```
Then, create a `.env` file in this directory and put your API key with `OPENAI_API_KEY=sk-...`, or add the `api_key` argument to `OpenAIChat` and `OpenAIEmbedder`.
Then, simply run with `python app.py` in this directory.
### With Docker
First, make sure to have your OpenAI API key in the environment, you can create a `.env` file as mentioned above, or specify the `api_key` argument in the `OpenAIChat` and `OpenAIEmbedder`.
In order to let the pipeline get updated with each change in local files, you need to mount the `data` folder inside the docker. The following commands show how to do that.
The following commands will:
- mount the `data` folder inside the Docker
- build the image
- run the app and expose the port `8000`.
You can omit the ```-v `pwd`/data:/app/data``` part if you are not using local files as a data source.
```bash
# Make sure you are in the right directory.
cd templates/multimodal_rag/
# Build the image in this folder
docker build -t rag .
# Run the image, mount the `data` folder into image and expose the port `8000`
docker run -v `pwd`/data:/app/data -p 8000:8000 rag
```
## Querying the pipeline
Follow the [steps below](#running-the-app) to set up the service. This will create a REST endpoint on your selected host and port, running a service that is connected to your file folder, and ready to answer your questions. There are no extra dependencies.
In this demo, we run the service on localhost (`0.0.0.0:8000`). You can connect your own front end or application to this endpoint. Here, we test the service with `curl`.
First, let's check the files contained in your folder are currently indexed:
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/list_documents' -H 'accept: */*' -H 'Content-Type: application/json'
```
This will return the list of files e.g. if you start with the [data folder](./data) provided in the demo, the answer will be as follows:
> `[{"modified_at": 1715765613, "owner": "berke", "path": "data/20230203_alphabet_10K.pdf", "seen_at": 1715768762}]`
In the default app setup, the connected folder is a local file folder. You can add more folders and file sources, such as [Google Drive](https://pathway.com/developers/user-guide/connectors/gdrive-connector#google-drive-connector) or [Sharepoint](https://pathway.com/developers/user-guide/connecting-to-data/connectors#tutorials), by adding a line of code to the template.
If you now add or remove files from your connected folder, you can repeat the request and see the index file list has been updated automatically. You can look into the logs of the service to see the progress of the indexing of new and modified files. PDF files of 100 pages should normally take under 10 seconds to sync, and the indexing parallelizes if multiple files are added at a single time.
Now, let's ask a question from one of the tables inside the report. In our tests, regular RAG applications struggled with the tables and couldn't answer to this question correctly.
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "How much was Operating lease cost in 2021?"
}'
```
> `{"response": "$2,699 million"}`
This response was correct thanks to the initial LLM parsing step.
When we check the context that is sent to the LLM, we see that Pathway included the table in the context where as other RAG applications failed to include the table.
Let's try another one,
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "What is the operating income for the fiscal year of 2022?"
}'
```
> `{"response": "$74,842 million"}`
Another example, let's ask a question that can be answered from the table on the 48th page of the PDF.
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "How much was Marketable securities worth in 2021 in the consolidated balance sheets?"
}'
```
> `{"response": "$118,704 million"}`
Now, let's also fetch the context documents,
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "How much was Operating lease cost in 2021?", "return_context_docs": true
}'
```
> `{"response": "$2,699 million", "context_docs": [{"text": "..."}, ...]`
Looking good!
## Conclusion
This showcase demonstrates setting up a powerful RAG pipeline with advanced table parsing capabilities, unlocking new finance use cases. While we've only scratched the surface, there's more to explore:
- Re-ranking: Prioritize the most relevant results for your specific query.
- Knowledge graphs: Leverage relationships between entities to improve understanding.
- Hybrid indexing: Combine different indexing strategies for optimal retrieval.
- Adaptive reranking: Iteratively enlarge the context for optimal accuracy, see [our example](../adaptive_rag/README.md).
Stay tuned for future examples exploring these advanced techniques with Pathway!
RAG applications are most effective when tailored to your specific use case. Here's how you can customize yours:
- Document parsers and splitters: Fine-tune how documents are processed and broken down for analysis.
- Indexing and retrieval strategies: Choose the most efficient approach for your data and search needs.
- User Interface (UI): Design a user-friendly interface that caters to your end users' workflows.
Ready to Get Started?
Let's discuss how we can help you build a powerful, customized RAG application. [Reach us here to talk or request a demo!](https://pathway.com/solutions/slides-ai-search?modal=requestdemo)
## Quick Links:
- [Pathway Developer Documentation](https://pathway.com/developers/user-guide/introduction/welcome)
- [Pathway App Templates](https://pathway.com/developers/templates)
- [Discord Community of Pathway](https://discord.gg/pathway)
- [Pathway Issue Tracker](https://github.com/pathwaycom/pathway/issues)
- [End-to-end dynamic RAG pipeline with Pathway](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag)
- [Using Pathway as a retriever with LlamaIndex](https://docs.llamaindex.ai/en/stable/examples/retrievers/pathway_retriever/)
Make sure to drop a "Star" to our repositories if you found this resource helpful!
================================================
FILE: templates/multimodal_rag/app.py
================================================
import logging
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.question_answering import SummaryQuestionAnswerer
from pathway.xpacks.llm.servers import QASummaryRestServer
from pydantic import BaseModel, ConfigDict, InstanceOf
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
question_answerer: InstanceOf[SummaryQuestionAnswerer]
host: str = "0.0.0.0"
port: int = 8000
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
server = QASummaryRestServer( # noqa: F841
self.host, self.port, self.question_answerer
)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/multimodal_rag/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Multimodal RAG template.
# It defines various components such as data sources, language models, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for generating responses.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: data
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Configures the LLM model settings for generating responses.
# The list of available Pathway LLM wrappers is available here:
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
# You can learn more about those in our documentation:
# https://pathway.com/developers/templates/rag-customization/llm-chats
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-4.1-mini"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
capacity: 8
async_mode: "fully_async"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
model: "text-embedding-3-small"
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
# Defines the llm to be used for parsing images.
$parsing_llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-4o"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
async_mode: "fully_async"
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
multimodal_llm: $parsing_llm
image_parsing_strategy: "llm"
table_parsing_strategy: "llm"
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the splitter for chunking the documents.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Configures the question-answering component using the RAG approach.
question_answerer: !pw.xpacks.llm.question_answering.BaseRAGQuestionAnswerer
llm: $llm
indexer: $document_store
# You can set the number of documents to be included as the context of the query
# search_topk: 6
# You can use your own prompt for querying.
# For that set prompt_template to string with `{query}` used as a placeholder for the question,
# and `{context}` as a placeholder for context documents.
# prompt_template: "Given these documents: {context}, please answer the question: {query}"
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: 8000
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/multimodal_rag/requirements.txt
================================================
pathway[all]
python-dotenv~=1.0
mpmath~=1.3
================================================
FILE: templates/private_rag/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8000
CMD ["python", "app.py"]
================================================
FILE: templates/private_rag/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# Fully private RAG with Pathway
## **Overview**
Retrieval-Augmented Generation (RAG) is a powerful method for answering questions using a private knowledge database. Ensuring data security is essential, especially for sensitive information like trade secrets, confidential IP, GDPR-protected data, and internal documents. This showcase demonstrates setting up a private RAG pipeline with adaptive retrieval using Pathway, Mistral, and Ollama. The provided code deploys this adaptive RAG technique with Pathway, ensuring no API access or data leaves the local machine.
The app utilizes modules under `pathway.xpacks.llm`. The `BaseRAGQuestionAnswerer` class is the foundation for building RAG applications with the Pathway vector store and xpack components, enabling a quick start with RAG applications.
This example uses `AdaptiveRAGQuestionAnswerer`, an extension of `BaseRAGQuestionAnswerer` with adaptive retrieval. For more on building and deploying RAG applications with Pathway, including containerization, refer to the demo on question answering.
The application responds to requests at the `/v2/answer` endpoint. The `answer_query` function takes the `pw_ai_queries` table as input, containing prompts and other arguments from the post request. This table's data is used to call the adaptive retrieval logic.
The `AdaptiveRAGQuestionAnswerer` implementation under `pathway.xpacks.llm.question_answering` builds a RAG app with the Pathway vector store and components. It supports two question answering strategies, short (concise) and long (detailed) responses, set during the post request. It allows LLM agnosticity, giving the freedom to choose between proprietary or open-source LLMs. It adapts the number of chunks used as a context, starting with `n_starting_documents` chunks and increasing until an answer is found.
To learn more about building & deploying RAG applications with Pathway, including containerization, refer to [demo question answering](../question_answering_rag/README.md).
## Table of contents
This includes the technical details to the steps to create a REST Endpoint to run the application via Docker and modify it for your use-cases.
- [Overview](#Overview)
- [Architecture](#Architecture)
- [Deploying and using a local LLM](#Deploying-and-using-a-local-LLM)
- [Running the app](#Running-the-app)
- [Querying the app/pipeline](#Querying-the-app/pipeline)
- [Modifying the code](#Modifying-the-code)
- [Conclusion](#Conclusion)
## Architecture
The architecture consists of two connected technology bricks, which will run as services on your machine:
- Pathway brings support for real-time data synchronization pipelines out of the box, and the possibility of secure private document handling with enterprise connectors for synchronizing Sharepoint and Google Drive incrementally. The Pathway service you'll run performs live document indexing pipeline, and will use Pathway’s built-in vector store.
- The language model you use will be a Mistral 7B, which you will locally deploy as an Ollama service. This model was chosen for its performance and compact size.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/templates/configure-yaml).
In the `app.yaml` file we define:
- input connectors
- LLM
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
You can also check our other templates - [Question Answering RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag),
[Multimodal RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/multimodal_rag) or
[Private RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/private_rag). As all of these only differ
in the YAML configuration file, you can also use them as an inspiration for your custom pipeline.
Here some examples of what can be modified.
### LLM Model
This template is prepared to run by default locally. However, the pipeline is LLM model agnostic, so you can change them to use other locally deployed model, or even
use LLM model available through API calls. For discussion on models used in this template check [the dedicated Section](#deploying-and-using-a-local-LLM).
### Webserver
You can configure the host and the port of the webserver.
Here is the default configuration:
```yaml
host: "0.0.0.0"
port: 8000
```
### Cache
You can configure whether you want to enable cache or persistence, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
persistence_mode: !pw.PersistenceMode.UDF_CACHING
persistence_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `data` folder.
#### Local Data Source
The local data source is configured by using map with tag `!pw.io.fs.read`. Then set `path` to denote the path to a folder with files to be indexed.
#### Google Drive Data Source
The Google Drive data source is enabled by using map with tag `!pw.io.gdrive.read`. The map must contain two main parameters:
- `object_id`, containing the ID of the folder that needs to be indexed. It can be found from the URL in the web interface, where it's the last part of the address. For example, the publicly available demo folder in Google Drive has the URL `https://drive.google.com/drive/folders/1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`. Consequently, the last part of this address is `1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`, hence this is the `object_id` you would need to specify.
- `service_user_credentials_file`, containing the path to the credentials files for the Google [service account](https://cloud.google.com/iam/docs/service-account-overview). To get more details on setting up the service account and getting credentials, you can also refer to [this tutorial](https://pathway.com/developers/user-guide/connectors/gdrive-connector#setting-up-google-drive).
Besides, to speed up the indexing process you may want to specify the `refresh_interval` parameter, denoted by an integer number of seconds. It corresponds to the frequency between two sequential folder scans. If unset, it defaults to 30 seconds.
For the full list of the available parameters, please refer to the Google Drive connector [documentation](https://pathway.com/developers/api-docs/pathway-io/gdrive#pathway.io.gdrive.read).
#### SharePoint Data Source
This data source requires Scale or Enterprise [license key](https://pathway.com/pricing) - you can obtain free Scale key on [Pathway website](https://pathway.com/get-license).
To use it, set the map tag to be `!pw.xpacks.connectors.sharepoint.read`, and then provide values of `url`, `tenant`, `client_id`, `cert_path`, `thumbprint` and `root_path`. To read about the meaning of these arguments, check the Sharepoint connector [documentation](https://pathway.com/developers/api-docs/pathway-xpacks-sharepoint#pathway.xpacks.connectors.sharepoint.read).
## Deploying and using a local LLM
### Embedding Model Selection
You will use `pathway.xpacks.llm.embedders` module to load open-source embedding models from the HuggingFace model library. For this showcase, pick the `avsolatorio/GIST-small-Embedding-v0` model which has a dimension of 384 as it is compact and performed well in our tests.
```yaml
$embedding_model: "avsolatorio/GIST-small-Embedding-v0"
$embedder: !pw.xpacks.llm.embedders.SentenceTransformerEmbedder
model: $embedding_model
call_kwargs:
show_progress_bar: False
```
If you would like to use a higher-dimensional model, here are some possible alternatives you could use instead:
- mixedbread-ai/mxbai-embed-large-v1
- avsolatorio/GIST-Embedding-v0
For other possible choices, take a look at the [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) managed by HuggingFace.
### Local LLM Deployment
Due to its size and performance it is best to run the `Mistral 7B` LLM. Here you would deploy it as a service running on GPU, using `Ollama`.
To run local LLM, you can refer to these steps:
- Download Ollama from [ollama.com/download](https://ollama.com/download)
- In your terminal, run `ollama serve`
- In another terminal, run `ollama run mistral`
You can now test it with the following request:
```bash
curl -X POST http://localhost:11434/api/generate -d '{
"model": "mistral",
"prompt":"Here is a story about llamas eating grass"
}'
```
### LLM Initialization
Now you will initialize the LLM instance that will call the local model.
```yaml
$llm_model: "ollama/mistral"
$llm: !pw.xpacks.llm.llms.LiteLLMChat
model: $llm_model
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
api_base: "http://localhost:11434"
# api_base: "http://host.docker.internal:11434" # use this when you are running the app in the Docker on Mac or Windows
async_mode: "fully_async"
```
## Running the app
First, make sure your local LLM is up and running. By default, the pipeline tries to access the LLM at `http://localhost:11434`. You can change that by setting `api_base` value in the app.yaml file.
### With Docker
In order to let the pipeline get updated with each change in local files, you need to mount the folder onto the docker. The following commands show how to do that.
#### Linux
```bash
# Build the image in this folder
docker build -t privaterag .
# Run the image, mount the `data` folder into image
docker run --net host -v ./data:/app/data privaterag
```
#### Mac or Windows
In the `app.yaml` change `api_base` to be `http://host.docker.internal:11434`. Then run:
```bash
# Build the image in this folder
docker build -t privaterag .
# Run the image, mount the `data` folder into image
docker run -v ./data:/app/data -p 8000:8000 privaterag
```
### Locally
To run locally you need to install the Pathway app with LLM dependencies using:
```bash
pip install pathway[all]
```
Then change your directory in the terminal to this folder and run the app:
```bash
python app.py
```
## Querying the app/pipeline
Finally, query the application with;
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "What is the start date of the contract?"
}'
```
> `{"response": "December 21, 2015 [6]"}`
## Conclusion:
Now you have a fully private RAG set up with Pathway and Ollama. All your data remains safe on your system. Moreover, the set-up is optimized for speed, thanks to how Ollama runs the LLM, and how Pathway’s adaptive retrieval mechanism reduces token consumption while preserving the accuracy of the RAG.
This is a full production-ready set-up which includes reading your data sources, parsing the data, and serving the endpoint.
This private RAG setup can be run entirely locally with open-source LLMs, making it ideal for organizations with sensitive data and explainable AI needs.
## Quick Links:
- [Pathway Developer Documentation](https://pathway.com/developers/user-guide/introduction/welcome)
- [Pathway App Templates](https://pathway.com/developers/templates)
- [Discord Community of Pathway](https://discord.gg/pathway)
- [Pathway Issue Tracker](https://github.com/pathwaycom/pathway/issues)
- [End-to-end dynamic RAG pipeline with Pathway](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag)
- [Using Pathway as a vector store with Langchain](https://python.langchain.com/v0.2/docs/integrations/vectorstores/pathway/)
- [Using Pathway as a retriever with LlamaIndex](https://docs.llamaindex.ai/en/stable/examples/retrievers/pathway_retriever/)
Make sure to drop a “Star” to our repositories if you found this resource helpful!
================================================
FILE: templates/private_rag/app.py
================================================
import logging
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.question_answering import SummaryQuestionAnswerer
from pathway.xpacks.llm.servers import QASummaryRestServer
from pydantic import BaseModel, ConfigDict, InstanceOf
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
question_answerer: InstanceOf[SummaryQuestionAnswerer]
host: str = "0.0.0.0"
port: int = 8000
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
server = QASummaryRestServer( # noqa: F841
self.host, self.port, self.question_answerer
)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/private_rag/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Private RAG template.
# It defines various components such as data sources, language models, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for generating responses.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: data
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Configures the LLM model settings for generating responses.
# The list of available Pathway LLM wrappers is available here:
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
# You can learn more about those in our documentation:
# https://pathway.com/developers/templates/rag-customization/llm-chats
$llm_model: "ollama/mistral"
$llm: !pw.xpacks.llm.llms.LiteLLMChat
model: $llm_model
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
api_base: "http://localhost:11434"
# api_base: "http://host.docker.internal:11434" # use this when you are running the app in the Docker on Mac or Windows
async_mode: "fully_async"
$embedding_model: "avsolatorio/GIST-small-Embedding-v0"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.SentenceTransformerEmbedder
model: $embedding_model
call_kwargs:
show_progress_bar: False
# Sets up the splitter for chunking the documents.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
table_parsing_strategy: "llm"
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Configures the question-answering component using the RAG approach.
question_answerer: !pw.xpacks.llm.question_answering.AdaptiveRAGQuestionAnswerer
llm: $llm
indexer: $document_store
n_starting_documents: 2
factor: 2
max_iterations: 4
strict_prompt: true
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: 8000
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/private_rag/requirements.txt
================================================
pathway[all]
python-dotenv~=1.0
mpmath~=1.3
================================================
FILE: templates/question_answering_rag/Dockerfile
================================================
ARG PATHWAY_SRC_IMAGE=pathwaycom/pathway:latest
FROM ${PATHWAY_SRC_IMAGE}
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv tesseract-ocr-eng poppler-utils libreoffice \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
CMD ["python", "app.py"]
================================================
FILE: templates/question_answering_rag/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# Pathway RAG app with always up-to-date knowledge
This demo shows how to create a real-time RAG application using [Pathway](https://github.com/pathwaycom/pathway) that provides always up-to-date knowledge to your LLM without the need for a separate ETL.
You will see a running example of how to get started with the Pathway vector store that eliminates the need for ETL pipelines which are needed in the regular VectorDBs.
This significantly reduces the developer's workload.
This demo allows you to:
- Create a Document store with real-time document indexing from Google Drive, Microsoft 365 SharePoint, or a local directory;
- Connect an OpenAI LLM model of choice to your knowledge base;
- Get quality, accurate, and precise responses to your questions;
- Ask questions about folders, files or all your documents easily, with the help of filtering options;
- Use LLMs over API to summarize texts;
- Get an executive outlook for a question on different files to easily access available knowledge in your documents;
Note: This app relies on [Document Store](https://pathway.com/developers/api-docs/pathway-xpacks-llm/document_store) to learn more, you can check out [this blog post](https://pathway.com/developers/user-guide/llm-xpack/docs-indexing).
## Table of contents
- [Summary of available endpoints](#Summary-of-available-endpoints)
- [How it works](#How-it-works)
- [Customizing the pipeline](#Customizing-the-pipeline)
- [How to run the project](#How-to-run-the-project)
- [Using the app](#Query-the-documents)
## Summary of available endpoints
This example spawns a lightweight webserver using Pathway’s [`QASummaryRestServer`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/servers#pathway.xpacks.llm.servers.QASummaryRestServer) that accepts queries on five possible endpoints, divided into two categories: document indexing and RAG with LLM.
### Document Indexing capabilities
- `/v1/retrieve` to perform similarity search;
- `/v1/statistics` to get the basic stats about the indexer's health;
- `/v2/list_documents` to retrieve the metadata of all files currently processed by the indexer.
### LLM and RAG capabilities
- `/v2/answer` to ask questions about your documents, or directly talk with your LLM;
- `/v2/summarize` to summarize a list of texts;
See the [using the app section](###Using-the-app) to learn how to use the provided endpoints.
## How it works
1. **Data Ingestion**
We define one or more sources in `app.yaml` (local directories, Google Drive, Microsoft SharePoint, etc.).
- The provided demo references a local folder `data/` by default.
- The code can poll these sources at configured intervals, so when new documents appear or existing ones change, they are automatically parsed and re-indexed in real-time.
2. **Parsing & Splitting**
Using [Docling](https://www.docling.ai/) (through Pathway’s [`DoclingParser`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/parsers#pathway.xpacks.llm.parsers.DoclingParser)) and [TokenCountSplitter](https://pathway.com/developers/api-docs/pathway-xpacks-llm/splitters#pathway.xpacks.llm.splitters.TokenCountSplitter), documents are parsed and chunked into smaller parts.
3. **Embedding**
Via [`OpenAIEmbedder`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/embedders#pathway.xpacks.llm.embedders.OpenAIEmbedder), the chunks get turned into embeddings. You can substitute your own embedder if you wish.
4. **Indexing**
Using [`USearchKnnFactory`](https://pathway.com/developers/api-docs/indexing#pathway.stdlib.indexing.UsearchKnnFactory), the embeddings are stored in a vector index. This is all streaming as well, so new embeddings are added or updated automatically.
5. **Serving**
- We create a [`SummaryQuestionAnswerer`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/question_answering#pathway.xpacks.llm.question_answering.SummaryQuestionAnswerer) (as specified in `app.py`), which can handle both question-answering and summarization requests.
- A web server, [`QASummaryRestServer`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/servers#pathway.xpacks.llm.servers.QASummaryRestServer), exposes multiple endpoints for retrieval, Q&A, summarization, and more.
Because Pathway is fully incremental, any changes to your source files immediately flow through parsing, embedding, indexing, and ultimately get reflected in the answers from the LLM. The user can then query the created index with simple HTTP requests to the endpoints mentioned above.
## Pipeline Organization
This folder contains several objects:
- `app.py`, the application code using Pathway and written in Python;
- `app.yaml`, the file containing configuration of the pipeline, like LLM models, sources or server address;
- `requirements.txt`, the dependencies for the pipeline. It can be passed to `pip install -r requirements.txt` to install everything that is needed to launch the pipeline locally;
- `Dockerfile`, the Docker configuration for running the pipeline in the container;
- `.env`, a short environment variables configuration file where the OpenAI key must be stored;
- `data/`, a folder with exemplary files that can be used for the test runs.
- `ui/`, a simple ui written in Streamlit for asking questions.
## Pathway tooling
### Prompts and helpers
Pathway allows you to define custom prompts in addition to the ones provided in [`pathway.xpacks.llm`](https://pathway.com/developers/user-guide/llm-xpack/overview).
You can also use user-defined functions using the [`@pw.udf`](https://pathway.com/developers/api-docs/pathway#pathway.udf) decorator to define custom functions that will run on streaming data.
### RAG
Pathway provides all the tools to create a RAG application and query it: a [Pathway Document store](https://pathway.com/developers/api-docs/pathway-xpacks-llm/document_store#pathway.xpacks.llm.document_store.DocumentStore) and a web server (defined with the [REST connector](https://pathway.com/developers/api-docs/pathway-io/http#pathway.io.http.rest_connector)).
For the sake of the demo, we kept the app simple, consisting of the main components you would find in a regular RAG application. It can be further enhanced with query writing methods, re-ranking layer and custom splitting steps.
Don't hesitate to take a look at our [documentation](https://pathway.com/developers/user-guide/introduction/welcome) to learn how Pathway works.
## OpenAI API Key Configuration
Default LLM provider in this template is OpenAI, so, unless you change the configuration, you need to provide OpenAI API key. Please configure your key in a `.env` file by providing it as follows: `OPENAI_API_KEY=sk-*******`. You can refer to the stub file `.env` in this repository, where you will need to paste your key instead of `sk-*******`.
## Customizing the pipeline
The code can be modified by changing the `app.yaml` configuration file. To read more about YAML files used in Pathway templates, read [our guide](https://pathway.com/developers/templates/configure-yaml).
In the `app.yaml` file we define:
- input connectors
- LLM
- embedder
- index
and any of these can be replaced or, if no longer needed, removed. For components that can be used check
Pathway [LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview), or you can implement your own.
You can also check our other templates - [Question Answering RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag),
[Multimodal RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/multimodal_rag) or
[Private RAG](https://github.com/pathwaycom/llm-app/tree/main/templates/private_rag). As all of these only differ
in the YAML configuration file, you can also use them as an inspiration for your custom pipeline.
Here some examples of what can be modified.
### LLM Model
You can choose any of the models offered by Open AI, like GPT-5, GPT-4.1, or GPT-4o.
You can find the whole list on their [models page](https://platform.openai.com/docs/models).
You simply need to change the `model` to the one you want to use, e.g., to use GPT-5:
```yaml
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-5"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache
capacity: 8
```
The default model is `gpt-4.1-mini`.
You can also use different provider, by using different class from [Pathway LLM xpack](https://pathway.com/developers/user-guide/llm-xpack/overview),
e.g. here is configuration for locally run Mistral model.
```yaml
$llm: !pw.xpacks.llm.llms.LiteLLMChat
model: "ollama/mistral"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DiskCache
temperature: 0
top_p: 1
api_base: "http://localhost:11434"
```
### Index of your choice
The [`DocumentStore`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/document_store#pathway.xpacks.llm.document_store.DocumentStore) makes building and customizing a document indexing pipeline straightforward. It processes documents and enables querying the closest documents to a given query based on your chosen indexing strategy. Here's how you can use a hybrid indexing approach, combining vector-based and text-based retrieval:
Example: Hybrid Indexing with `USearchKNN` and `TantivyBM25`
The following example demonstrates how to configure and use the [HybridIndex](https://pathway.com/developers/api-docs/indexing#pathway.stdlib.indexing.HybridIndex) that combines:
- **`USearchKNN`**: A vector-based index leveraging embeddings for semantic similarity search.
- **[`TantivyBM25`](https://pathway.com/developers/api-docs/indexing#pathway.stdlib.indexing.TantivyBM25)**: A text-based index using BM25 for keyword matching.
```yaml
$knn_index: !pw.indexing.USearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchKnnMetricKind.COS
dimensions: 1536
$bm25_index: !pw.indexing.TantivyBM25Factory
$hybrid_index_factory: !pw.indexing.HybridIndexFactory
retriever_factories:
- $knn_index
- $bm25_index
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $hybrid_index_factory
```
Choose the indexing strategy that fits your requirements with `DocumentStore`
### Webserver
You can configure the host and the port of the webserver.
Here is the default configuration:
```yaml
host: "0.0.0.0"
port: 8000
```
### Cache
You can configure whether you want to enable cache or persistence, to avoid repeated API accesses, and where the cache is stored.
Default values:
```yaml
persistence_mode: !pw.PersistenceMode.UDF_CACHING
persistence_backend: !pw.persistence.Backend.filesystem
path: ".Cache"
```
### Data sources
You can configure the data sources by changing `$sources` in `app.yaml`.
You can add as many data sources as you want. You can have several sources of the same kind, for instance, several local sources from different folders.
The sections below describe how to configure local, Google Drive and Sharepoint source, and you can check the examples of YAML configuration in our [user guide](https://pathway.com/developers/templates/yaml-snippets/data-sources-examples/). While these are not described in this Section, you can also use any input [connector](https://pathway.com/developers/user-guide/connecting-to-data/connectors) from Pathway package.
By default, the app uses a local data source to read documents from the `data` folder.
#### Local Data Source
The local data source is configured by using map with tag `!pw.io.fs.read`. Then set `path` to denote the path to a folder with files to be indexed.
#### Google Drive Data Source
The Google Drive data source is enabled by using map with tag `!pw.io.gdrive.read`. The map must contain two main parameters:
- `object_id`, containing the ID of the folder that needs to be indexed. It can be found from the URL in the web interface, where it's the last part of the address. For example, the publicly available demo folder in Google Drive has the URL `https://drive.google.com/drive/folders/1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs`. The last part of this address is `1cULDv2OaViJBmOfG5WB0oWcgayNrGtVs` and this is the `object_id` you would need to specify.
- `service_user_credentials_file`, containing the path to the credentials files for the Google [service account](https://cloud.google.com/iam/docs/service-account-overview). To get more details on setting up the service account and getting credentials, you can also refer to [this tutorial](https://pathway.com/developers/user-guide/connectors/gdrive-connector#setting-up-google-drive).
Besides, to speed up the indexing process you may want to specify the `refresh_interval` parameter, denoted by an integer number of seconds. It corresponds to the frequency between two sequential folder scans. If unset, it defaults to 30 seconds.
For the full list of the available parameters, please refer to the Google Drive connector [documentation](https://pathway.com/developers/api-docs/pathway-io/gdrive#pathway.io.gdrive.read).
#### SharePoint Data Source
This data source requires Scale or Enterprise [license key](https://pathway.com/pricing) - you can obtain free Scale key on [Pathway website](https://pathway.com/get-license).
To use it, set the map tag to be `!pw.xpacks.connectors.sharepoint.read`, and then provide values of `url`, `tenant`, `client_id`, `cert_path`, `thumbprint` and `root_path`. To read about the meaning of these arguments, check the Sharepoint connector [documentation](https://pathway.com/developers/api-docs/pathway-xpacks-sharepoint#pathway.xpacks.connectors.sharepoint.read).
## How to run the project
Clone the llm-app repository from GitHub. This repository contains all the files you’ll need.
```bash
git clone https://github.com/pathwaycom/llm-app.git
```
### Locally
If you are on Windows, please refer to [running with docker](#With-Docker) section below.
To run locally, change your directory in the terminal to this folder. Then, run the app with `python`.
```bash
cd templates/question_answering_rag
python app.py
```
Please note that the local run requires the dependencies to be installed. It can be done with a simple pip command:
`pip install -r requirements.txt`
### With Docker
Build the Docker with:
```bash
docker compose build
```
And, run with:
```bash
docker compose up
```
This will start the pipeline and the ui for asking questions.
### Query the documents
You will see the logs for parsing & embedding documents in the Docker image logs.
Give it a few minutes to finish up on embeddings, you will see `0 entries (x minibatch(es)) have been...` message.
If there are no more updates, this means the app is ready for use!
To test it, let's query the stats:
```bash
curl -X 'POST' 'http://localhost:8000/v1/statistics' -H 'accept: */*' -H 'Content-Type: application/json'
```
For more information on available endpoints by default, see [above](#Summary-of-available-endpoints).
We provide some example `curl` queries to start with.
The general structure is sending a request to `http://{HOST}:{PORT}/{ENDPOINT}`.
Where HOST is the `host` variable you specify in your app configuration. PORT is the `port` number you are running your app on, and ENDPOINT is the specific extension for endpoints. They are specified in the application code, and they are listed with the versioning as `/v1/...`.
Note that, if you are using the Pathway hosted version, you should send requests to `https://...` rather than `http://...` and emit the `:{PORT}` part of the URL.
You need to add two headers, `-H 'accept: */*' -H 'Content-Type: application/json'`.
Finally, for endpoints that expect data in the query, you can pass it with `-d '{key: value}'` format.
#### Listing inputs
Get the list of available inputs and associated metadata.
```bash
curl -X 'POST' 'http://localhost:8000/v2/list_documents' -H 'accept: */*' -H 'Content-Type: application/json'
```
#### Searching in your documents
Search API gives you the ability to search in available inputs and get up-to-date knowledge.
`query` is the search query you want to execute.
`k` (optional) is an integer, the number of documents to be retrieved. Documents in this case means small chunks that are stored in your vector store.
`metadata_filter` (optional) String to filter results with Jmespath query.
`filepath_globpattern` (optional) String to filter results with globbing pattern. For example `"*"` would result in no filter, `"*.docx"` would result in only `docx` files being retrieved.
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v1/retrieve' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"query": "Which articles of General Data Protection Regulation are relevant for clinical trials?",
"k": 6
}'
```
#### Asking questions to LLM (With and without RAG)
- Note: The local version of this app does not require `openai_api_key` parameter in the payload of the query. Embedder and LLM will use the API key in the `.env` file. However, Pathway hosted public demo available on the [website](https://pathway.com/solutions/ai-pipelines) requires a valid `openai_api_key` to execute the query.
- Note: All of the RAG endpoints use the `model` provided in the config by default, however, you can specify another model with the `model` parameter in the payload to use a different one for generating the response.
For question answering without any context, simply omit `filters` key in the payload and send the following request.
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v2/answer' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"prompt": "What is the start date of the contract?"
}'
```
Question answering with the knowledge from a specific file, based on the path.
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v2/answer' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"prompt": "What is the start date of the contract?",
"filters": "globmatch(`"IdeanomicsInc_20160330_10-K_EX-10.26_9512211_EX-10.26_Content License Agreement.pdf"`, path)"
}'
```
Alternatively, with the knowledge from files that have the word `Ide` in their paths.
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v2/answer' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"prompt": "What is the start date of the contract?",
"filters": "contains(path, `"Ide"`)"
}'
```
You can also retrieve the context documents that were used to answer the question,
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v2/answer' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"prompt": "What is the start date of the contract?",
"return_context_docs": true
}'
```
- Note: You can limit the knowledge to a folder or, to only Word documents by using ```"contains(path, `docx`)"```
- Note: You could also use a few filters separated with `||` (`or` clause) or with `&&` (`and` clause).
You can further modify behavior in the payload by defining keys and values in `-d '{key: value}'`.
If you wish to use another model, specify in the payload as `"model": "gpt-4"`.
For more detailed responses add `"response_type": "long"` to payload.
#### Summarization
To summarize a list of texts, use the following `curl` command.
```bash
curl -X 'POST' \
'http://0.0.0.0:8000/v2/summarize' \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-d '{
"text_list": [
"I love apples.",
"I love oranges."
]
}'
```
Specifying the GPT model with `"model": "gpt-4"` is also possible.
This endpoint also supports setting different models in the query by default.
To execute similar curl queries as above, you can visit [ai-pipelines page](https://pathway.com/solutions/ai-pipelines/) and try out the queries from the Swagger UI.
#### Adding Files to Index
First, you can try adding your files and seeing changes in the index. To test index updates, simply add more files to the `data` folder.
If you are using Google Drive or other sources, simply upload your files there.
### Using the UI
This pipeline includes a simple ui written in Streamlit. After you run the pipeline with `docker compose up`, you can access the UI at `http://localhost:8501`. This UI uses the `/v2/answer` endpoint to answer your questions.
================================================
FILE: templates/question_answering_rag/app.py
================================================
import logging
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks.llm.question_answering import SummaryQuestionAnswerer
from pathway.xpacks.llm.servers import QASummaryRestServer
from pydantic import BaseModel, ConfigDict, InstanceOf
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
load_dotenv()
class App(BaseModel):
question_answerer: InstanceOf[SummaryQuestionAnswerer]
host: str = "0.0.0.0"
port: int = 8000
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
server = QASummaryRestServer( # noqa: F841
self.host, self.port, self.question_answerer
)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
with open("app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/question_answering_rag/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Question Answering RAG template.
# It defines various components such as data sources, language models, embedders, splitters, parsers, and retrievers.
# Each section is configured to specify how the template should process and handle data for generating responses.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
$sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: data
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Configures the LLM model settings for generating responses.
# The list of available Pathway LLM wrappers is available here:
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
# You can learn more about those in our documentation:
# https://pathway.com/developers/templates/rag-customization/llm-chats
$llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-4.1-mini"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0
capacity: 8
async_mode: "fully_async"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
model: "text-embedding-3-small"
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
# Defines the splitter settings for dividing text into smaller chunks.
$splitter: !pw.xpacks.llm.splitters.TokenCountSplitter
max_tokens: 400
# Configures the parser for processing and extracting information from documents.
$parser: !pw.xpacks.llm.parsers.DoclingParser
table_parsing_strategy: "llm"
async_mode: "fully_async"
chunk: false
cache_strategy: !pw.udfs.DefaultCache {}
# Sets up the retriever factory for indexing and retrieving documents.
$retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Manages the storage and retrieval of documents for the RAG template.
$document_store: !pw.xpacks.llm.document_store.DocumentStore
docs: $sources
parser: $parser
splitter: $splitter
retriever_factory: $retriever_factory
# Configures the question-answering component using the RAG approach.
# The component builds a RAG over an index.
# You can interact with obtained RAG using a REST API.
# You can learn more about the available operations here:
# https://pathway.com/developers/templates/rag-customization/rest-api
question_answerer: !pw.xpacks.llm.question_answering.BaseRAGQuestionAnswerer
llm: $llm
indexer: $document_store
# You can set the number of documents to be included as the context of the query
# search_topk: 6
# You can use your own prompt for querying.
# For that set prompt_template to string with `{query}` used as a placeholder for the question,
# and `{context}` as a placeholder for context documents.
# prompt_template: "Given these documents: {context}, please answer the question: {query}"
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: $PATHWAY_PORT
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/question_answering_rag/docker-compose.yml
================================================
services:
app:
build:
context: .
args:
PATHWAY_SRC_IMAGE: ${PATHWAY_SRC_IMAGE:-pathwaycom/pathway:latest}
ports:
- "${PATHWAY_PORT:-8000}:${PATHWAY_PORT:-8000}"
networks:
- network
environment:
PATHWAY_PORT: "${PATHWAY_PORT:-8000}"
PATHWAY_LICENSE_KEY: $PATHWAY_LICENSE_KEY
volumes:
- ./data:/app/data
- ./Cache:/app/Cache
ui:
build:
context: ui
args:
PATHWAY_SRC_IMAGE: ${PATHWAY_SRC_IMAGE:-pathwaycom/pathway:latest}
networks:
- network
environment:
PATHWAY_HOST: "app"
PATHWAY_PORT: "${PATHWAY_PORT:-8000}"
UI_PORT: 8501
ports:
- "8501:8501"
networks:
network:
driver: bridge
================================================
FILE: templates/question_answering_rag/requirements.txt
================================================
pathway[all]
python-dotenv~=1.0
mpmath~=1.3
================================================
FILE: templates/question_answering_rag/ui/.streamlit/config.toml
================================================
[client]
toolbarMode = "minimal"
[server]
enableStaticServing = true
[theme]
base = "light"
================================================
FILE: templates/question_answering_rag/ui/Dockerfile
================================================
ARG PATHWAY_SRC_IMAGE=pathwaycom/pathway:latest
FROM ${PATHWAY_SRC_IMAGE}
ENV PYTHONUNBUFFERED=1
WORKDIR /ui
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
CMD exec streamlit run ui.py --server.port ${UI_PORT}
================================================
FILE: templates/question_answering_rag/ui/requirements.txt
================================================
streamlit==1.37.0
load_dotenv==0.1.0
================================================
FILE: templates/question_answering_rag/ui/ui.py
================================================
# Copyright © 2026 Pathway
import logging
import os
import streamlit as st
from dotenv import load_dotenv
from pathway.xpacks.llm.document_store import IndexingStatus
from pathway.xpacks.llm.question_answering import RAGClient
load_dotenv()
PATHWAY_HOST = os.environ.get("PATHWAY_HOST", "app")
PATHWAY_PORT = os.environ.get("PATHWAY_PORT", 8000)
st.set_page_config(page_title="Pathway RAG App", page_icon="favicon.ico")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
logger = logging.getLogger("streamlit")
logger.setLevel(logging.INFO)
conn = RAGClient(url=f"http://{PATHWAY_HOST}:{PATHWAY_PORT}")
note = """
Ask a question"""
st.markdown(note, unsafe_allow_html=True)
st.markdown(
"""
""",
unsafe_allow_html=True,
)
question = st.text_input(label="", placeholder="Ask your question?")
def get_indexed_files(metadata_list: list[dict], opt_key: str) -> list:
"""Get all available options in a specific metadata key."""
only_indexed_files = [
file
for file in metadata_list
if file["_indexing_status"] == IndexingStatus.INDEXED
]
options = set(map(lambda x: x[opt_key], only_indexed_files))
return list(options)
def get_ingested_files(metadata_list: list[dict], opt_key: str) -> list:
"""Get all available options in a specific metadata key."""
not_indexed_files = [
file
for file in metadata_list
if file["_indexing_status"] == IndexingStatus.INGESTED
]
options = set(map(lambda x: x[opt_key], not_indexed_files))
return list(options)
logger.info("Requesting list_documents...")
document_meta_list = conn.list_documents(keys=[])
logger.info("Received response list_documents")
st.session_state["document_meta_list"] = document_meta_list
indexed_files = get_indexed_files(st.session_state["document_meta_list"], "path")
ingested_files = get_ingested_files(st.session_state["document_meta_list"], "path")
logo_htm = """
"""
with st.sidebar:
st.markdown(logo_htm, unsafe_allow_html=True)
st.info(
body="See the source code [here](https://github.com/pathwaycom/llm-app/tree/main/templates/question_answering_rag).", # noqa: E501
icon=":material/code:",
)
indexed_file_names = [i.split("/")[-1] for i in indexed_files]
ingested_file_names = [i.split("/")[-1] for i in ingested_files]
markdown_table = "| Indexed files |\n| --- |\n"
for file_name in indexed_file_names:
markdown_table += f"| {file_name} |\n"
if len(ingested_file_names) > 0:
markdown_table += "| Files being processed |\n| --- |\n"
for file_name in ingested_file_names:
markdown_table += f"| {file_name} |\n"
st.markdown(markdown_table, unsafe_allow_html=True)
st.button("⟳ Refresh", use_container_width=True)
css = """
"""
st.markdown(css, unsafe_allow_html=True)
if question:
logger.info(
{
"_type": "search_request_event",
"query": question,
}
)
with st.spinner("Retrieving response..."):
api_response = conn.answer(question, return_context_docs=True)
response = api_response["response"]
context_docs = api_response["context_docs"]
logger.info(
{
"_type": "search_response_event",
"query": question,
"response": type(response),
}
)
logger.info(type(response))
st.markdown(f"**Answering question:** {question}")
st.markdown(f"""{response}""")
with st.expander(label="Context documents"):
st.markdown("Documents sent to LLM as context:\n")
for i, doc in enumerate(context_docs):
st.markdown(
f"{i+1}. Path: {doc['metadata']['path']}\n ```\n{doc['text']}\n```"
)
================================================
FILE: templates/slides_ai_search/.dockerignore
================================================
Cache
================================================
FILE: templates/slides_ai_search/.gitignore
================================================
data
.env
storage
================================================
FILE: templates/slides_ai_search/Dockerfile
================================================
ARG PATHWAY_SRC_IMAGE=pathwaycom/pathway:latest
FROM ${PATHWAY_SRC_IMAGE}
ENV PYTHONUNBUFFERED=1
WORKDIR /app
RUN apt-get update && apt-get install -y \
poppler-utils \
libreoffice \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
CMD [ "python", "app.py" ]
================================================
FILE: templates/slides_ai_search/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# **Slides AI Search App**
## **Overview**
This app template will help you build a multi-modal search service using `GPT-4o` with Metadata Extraction and Vector Index. It uses [Pathway](https://github.com/pathwaycom/llm-app) for indexing and retrieving slides from PowerPoint and PDF presentations.
How is this different?
* Build highly accurate RAG pipelines powered by indexes that are updated in real-time.
* All of the steps, including parsing, embedding and indexing happen locally on your machine (local or cloud).
* Pathway uses vision language models to understand and index your presentations and PDFs, automatically updating as changes are made.
* Get started with a minimalistic and production-ready approach.
Boost productivity with accurate search across your PowerPoints, PDFs, and Slides all within your work environment. Try out the [demo](https://sales-rag-chat.demo.pathway.com/#search-your-slide-decks) here.
## Quickstart
Check the `.env.example`, create a new `.env` file and fill in the template.
For a quick start, you need to only change the following fields:
- `PATHWAY_LICENSE_KEY`
- `OPENAI_API_KEY`
This app template is available for free via [Pathway Scale](https://pathway.com/features). Get your [license key here](https://pathway.com/user/license) and fill in the `PATHWAY_LICENSE_KEY` here in the `.env` file.
To learn more about configuring the input sources, how to overcome OpenAI limits and other information, check out the [configuration section below](#prerequisitesconfiguration).
> **Note:** Pathway API is only used for logging basic statistics, everything happens and stays in your computer, except the OpenAI API calls. No personal or private data will be sent to Pathway servers. Handling of the data, processing, parsing and indexing are done locally.
## How it Helps
**1) Improved Efficiency:**
* **Save Efforts:** You no longer need to manually sift through countless presentations.
* **Faster Information Retrieval:** Instantly find specific information with a few keywords or descriptive prompts, saving you time when preparing for presentations or reviewing past projects.
**2) Enhanced Organization**
* **Automated Categorization:** You can organize your slide library by topic, project, or other criteria. Configure the schema file to customize the parsed fields.
**3) Enhanced Reliability**
* **Automatic Updates:** Hybrid indexes update automatically whenever a new slide is added or removed, ensuring your information is always current and accurate.
**4) Automated Slide Parsing:**
* Process PPTX and PDF slide decks with vision language models to extract the content. (The default setup loads PDF's).
**5) Flexible Data Sources:**
* Compatible with local directories, SharePoint, Google Drive, and other [Pathway connectors](https://pathway.com/developers/user-guide/connect/pathway-connectors), ensuring a wide range of application scenarios can be supported.
By automating the extraction and retrieval of slide information, this app addresses the critical pain point of managing and utilizing extensive slide decks efficiently, enhancing productivity and information accuracy for sales teams.
## Architecture:
The architecture of the Slides AI Search App is designed to connect various local or cloud repositories, transforming and indexing slides for efficient querying. It supports integration with closed and open-source LLMs for enhanced search capabilities.
This demo consists of three parts:
* `app.py`: Pathway app that handles parsing, indexing and backend.
* `nginx`: File server that hosts images to be consumed by the UI.
* `UI`: A Streamlit UI for interacting with the app.
## How it works:
### **Data Ingestion**
1. **Data Sources**:
* The application reads slide files (PPTX and PDF) from a specified directory. The directory is set to `./data/`in the `app.py` file.
* In the default app setup, the connected folder is a local file folder. You can add more folders and file sources, such as [Google Drive](https://pathway.com/developers/user-guide/connectors/gdrive-connector#google-drive-connector) or [Sharepoint](https://pathway.com/developers/user-guide/connecting-to-data/connectors#tutorials), by changing configuration in `app.yaml`.
* More inputs can be added by configuring the `sources` list in the `app.yaml`.
### **Slide Parsing and Indexing**
1. **Parsing**:
* The [`SlideParser`](https://pathway.com/developers/api-docs/pathway-xpacks-llm/parsers#pathway.xpacks.llm.parsers.SlideParser) from Pathway is used to parse the slides. The parser is configured to parse a text description and schema that is defined in the `app.yaml`.
* Our example schema includes fields such as `category`, `tags`, `title`, `main_color`, `language`, and `has_images`. This can be modified for specific use cases.
* Note that, UI is configured to make use of two extracted fields `category` and `language`, these need to be kept for the UI to work. However, the app can still be used without the UI with different schemas or no parsed schema.
2. **Embedding**:
* Parsed slide content is embedded with the OpenAI's `text-embedding-3-small` embedder.
* The embeddings are then stored in Pathway's vector store using the `SlidesVectorStoreServer`.
3. **Metadata Handling**:
* Images and files are dumped into local directories (`storage/pw_dump_images` and `storage/pw_dump_files`).
* Each slide gets a unique ID. This helps with opening files and images from the UI.
### **Query Handling**
1. **Retrieval Augmented Generation (RAG)**:
* The `DeckRetriever` class builds the backend, handling all steps of the application from parsing files to serving the endpoints. Refer to the [API docs](https://pathway.com/developers/api-docs/pathway-xpacks-llm/question_answering#pathway.xpacks.llm.question_answering.DeckRetriever) for more information.
## Pipeline Organization
This folder contains several components necessary for setting up and running the Sales Slide RAG application:
1. **app.py**:
* The main application that sets up the slide search functionality. It initializes the OpenAI vision-language model, slide parser, vector store, and initializes the DeckRetriever for handling queries.
2. **app.yaml**:
* Defines data sources, OpenAI vision-language model configuration, and other key settings.
* Defines the schema for parsing the slides and including fields such as `category`, `tags`, `title`, `main_color`, `language`, and `has_images`. These fields will be appended to the `metadata`, if you prefer to also add them to `text` field, set `include_schema_in_text` of `SlideParser` to `True`.
* **Note:** If you intend the use the default UI, `category` and the `language` fields in the schema are needed for the filtering options in the UI. The UI will not function properly without them.
3. **.env**:
* Config file for the environment variables, such as the OpenAI API key and Pathway key.
## **Prerequisites/Configuration**
### **Environment Setup**
1. **OpenAI API Key**:
* Get an API key from the [OpenAI’s API Key Management page](https://platform.openai.com/account/api-keys). Keep this API key secure.
* Configure your key in the `.env` file.
* You can refer to the stub file `.env.example` in this repository.
* Note: This is only needed in OpenAI LLMs and embedders. It is also possible to use other multi-modal, local LLMs and embedders.
**OpenAI API Usage**:
* This app relies on `gpt-4o` model for image parsing. OpenAI currently limits the usage to paid users only. It is possible to use any other model (including local models) with the modules under the `pathway.xpacks`.
* If you are experiencing API throttle, you can set the `capacity` parameter of the LLM instance `llms.OpenAIChat` to be lower. This parameter defines the number of parallel requests. Or, it is possible to disable parallel requests and only parse sequentially by changing the `run_mode` in the `SlideParser` to `run_mode="sequential"` instead of the `"parallel"`.
* Update: You can also change the model to any of the newer multimodal models, like GPT-5 or GPT-5.1, or their smaller variants.
2. **Pathway’s License Key**:
* This app template is available for free via [Pathway Scale](https://pathway.com/features).
* Get your [license key here](https://pathway.com/user/license).
* **Note:** Pathway API is only used for logging basic statistics, everything happens and stays in your computer except the OpenAI API calls. No personal or private data will be sent to Pathway servers.
### **Configuring the Inputs**
By default, the app takes the files under the `./data/` folder as input. Inputs can be set by adding more entries to the `sources` list under the `app.yaml`.
It is possible to configure the app to use any kind of input, `Google Drive`, `Microsoft 365 SharePoint`, or a `local directory` to name a few.
You can also use other kind of data sources using the [connectors](https://pathway.com/developers/user-guide/connecting-to-data/connectors) provided by Pathway.
Pathway polls the changes with low latency. So, if something changes in the tracked files, the corresponding change is reflected in real-time, and search results are updated accordingly.
To learn more about the data sources, you can check out [demo question answering](../question_answering_rag/README.md#data-sources)
## How to run the project on your machine
First, clone the Pathway LLM App Repository
```bash
git clone https://github.com/pathwaycom/llm-app.git
```
Make sure you are in the right directory:
```bash
cd templates/slides_ai_search
```
> Note: If your OpenAI API usage is throttled, you may want to change the `run_mode` in the `SlideParser` to `run_mode="sequential"` instead of the `"parallel"`.
## Deploying and running with Docker
Build the Docker with:
```bash
docker compose build
```
And, run with:
```bash
docker compose up
```
This will start all three components of the demo. This deployment method is recommended for production use.
## Using the app
After Docker is running, you will see a stream of logs of your files being parsed.
### Accessing the UI
On your browser, visit [`http://localhost:8501`](http://localhost:8501/) to access the UI.
Here, you will see a search bar, some filters, and information about the indexed documents on the left side.
### Sending requests to the server
#### With CURL
UI is not a necessary component, especially for developers. If you are interested in building your own app, check out the following ways to use the app:
First, let's check the indexed files:
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/list_documents' -H 'accept: */*' -H 'Content-Type: application/json'
```
This will return a list of metadata from the indexed files.
Now, let's search through our slides:
```bash
curl -X 'POST' 'http://0.0.0.0:8000/v2/answer' -H 'accept: */*' -H 'Content-Type: application/json' -d '{
"prompt": "diagrams that contain value propositions"
}'
```
This will search through our files, and return parsed slides with the `text`, `slide_id` and other `metadata` (also including the parsed schema).
#### With the Pathway RAG Client
Import RAGClient with:
```python
from pathway.xpacks.llm.question_answering import RAGClient
```
Initialize the client:
```python
# conn = RAGClient(url=f"http://{PATHWAY_HOST}:{PATHWAY_PORT}")
# with the default config
conn = RAGClient(url=f"http://localhost:8000")
```
List the indexed files:
```python
conn.pw_list_documents()
```
> `[{'path': 'data/slide.pdf'}, ...`
Query the app:
```python
conn.pw_ai_answer("introduction slide")
```
> `[{'dist': 0.47761982679367065, 'metadata': ...`
## Advanced variant: Run locally without Open AI, using a local LLM and Embedder
To run the app fully locally, without needing any API access, we use [vLLM](https://github.com/vllm-project/vllm) and open source embedder from the HuggingFace.
### Running the local LLM
We use Phi 3 Vision for its relatively small size and good performance. It is possible to use any other VLM.
1. **Download and Install vLLM:**
See the [installation page](https://docs.vllm.ai/en/latest/getting_started/installation.html).
2. **Run the model:**
The following command will run the Phi 3 vision model and mimic the OpenAI API.
```bash
python -m vllm.entrypoints.openai.api_server --model microsoft/Phi-3-vision-128k-instruct --trust-remote-code --dtype=half --max-model-len=42500 --gpu-memory-utilization 0.9 --swap-space 16 --max-num-seqs 65
```
Check if the model is available with:
```bash
curl http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d '{
"model": "microsoft/Phi-3-vision-128k-instruct",
"prompt": "San Francisco is a",
"max_tokens": 7,
"temperature": 0
}'
```
### Set the LLM Instance in the configuration file
```yaml
llm: !pw.xpacks.llm.llms.OpenAIChat
model: "microsoft/Phi-3-vision-128k-instruct"
temperature: 0.0
capacity: 1
base_url: "http://localhost:8000/v1"
api_key: "ignore the key, not needed"
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 3
```
This will use your local Phi 3 vision model as the LLM for parsing the slides.
### Set an open-source embedder model for embeddings
Here, we can check [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard) to find a good-performing embedder model.
From performance/computational-cost standpoint, `avsolatorio/GIST-Embedding-v0`, `avsolatorio/GIST-small-Embedding-v0`, `mixedbread-ai/mxbai-embed-large-v1`, `Alibaba-NLP/gte-large-en-v1.5` are some of the better models.
Here, we go with the `avsolatorio/GIST-small-Embedding-v0`.
Note that, larger models may take longer to process the inputs.
We replace the `embedder` with the following embedding model in `app.yaml`:
```yaml
$embedding_model: "avsolatorio/GIST-small-Embedding-v0"
embedder: !pw.xpacks.llm.embedders.SentenceTransformerEmbedder
model: $embedding_model
call_kwargs:
show_progress_bar: false
```
## Advanced variant: Running without Docker
Running the whole demo without Docker is a bit tricky as there are three components.
1. **Download and Install LibreOffice:**
* Download LibreOffice from the [LibreOffice website](https://www.libreoffice.org/download/download-libreoffice).
* Follow the installation instructions specific to your operating system. \
2. **Verify LibreOffice Installation:**
* Download LibreOffice from the LibreOffice website.
* Open a terminal or command prompt and run the following command:
* You should see the LibreOffice version information, indicating LibreOffice is installed correctly.
**Purpose:** LibreOffice helps with converting PPTX files into PDFs, which is essential for the document processing workflow in the Slides AI Search App.
If you are on Windows, please refer to the [running with Docker](#Running-with-docker) section below.
To run the Pathway app without the UI,
```bash
python app.py
```
## Not sure how to get started?
Let's discuss how we can help you build a powerful, customized RAG application. [Reach us here to talk or request a demo!](https://pathway.com/solutions/slides-ai-search?modal=requestdemo)
## FAQ
> I am getting OpenAI API throttle limit errors.
- You can change `run_mode` in `SlideParser` to `run_mode="sequential"`. This will parse images one by one, however, this will significantly slow down the parsing stage.
> UI shows that my file is being indexed, but I don't have that file in the inputs.
- App mounts `storage` folder from the Docker to the local file system. This helps the file server serve the content. This folder is not cleaned between the runs, files from the previous runs will be staying here. You can remove the folder after closing the app to get rid of these.
> Can I use other vision LMs or LLMs?
- Yes, you can configure the `OpenAIChat` to reach local LLMs or swap it with any other LLM wrappers (such as `LiteLLMChat`) to use other models. Make sure your model of choice supports vision inputs.
> Can I persist the cache between the runs?
- Yes, you can uncomment the `- ./Cache:/app/Cache` under the `app:/volumes:` section inside the `docker-compose.yml` to allow caching between the runs. You will see that requests are not repeated in the next runs.
================================================
FILE: templates/slides_ai_search/app.py
================================================
#!/usr/bin/env python
# Copyright © 2026 Pathway
# Copied and adapted from templates/slides_ai_search/app.py
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it in the `.env` file (check `.env.example`).
from pathlib import Path
from typing import Any
from warnings import warn
import pathway as pw
from dotenv import load_dotenv
from pathway.xpacks import llm
from pathway.xpacks.llm.document_store import SlidesDocumentStore
from pathway_slides_ai_search import DeckRetrieverWithFileSave, add_slide_id, get_model
from pydantic import BaseModel, ConfigDict, FilePath, InstanceOf
class App(BaseModel):
host: str = "0.0.0.0"
port: int = 8000
sources: list[InstanceOf[pw.Table]]
llm: InstanceOf[pw.UDF]
retriever_factory: InstanceOf[pw.indexing.AbstractRetrieverFactory]
search_topk: int = 6
details_schema: FilePath | dict[str, Any] | None = None
with_cache: bool | None = None # deprecated
persistence_backend: pw.persistence.Backend | None = None
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
terminate_on_error: bool = False
def run(self) -> None:
if self.details_schema is not None:
detail_schema = get_model(self.details_schema)
else:
detail_schema = None
parser = llm.parsers.SlideParser(
detail_parse_schema=detail_schema,
run_mode="parallel",
include_schema_in_text=False,
llm=self.llm,
cache_strategy=pw.udfs.DefaultCache(),
async_mode="fully_async",
)
doc_store = SlidesDocumentStore(
self.sources,
retriever_factory=self.retriever_factory,
splitter=None,
parser=parser,
doc_post_processors=[add_slide_id],
)
app = DeckRetrieverWithFileSave(
indexer=doc_store,
search_topk=self.search_topk,
)
app.build_server(host=self.host, port=self.port)
if self.persistence_mode is None:
if self.with_cache is True:
warn(
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
DeprecationWarning,
)
persistence_mode = pw.PersistenceMode.UDF_CACHING
else:
persistence_mode = None
else:
persistence_mode = self.persistence_mode
if persistence_mode is not None:
if self.persistence_backend is None:
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
else:
persistence_backend = self.persistence_backend
persistence_config = pw.persistence.Config(
persistence_backend,
persistence_mode=persistence_mode,
)
else:
persistence_config = None
pw.run(
persistence_config=persistence_config,
terminate_on_error=self.terminate_on_error,
monitoring_level=pw.MonitoringLevel.NONE,
)
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
if __name__ == "__main__":
base_dir = Path(__file__).resolve().parent
load_dotenv(base_dir / ".env")
with open(base_dir / "app.yaml") as f:
config = pw.load_yaml(f)
app = App(**config)
app.run()
================================================
FILE: templates/slides_ai_search/app.yaml
================================================
# This YAML configuration file is used to set up and configure the Slides Search RAG template.
# It defines various components such as data sources, language models, embedders, indices, and data schema.
# Each section is configured to specify how the template should process and handle data for generating responses.
# You can learn more about the YAML syntax here: https://pathway.com/developers/templates/configure-yaml
# $sources defines the data sources used to read the data which will be indexed in the RAG.
# You can learn more how to configure data sources here:
# https://pathway.com/developers/templates/yaml-examples/data-sources-examples
sources:
# File System connector, reading data locally.
- !pw.io.fs.read
path: data
format: binary
with_metadata: true
# Uncomment to use the SharePoint connector
# - !pw.xpacks.connectors.sharepoint.read
# url: $SHAREPOINT_URL
# tenant: $SHAREPOINT_TENANT
# client_id: $SHAREPOINT_CLIENT_ID
# cert_path: sharepointcert.pem
# thumbprint: $SHAREPOINT_THUMBPRINT
# root_path: $SHAREPOINT_ROOT
# with_metadata: true
# refresh_interval: 30
# Uncomment to use the Google Drive connector
# - !pw.io.gdrive.read
# object_id: $DRIVE_ID
# service_user_credentials_file: gdrive_indexer.json
# file_name_pattern:
# - "*.pdf"
# - "*.pptx"
# object_size_limit: null
# with_metadata: true
# refresh_interval: 30
# Configures the LLM model settings for generating responses.
# The list of available Pathway LLM wrappers is available here:
# https://pathway.com/developers/api-docs/pathway-xpacks-llm/llms
# You can learn more about those in our documentation:
# https://pathway.com/developers/templates/rag-customization/llm-chats
llm: !pw.xpacks.llm.llms.OpenAIChat
model: "gpt-4o"
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy
max_retries: 6
initial_delay: 2500
backoff_factor: 2.5
cache_strategy: !pw.udfs.DefaultCache {}
temperature: 0.0
capacity: 8 # reduce this in case you are hitting API throttle limits
async_mode: "fully_async"
# Specifies the embedder model for converting text into embeddings.
$embedder: !pw.xpacks.llm.embedders.OpenAIEmbedder
cache_strategy: !pw.udfs.DefaultCache {}
retry_strategy: !pw.udfs.ExponentialBackoffRetryStrategy {}
model: "text-embedding-3-small"
# Sets up the retriever factory for indexing and retrieving documents.
retriever_factory: !pw.indexing.UsearchKnnFactory
reserved_space: 1000
embedder: $embedder
metric: !pw.indexing.USearchMetricKind.COS
# Defines the schema used for the data extraction of each slide.
details_schema:
category:
type: option
values:
- "Problem Statement"
- "Solution Overview"
- "Product/Service Demo"
- "Benefits and Value Proposition"
- "Competitive Landscape"
- "Case Studies and Testimonials"
- "Features and Specifications"
- "How it Works"
- "Return on Investment (ROI) and Cost Savings"
- "Call to Action (CTA) and Next Steps"
- "Company Overview and Credentials"
- "Market Opportunity and Trends"
- "Customer Success Stories and Use Cases"
- "Pricing and Packaging"
- "Implementation and Support"
tags:
type: list[str]
description: "Tags associated with the slide. For example ['price', 'shopping', 'consumer'] or ['beverage', 'cola', 'manufacturing']"
title:
type: str
description: "Title of the slide"
main_color:
type: str
description: "Most common color. For example 'black', 'blue', 'yellow'"
language:
type: str
description: "Language of the slide. For example 'fr', 'en'"
has_images:
type: bool
description: "Whether the slide contains photographs"
# Change host and port of the webserver by uncommenting these lines
# host: "0.0.0.0"
# port: $PATHWAY_PORT
# By default, caching is enabled for UDFs with cache_strategy set.
# You can disable it by uncommenting the following line.
# persistence_mode: null
# You can also set persistence_mode to !pw.PersistenceMode.PERSISTING to enable persistence
# across restarts.
# By default, when enabled, Cache is stored in .Cache directory.
# You can customize the location by uncommenting and modifying the following lines:
# persistence_backend: !pw.persistence.Backend.filesystem
# path: ".Cache"
# If `terminate_on_error` is true then the program will terminate whenever any error is encountered.
# Defaults to false, uncomment the following line if you want to set it to true
# terminate_on_error: true
================================================
FILE: templates/slides_ai_search/docker-compose.yml
================================================
services:
app:
build:
context: .
env_file:
- .env
environment:
PATHWAY_PORT: "${PATHWAY_PORT:-8000}"
PATHWAY_LICENSE_KEY: $PATHWAY_LICENSE_KEY
ports:
- "${PATHWAY_PORT:-8000}:${PATHWAY_PORT:-8000}"
networks:
- network
volumes:
- ./data:/app/data
- ./storage/pw_dump_files:/app/storage/pw_dump_files
- ./storage/pw_dump_images:/app/storage/pw_dump_images
- ./Cache:/app/Cache
nginx:
build:
context: nginx
ports:
- "8080:8080"
- "8443:8443"
networks:
- network
volumes:
- ./storage/pw_dump_files:/app/pw_dump_files
- ./storage/pw_dump_images:/app/pw_dump_images
ui:
build:
context: ui
environment:
PATHWAY_HOST: "app"
PATHWAY_PORT: "${PATHWAY_PORT:-8000}"
UI_PORT: 8501
ports:
- "8501:8501"
networks:
- network
networks:
network:
driver: bridge
================================================
FILE: templates/slides_ai_search/nginx/Dockerfile
================================================
FROM nginx:latest
COPY nginx.conf /etc/nginx/conf.d/default.conf
RUN mkdir -p /app/pw_dump_images /app/pw_dump_files
EXPOSE 8080 8443
================================================
FILE: templates/slides_ai_search/nginx/nginx.conf
================================================
server {
listen 8080;
listen 8443;
location = / {
deny all;
}
location /images {
alias /app/pw_dump_images;
autoindex on;
}
location /documents {
alias /app/pw_dump_files;
default_type application/pdf;
add_header Content-Disposition 'inline';
autoindex on;
}
}
================================================
FILE: templates/slides_ai_search/pathway_slides_ai_search/__init__.py
================================================
# Copyright © 2026 Pathway
import base64
import logging
import os
from pathlib import Path
from typing import Any, Literal
import pathway as pw
import yaml
from pathway.xpacks.llm.question_answering import DeckRetriever
from pydantic import BaseModel, Field, create_model
CUSTOM_FIELDS = {"option": Literal}
def get_model_from_file(file_path: str | os.PathLike[str]) -> type[BaseModel]:
"""
Return Pydantic schema from a YAML file.
Replaces types of `CUSTOM_FIELDS` with the definitions, other types are evaluated
as primitive Python type.
Args:
- file_path: Path of the YAML file.
"""
with open(file_path, "r") as file:
schema = yaml.safe_load(file)
return get_model_from_dict(schema)
def get_model_from_dict(schema: dict[str, Any]) -> type[BaseModel]:
fields: dict[str, Any] = {}
if schema.keys() == {"fields"} and "type" not in schema["fields"]:
schema = schema["fields"]
for field_name, field_info in schema.items():
field_type: object
match field_info.pop("type"):
case "option":
field_type = Literal[tuple(field_info.pop("values"))]
case f_type:
field_type = f_type
fields[field_name] = (field_type, Field(**field_info))
return create_model("ParsePydanticSchema", **fields)
def get_model(source: str | os.PathLike[str] | dict[str, Any]) -> type[BaseModel]:
if isinstance(source, dict):
return get_model_from_dict(source)
else:
return get_model_from_file(source)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True, # instructor library overrides `basicConfig`, this fixes logging
)
STORAGE_FOLDER = Path("storage")
IMAGE_DUMP_FOLDER = STORAGE_FOLDER / "pw_dump_images"
FILE_DUMP_FOLDER = STORAGE_FOLDER / "pw_dump_files"
def encode_str(original_string: str) -> str:
return base64.urlsafe_b64encode(original_string.encode("utf-8")).decode("us-ascii")
def add_slide_id(text: str, metadata: dict) -> tuple[str, dict]:
encoded_name = encode_str(metadata["path"])
page = metadata["image_page"]
page_count = metadata["tot_pages"]
slide_id = f"{encoded_name}_{page}_{page_count}.png"
logging.info(f"`add_slide_id` for {slide_id}...")
metadata["slide_id"] = slide_id
return (text, metadata)
class DeckRetrieverWithFileSave(DeckRetriever):
def dump_img_callback(self, key, row, time, is_addition):
# save images parsed by the Pathway
metadata = row["data"]
slide_id = metadata["slide_id"].value
file_path = IMAGE_DUMP_FOLDER / slide_id
if is_addition:
data = base64.b64decode(metadata["b64_image"].value)
file_path.write_bytes(data)
else:
try:
file_path.unlink()
logging.info("Removed %s", file_path)
except Exception as e:
logging.info("Error removing %s: %s", file_path, e)
def dump_file_callback(self, key, row, time, is_addition):
# save parsed files
file_name = row["path"].value.rpartition("/")[-1] # XXX
file_path = FILE_DUMP_FOLDER / file_name
if is_addition:
file_path.write_bytes(row["data"])
else:
try:
file_path.unlink()
logging.info("Removed %s", file_path)
except Exception as e:
logging.info("Error removing %s: %s", file_path, e)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
IMAGE_DUMP_FOLDER.mkdir(parents=True, exist_ok=True)
FILE_DUMP_FOLDER.mkdir(parents=True, exist_ok=True)
chunked_docs = self.indexer.chunked_docs
t = chunked_docs.select(
data=pw.this.metadata,
)
pw.io.subscribe(t, on_change=self.dump_img_callback)
docs = self.indexer.input_docs
t = docs.select(data=docs.text, path=docs.metadata["path"])
pw.io.subscribe(t, on_change=self.dump_file_callback)
================================================
FILE: templates/slides_ai_search/requirements.txt
================================================
python-dotenv~=1.0
================================================
FILE: templates/slides_ai_search/ui/.streamlit/config.toml
================================================
[server]
enableStaticServing = true
[theme]
base = "light"
================================================
FILE: templates/slides_ai_search/ui/Dockerfile
================================================
ARG PATHWAY_SRC_IMAGE=pathwaycom/pathway:latest
FROM ${PATHWAY_SRC_IMAGE}
ENV PYTHONUNBUFFERED=1
WORKDIR /ui
COPY requirements.txt .
RUN pip install -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8501
CMD [ "streamlit", "run", "ui.py" ]
================================================
FILE: templates/slides_ai_search/ui/requirements.txt
================================================
streamlit==1.35.0
load_dotenv==0.1.0
nest_asyncio==1.6.0
aiohttp==3.9.5
beautifulsoup4==4.12.3
openai==1.35.10
================================================
FILE: templates/slides_ai_search/ui/ui.py
================================================
# Copyright © 2026 Pathway
import logging
import os
import urllib.parse
from itertools import cycle
from pathlib import PurePosixPath
import requests
import streamlit as st
from bs4 import BeautifulSoup
from dotenv import load_dotenv
from pathway.xpacks.llm.question_answering import RAGClient
load_dotenv()
PATHWAY_HOST = os.environ.get("PATHWAY_HOST", "app")
PATHWAY_PORT = os.environ.get("PATHWAY_PORT", 8000)
st.set_page_config(page_title="Find the right slide", page_icon="favicon.ico")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(name)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
force=True,
)
logger = logging.getLogger("streamlit")
logger.setLevel(logging.INFO)
conn = RAGClient(url=f"http://{PATHWAY_HOST}:{PATHWAY_PORT}")
file_server_base_url = os.environ.get("FILE_SERVER_URL", "http://localhost:8080/")
file_server_image_base_url = f"{file_server_base_url}images"
file_server_pdf_base_url = f"{file_server_base_url}documents"
internal_file_server_pdf_base_url = "http://nginx:8080/"
note = """
Search your slide decks"""
st.markdown(note, unsafe_allow_html=True)
st.markdown(
"""
""",
unsafe_allow_html=True,
)
question = st.text_input(label="", placeholder="What are you searching for?")
def get_options_list(metadata_list: list[dict], opt_key: str) -> list:
"""Get all available options in a specific metadata key."""
options = set(map(lambda x: x[opt_key], metadata_list))
return list(options)
def parse_slide_id_components(slide_id: str) -> tuple[str, int, int]:
stem = PurePosixPath(slide_id).stem
(name_page, _, page_count) = stem.rpartition("_")
(name, _, page) = name_page.rpartition("_")
return (name, int(page), int(page_count))
def create_slide_url(name: str, page: int, page_count: int) -> str:
return f"{file_server_image_base_url}/{name}_{page}_{page_count}.png"
def get_image_serve_url(metadata: dict) -> str:
name, page, page_count = parse_slide_id_components(metadata["slide_id"])
return create_slide_url(name, page, page_count)
def get_adjacent_image_urls(metadata: dict) -> list[str]:
logger.info(
{"_type": "create_adjacent_image_urls", "slide_id": metadata["slide_id"]}
)
name, page, page_count = parse_slide_id_components(metadata["slide_id"])
ret_images = []
for page in range(page - 2, page + 3):
if page < 0 or page >= page_count:
continue
ret_images.append(create_slide_url(name, page, page_count))
return ret_images
st.session_state["available_categories"] = None
st.session_state["available_languages"] = None
logger.info("Requesting list_documents...")
document_meta_list = conn.list_documents(keys=[])
logger.info("Received response list_documents")
st.session_state["document_meta_list"] = document_meta_list
available_categories = get_options_list(document_meta_list, "category")
st.session_state["available_categories"] = available_categories
available_languages = get_options_list(document_meta_list, "language")
st.session_state["available_languages"] = available_languages
available_files = get_options_list(st.session_state["document_meta_list"], "path")
def get_slide_link(file_name, page_num=None) -> str:
filename_encoded = urllib.parse.quote(file_name)
image_url = f"{file_server_pdf_base_url}/{filename_encoded}"
if page_num is not None:
image_url += f"#page={page_num}"
return image_url
def get_all_index_files() -> list[str]:
logger.info("request get_all_index_files")
response = requests.get(internal_file_server_pdf_base_url + "/")
logger.info("response get_all_index_files")
if response.status_code == 200:
soup = BeautifulSoup(response.content, "html.parser")
file_links = [a["href"] for a in soup.find_all("a", href=True)]
file_links = [link for link in file_links if not link.endswith("/")]
else:
file_links = []
return file_links
with st.sidebar:
st.info(
"""This demo app only allows `PDF` and `PPTX` documents.
For other file types, convert to `PDF` or contact **Pathway**."""
)
st.info(
body="See the source code [here](https://github.com/pathwaycom/llm-app/tree/main/templates/slides_ai_search).", # noqa: E501
icon=":material/code:",
)
file_names = [i.split("/")[-1] for i in available_files]
links = [get_slide_link(i) for i in file_names]
markdown_table = "| Slides Ready for Search |\n| --- |\n"
for file_name, link in zip(file_names, links):
markdown_table += f"| [{file_name}]({link}) |\n"
st.markdown(markdown_table, unsafe_allow_html=True)
all_drive_files = get_all_index_files()
all_drive_files = [urllib.parse.unquote(i) for i in all_drive_files]
all_drive_files = [
i for i in all_drive_files if i.endswith(".pdf") or i.endswith(".pptx")
]
logger.info(f"All source files: {all_drive_files}\nIndexed files: {file_names}")
currently_processing_files = set(all_drive_files) - set(file_names)
st.markdown("\n\n", unsafe_allow_html=True)
if currently_processing_files:
markdown_table = "| Indexing in Progress |\n| --- |\n"
links = [get_slide_link(i) for i in currently_processing_files]
for file_name, link in zip(currently_processing_files, links):
markdown_table += f"| [{file_name}]({link}) |\n"
st.markdown(markdown_table, unsafe_allow_html=True)
else:
st.markdown("## All files indexed.")
st.button("⟳ Refresh", use_container_width=True)
category_options = st.session_state["available_categories"]
lang_options = st.session_state["available_languages"]
cols = cycle(st.columns(2))
with next(cols):
cat_options = st.multiselect(
"Filtered Categories",
category_options or [],
[],
key="cat_selection",
label_visibility="hidden",
placeholder="Filtered Categories",
)
with next(cols):
language_options = st.multiselect(
"Languages",
lang_options or [],
[],
key="lang_selection",
label_visibility="hidden",
placeholder="Filtered Languages",
)
with st.sidebar:
cat_prefix = "cat_"
logger.info("All category options: %s", category_options)
selected_categories = category_options if len(cat_options) == 0 else cat_options
logger.info("Selected categories: %s", selected_categories)
lang_prefix = "lang_"
selected_languages = (
lang_options if len(language_options) == 0 else language_options
)
st.session_state.category_filter = selected_categories
st.session_state.language_filter = selected_languages
def get_category_filter(category: str) -> str:
return f"contains(`{str(category)}`, category)"
# TODO: merge these
def get_language_filter(lang: str) -> str:
return f"contains(`{str(lang)}`, language)"
def combine_filters(*args: str | None) -> str:
"""Construct single jmespath filter with `&&` from number of filters."""
return " && ".join([arg for arg in args if arg is not None])
css = """
"""
st.markdown(css, unsafe_allow_html=True)
def get_ext_img_with_href(url, target_url, *args) -> str:
width: int = 600
margin = 20
def get_img_html(dc):
return f"""
"""
slider_images = "\n".join([get_img_html(dc) for dc in args])
html_code = f"""
""" # noqa: E501
return html_code
def log_rate_answer(event, idx, kwargs):
logger.info({"_type": "rate_event", "rating": event, "rank": idx, **kwargs})
if question:
select_cat = st.session_state.category_filter
select_lang = st.session_state.language_filter
filter_ls = [get_category_filter(select_cat), get_language_filter(select_lang)]
combined_query_filter = combine_filters(*filter_ls)
logger.info(
{
"_type": "search_request_event",
"filter": combined_query_filter,
"query": question,
}
)
response = conn.answer(question, filters=combined_query_filter)
logger.info(
{
"_type": "search_response_event",
"filter": combined_query_filter,
"query": question,
"response": type(response),
}
)
if response:
logger.info(type(response[0]))
text_responses = [r["text"] for r in response]
image_metadatas = [r["metadata"] for r in response]
for m in image_metadatas:
logger.info("Retrieved metadatas: %s || %s", m["language"], m["category"])
st.markdown(f"**Searched for:** {question}")
for idx, cur_metadata in enumerate(image_metadatas):
file_name = cur_metadata["path"].split("/")[-1]
select_page = cur_metadata["image_page"] + 1
adjacent_urls = get_adjacent_image_urls(cur_metadata)
args = [{"url": i} for i in adjacent_urls]
image_html = get_ext_img_with_href(
get_image_serve_url(cur_metadata),
get_slide_link(file_name, select_page),
*args,
)
image_url = get_slide_link(file_name, select_page)
slide_id = cur_metadata["slide_id"]
st.markdown(f"Page `{select_page}` of [`{file_name}`]({image_url})")
st.markdown(image_html, unsafe_allow_html=True)
log_args = (
idx,
{
"slide_id": slide_id,
"filter": combined_query_filter,
"query": question,
"file_name": file_name,
"selected_cat": select_cat,
"selected_lang": select_lang,
},
)
col1, col2, col3 = st.columns([12, 1, 1])
else:
st.markdown(
f"""No results were found for search query: `{question}`
and filter criteria: `{combined_query_filter}`"""
)
================================================
FILE: templates/unstructured_to_sql_on_the_fly/Dockerfile
================================================
FROM pathwaycom/pathway:latest
WORKDIR /app
RUN apt-get update \
&& apt-get install -y python3-opencv \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY requirements.txt .
RUN pip install --pre -U --no-cache-dir -r requirements.txt
COPY . .
EXPOSE 8080
CMD ["python", "app.py"]
================================================
FILE: templates/unstructured_to_sql_on_the_fly/README.md
================================================
Deploy with GCP |
Deploy with AWS |
Deploy with Azure |
Deploy with Render
# Pathway + PostgreSQL + LLM: app for querying financial reports with live document structuring pipeline
The aim of this pipeline is to extract and structure the data out of unstructured data (PDFs, queries)
on the fly.
This example consists of two separate parts that can be used independently.
1 - Pipeline 1: Proactive data pipeline that is always live and tracking file changes,
it reads documents, structures them and writes results to PostgreSQL.
2 - Pipeline 2: Query answering pipeline that reads user queries, and answers them by
generating SQL queries that are run on the data stored in PostgreSQL.
Specifically, Pipeline 1 reads in a collection of financial PDF documents from a local directory
(that can be synchronized with a Dropbox account), tokenizes each document using the tiktoken encoding,
then extracts, using the OpenAI API, the wanted fields.
The values are stored in a Pathway table which is then output to a PostgreSQL instance.
Pipeline 2 then starts a REST API endpoint serving queries about programming in Pathway.
Each query text is converted into a SQL query using the OpenAI API.
Architecture diagram and description are at
https://pathway.com/developers/templates/rag/unstructured-to-structured
⚠️ This project requires a running PostgreSQL instance.
🔵 The extracted fields from the PDFs documents are the following:
- company_symbol: str
- year: int
- quarter: str
- revenue_md: float
- eps: float
- net_income_md: float
⚠️ The revenue and net income are expressed in millions of dollars, the eps is in dollars.
🔵 The script uses a prompt to instruct the Language Model and generate SQL queries that adhere to the specified format.
The allowed queries follow a particular pattern:
1. The SELECT clause should specify columns or standard aggregator operators (SUM, COUNT, MIN, MAX, AVG).
2. The WHERE clause should include conditions using standard binary operators (<, >, =, etc.),
with support for AND and OR logic.
3. To prevent 'psycopg2.errors.GroupingError', relevant columns from the WHERE clause are included
in the GROUP BY clause.
4. For readability, if no aggregator is used, the company_symbol, year,
and quarter are included in addition to the wanted columns.
Example:
"What is the net income of all companies?" should return:
Response:
'SELECT company_symbol, net_income_md, quarter, net_income_md FROM table;'
## Project architecture:
```
.
├── postgresql/
│ └── init-db.sql
├── ui/
│ └── server.py
├── __init__.py
├── app.py
└── docker-compose.yml
```
## Running the Pipeline
### Setup environment:
Set your env variables in the .env file placed in this directory.
```bash
OPENAI_API_KEY=sk-...
PATHWAY_PERSISTENT_STORAGE= # Set this variable if you want to use caching
```
### With Docker
To run jointly the Unstructured to SQL on the fly pipeline and a simple UI please execute:
```bash
docker compose up --build
```
Then, the UI will run at http://127.0.0.1:8501 by default. You can access it by following this URL in your web browser.
The `docker-compose.yml` file declares a [volume bind mount](https://docs.docker.com/reference/cli/docker/container/run/#volume) that makes changes to files under `data/` made on your host computer visible inside the docker container. The files in `data/quarterly_earnings` are indexed by the pipeline - you can paste new files there and they will impact the computations.
### Manually
Alternatively, you can run each service separately. To run PostgreSQL use Docker. To run it, run:
`docker compose up -d postgres`.
To install the dependencies, run:
`pip install -r requirements.txt`
Then run:
`python app.py`
This will run the pipeline, which you can access through REST API at `localhost:8080`. For example, you can send questions using curl:
```
curl --data '{
"user": "user",
"query": "What is the maximum quarterly revenue achieved by Apple?"
}' http://localhost:8080/ | jq
```
You can also run the Streamlit interface:
`streamlit run ui/server.py`
================================================
FILE: templates/unstructured_to_sql_on_the_fly/__init__.py
================================================
from .app import run
__all__ = ["run"]
================================================
FILE: templates/unstructured_to_sql_on_the_fly/app.py
================================================
"""
Microservice for accounting assistant.
The aim of this project is to extract and structure the data out of unstructured data (PDFs, queries)
on the fly.
This example consists of two separate parts that can be used independently.
1 - Pipeline 1: Proactive data pipeline that is always live and tracking file changes,
it reads documents, structures them and writes results to PostgreSQL.
2 - Pipeline 2: Query answering pipeline that reads user queries, and answers them by
generating SQL queries that are run on the data stored in PostgreSQL.
Specifically, Pipeline 1 reads in a collection of financial PDF documents from a local directory
(that can be synchronized with a Dropbox account), tokenizes each document using the tiktoken encoding,
then extracts, using the OpenAI API, the wanted fields.
The values are stored in a Pathway table which is then output to a PostgreSQL instance.
Pipeline 2 then starts a REST API endpoint serving queries about programming in Pathway.
Each query text is converted into a SQL query using the OpenAI API.
Architecture diagram and description are at
https://pathway.com/developers/templates/rag/unstructured-to-structured
⚠️ This project requires a running PostgreSQL instance.
🔵 The extracted fields from the PDFs documents are the following:
- company_symbol: str
- year: int
- quarter: str
- revenue_md: float
- eps: float
- net_income_md: float
⚠️ The revenue and net income are expressed in millions of dollars, the eps is in dollars.
🔵 The script uses a prompt to instruct the Language Model and generate SQL queries that adhere to the specified format.
The allowed queries follow a particular pattern:
1. The SELECT clause should specify columns or standard aggregator operators (SUM, COUNT, MIN, MAX, AVG).
2. The WHERE clause should include conditions using standard binary operators (<, >, =, etc.),
with support for AND and OR logic.
3. To prevent 'psycopg2.errors.GroupingError', relevant columns from the WHERE clause are included
in the GROUP BY clause.
4. For readability, if no aggregator is used, the company_symbol, year,
and quarter are included in addition to the wanted columns.
Example:
"What is the net income of all companies?" should return:
Response:
'SELECT company_symbol, net_income_md, quarter, net_income_md FROM table;'
Please check the README.md in this directory for how-to-run instructions.
"""
import json
import logging
import os
import dotenv
import pathway as pw
import psycopg
import tiktoken
from pathway.stdlib.utils.col import unpack_col
from pathway.xpacks.llm.llms import OpenAIChat, prompt_chat_single_qa
from pathway.xpacks.llm.parsers import UnstructuredParser
# To use advanced features with Pathway Scale, get your free license key from
# https://pathway.com/features and paste it below.
# To use Pathway Community, comment out the line below.
pw.set_license_key("demo-license-key-with-telemetry")
dotenv.load_dotenv()
class FinancialStatementSchema(pw.Schema):
company_symbol: str
year: int
quarter: str
revenue_md: float
eps: float
net_income_md: float
class NLQuerySchema(pw.Schema):
query: str
user: str
@pw.udf
def build_prompt_structure(
texts: list[str],
max_tokens: int = 8000,
encoding_name: str = "cl100k_base",
):
"""
Insert instructions for the LLM here.
max_tokens for the context. If gpt-3.5-turbo-16k is used, set it to 16k.
"""
docs_str = " ".join(texts)
encoding = tiktoken.get_encoding(encoding_name)
prompt_prefix = "Given the following quarterly earnings release : \n"
prompt_suffix = (
f" \nfill in this schema for the quarter in question {FinancialStatementSchema.typehints()}\n"
+ """while respecting the instructions:
- amounts should be in millions of dollars.
- Parse quarterly data and ignore yearly records if present.
- Your answer should be parseable by json. i.e. json.loads(response) doesn't throw any errors."""
)
prefix_tokens = len(list(encoding.encode_ordinary(prompt_prefix)))
suffix_tokens = len(list(encoding.encode_ordinary(prompt_suffix)))
# Calculate available tokens for docs_str
available_tokens = max_tokens - (prefix_tokens + suffix_tokens)
# Tokenize docs_str and truncate if needed
doc_tokens = list(encoding.encode_ordinary(docs_str))
if len(doc_tokens) > available_tokens:
logging.warning("Document is too large for one query.")
docs_str = encoding.decode(doc_tokens[:available_tokens])
prompt = prompt_prefix + docs_str + prompt_suffix
return prompt
@pw.udf
def build_prompt_query(postresql_table: str, query: str) -> str:
prompt = f"""Transform the given query '{query}' into a specific SQL SELECT statement format.
For invalid queries, return the string 'None'. The result should be executable in PostgreSQL.
The query should include the following components:
The SELECT clause should specify one or more columns from the table {postresql_table}.
You can use column names or standard aggregator operators such as SUM, COUNT, MIN, MAX, or AVG
to retrieve data from the columns.
The WHERE clause should include conditions that use standard binary operators (e.g., <, >, =) to filter the data.
You can use AND and OR operators to combine multiple conditions.
If any columns from the WHERE clause are used in the conditions, please ensure that those columns are included
in the GROUP BY clause to prevent the 'psycopg2.errors.GroupingError.'
You may use logical reasoning to decide which columns should be part of the GROUP BY clause.
The columns are from {postresql_table} table whose schema is:
company_symbol (str)
year (int)
quarter (str)
revenue_md (float)
eps (float)
net_income_md (float)
Quarter values are Q1, Q2, Q3 or Q4.
The company_symbol is the stock name: for example AAPL for Apple and GOOG for Google.
If no aggregator are used, please always add the company_symbol, year,
and quarter in addition of the wanted columns:
"What is the net income of all companies?" should return:
'SELECT company_symbol, net_income_md, quarter, net_income_md FROM {postresql_table};'
Please ensure that the generated SQL query follows this structure and constraints.
For example, a valid query might look like:
'SELECT company_symbol, SUM(net_income_md) FROM {postresql_table}
WHERE year = 2022 AND eps > 1.0 GROUP BY company_symbol;'
Make sure the query adheres to the specified format, and that it includes GROUP BY clause
if you aggregate results
and do not include any other SQL commands or clauses besides the SELECT statement.
Thank you!"""
return prompt
@pw.udf
def parse_str_to_list(response: str) -> list:
dct = json.loads(response)
return [dct[k] for k in sorted(dct)]
def structure_on_the_fly(
documents: pw.Table,
api_key: str,
model_locator: str,
max_tokens: int,
temperature: float,
):
prompt = documents.select(prompt=build_prompt_structure(pw.this.texts))
model = OpenAIChat(
api_key=api_key,
model=model_locator,
temperature=temperature,
max_tokens=max_tokens,
retry_strategy=pw.udfs.ExponentialBackoffRetryStrategy(),
cache_strategy=pw.udfs.DefaultCache(),
)
responses = prompt.select(
result=model(prompt_chat_single_qa(pw.this.prompt)),
)
responses = responses.select(values=parse_str_to_list(pw.this.result))
result = unpack_col(responses.values, *sorted(FinancialStatementSchema.keys()))
result = result.with_columns(
eps=pw.apply(float, pw.this.eps),
net_income_md=pw.apply(float, pw.this.net_income_md),
revenue_md=pw.apply(float, pw.this.revenue_md),
)
return result
def unstructured_query(
postgreSQL_settings,
postgreSQL_table,
api_key: str,
model_locator: str,
max_tokens: int,
temperature: float,
host: str,
port: int,
):
query, response_writer = pw.io.http.rest_connector(
host=host,
port=port,
schema=NLQuerySchema,
autocommit_duration_ms=50,
delete_completed_queries=True,
)
query += query.select(prompt=build_prompt_query(postgreSQL_table, pw.this.query))
model = OpenAIChat(
api_key=api_key,
model=model_locator,
temperature=temperature,
max_tokens=max_tokens,
retry_strategy=pw.udfs.ExponentialBackoffRetryStrategy(),
cache_strategy=pw.udfs.DefaultCache(),
)
query += query.select(
sql_query=model(prompt_chat_single_qa(pw.this.prompt)),
)
# Connecting to the document database for queries
connection_string = psycopg.conninfo.make_conninfo(**postgreSQL_settings)
conn = psycopg.connect(connection_string)
cursor = conn.cursor()
@pw.udf
def execute_sql_query(sql_query):
cursor.execute(sql_query)
answer = cursor.fetchall()
# answer = answer[0][0]
conn.commit()
return answer
query = query.select(
pw.this.query,
pw.this.sql_query,
result=execute_sql_query(
pw.this.sql_query,
),
)
answers = query.select(result=pw.make_tuple(pw.this.sql_query, pw.this.result))
response_writer(answers)
@pw.udf
def strip_metadata(docs: list[tuple[str, dict]]) -> list[str]:
return [doc[0] for doc in docs]
def run(
*,
data_dir: str = os.environ.get("PATHWAY_DATA_DIR", "./data/quarterly_earnings"),
api_key: str = os.environ.get("OPENAI_API_KEY", ""),
host: str = os.environ.get("PATHWAY_REST_CONNECTOR_HOST", "0.0.0.0"),
port: int = int(os.environ.get("PATHWAY_REST_CONNECTOR_PORT", "8080")),
model_locator: str = "gpt-3.5-turbo", # "gpt-4", # gpt-3.5-turbo-16k
max_tokens: int = 120,
temperature: float = 0.0,
postresql_host: str = os.environ.get("POSTGRESQL_HOST", "localhost"),
postresql_port: str = os.environ.get("POSTGRESQL_PORT", "5432"),
postresql_db: str = os.environ.get("POSTGRESQL_DB", "STRUCTUREDDB"),
postresql_user: str = os.environ.get("POSTGRESQL_USER", "user"),
postresql_password: str = os.environ.get("POSTGRESQL_PASSWORD", "password"),
postresql_table: str = os.environ.get("POSTGRESQL_TABLE", "quarterly_earnings"),
**kwargs,
):
#
# # Pipeline 1 - parsing documents into a PostgreSql table
#
postgreSQL_settings = {
"host": postresql_host,
"port": postresql_port,
"dbname": postresql_db,
"user": postresql_user,
"password": postresql_password,
}
files = pw.io.fs.read(
data_dir,
format="binary",
)
parser = UnstructuredParser()
unstructured_documents = files.select(texts=parser(pw.this.data)).select(
texts=strip_metadata(pw.this.texts)
)
structured_table = structure_on_the_fly(
unstructured_documents, api_key, model_locator, max_tokens, temperature
)
pw.io.postgres.write(structured_table, postgreSQL_settings, postresql_table)
pw.io.csv.write(structured_table, "./data/quarterly_earnings.csv")
#
# # Pipeline 2 - query answering using PostgreSql
#
unstructured_query(
postgreSQL_settings,
postresql_table,
api_key,
model_locator,
max_tokens,
temperature,
host,
port,
)
pw.run(monitoring_level=pw.MonitoringLevel.NONE)
if __name__ == "__main__":
run()
================================================
FILE: templates/unstructured_to_sql_on_the_fly/docker-compose.yml
================================================
version: "3.8"
services:
postgres:
container_name: postgres
image: postgres
ports:
- 5432:5432
environment:
- POSTGRES_USER=user
- POSTGRES_PASSWORD=password
- POSTGRES_DB=STRUCTUREDDB
- PGPASSWORD=password
volumes:
- ./postgres/init-db.sql:/docker-entrypoint-initdb.d/init-db.sql
expose:
- 5432
healthcheck:
test:
[
"CMD-SHELL",
"pg_isready -h 0.0.0.0 -p 5432 -U user -d STRUCTUREDDB"
]
interval: 5s
timeout: 5s
retries: 5
pathway:
depends_on:
postgres:
condition: service_healthy
build:
context: .
ports:
- "8080:8080"
environment:
OPENAI_API_KEY: "${OPENAI_API_KEY}"
PATHWAY_REST_CONNECTOR_HOST:
PATHWAY_REST_CONNECTOR_PORT:
PATHWAY_PERSISTENT_STORAGE:
POSTGRESQL_HOST: "postgres"
volumes:
- "./data:/app/data"
streamlit_ui:
depends_on:
- pathway
build:
context: ./ui
ports:
- "8501:8501"
environment:
PATHWAY_HOST: "pathway"
PATHWAY_PORT: "8080"
================================================
FILE: templates/unstructured_to_sql_on_the_fly/postgres/init-db.sql
================================================
CREATE TABLE IF NOT EXISTS quarterly_earnings (
company_symbol TEXT NOT NULL,
eps FLOAT NOT NULL,
net_income_md FLOAT NOT NULL,
quarter TEXT NOT NULL,
revenue_md FLOAT NOT NULL,
year BIGINT NOT NULL,
time BIGINT NOT NULL,
diff INTEGER NOT NULL
);
================================================
FILE: templates/unstructured_to_sql_on_the_fly/requirements.txt
================================================
python-dotenv~=1.0
psycopg~=3.1
================================================
FILE: templates/unstructured_to_sql_on_the_fly/ui/Dockerfile
================================================
FROM python:3.11
WORKDIR /app
RUN pip install streamlit
COPY . .
EXPOSE 8501
CMD ["streamlit", "run", "server.py", "--server.port", "8501", "--server.address", "0.0.0.0"]
================================================
FILE: templates/unstructured_to_sql_on_the_fly/ui/server.py
================================================
import os
import pandas as pd
import requests
import streamlit as st
api_host = os.environ.get("PATHWAY_HOST", "localhost")
api_port = os.environ.get("PATHWAY_PORT", 8000)
with st.sidebar:
st.markdown(
"## How to query your data\n"
"Enter your question about financial data\n"
"Example: What is the average quarterly net income achieved by all companies in the fourth quarter?"
)
st.markdown("---")
st.markdown("# Units")
st.markdown(
"⚠️ The revenue and net income are expressed in millions of dollars,"
" the eps is in dollars per share."
)
st.markdown("---")
st.markdown("# About")
st.markdown(
"Financial LLM app to synthesize on the fly the data in your financial documents."
"It uses Pathway's [LLM App features](https://github.com/pathwaycom/llm-app) "
"to build and maintain a real-time database and synthesize your documents "
" using LLM(Large Language Model) and store the data in PostgreSQL.\n"
)
st.markdown(
"""[View the source code on GitHub](
https://github.com/pathwaycom/llm-app/templates/unstructured_to_sql_on_the_fly/app.py)"""
)
# Streamlit UI elements
st.title("📈 Financial summary with LLM App")
question = st.text_input(
"Search for something",
placeholder="What summary are looking for?",
)
def json_to_table(json_value):
result = ""
for row in json_value:
for col_value in row:
result = result + str(col_value) + "\t"
result = result + "\n"
return result
# Handle Discounts API request if data source is selected and a question is provided
if question:
url = f"http://{api_host}:{api_port}/"
try:
data = {"user": "user", "query": question}
response = requests.post(url, json=data)
if response.status_code == 200:
response_JSON = response.json()
sql_query = response_JSON[0]
answer = response_JSON[1]
dataframe = pd.DataFrame.from_records(answer)
st.write("### Answer")
st.write("Resulting SQL query:")
st.write(sql_query)
st.write("SQL Answer:")
st.dataframe(dataframe)
else:
st.error(
f"Failed to send data to Finance API. Status code: {response.status_code}"
)
except Exception as e:
st.write("### Parsing error:")
st.write("Couldn't parse the entry.")
st.write(str(e))
