[ { "path": ".git-blame-ignore-revs", "content": "# Initial Ruff Linting\n70d7725e5c89bccfe7d4e5a3ccd87e05c642d74b\n# Change line-length and ruff format\n39bbfdb8298b5faa32e4bc052080d240f6140bea\n# pre-commit hooks and ruff\n6ed123ecc4aec9da26bd48748df670cd5b42b3cd\n" }, { "path": ".gitattributes", "content": "*.ipynb linguist-documentation\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: \"\\U0001F41B Bug Report\"\ndescription: Report your bug here.\nlabels: [\"bug\"]\nbody:\n\n - type: markdown\n attributes:\n value: |\n Thanks for taking the time to fill out this bug report! Any information you can provide about your system and the issue you encountered will help to resolve it faster.\n\n - type: checkboxes\n attributes:\n label: Have you searched existing issues? πŸ”Ž\n description: Please search to see if an [issue](https://github.com/MaartenGr/BERTopic/issues) already exists for the issue you encountered.\n options:\n - label: I have searched and found no existing issues\n required: true\n\n - type: textarea\n id: describe_the_bug\n attributes:\n label: Desribe the bug\n description: Please provide a concise description of the bug. If there is an error, make sure to provide the **full** error log.\n placeholder: Describe the bug\n validations:\n required: true\n\n - type: textarea\n id: reproduction\n attributes:\n label: Reproduction\n description: Please provide a minimal example, with code, that can be run to reproduce the issue.\n value: |\n ```python\n from bertopic import BERTopic\n\n ```\n\n - type: input\n id: bertopic_version\n attributes:\n label: BERTopic Version\n description: What version of BERTopic are you using?\n validations:\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: true\ncontact_links:\n - name: πŸ’‘ General questions\n url: https://github.com/MaartenGr/BERTopic/discussions\n about: Ask a question there!\n - name: Want to contribute?\n url: https://github.com/MaartenGr/BERTopic/blob/master/CONTRIBUTING.md\n about: Head to the contributing guidelines\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: \"\\U0001F680 Feature request\"\ndescription: Submit a proposal/request for a new BERTopic feature\nlabels: [\"Feature request\"]\nbody:\n - type: textarea\n id: feature-request\n validations:\n required: true\n attributes:\n label: Feature request\n description: |\n A clear and concise description of the feature proposal.\n\n - type: textarea\n id: motivation\n validations:\n required: true\n attributes:\n label: Motivation\n description: |\n Please outline the motivation for the proposal. If this is related to another GitHub issue, please link here too.\n\n - type: textarea\n id: contribution\n validations:\n required: true\n attributes:\n label: Your contribution\n description: |\n Any help on the implementation of this feature would be greatly appreciated. If you are interested in working on this, make sure to read the [CONTRIBUTING.MD guide](https://github.com/MaartenGr/BERTopic/blob/master/CONTRIBUTING.md)\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "# What does this PR do?\n\n\n\n\n\nFixes # (issue)\n\n\n## Before submitting\n- [ ] This PR fixes a typo or improves the docs (if yes, ignore all other checks!).\n- [ ] Did you read the [contributor guideline](https://github.com/MaartenGr/BERTopic/blob/master/CONTRIBUTING.md)?\n- [ ] Was this discussed/approved via a Github issue? Please add a link to it if that's the case.\n- [ ] Did you make sure to update the documentation with your changes (if applicable)?\n- [ ] Did you write any new necessary tests?\n" }, { "path": ".github/workflows/testing.yml", "content": "name: Code Checks\n\non:\n push:\n branches:\n - master\n - dev\n pull_request:\n branches:\n - master\n - dev\n\njobs:\n lint:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v5\n - uses: actions/setup-python@v6\n with:\n python-version: \"3.13\"\n # Ref: https://github.com/tox-dev/action-pre-commit-uv\n - uses: tox-dev/action-pre-commit-uv@246b66536e366bb885f52d61983bf32f7c95e8b1 # v1.0.3\n\n build:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n python-version: [\"3.10\", \"3.11\", \"3.12\", \"3.13\"]\n\n steps:\n - uses: actions/checkout@v5\n - name: Install latest version of uv and set up Python ${{ matrix.python-version }}\n uses: astral-sh/setup-uv@v7\n with:\n python-version: ${{ matrix.python-version }}\n activate-environment: \"true\"\n - name: Install dependencies (including test group)\n run: uv sync --extra test\n - name: Display the project's dependency tree (including information on outdated packages)\n run: uv tree --outdated\n - name: Run Checking Mechanisms\n run: make check\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\nmodel_dir\nmodel_dir/\ntest\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Sphinx documentation\ndocs/_build/\n\n\n# Jupyter Notebook\n.ipynb_checkpoints\nnotebooks/\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n*.lock\n\n# Artifacts\n.idea\n.idea/\n.vscode\n.DS_Store\n\n# mkdocs\nsite/\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\n- repo: https://github.com/pre-commit/pre-commit-hooks\n rev: v6.0.0\n hooks:\n - id: trailing-whitespace\n exclude: |\n (?x)^(\n README.md|\n docs/\n )$\n - id: end-of-file-fixer\n exclude_types: [html, svg]\n - id: check-yaml\n - id: check-added-large-files\n- repo: https://github.com/astral-sh/ruff-pre-commit\n rev: v0.14.0\n hooks:\n - id: ruff-check\n args: [--fix, --show-fixes, --exit-non-zero-on-fix]\n - id: ruff-format\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to BERTopic\n\nHi! Thank you for considering contributing to BERTopic. With the modular nature of BERTopic, many new add-ons, backends, representation models, sub-models, and LLMs, can quickly be added to keep up with the incredibly fast-pacing field.\n\nWhether contributions are new features, better documentation, bug fixes, or improvement on the repository itself, anything is appreciated!\n\n## πŸ“š Guidelines\n\n### πŸ€– Contributing Code\n\nTo contribute to this project, we follow an `issue -> pull request` approach for main features and bug fixes. This means that any new feature, bug fix, or anything else that touches on code directly needs to start from an issue first. That way, the main discussion about what needs to be added/fixed can be done in the issue before creating a pull request. This makes sure that we are on the same page before you start coding your pull request. If you start working on an issue, please assign it to yourself but do so after there is an agreement with the maintainer, [@MaartenGr](https://github.com/MaartenGr).\n\nWhen there is agreement on the assigned approach, a pull request can be created in which the fix/feature can be added. This follows a [\"fork and pull request\"](https://docs.github.com/en/get-started/quickstart/contributing-to-projects) workflow.\nPlease do not try to push directly to this repo unless you are a maintainer.\n\nThere are exceptions to the `issue -> pull request` approach that are typically small changes that do not need agreements, such as:\n* Documentation\n* Spelling/grammar issues\n* Docstrings\n* etc.\n\nThere is a large focus on documentation in this repository, so please make sure to add extensive descriptions of features when creating the pull request.\n\nNote that the main focus of pull requests and code should be:\n* Easy readability\n* Clear communication\n* Sufficient documentation\n\n## πŸš€ Quick Start\n\nTo start contributing, make sure to first start from a fresh environment. Using an environment manager, such as `conda` or `pyenv` helps in making sure that your code is reproducible and tracks the versions you have in your environment.\n\nIf you are using conda, you can approach it as follows:\n\n1. Create and activate a new conda environment (e.g., `conda create -n bertopic python=3.10`)\n2. Install requirements (e.g., `pip install .[dev]`)\n * This makes sure to also install documentation and testing packages\n3. (Optional) Run `make docs` to build your documentation\n4. (Optional) Run `make test` to run the unit tests and `make coverage` to check the coverage of unit tests\n\n❗Note: Unit testing the package can take quite some time since it needs to run several variants of the BERTopic pipeline.\n\n## 🧹 Linting and Formatting\n\nWe use [Ruff](https://docs.astral.sh/ruff/) to ensure code is uniformly formatted and to avoid common mistakes and bad practices.\n\n* To automatically re-format code, run `make format`\n* To check for linting issues, run `make lint` - some issues may be automatically fixed, some will not be\n\nWhen a pull request is made, the CI will automatically check for linting and formatting issues. However, it will not automatically apply any fixes, so it is easiest to run locally.\n\nIf you believe an error is incorrectly flagged, use a [`# noqa:` comment to suppress](https://docs.astral.sh/ruff/linter/#error-suppression), but this is discouraged unless strictly necessary.\n\n## πŸ€“ Collaborative Efforts\n\nWhen you run into any issue with the above or need help to start with a pull request, feel free to reach out in the issues! As with all repositories, this one has its particularities as a result of the maintainer's view. Each repository is quite different and so will their processes.\n\n## πŸ† Recognition\n\nIf your contribution has made its way into a new release of BERTopic, you will be given credit in the changelog of the new release! Regardless of the size of the contribution, any help is greatly appreciated.\n\n## 🎈 Release\n\nBERTopic tries to mostly follow [semantic versioning](https://semver.org/) for its new releases. Even though BERTopic has been around for a few years now, it is still pre-1.0 software. With the rapid chances in the field and as a way to keep up, this versioning is on purpose. Backwards-compatibility is taken into account but integrating new features and thereby keeping up with the field takes priority. Especially since BERTopic focuses on modularity, flexibility is necessary.\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2024, Maarten P. Grootendorst\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "test:\n\tpytest\n\ntest-no-plotly:\n\tuv sync --extra test\n\tuv pip uninstall plotly\n\tpytest tests/test_other.py -k plotly --pdb\n\tuv sync --extra test\n\tpytest tests/test_other.py -k plotly\n\ncoverage:\n\tpytest --cov\n\nformat:\n\truff format\n\nlint:\n\truff check --fix\n\ninstall:\n\tpython -m pip install -e .\n\ninstall-test:\n\tpython -m pip install -e \".[dev]\"\n\ndocs:\n\tmkdocs serve\n\npypi:\n\tpython -m build\n\ttwine upload dist/*\n\nclean:\n\trm -rf **/.ipynb_checkpoints **/.pytest_cache **/__pycache__ **/**/__pycache__ .ipynb_checkpoints .pytest_cache\n\ncheck: test clean\n" }, { "path": "README.md", "content": "[![PyPI Downloads](https://static.pepy.tech/badge/bertopic)](https://pepy.tech/projects/bertopic)\n[![PyPI - Python](https://img.shields.io/badge/python-v3.10+-blue.svg)](https://pypi.org/project/bertopic/)\n[![Build](https://img.shields.io/github/actions/workflow/status/MaartenGr/BERTopic/testing.yml?branch=master)](https://github.com/MaartenGr/BERTopic/actions)\n[![docs](https://img.shields.io/badge/docs-Passing-green.svg)](https://maartengr.github.io/BERTopic/)\n[![PyPI - PyPi](https://img.shields.io/pypi/v/BERTopic)](https://pypi.org/project/bertopic/)\n[![PyPI - License](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/MaartenGr/VLAC/blob/master/LICENSE)\n[![arXiv](https://img.shields.io/badge/arXiv-2203.05794-.svg)](https://arxiv.org/abs/2203.05794)\n\n\n# BERTopic\n\n \n\nBERTopic is a topic modeling technique that leverages πŸ€— transformers and c-TF-IDF to create dense clusters\nallowing for easily interpretable topics whilst keeping important words in the topic descriptions.\n\nBERTopic supports all kinds of topic modeling techniques: \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
GuidedSupervisedSemi-supervised
ManualMulti-topic distributionsHierarchical
Class-basedDynamicOnline/Incremental
MultimodalMulti-aspectText Generation/LLM
Zero-shot (new!)Merge Models (new!)Seed Words (new!)
\n\nCorresponding medium posts can be found [here](https://medium.com/data-science/topic-modeling-with-bert-779f7db187e6?sk=0b5a470c006d1842ad4c8a3057063a99\n), [here](https://medium.com/data-science/using-whisper-and-bertopic-to-model-kurzgesagts-videos-7d8a63139bdf?sk=b1e0fd46f70cb15e8422b4794a81161d\n) and [here](https://medium.com/data-science/interactive-topic-modeling-with-bertopic-1ea55e7d73d8?sk=03c2168e9e74b6bda2a1f3ed953427e4\n). For a more detailed overview, you can read the [paper](https://arxiv.org/abs/2203.05794) or see a [brief overview](https://maartengr.github.io/BERTopic/algorithm/algorithm.html). \n\n## Installation\n\nInstallation, with sentence-transformers, can be done using [uv](https://docs.astral.sh/uv/):\n\n```bash\nuv add bertopic\n```\n\nor with [pip](https://github.com/pypa/pip):\n\n```bash\npip install bertopic\n```\n\nIf you want to install BERTopic with other embedding models, you can choose one of the following:\n\n```bash\n# Choose an embedding backend\npip install bertopic[flair,gensim,spacy,use]\n\n# Topic modeling with images\npip install bertopic[vision]\n```\n\nFor a *light-weight installation* without transformers, UMAP and/or HDBSCAN (for training with Model2Vec or inference), see [this tutorial](https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html#lightweight-installation).\n\n## Getting Started\nFor an in-depth overview of the features of BERTopic \nyou can check the [**full documentation**](https://maartengr.github.io/BERTopic/) or you can follow along \nwith one of the examples below:\n\n| Name | Link |\n|---|---|\n| Start Here - **Best Practices in BERTopic** | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1BoQ_vakEVtojsd2x_U6-_x52OOuqruj2?usp=sharing) |\n| **πŸ†• New!** - Topic Modeling on Large Data (GPU Acceleration) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1W7aEdDPxC29jP99GGZphUlqjMFFVKtBC?usp=sharing) |\n| **πŸ†• New!** - Topic Modeling with Llama 2 πŸ¦™ | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1QCERSMUjqGetGGujdrvv_6_EeoIcd_9M?usp=sharing) |\n| **πŸ†• New!** - Topic Modeling with Quantized LLMs | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DdSHvVPJA3rmNfBWjCo2P1E9686xfxFx?usp=sharing) |\n| Topic Modeling with BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1FieRA9fLdkQEGDIMYl0I3MCjSUKVF8C-?usp=sharing) |\n| (Custom) Embedding Models in BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/18arPPe50szvcCp_Y6xS56H2tY0m-RLqv?usp=sharing) |\n| Advanced Customization in BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1ClTYut039t-LDtlcd-oQAdXWgcsSGTw9?usp=sharing) |\n| (semi-)Supervised Topic Modeling with BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bxizKzv5vfxJEB29sntU__ZC7PBSIPaQ?usp=sharing) |\n| Dynamic Topic Modeling with Trump's Tweets | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1un8ooI-7ZNlRoK0maVkYhmNRl0XGK88f?usp=sharing) |\n| Topic Modeling arXiv Abstracts | [![Kaggle](https://img.shields.io/static/v1?style=for-the-badge&message=Kaggle&color=222222&logo=Kaggle&logoColor=20BEFF&label=)](https://www.kaggle.com/maartengr/topic-modeling-arxiv-abstract-with-bertopic) |\n\n\n## Quick Start\nWe start by extracting topics from the well-known 20 newsgroups dataset containing English documents:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n \ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nAfter generating topics and their probabilities, we can access all of the topics together with their topic representations:\n\n```python\n>>> topic_model.get_topic_info()\n\nTopic\tCount\tName\n-1\t4630\t-1_can_your_will_any\n0\t693\t49_windows_drive_dos_file\n1\t466\t32_jesus_bible_christian_faith\n2\t441\t2_space_launch_orbit_lunar\n3\t381\t22_key_encryption_keys_encrypted\n...\n```\n\nThe `-1` topic refers to all outlier documents and are typically ignored. Each word in a topic describes the underlying theme of that topic and can be used \nfor interpreting that topic. Next, let's take a look at the most frequent topic that was generated:\n\n```python\n>>> topic_model.get_topic(0)\n\n[('windows', 0.006152228076250982),\n ('drive', 0.004982897610645755),\n ('dos', 0.004845038866360651),\n ('file', 0.004140142872194834),\n ('disk', 0.004131678774810884),\n ('mac', 0.003624848635985097),\n ('memory', 0.0034840976976789903),\n ('software', 0.0034415334250699077),\n ('email', 0.0034239554442333257),\n ('pc', 0.003047105930670237)]\n``` \n\nUsing `.get_document_info`, we can also extract information on a document level, such as their corresponding topics, probabilities, whether they are representative documents for a topic, etc.:\n\n```python\n>>> topic_model.get_document_info(docs)\n\nDocument Topic\tName\t Top_n_words Probability ...\nI am sure some bashers of Pens...\t0\t0_game_team_games_season\tgame - team - games...\t 0.200010 ...\nMy brother is in the market for... -1 -1_can_your_will_any\t can - your - will...\t 0.420668 ...\nFinally you said what you dream...\t-1 -1_can_your_will_any\t can - your - will... 0.807259 ...\nThink! It's the SCSI card doing...\t49 49_windows_drive_dos_file\twindows - drive - docs...\t0.071746 ...\n1) I have an old Jasmine drive...\t49 49_windows_drive_dos_file\twindows - drive - docs...\t0.038983 ...\n```\n\n**`πŸ”₯ Tip`**: Use `BERTopic(language=\"multilingual\")` to select a model that supports 50+ languages. \n\n## Fine-tune Topic Representations\n\nIn BERTopic, there are a number of different [topic representations](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) that we can choose from. They are all quite different from one another and give interesting perspectives and variations of topic representations. A great start is `KeyBERTInspired`, which for many users increases the coherence and reduces stopwords from the resulting topic representations:\n\n ```python\nfrom bertopic.representation import KeyBERTInspired\n\n# Fine-tune your topic representations\nrepresentation_model = KeyBERTInspired()\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nHowever, you might want to use something more powerful to describe your clusters. You can even use ChatGPT or other models from OpenAI to generate labels, summaries, phrases, keywords, and more:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\n\n# Fine-tune topic representations with GPT\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(client, model=\"gpt-4o-mini\", chat=True)\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n**`πŸ”₯ Tip`**: Instead of iterating over all of these different topic representations, you can model them simultaneously with [multi-aspect topic representations](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) in BERTopic. \n\n\n## Visualizations\nAfter having trained our BERTopic model, we can iteratively go through hundreds of topics to get a good \nunderstanding of the topics that were extracted. However, that takes quite some time and lacks a global representation. Instead, we can use one of the [many visualization options](https://maartengr.github.io/BERTopic/getting_started/visualization/visualization.html) in BERTopic. \nFor example, we can visualize the topics that were generated in a way very similar to \n[LDAvis](https://github.com/cpsievert/LDAvis):\n\n```python\ntopic_model.visualize_topics()\n``` \n\n\n\n## Modularity\nBy default, the [main steps](https://maartengr.github.io/BERTopic/algorithm/algorithm.html) for topic modeling with BERTopic are sentence-transformers, UMAP, HDBSCAN, and c-TF-IDF run in sequence. However, it assumes some independence between these steps which makes BERTopic quite modular. In other words, BERTopic not only allows you to build your own topic model but to explore several topic modeling techniques on top of your customized topic model:\n\nhttps://user-images.githubusercontent.com/25746895/218420473-4b2bb539-9dbe-407a-9674-a8317c7fb3bf.mp4\n\nYou can swap out any of these models or even remove them entirely. The following steps are completely modular:\n\n1. [Embedding](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html) documents\n2. [Reducing dimensionality](https://maartengr.github.io/BERTopic/getting_started/dim_reduction/dim_reduction.html) of embeddings\n3. [Clustering](https://maartengr.github.io/BERTopic/getting_started/clustering/clustering.html) reduced embeddings into topics\n4. [Tokenization](https://maartengr.github.io/BERTopic/getting_started/vectorizers/vectorizers.html) of topics\n5. [Weight](https://maartengr.github.io/BERTopic/getting_started/ctfidf/ctfidf.html) tokens\n6. [Represent topics](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) with one or [multiple](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) representations\n\n\n## Functionality\nBERTopic has many functions that quickly can become overwhelming. To alleviate this issue, you will find an overview \nof all methods and a short description of its purpose. \n\n### Common\nBelow, you will find an overview of common functions in BERTopic. \n\n| Method | Code | \n|-----------------------|---|\n| Fit the model | `.fit(docs)` |\n| Fit the model and predict documents | `.fit_transform(docs)` |\n| Predict new documents | `.transform([new_doc])` |\n| Access single topic | `.get_topic(topic=12)` | \n| Access all topics | `.get_topics()` |\n| Get topic freq | `.get_topic_freq()` |\n| Get all topic information| `.get_topic_info()` |\n| Get all document information| `.get_document_info(docs)` |\n| Get representative docs per topic | `.get_representative_docs()` |\n| Update topic representation | `.update_topics(docs, n_gram_range=(1, 3))` |\n| Generate topic labels | `.generate_topic_labels()` |\n| Set topic labels | `.set_topic_labels(my_custom_labels)` |\n| Merge topics | `.merge_topics(docs, topics_to_merge)` |\n| Reduce nr of topics | `.reduce_topics(docs, nr_topics=30)` |\n| Reduce outliers | `.reduce_outliers(docs, topics)` |\n| Find topics | `.find_topics(\"vehicle\")` |\n| Save model | `.save(\"my_model\", serialization=\"safetensors\")` |\n| Load model | `BERTopic.load(\"my_model\")` |\n| Get parameters | `.get_params()` |\n\n\n### Attributes\nAfter having trained your BERTopic model, several attributes are saved within your model. These attributes, in part, \nrefer to how model information is stored on an estimator during fitting. The attributes that you see below all end in `_` and are \npublic attributes that can be used to access model information. \n\n| Attribute | Description |\n|------------------------|---------------------------------------------------------------------------------------------|\n| `.topics_` | The topics that are generated for each document after training or updating the topic model. |\n| `.probabilities_` | The probabilities that are generated for each document if HDBSCAN is used. |\n| `.topic_sizes_` | The size of each topic |\n| `.topic_mapper_` | A class for tracking topics and their mappings anytime they are merged/reduced. |\n| `.topic_representations_` | The top *n* terms per topic and their respective c-TF-IDF values. |\n| `.c_tf_idf_` | The topic-term matrix as calculated through c-TF-IDF. |\n| `.topic_aspects_` | The different aspects, or representations, of each topic. |\n| `.topic_labels_` | The default labels for each topic. |\n| `.custom_labels_` | Custom labels for each topic as generated through `.set_topic_labels`. |\n| `.topic_embeddings_` | The embeddings for each topic if `embedding_model` was used. |\n| `.representative_docs_` | The representative documents for each topic if HDBSCAN is used. |\n\n\n### Variations\nThere are many different use cases in which topic modeling can be used. As such, several variations of BERTopic have been developed such that one package can be used across many use cases.\n\n| Method | Code | \n|-----------------------|---|\n| [Topic Distribution Approximation](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html) | `.approximate_distribution(docs)` |\n| [Online Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/online/online.html) | `.partial_fit(doc)` |\n| [Semi-supervised Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/semisupervised/semisupervised.html) | `.fit(docs, y=y)` |\n| [Supervised Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/supervised/supervised.html) | `.fit(docs, y=y)` |\n| [Manual Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/manual/manual.html) | `.fit(docs, y=y)` |\n| [Multimodal Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/multimodal/multimodal.html) | ``.fit(docs, images=images)`` |\n| [Topic Modeling per Class](https://maartengr.github.io/BERTopic/getting_started/topicsperclass/topicsperclass.html) | `.topics_per_class(docs, classes)` |\n| [Dynamic Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/topicsovertime/topicsovertime.html) | `.topics_over_time(docs, timestamps)` |\n| [Hierarchical Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html) | `.hierarchical_topics(docs)` |\n| [Guided Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/guided/guided.html) | `BERTopic(seed_topic_list=seed_topic_list)` |\n| [Zero-shot Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/zeroshot/zeroshot.html) | `BERTopic(zeroshot_topic_list=zeroshot_topic_list)` |\n| [Merge Multiple Models](https://maartengr.github.io/BERTopic/getting_started/merge/merge.html) | `BERTopic.merge_models([topic_model_1, topic_model_2])` |\n\n\n### Visualizations\nEvaluating topic models can be rather difficult due to the somewhat subjective nature of evaluation. \nVisualizing different aspects of the topic model helps in understanding the model and makes it easier \nto tweak the model to your liking. \n\n| Method | Code | \n|-----------------------|---|\n| Visualize Topics | `.visualize_topics()` |\n| Visualize Documents | `.visualize_documents()` |\n| Visualize Document Hierarchy | `.visualize_hierarchical_documents()` |\n| Visualize Topic Hierarchy | `.visualize_hierarchy()` |\n| Visualize Topic Tree | `.get_topic_tree(hierarchical_topics)` |\n| Visualize Topic Terms | `.visualize_barchart()` |\n| Visualize Topic Similarity | `.visualize_heatmap()` |\n| Visualize Term Score Decline | `.visualize_term_rank()` |\n| Visualize Topic Probability Distribution | `.visualize_distribution(probs[0])` |\n| Visualize Topics over Time | `.visualize_topics_over_time(topics_over_time)` |\n| Visualize Topics per Class | `.visualize_topics_per_class(topics_per_class)` | \n\n\n## Citation\nTo cite the [BERTopic paper](https://arxiv.org/abs/2203.05794), please use the following bibtex reference:\n\n```bibtext\n@article{grootendorst2022bertopic,\n title={BERTopic: Neural topic modeling with a class-based TF-IDF procedure},\n author={Grootendorst, Maarten},\n journal={arXiv preprint arXiv:2203.05794},\n year={2022}\n}\n```\n" }, { "path": "bertopic/__init__.py", "content": "from importlib.metadata import version\n\nfrom bertopic._bertopic import BERTopic\n\n__version__ = version(\"bertopic\")\n\n__all__ = [\n \"BERTopic\",\n]\n" }, { "path": "bertopic/_bertopic.py", "content": "# ruff: noqa: E402\nimport yaml\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n\ntry:\n yaml._warnings_enabled[\"YAMLLoadWarning\"] = False\nexcept (KeyError, AttributeError, TypeError):\n pass\n\nimport re\nimport math\nimport joblib\nimport inspect\nimport collections\nimport numpy as np\nimport pandas as pd\nimport scipy.sparse as sp\nfrom copy import deepcopy\n\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom packaging import version\nfrom tempfile import TemporaryDirectory\nfrom collections import defaultdict, Counter\nfrom scipy.sparse import csr_matrix\nfrom scipy.cluster import hierarchy as sch\nfrom importlib.util import find_spec\n\nfrom typing import List, Tuple, Union, Mapping, Any, Callable, Iterable, TYPE_CHECKING, Literal\n\n# Plotting\nif find_spec(\"plotly\") is None:\n from bertopic._utils import MockPlotlyModule\n\n plotting = MockPlotlyModule()\n\nelse:\n from bertopic import plotting\n\n if TYPE_CHECKING:\n import plotly.graph_objs as go\n import matplotlib.figure as fig\n\n\n# Models\ntry:\n from hdbscan import HDBSCAN\n\n HAS_HDBSCAN = True\nexcept (ImportError, ModuleNotFoundError):\n HAS_HDBSCAN = False\n from sklearn.cluster import HDBSCAN as SK_HDBSCAN\n\nfrom sklearn.preprocessing import normalize\nfrom sklearn import __version__ as sklearn_version\nfrom sklearn.cluster import AgglomerativeClustering\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n\n# BERTopic\nfrom bertopic.cluster import BaseCluster\nfrom bertopic.backend import BaseEmbedder\nfrom bertopic.representation._mmr import mmr\nfrom bertopic.backend._utils import select_backend\nfrom bertopic.vectorizers import ClassTfidfTransformer\nfrom bertopic.representation import BaseRepresentation, KeyBERTInspired\nfrom bertopic.dimensionality import BaseDimensionalityReduction\nfrom bertopic.cluster._utils import hdbscan_delegator, is_supported_hdbscan\nfrom bertopic._utils import (\n MyLogger,\n check_documents_type,\n check_embeddings_shape,\n check_is_fitted,\n validate_distance_matrix,\n select_topic_representation,\n get_unique_distances,\n)\nimport bertopic._save_utils as save_utils\n\nlogger = MyLogger()\nlogger.configure(\"WARNING\")\n\n\nclass BERTopic:\n \"\"\"BERTopic is a topic modeling technique that leverages BERT embeddings and\n c-TF-IDF to create dense clusters allowing for easily interpretable topics\n whilst keeping important words in the topic descriptions.\n\n The default embedding model is `all-MiniLM-L6-v2` when selecting `language=\"english\"`\n and `paraphrase-multilingual-MiniLM-L12-v2` when selecting `language=\"multilingual\"`.\n\n Attributes:\n topics_ (List[int]) : The topics that are generated for each document after training or updating\n the topic model. The most recent topics are tracked.\n probabilities_ (List[float]): The probability of the assigned topic per document. These are\n only calculated if a HDBSCAN model is used for the clustering step.\n When `calculate_probabilities=True`, then it is the probabilities\n of all topics per document.\n topic_sizes_ (Mapping[int, int]) : The size of each topic.\n topic_mapper_ (TopicMapper) : A class for tracking topics and their mappings anytime they are\n merged, reduced, added, or removed.\n topic_representations_ (Mapping[int, Tuple[int, float]]) : The top n terms per topic and their respective\n c-TF-IDF values.\n c_tf_idf_ (csr_matrix) : The topic-term matrix as calculated through c-TF-IDF. To access its respective\n words, run `.vectorizer_model.get_feature_names()` or\n `.vectorizer_model.get_feature_names_out()`\n topic_labels_ (Mapping[int, str]) : The default labels for each topic.\n custom_labels_ (List[str]) : Custom labels for each topic.\n topic_embeddings_ (np.ndarray) : The embeddings for each topic. They are calculated by taking the\n centroid embedding of each cluster.\n representative_docs_ (Mapping[int, str]) : The representative documents for each topic.\n\n Examples:\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n\n docs = fetch_20newsgroups(subset='all')['data']\n topic_model = BERTopic()\n topics, probabilities = topic_model.fit_transform(docs)\n ```\n\n If you want to use your own embedding model, use it as follows:\n\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n\n docs = fetch_20newsgroups(subset='all')['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n topic_model = BERTopic(embedding_model=sentence_model)\n ```\n\n Due to the stochastic nature of UMAP, the results from BERTopic might differ\n and the quality can degrade. Using your own embeddings allows you to\n try out BERTopic several times until you find the topics that suit\n you best.\n \"\"\"\n\n def __init__(\n self,\n language: str = \"english\",\n top_n_words: int = 10,\n n_gram_range: Tuple[int, int] = (1, 1),\n min_topic_size: int = 10,\n nr_topics: Union[int, str] | None = None,\n low_memory: bool = False,\n calculate_probabilities: bool = False,\n seed_topic_list: List[List[str]] | None = None,\n zeroshot_topic_list: List[str] | None = None,\n zeroshot_min_similarity: float = 0.7,\n embedding_model=None,\n umap_model=None,\n hdbscan_model=None,\n vectorizer_model: CountVectorizer = None,\n ctfidf_model: TfidfTransformer = None,\n representation_model: BaseRepresentation = None,\n verbose: bool = False,\n ):\n \"\"\"BERTopic initialization.\n\n Arguments:\n language: The main language used in your documents. The default sentence-transformers\n model for \"english\" is `all-MiniLM-L6-v2`. For a full overview of\n supported languages see bertopic.backend.languages. Select\n \"multilingual\" to load in the `paraphrase-multilingual-MiniLM-L12-v2`\n sentence-transformers model that supports 50+ languages.\n NOTE: This is not used if `embedding_model` is used.\n top_n_words: The number of words per topic to extract. Setting this\n too high can negatively impact topic embeddings as topics\n are typically best represented by at most 10 words.\n n_gram_range: The n-gram range for the CountVectorizer.\n Advised to keep high values between 1 and 3.\n More would likely lead to memory issues.\n NOTE: This param will not be used if you pass in your own\n CountVectorizer.\n min_topic_size: The minimum size of the topic. Increasing this value will lead\n to a lower number of clusters/topics and vice versa.\n It is the same parameter as `min_cluster_size` in HDBSCAN.\n NOTE: This param will not be used if you are using `hdbscan_model`.\n nr_topics: Specifying the number of topics will reduce the initial\n number of topics to the value specified. This reduction can take\n a while as each reduction in topics (-1) activates a c-TF-IDF\n calculation. If this is set to None, no reduction is applied. Use\n \"auto\" to automatically reduce topics using HDBSCAN.\n NOTE: Controlling the number of topics is best done by adjusting\n `min_topic_size` first before adjusting this parameter.\n low_memory: Sets UMAP low memory to True to make sure less memory is used.\n NOTE: This is only used in UMAP. For example, if you use PCA instead of UMAP\n this parameter will not be used.\n calculate_probabilities: Calculate the probabilities of all topics\n per document instead of the probability of the assigned\n topic per document. This could slow down the extraction\n of topics if you have many documents (> 100_000).\n NOTE: If false you cannot use the corresponding\n visualization method `visualize_probabilities`.\n NOTE: This is an approximation of topic probabilities\n as used in HDBSCAN and not an exact representation.\n seed_topic_list: A list of seed words per topic to converge around\n zeroshot_topic_list: A list of topic names to use for zero-shot classification\n zeroshot_min_similarity: The minimum similarity between a zero-shot topic and\n a document for assignment. The higher this value, the more\n confident the model needs to be to assign a zero-shot topic to a document.\n verbose: Changes the verbosity of the model, Set to True if you want\n to track the stages of the model.\n embedding_model: Use a custom embedding model.\n The following backends are currently supported\n * SentenceTransformers\n * Flair\n * Spacy\n * Gensim\n * USE (TF-Hub)\n You can also pass in a string that points to one of the following\n sentence-transformers models:\n * https://www.sbert.net/docs/pretrained_models.html\n umap_model: Pass in a UMAP model to be used instead of the default.\n NOTE: You can also pass in any dimensionality reduction algorithm as long\n as it has `.fit` and `.transform` functions.\n hdbscan_model: Pass in a hdbscan.HDBSCAN model to be used instead of the default\n NOTE: You can also pass in any clustering algorithm as long as it has\n `.fit` and `.predict` functions along with the `.labels_` variable.\n vectorizer_model: Pass in a custom `CountVectorizer` instead of the default model.\n ctfidf_model: Pass in a custom ClassTfidfTransformer instead of the default model.\n representation_model: Pass in a model that fine-tunes the topic representations\n calculated through c-TF-IDF. Models from `bertopic.representation`\n are supported.\n \"\"\"\n # Topic-based parameters\n if top_n_words > 100:\n logger.warning(\n \"Note that extracting more than 100 words from a sparse can slow down computation quite a bit.\"\n )\n\n self.top_n_words = top_n_words\n self.min_topic_size = min_topic_size\n self.nr_topics = nr_topics\n self.low_memory = low_memory\n self.calculate_probabilities = calculate_probabilities\n self.verbose = verbose\n self.seed_topic_list = seed_topic_list\n self.zeroshot_topic_list = zeroshot_topic_list\n self.zeroshot_min_similarity = zeroshot_min_similarity\n\n # Embedding model\n self.language = language if not embedding_model else None\n self.embedding_model = embedding_model\n\n # Vectorizer\n self.n_gram_range = n_gram_range\n self.vectorizer_model = vectorizer_model or CountVectorizer(ngram_range=self.n_gram_range)\n self.ctfidf_model = ctfidf_model or ClassTfidfTransformer()\n\n # Representation model\n self.representation_model = representation_model\n\n # UMAP or another algorithm that has .fit and .transform functions\n if umap_model is not None:\n self.umap_model = umap_model\n else:\n try:\n from umap import UMAP\n\n self.umap_model = UMAP(\n n_neighbors=15,\n n_components=5,\n min_dist=0.0,\n metric=\"cosine\",\n low_memory=self.low_memory,\n )\n except (ImportError, ModuleNotFoundError):\n self.umap_model = PCA(n_components=5)\n\n # HDBSCAN or another clustering algorithm that has .fit and .predict functions and\n # the .labels_ variable to extract the labels\n\n if hdbscan_model is not None:\n self.hdbscan_model = hdbscan_model\n elif HAS_HDBSCAN:\n self.hdbscan_model = HDBSCAN(\n min_cluster_size=self.min_topic_size,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n )\n else:\n self.hdbscan_model = SK_HDBSCAN(\n min_cluster_size=self.min_topic_size, metric=\"euclidean\", cluster_selection_method=\"eom\", n_jobs=-1\n )\n\n # Public attributes\n self.topics_ = None\n self.probabilities_ = None\n self.topic_sizes_ = None\n self.topic_mapper_ = None\n self.topic_representations_ = None\n self.topic_embeddings_ = None\n self._topic_id_to_zeroshot_topic_idx = {}\n self.custom_labels_ = None\n self.c_tf_idf_ = None\n self.representative_images_ = None\n self.representative_docs_ = {}\n self.topic_aspects_ = {}\n\n # Private attributes for internal tracking purposes\n self._merged_topics = None\n\n if verbose:\n logger.set_level(\"DEBUG\")\n else:\n logger.set_level(\"WARNING\")\n\n @property\n def _outliers(self):\n \"\"\"Some algorithms have outlier labels (-1) that can be tricky to work\n with if you are slicing data based on that labels. Therefore, we\n track if there are outlier labels and act accordingly when slicing.\n\n Returns:\n An integer indicating whether outliers are present in the topic model\n \"\"\"\n return 1 if -1 in self.topic_sizes_ else 0\n\n @property\n def topic_labels_(self):\n \"\"\"Map topic IDs to their labels.\n A label is the topic ID, along with the first four words of the topic representation, joined using '_'.\n Zeroshot topic labels come from self.zeroshot_topic_list rather than the calculated representation.\n\n Returns:\n topic_labels: a dict mapping a topic ID (int) to its label (str)\n \"\"\"\n topic_labels = {\n key: f\"{key}_\" + \"_\".join([word[0] for word in values[:4]])\n for key, values in self.topic_representations_.items()\n }\n if self._is_zeroshot():\n # Need to correct labels from zero-shot topics\n topic_id_to_zeroshot_label = {\n topic_id: self.zeroshot_topic_list[zeroshot_topic_idx]\n for topic_id, zeroshot_topic_idx in self._topic_id_to_zeroshot_topic_idx.items()\n }\n topic_labels.update(topic_id_to_zeroshot_label)\n return topic_labels\n\n def fit(\n self,\n documents: List[str],\n embeddings: np.ndarray = None,\n images: List[str] | None = None,\n y: Union[List[int], np.ndarray] = None,\n ):\n \"\"\"Fit the models on a collection of documents and generate topics.\n\n Arguments:\n documents: A list of documents to fit on\n embeddings: Pre-trained document embeddings. These can be used\n instead of the sentence-transformer model\n images: A list of paths to the images to fit on or the images themselves\n y: The target class for (semi)-supervised modeling. Use -1 if no class for a\n specific instance is specified.\n\n Examples:\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n\n docs = fetch_20newsgroups(subset='all')['data']\n topic_model = BERTopic().fit(docs)\n ```\n\n If you want to use your own embeddings, use it as follows:\n\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n\n # Create embeddings\n docs = fetch_20newsgroups(subset='all')['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=True)\n\n # Create topic model\n topic_model = BERTopic().fit(docs, embeddings)\n ```\n \"\"\"\n self.fit_transform(documents=documents, embeddings=embeddings, y=y, images=images)\n return self\n\n def fit_transform(\n self,\n documents: List[str],\n embeddings: np.ndarray = None,\n images: List[str] | None = None,\n y: Union[List[int], np.ndarray] = None,\n ) -> Tuple[List[int], Union[np.ndarray, None]]:\n \"\"\"Fit the models on a collection of documents, generate topics,\n and return the probabilities and topic per document.\n\n Arguments:\n documents: A list of documents to fit on\n embeddings: Pre-trained document embeddings. These can be used\n instead of the sentence-transformer model\n images: A list of paths to the images to fit on or the images themselves\n y: The target class for (semi)-supervised modeling. Use -1 if no class for a\n specific instance is specified.\n\n Returns:\n predictions: Topic predictions for each documents\n probabilities: The probability of the assigned topic per document.\n If `calculate_probabilities` in BERTopic is set to True, then\n it calculates the probabilities of all topics across all documents\n instead of only the assigned topic. This, however, slows down\n computation and may increase memory usage.\n\n Examples:\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n\n docs = fetch_20newsgroups(subset='all')['data']\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n ```\n\n If you want to use your own embeddings, use it as follows:\n\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n\n # Create embeddings\n docs = fetch_20newsgroups(subset='all')['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=True)\n\n # Create topic model\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs, embeddings)\n ```\n \"\"\"\n if documents is not None:\n check_documents_type(documents)\n check_embeddings_shape(embeddings, documents)\n\n doc_ids = range(len(documents)) if documents is not None else range(len(images))\n documents = pd.DataFrame({\"Document\": documents, \"ID\": doc_ids, \"Topic\": None, \"Image\": images})\n\n # Extract embeddings\n if embeddings is None:\n logger.info(\"Embedding - Transforming documents to embeddings.\")\n self.embedding_model = select_backend(self.embedding_model, language=self.language, verbose=self.verbose)\n embeddings = self._extract_embeddings(\n documents.Document.to_numpy().tolist(),\n images=images,\n method=\"document\",\n verbose=self.verbose,\n )\n logger.info(\"Embedding - Completed \\u2713\")\n else:\n if self.embedding_model is not None:\n self.embedding_model = select_backend(\n self.embedding_model, language=self.language, verbose=self.verbose\n )\n\n # Guided Topic Modeling\n if self.seed_topic_list is not None and self.embedding_model is not None:\n y, embeddings = self._guided_topic_modeling(embeddings)\n\n # Reduce dimensionality and fit UMAP model\n umap_embeddings = self._reduce_dimensionality(embeddings, y)\n\n # Zero-shot Topic Modeling\n if self._is_zeroshot():\n documents, embeddings, assigned_documents, assigned_embeddings = self._zeroshot_topic_modeling(\n documents, embeddings\n )\n\n # Filter UMAP embeddings to only non-assigned embeddings to be used for clustering\n if len(documents) > 0:\n umap_embeddings = self.umap_model.transform(embeddings)\n\n if len(documents) > 0:\n # Cluster reduced embeddings\n documents, probabilities = self._cluster_embeddings(umap_embeddings, documents, y=y)\n if self._is_zeroshot() and len(assigned_documents) > 0:\n documents, embeddings = self._combine_zeroshot_topics(\n documents, embeddings, assigned_documents, assigned_embeddings\n )\n else:\n # All documents matches zero-shot topics\n documents = assigned_documents\n embeddings = assigned_embeddings\n\n # Sort and Map Topic IDs by their frequency\n if not self.nr_topics:\n documents = self._sort_mappings_by_frequency(documents)\n\n # Create documents from images if we have images only\n if documents.Document.to_numpy()[0] is None:\n custom_documents = self._images_to_text(documents, embeddings)\n\n # Extract topics by calculating c-TF-IDF, reduce topics if needed, and get representations.\n self._extract_topics(custom_documents, embeddings=embeddings, fine_tune_representation=not self.nr_topics)\n if self.nr_topics:\n custom_documents = self._reduce_topics(custom_documents)\n self._create_topic_vectors(documents=documents, embeddings=embeddings)\n\n # Save the top 3 most representative documents per topic\n self._save_representative_docs(custom_documents)\n\n else:\n # Extract topics by calculating c-TF-IDF, reduce topics if needed, and get representations.\n self._extract_topics(\n documents, embeddings=embeddings, verbose=self.verbose, fine_tune_representation=not self.nr_topics\n )\n if self.nr_topics:\n documents = self._reduce_topics(documents)\n\n # Save the top 3 most representative documents per topic\n self._save_representative_docs(documents)\n\n # In the case of zero-shot topics, probability will come from cosine similarity,\n # and the HDBSCAN model will be removed\n if self._is_zeroshot() and len(assigned_documents) > 0:\n self.hdbscan_model = BaseCluster()\n sim_matrix = cosine_similarity(embeddings, np.array(self.topic_embeddings_))\n\n if self.calculate_probabilities:\n self.probabilities_ = sim_matrix\n else:\n self.probabilities_ = np.max(sim_matrix, axis=1)\n else:\n self.probabilities_ = self._map_probabilities(probabilities, original_topics=True)\n predictions = documents.Topic.to_list()\n\n return predictions, self.probabilities_\n\n def transform(\n self,\n documents: Union[str, List[str]],\n embeddings: np.ndarray = None,\n images: List[str] | None = None,\n ) -> Tuple[List[int], np.ndarray]:\n \"\"\"After having fit a model, use transform to predict new instances.\n\n Arguments:\n documents: A single document or a list of documents to predict on\n embeddings: Pre-trained document embeddings. These can be used\n instead of the sentence-transformer model.\n images: A list of paths to the images to predict on or the images themselves\n\n Returns:\n predictions: Topic predictions for each documents\n probabilities: The topic probability distribution which is returned by default.\n If `calculate_probabilities` in BERTopic is set to False, then the\n probabilities are not calculated to speed up computation and\n decrease memory usage.\n\n Examples:\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n\n docs = fetch_20newsgroups(subset='all')['data']\n topic_model = BERTopic().fit(docs)\n topics, probs = topic_model.transform(docs)\n ```\n\n If you want to use your own embeddings:\n\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n\n # Create embeddings\n docs = fetch_20newsgroups(subset='all')['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=True)\n\n # Create topic model\n topic_model = BERTopic().fit(docs, embeddings)\n topics, probs = topic_model.transform(docs, embeddings)\n ```\n \"\"\"\n check_is_fitted(self)\n check_embeddings_shape(embeddings, documents)\n\n if isinstance(documents, str) or documents is None:\n documents = [documents]\n\n if embeddings is None:\n embeddings = self._extract_embeddings(documents, images=images, method=\"document\", verbose=self.verbose)\n\n # Check if an embedding model was found\n if embeddings is None:\n raise ValueError(\n \"No embedding model was found to embed the documents.\"\n \"Make sure when loading in the model using BERTopic.load()\"\n \"to also specify the embedding model.\"\n )\n\n # Transform without hdbscan_model and umap_model using only cosine similarity\n elif type(self.hdbscan_model) is BaseCluster:\n logger.info(\"Predicting topic assignments through cosine similarity of topic and document embeddings.\")\n sim_matrix = cosine_similarity(embeddings, np.array(self.topic_embeddings_))\n predictions = np.argmax(sim_matrix, axis=1) - self._outliers\n\n if self.calculate_probabilities:\n probabilities = sim_matrix\n else:\n probabilities = np.max(sim_matrix, axis=1)\n\n # Transform with full pipeline\n else:\n logger.info(\"Dimensionality - Reducing dimensionality of input embeddings.\")\n umap_embeddings = self.umap_model.transform(embeddings)\n logger.info(\"Dimensionality - Completed \\u2713\")\n\n # Extract predictions and probabilities if it is a HDBSCAN-like model\n logger.info(\"Clustering - Approximating new points with `hdbscan_model`\")\n if is_supported_hdbscan(self.hdbscan_model):\n predictions, probabilities = hdbscan_delegator(\n self.hdbscan_model, \"approximate_predict\", umap_embeddings\n )\n\n # Calculate probabilities\n if self.calculate_probabilities:\n logger.info(\"Probabilities - Start calculation of probabilities with HDBSCAN\")\n probabilities = hdbscan_delegator(self.hdbscan_model, \"membership_vector\", umap_embeddings)\n logger.info(\"Probabilities - Completed \\u2713\")\n else:\n predictions = self.hdbscan_model.predict(umap_embeddings)\n probabilities = None\n logger.info(\"Cluster - Completed \\u2713\")\n\n # Map probabilities and predictions\n probabilities = self._map_probabilities(probabilities, original_topics=True)\n predictions = self._map_predictions(predictions)\n return predictions, probabilities\n\n def partial_fit(\n self,\n documents: List[str],\n embeddings: np.ndarray = None,\n y: Union[List[int], np.ndarray] = None,\n ):\n \"\"\"Fit BERTopic on a subset of the data and perform online learning\n with batch-like data.\n\n Online topic modeling in BERTopic is performed by using dimensionality\n reduction and cluster algorithms that support a `partial_fit` method\n in order to incrementally train the topic model.\n\n Likewise, the `bertopic.vectorizers.OnlineCountVectorizer` is used\n to dynamically update its vocabulary when presented with new data.\n It has several parameters for modeling decay and updating the\n representations.\n\n In other words, although the main algorithm stays the same, the training\n procedure now works as follows:\n\n For each subset of the data:\n\n 1. Generate embeddings with a pre-trained language model\n 2. Incrementally update the dimensionality reduction algorithm with `partial_fit`\n 3. Incrementally update the cluster algorithm with `partial_fit`\n 4. Incrementally update the OnlineCountVectorizer and apply some form of decay\n\n Note that it is advised to use `partial_fit` with batches and\n not single documents for the best performance.\n\n Arguments:\n documents: A list of documents to fit on\n embeddings: Pre-trained document embeddings. These can be used\n instead of the sentence-transformer model\n y: The target class for (semi)-supervised modeling. Use -1 if no class for a\n specific instance is specified.\n\n Examples:\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sklearn.cluster import MiniBatchKMeans\n from sklearn.decomposition import IncrementalPCA\n from bertopic.vectorizers import OnlineCountVectorizer\n from bertopic import BERTopic\n\n # Prepare documents\n docs = fetch_20newsgroups(subset=subset, remove=('headers', 'footers', 'quotes'))[\"data\"]\n\n # Prepare sub-models that support online learning\n umap_model = IncrementalPCA(n_components=5)\n cluster_model = MiniBatchKMeans(n_clusters=50, random_state=0)\n vectorizer_model = OnlineCountVectorizer(stop_words=\"english\", decay=.01)\n\n topic_model = BERTopic(umap_model=umap_model,\n hdbscan_model=cluster_model,\n vectorizer_model=vectorizer_model)\n\n # Incrementally fit the topic model by training on 1000 documents at a time\n for index in range(0, len(docs), 1000):\n topic_model.partial_fit(docs[index: index+1000])\n ```\n \"\"\"\n # Checks\n check_embeddings_shape(embeddings, documents)\n if not hasattr(self.hdbscan_model, \"partial_fit\"):\n raise ValueError(\"In order to use `.partial_fit`, the cluster model should have a `.partial_fit` function.\")\n\n # Prepare documents\n if isinstance(documents, str):\n documents = [documents]\n documents = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": None})\n\n # Extract embeddings\n if embeddings is None:\n if self.topic_representations_ is None:\n self.embedding_model = select_backend(\n self.embedding_model, language=self.language, verbose=self.verbose\n )\n embeddings = self._extract_embeddings(\n documents.Document.to_numpy().tolist(),\n method=\"document\",\n verbose=self.verbose,\n )\n else:\n if self.embedding_model is not None and self.topic_representations_ is None:\n self.embedding_model = select_backend(\n self.embedding_model, language=self.language, verbose=self.verbose\n )\n\n # Reduce dimensionality\n if self.seed_topic_list is not None and self.embedding_model is not None:\n y, embeddings = self._guided_topic_modeling(embeddings)\n umap_embeddings = self._reduce_dimensionality(embeddings, y, partial_fit=True)\n\n # Cluster reduced embeddings\n documents, self.probabilities_ = self._cluster_embeddings(umap_embeddings, documents, partial_fit=True)\n topics = documents.Topic.to_list()\n\n # Map and find new topics\n if not self.topic_mapper_:\n self.topic_mapper_ = TopicMapper(topics)\n mappings = self.topic_mapper_.get_mappings()\n new_topics = set(topics).difference(set(mappings.keys()))\n new_topic_ids = {topic: max(mappings.values()) + index + 1 for index, topic in enumerate(new_topics)}\n self.topic_mapper_.add_new_topics(new_topic_ids)\n updated_mappings = self.topic_mapper_.get_mappings()\n updated_topics = [updated_mappings[topic] for topic in topics]\n documents[\"Topic\"] = updated_topics\n\n # Add missing topics (topics that were originally created but are now missing)\n if self.topic_representations_:\n missing_topics = set(self.topic_representations_.keys()).difference(set(updated_topics))\n for missing_topic in missing_topics:\n documents.loc[len(documents), :] = [\" \", len(documents), missing_topic]\n else:\n missing_topics = {}\n\n # Prepare documents\n documents_per_topic = documents.sort_values(\"Topic\").groupby([\"Topic\"], as_index=False)\n updated_topics = documents_per_topic.first().Topic.astype(int)\n documents_per_topic = documents_per_topic.agg({\"Document\": \" \".join})\n\n # Update topic representations\n self.c_tf_idf_, updated_words = self._c_tf_idf(documents_per_topic, partial_fit=True)\n self.topic_representations_ = self._extract_words_per_topic(\n updated_words, documents, self.c_tf_idf_, calculate_aspects=False\n )\n self._create_topic_vectors()\n\n # Update topic sizes\n if len(missing_topics) > 0:\n documents = documents.iloc[: -len(missing_topics)]\n\n if self.topic_sizes_ is None:\n self._update_topic_size(documents)\n else:\n sizes = documents.groupby([\"Topic\"], as_index=False).count()\n for _, row in sizes.iterrows():\n topic = int(row.Topic)\n if self.topic_sizes_.get(topic) is not None and topic not in missing_topics:\n self.topic_sizes_[topic] += int(row.Document)\n elif self.topic_sizes_.get(topic) is None:\n self.topic_sizes_[topic] = int(row.Document)\n self.topics_ = documents.Topic.astype(int).tolist()\n\n return self\n\n def topics_over_time(\n self,\n docs: List[str],\n timestamps: Union[List[str], List[int]],\n topics: List[int] | None = None,\n nr_bins: int | None = None,\n datetime_format: str | None = None,\n evolution_tuning: bool = True,\n global_tuning: bool = True,\n ) -> pd.DataFrame:\n \"\"\"Create topics over time.\n\n To create the topics over time, BERTopic needs to be already fitted once.\n From the fitted models, the c-TF-IDF representations are calculate at\n each timestamp t. Then, the c-TF-IDF representations at timestamp t are\n averaged with the global c-TF-IDF representations in order to fine-tune the\n local representations.\n\n Note:\n Make sure to use a limited number of unique timestamps (<100) as the\n c-TF-IDF representation will be calculated at each single unique timestamp.\n Having a large number of unique timestamps can take some time to be calculated.\n Moreover, there aren't many use-cases where you would like to see the difference\n in topic representations over more than 100 different timestamps.\n\n Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n timestamps: The timestamp of each document. This can be either a list of strings or ints.\n If it is a list of strings, then the datetime format will be automatically\n inferred. If it is a list of ints, then the documents will be ordered in\n ascending order.\n topics: A list of topics where each topic is related to a document in `docs` and\n a timestamp in `timestamps`. You can use this to apply topics_over_time on\n a subset of the data. Make sure that `docs`, `timestamps`, and `topics`\n all correspond to one another and have the same size.\n nr_bins: The number of bins you want to create for the timestamps. The left interval will\n be chosen as the timestamp. An additional column will be created with the\n entire interval.\n datetime_format: The datetime format of the timestamps if they are strings, eg \"%d/%m/%Y\".\n Set this to None if you want to have it automatically detect the format.\n See strftime documentation for more information on choices:\n https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior.\n evolution_tuning: Fine-tune each topic representation at timestamp *t* by averaging its\n c-TF-IDF matrix with the c-TF-IDF matrix at timestamp *t-1*. This creates\n evolutionary topic representations.\n global_tuning: Fine-tune each topic representation at timestamp *t* by averaging its c-TF-IDF matrix\n with the global c-TF-IDF matrix. Turn this off if you want to prevent words in\n topic representations that could not be found in the documents at timestamp *t*.\n\n Returns:\n topics_over_time: A dataframe that contains the topic, words, and frequency of topic\n at timestamp *t*.\n\n Examples:\n The timestamps variable represents the timestamp of each document. If you have over\n 100 unique timestamps, it is advised to bin the timestamps as shown below:\n\n ```python\n from bertopic import BERTopic\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n topics_over_time = topic_model.topics_over_time(docs, timestamps, nr_bins=20)\n ```\n \"\"\"\n check_is_fitted(self)\n check_documents_type(docs)\n selected_topics = topics if topics else self.topics_\n documents = pd.DataFrame({\"Document\": docs, \"Topic\": selected_topics, \"Timestamps\": timestamps})\n global_c_tf_idf = normalize(self.c_tf_idf_, axis=1, norm=\"l1\", copy=False)\n\n all_topics = sorted(list(documents.Topic.unique()))\n all_topics_indices = {topic: index for index, topic in enumerate(all_topics)}\n\n if isinstance(timestamps[0], str):\n infer_datetime_format = True if not datetime_format else False\n documents[\"Timestamps\"] = pd.to_datetime(\n documents[\"Timestamps\"],\n infer_datetime_format=infer_datetime_format,\n format=datetime_format,\n )\n\n if nr_bins:\n documents[\"Bins\"] = pd.cut(documents.Timestamps, bins=nr_bins)\n documents[\"Timestamps\"] = documents.apply(lambda row: row.Bins.left, 1)\n\n # Sort documents in chronological order\n documents = documents.sort_values(\"Timestamps\")\n timestamps = documents.Timestamps.unique()\n if len(timestamps) > 100:\n logger.warning(\n f\"There are more than 100 unique timestamps (i.e., {len(timestamps)}) \"\n \"which significantly slows down the application. Consider setting `nr_bins` \"\n \"to a value lower than 100 to speed up calculation. \"\n )\n\n # For each unique timestamp, create topic representations\n topics_over_time = []\n for index, timestamp in tqdm(enumerate(timestamps), disable=not self.verbose):\n # Calculate c-TF-IDF representation for a specific timestamp\n selection = documents.loc[documents.Timestamps == timestamp, :]\n documents_per_topic = selection.groupby([\"Topic\"], as_index=False).agg(\n {\"Document\": \" \".join, \"Timestamps\": \"count\"}\n )\n c_tf_idf, words = self._c_tf_idf(documents_per_topic, fit=False)\n\n if global_tuning or evolution_tuning:\n c_tf_idf = normalize(c_tf_idf, axis=1, norm=\"l1\", copy=False)\n\n # Fine-tune the c-TF-IDF matrix at timestamp t by averaging it with the c-TF-IDF\n # matrix at timestamp t-1\n if evolution_tuning and index != 0:\n current_topics = sorted(list(documents_per_topic.Topic.values))\n overlapping_topics = sorted(\n list(set(previous_topics).intersection(set(current_topics))) # noqa: F821\n )\n\n current_overlap_idx = [current_topics.index(topic) for topic in overlapping_topics]\n previous_overlap_idx = [\n previous_topics.index(topic) # noqa: F821\n for topic in overlapping_topics\n ]\n\n c_tf_idf.tolil()[current_overlap_idx] = (\n (\n c_tf_idf[current_overlap_idx] + previous_c_tf_idf[previous_overlap_idx] # noqa: F821\n )\n / 2.0\n ).tolil()\n\n # Fine-tune the timestamp c-TF-IDF representation based on the global c-TF-IDF representation\n # by simply taking the average of the two\n if global_tuning:\n selected_topics = [all_topics_indices[topic] for topic in documents_per_topic.Topic.to_numpy()]\n c_tf_idf = (global_c_tf_idf[selected_topics] + c_tf_idf) / 2.0\n\n # Extract the words per topic\n words_per_topic = self._extract_words_per_topic(words, selection, c_tf_idf, calculate_aspects=False)\n topic_frequency = pd.Series(\n documents_per_topic.Timestamps.values, index=documents_per_topic.Topic\n ).to_dict()\n\n # Fill dataframe with results\n topics_at_timestamp = [\n (\n topic,\n \", \".join([words[0] for words in values][:5]),\n topic_frequency[topic],\n timestamp,\n )\n for topic, values in words_per_topic.items()\n ]\n topics_over_time.extend(topics_at_timestamp)\n\n if evolution_tuning:\n previous_topics = sorted(list(documents_per_topic.Topic.values)) # noqa: F841\n previous_c_tf_idf = c_tf_idf.copy() # noqa: F841\n\n return pd.DataFrame(topics_over_time, columns=[\"Topic\", \"Words\", \"Frequency\", \"Timestamp\"])\n\n def topics_per_class(\n self,\n docs: List[str],\n classes: Union[List[int], List[str]],\n global_tuning: bool = True,\n ) -> pd.DataFrame:\n \"\"\"Create topics per class.\n\n To create the topics per class, BERTopic needs to be already fitted once.\n From the fitted models, the c-TF-IDF representations are calculated at\n each class c. Then, the c-TF-IDF representations at class c are\n averaged with the global c-TF-IDF representations in order to fine-tune the\n local representations. This can be turned off if the pure representation is\n needed.\n\n Note:\n Make sure to use a limited number of unique classes (<100) as the\n c-TF-IDF representation will be calculated at each single unique class.\n Having a large number of unique classes can take some time to be calculated.\n\n Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n classes: The class of each document. This can be either a list of strings or ints.\n global_tuning: Fine-tune each topic representation for class c by averaging its c-TF-IDF matrix\n with the global c-TF-IDF matrix. Turn this off if you want to prevent words in\n topic representations that could not be found in the documents for class c.\n\n Returns:\n topics_per_class: A dataframe that contains the topic, words, and frequency of topics\n for each class.\n\n Examples:\n ```python\n from bertopic import BERTopic\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n topics_per_class = topic_model.topics_per_class(docs, classes)\n ```\n \"\"\"\n check_documents_type(docs)\n documents = pd.DataFrame({\"Document\": docs, \"Topic\": self.topics_, \"Class\": classes})\n global_c_tf_idf = normalize(self.c_tf_idf_, axis=1, norm=\"l1\", copy=False)\n\n # For each unique timestamp, create topic representations\n topics_per_class = []\n for _, class_ in tqdm(enumerate(set(classes)), disable=not self.verbose):\n # Calculate c-TF-IDF representation for a specific timestamp\n selection = documents.loc[documents.Class == class_, :]\n documents_per_topic = selection.groupby([\"Topic\"], as_index=False).agg(\n {\"Document\": \" \".join, \"Class\": \"count\"}\n )\n c_tf_idf, words = self._c_tf_idf(documents_per_topic, fit=False)\n\n # Fine-tune the timestamp c-TF-IDF representation based on the global c-TF-IDF representation\n # by simply taking the average of the two\n if global_tuning:\n c_tf_idf = normalize(c_tf_idf, axis=1, norm=\"l1\", copy=False)\n c_tf_idf = (global_c_tf_idf[documents_per_topic.Topic.to_numpy() + self._outliers] + c_tf_idf) / 2.0\n\n # Extract the words per topic\n words_per_topic = self._extract_words_per_topic(words, selection, c_tf_idf, calculate_aspects=False)\n topic_frequency = pd.Series(documents_per_topic.Class.to_numpy(), index=documents_per_topic.Topic).to_dict()\n\n # Fill dataframe with results\n topics_at_class = [\n (\n topic,\n \", \".join([words[0] for words in values][:5]),\n topic_frequency[topic],\n class_,\n )\n for topic, values in words_per_topic.items()\n ]\n topics_per_class.extend(topics_at_class)\n\n topics_per_class = pd.DataFrame(topics_per_class, columns=[\"Topic\", \"Words\", \"Frequency\", \"Class\"])\n\n return topics_per_class\n\n def hierarchical_topics(\n self,\n docs: List[str],\n use_ctfidf: bool = True,\n linkage_function: Callable[[csr_matrix], np.ndarray] | None = None,\n distance_function: Callable[[csr_matrix], csr_matrix] | None = None,\n ) -> pd.DataFrame:\n \"\"\"Create a hierarchy of topics.\n\n To create this hierarchy, BERTopic needs to be already fitted once.\n Then, a hierarchy is calculated on the distance matrix of the c-TF-IDF or topic embeddings\n representation using `scipy.cluster.hierarchy.linkage`.\n\n Based on that hierarchy, we calculate the topic representation at each\n merged step. This is a local representation, as we only assume that the\n chosen step is merged and not all others which typically improves the\n topic representation.\n\n Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the\n embeddings from the embedding model are used.\n linkage_function: The linkage function to use. Default is:\n `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`\n distance_function: The distance function to use on the c-TF-IDF matrix. Default is:\n `lambda x: 1 - cosine_similarity(x)`.\n You can pass any function that returns either a square matrix of\n shape (n_samples, n_samples) with zeros on the diagonal and\n non-negative values or condensed distance matrix of shape\n (n_samples * (n_samples - 1) / 2,) containing the upper\n triangular of the distance matrix.\n\n Returns:\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children\n\n Examples:\n ```python\n from bertopic import BERTopic\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n ```\n\n A custom linkage function can be used as follows:\n\n ```python\n from scipy.cluster import hierarchy as sch\n from bertopic import BERTopic\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n\n # Hierarchical topics\n linkage_function = lambda x: sch.linkage(x, 'ward', optimal_ordering=True)\n hierarchical_topics = topic_model.hierarchical_topics(docs, linkage_function=linkage_function)\n ```\n \"\"\"\n check_documents_type(docs)\n if distance_function is None:\n distance_function = lambda x: 1 - cosine_similarity(x)\n\n if linkage_function is None:\n linkage_function = lambda x: sch.linkage(x, \"ward\", optimal_ordering=True)\n\n # Calculate distance\n embeddings = select_topic_representation(self.c_tf_idf_, self.topic_embeddings_, use_ctfidf)[0][\n self._outliers :\n ]\n X = distance_function(embeddings)\n X = validate_distance_matrix(X, embeddings.shape[0])\n\n # Use the 1-D condensed distance matrix as an input instead of the raw distance matrix\n Z = linkage_function(X)\n\n # Ensuring that the distances between clusters are unique otherwise the flatting of the hierarchy with\n # `sch.fcluster(...)` would produce incorrect values for \"Topics\" for these clusters\n if len(Z[:, 2]) != len(np.unique(Z[:, 2])):\n Z[:, 2] = get_unique_distances(Z[:, 2])\n\n # Calculate basic bag-of-words to be iteratively merged later\n documents = pd.DataFrame({\"Document\": docs, \"ID\": range(len(docs)), \"Topic\": self.topics_})\n documents_per_topic = documents.groupby([\"Topic\"], as_index=False).agg({\"Document\": \" \".join})\n documents_per_topic = documents_per_topic.loc[documents_per_topic.Topic != -1, :]\n clean_documents = self._preprocess_text(documents_per_topic.Document.values)\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = self.vectorizer_model.get_feature_names_out()\n else:\n words = self.vectorizer_model.get_feature_names()\n\n bow = self.vectorizer_model.transform(clean_documents)\n\n # Extract clusters\n hier_topics = pd.DataFrame(\n columns=[\n \"Parent_ID\",\n \"Parent_Name\",\n \"Topics\",\n \"Child_Left_ID\",\n \"Child_Left_Name\",\n \"Child_Right_ID\",\n \"Child_Right_Name\",\n ]\n )\n for index in tqdm(range(len(Z))):\n # Find clustered documents\n clusters = sch.fcluster(Z, t=Z[index][2], criterion=\"distance\") - self._outliers\n nr_clusters = len(clusters)\n\n # Extract first topic we find to get the set of topics in a merged topic\n topic = None\n val = Z[index][0]\n while topic is None:\n if val - len(clusters) < 0:\n topic = int(val)\n else:\n val = Z[int(val - len(clusters))][0]\n clustered_topics = [i for i, x in enumerate(clusters) if x == clusters[topic]]\n\n # Group bow per cluster, calculate c-TF-IDF and extract words\n grouped = csr_matrix(bow[clustered_topics].sum(axis=0))\n c_tf_idf = self.ctfidf_model.transform(grouped)\n selection = documents.loc[documents.Topic.isin(clustered_topics), :]\n selection.Topic = 0\n words_per_topic = self._extract_words_per_topic(words, selection, c_tf_idf, calculate_aspects=False)\n\n # Extract parent's name and ID\n parent_id = index + len(clusters)\n parent_name = \"_\".join([x[0] for x in words_per_topic[0]][:5])\n\n # Extract child's name and ID\n Z_id = Z[index][0]\n child_left_id = Z_id if Z_id - nr_clusters < 0 else Z_id - nr_clusters\n\n if Z_id - nr_clusters < 0:\n child_left_name = \"_\".join([x[0] for x in self.get_topic(Z_id)][:5])\n else:\n child_left_name = hier_topics.iloc[int(child_left_id)].Parent_Name\n\n # Extract child's name and ID\n Z_id = Z[index][1]\n child_right_id = Z_id if Z_id - nr_clusters < 0 else Z_id - nr_clusters\n\n if Z_id - nr_clusters < 0:\n child_right_name = \"_\".join([x[0] for x in self.get_topic(Z_id)][:5])\n else:\n child_right_name = hier_topics.iloc[int(child_right_id)].Parent_Name\n\n # Save results\n hier_topics.loc[len(hier_topics), :] = [\n parent_id,\n parent_name,\n clustered_topics,\n int(Z[index][0]),\n child_left_name,\n int(Z[index][1]),\n child_right_name,\n ]\n\n hier_topics[\"Distance\"] = Z[:, 2]\n hier_topics = hier_topics.sort_values(\"Parent_ID\", ascending=False)\n hier_topics[[\"Parent_ID\", \"Child_Left_ID\", \"Child_Right_ID\"]] = hier_topics[\n [\"Parent_ID\", \"Child_Left_ID\", \"Child_Right_ID\"]\n ].astype(str)\n\n return hier_topics\n\n def approximate_distribution(\n self,\n documents: Union[str, List[str]],\n window: int = 4,\n stride: int = 1,\n min_similarity: float = 0.1,\n batch_size: int = 1000,\n padding: bool = False,\n use_embedding_model: bool = False,\n calculate_tokens: bool = False,\n separator: str = \" \",\n ) -> Tuple[np.ndarray, Union[List[np.ndarray], None]]:\n \"\"\"A post-hoc approximation of topic distributions across documents.\n\n In order to perform this approximation, each document is split into tokens\n according to the provided tokenizer in the `CountVectorizer`. Then, a\n sliding window is applied on each document creating subsets of the document.\n For example, with a window size of 3 and stride of 1, the sentence:\n\n `Solving the right problem is difficult.`\n\n can be split up into `solving the right`, `the right problem`, `right problem is`,\n and `problem is difficult`. These are called tokensets. For each of these\n tokensets, we calculate their c-TF-IDF representation and find out\n how similar they are to the previously generated topics. Then, the\n similarities to the topics for each tokenset are summed up in order to\n create a topic distribution for the entire document.\n\n We can also dive into this a bit deeper by then splitting these tokensets\n up into individual tokens and calculate how much a word, in a specific sentence,\n contributes to the topics found in that document. This can be enabled by\n setting `calculate_tokens=True` which can be used for visualization purposes\n in `topic_model.visualize_approximate_distribution`.\n\n The main output, `topic_distributions`, can also be used directly in\n `.visualize_distribution(topic_distributions[index])` by simply selecting\n a single distribution.\n\n Arguments:\n documents: A single document or a list of documents for which we\n approximate their topic distributions\n window: Size of the moving window which indicates the number of\n tokens being considered.\n stride: How far the window should move at each step.\n min_similarity: The minimum similarity of a document's tokenset\n with respect to the topics.\n batch_size: The number of documents to process at a time. If None,\n then all documents are processed at once.\n NOTE: With a large number of documents, it is not\n advised to process all documents at once.\n padding: Whether to pad the beginning and ending of a document with\n empty tokens.\n use_embedding_model: Whether to use the topic model's embedding\n model to calculate the similarity between\n tokensets and topics instead of using c-TF-IDF.\n calculate_tokens: Calculate the similarity of tokens with all topics.\n NOTE: This is computation-wise more expensive and\n can require more memory. Using this over batches of\n documents might be preferred.\n separator: The separator used to merge tokens into tokensets.\n\n Returns:\n topic_distributions: A `n` x `m` matrix containing the topic distributions\n for all input documents with `n` being the documents\n and `m` the topics.\n topic_token_distributions: A list of `t` x `m` arrays with `t` being the\n number of tokens for the respective document\n and `m` the topics.\n\n Examples:\n After fitting the model, the topic distributions can be calculated regardless\n of the clustering model and regardless of whether the documents were previously\n seen or not:\n\n ```python\n topic_distr, _ = topic_model.approximate_distribution(docs)\n ```\n\n As a result, the topic distributions are calculated in `topic_distr` for the\n entire document based on a token set with a specific window size and stride.\n\n If you want to calculate the topic distributions on a token-level:\n\n ```python\n topic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)\n ```\n\n The `topic_token_distr` then contains, for each token, the best fitting topics.\n As with `topic_distr`, it can contain multiple topics for a single token.\n \"\"\"\n if isinstance(documents, str):\n documents = [documents]\n\n if batch_size is None:\n batch_size = len(documents)\n batches = 1\n else:\n batches = math.ceil(len(documents) / batch_size)\n\n topic_distributions = []\n topic_token_distributions = []\n\n for i in tqdm(range(batches), disable=not self.verbose):\n doc_set = documents[i * batch_size : (i + 1) * batch_size]\n\n # Extract tokens\n analyzer = self.vectorizer_model.build_tokenizer()\n tokens = [analyzer(document) for document in doc_set]\n\n # Extract token sets\n all_sentences = []\n all_indices = [0]\n all_token_sets_ids = []\n\n for tokenset in tokens:\n if len(tokenset) < window:\n token_sets = [tokenset]\n token_sets_ids = [list(range(len(tokenset)))]\n else:\n # Extract tokensets using window and stride parameters\n stride_indices = list(range(len(tokenset)))[::stride]\n token_sets = []\n token_sets_ids = []\n for stride_index in stride_indices:\n selected_tokens = tokenset[stride_index : stride_index + window]\n\n if padding or len(selected_tokens) == window:\n token_sets.append(selected_tokens)\n token_sets_ids.append(\n list(\n range(\n stride_index,\n stride_index + len(selected_tokens),\n )\n )\n )\n\n # Add empty tokens at the beginning and end of a document\n if padding:\n padded = []\n padded_ids = []\n t = math.ceil(window / stride) - 1\n for i in range(math.ceil(window / stride) - 1):\n padded.append(tokenset[: window - ((t - i) * stride)])\n padded_ids.append(list(range(0, window - ((t - i) * stride))))\n\n token_sets = padded + token_sets\n token_sets_ids = padded_ids + token_sets_ids\n\n # Join the tokens\n sentences = [separator.join(token) for token in token_sets]\n all_sentences.extend(sentences)\n all_token_sets_ids.extend(token_sets_ids)\n all_indices.append(all_indices[-1] + len(sentences))\n\n # Calculate similarity between embeddings of token sets and the topics\n if use_embedding_model:\n embeddings = self._extract_embeddings(all_sentences, method=\"document\", verbose=True)\n similarity = cosine_similarity(embeddings, self.topic_embeddings_[self._outliers :])\n\n # Calculate similarity between c-TF-IDF of token sets and the topics\n else:\n bow_doc = self.vectorizer_model.transform(all_sentences)\n c_tf_idf_doc = self.ctfidf_model.transform(bow_doc)\n similarity = cosine_similarity(c_tf_idf_doc, self.c_tf_idf_[self._outliers :])\n\n # Only keep similarities that exceed the minimum\n similarity[similarity < min_similarity] = 0\n\n # Aggregate results on an individual token level\n if calculate_tokens:\n topic_distribution = []\n topic_token_distribution = []\n for index, token in enumerate(tokens):\n start = all_indices[index]\n end = all_indices[index + 1]\n\n if start == end:\n end = end + 1\n\n # Assign topics to individual tokens\n token_id = [i for i in range(len(token))]\n token_val = {index: [] for index in token_id}\n for sim, token_set in zip(similarity[start:end], all_token_sets_ids[start:end]):\n for token in token_set:\n if token in token_val:\n token_val[token].append(sim)\n\n matrix = []\n for _, value in token_val.items():\n matrix.append(np.add.reduce(value))\n\n # Take empty documents into account\n matrix = np.array(matrix)\n if len(matrix.shape) == 1:\n matrix = np.zeros((1, len(self.topic_labels_) - self._outliers))\n\n topic_token_distribution.append(np.array(matrix))\n topic_distribution.append(np.add.reduce(matrix))\n\n topic_distribution = normalize(topic_distribution, norm=\"l1\", axis=1)\n\n # Aggregate on a tokenset level indicated by the window and stride\n else:\n topic_distribution = []\n for index in range(len(all_indices) - 1):\n start = all_indices[index]\n end = all_indices[index + 1]\n\n if start == end:\n end = end + 1\n group = similarity[start:end].sum(axis=0)\n topic_distribution.append(group)\n topic_distribution = normalize(np.array(topic_distribution), norm=\"l1\", axis=1)\n topic_token_distribution = None\n\n # Combine results\n topic_distributions.append(topic_distribution)\n if topic_token_distribution is None:\n topic_token_distributions = None\n else:\n topic_token_distributions.extend(topic_token_distribution)\n\n topic_distributions = np.vstack(topic_distributions)\n\n return topic_distributions, topic_token_distributions\n\n def find_topics(\n self, search_term: str | None = None, image: str | None = None, top_n: int = 5\n ) -> Tuple[List[int], List[float]]:\n \"\"\"Find topics most similar to a search_term.\n\n Creates an embedding for a search query and compares that with\n the topic embeddings. The most similar topics are returned\n along with their similarity values.\n\n The query is specified using search_term for text queries or image for image queries.\n\n The search_term can be of any size but since it is compared\n with the topic representation it is advised to keep it\n below 5 words.\n\n Arguments:\n search_term: the term you want to use to search for topics.\n image: path to the image you want to use to search for topics.\n top_n: the number of topics to return\n\n Returns:\n similar_topics: the most similar topics from high to low\n similarity: the similarity scores from high to low\n\n Examples:\n You can use the underlying embedding model to find topics that\n best represent the search term:\n\n ```python\n topics, similarity = topic_model.find_topics(\"sports\", top_n=5)\n ```\n\n Note that the search query is typically more accurate if the\n search_term consists of a phrase or multiple words.\n \"\"\"\n if self.embedding_model is None:\n raise Exception(\"This method can only be used if you did not use custom embeddings.\")\n\n topic_list = list(self.topic_representations_.keys())\n topic_list.sort()\n\n # Extract search_term embeddings and compare with topic embeddings\n if search_term is not None:\n search_embedding = self._extract_embeddings([search_term], method=\"word\", verbose=False).flatten()\n elif image is not None:\n search_embedding = self._extract_embeddings(\n [None], images=[image], method=\"document\", verbose=False\n ).flatten()\n sims = cosine_similarity(search_embedding.reshape(1, -1), self.topic_embeddings_).flatten()\n\n # Extract topics most similar to search_term\n ids = np.argsort(sims)[-top_n:]\n similarity = [sims[i] for i in ids][::-1]\n similar_topics = [topic_list[index] for index in ids][::-1]\n\n return similar_topics, similarity\n\n def update_topics(\n self,\n docs: List[str],\n images: List[str] | None = None,\n topics: List[int] | None = None,\n top_n_words: int = 10,\n n_gram_range: Tuple[int, int] | None = None,\n vectorizer_model: CountVectorizer = None,\n ctfidf_model: ClassTfidfTransformer = None,\n representation_model: BaseRepresentation = None,\n ):\n \"\"\"Updates the topic representation by recalculating c-TF-IDF with the new\n parameters as defined in this function.\n\n When you have trained a model and viewed the topics and the words that represent them,\n you might not be satisfied with the representation. Perhaps you forgot to remove\n stop_words or you want to try out a different n_gram_range. This function allows you\n to update the topic representation after they have been formed.\n\n Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n images: The images you used when calling either `fit` or `fit_transform`\n topics: A list of topics where each topic is related to a document in `docs`.\n Use this variable to change or map the topics.\n NOTE: Using a custom list of topic assignments may lead to errors if\n topic reduction techniques are used afterwards. Make sure that\n manually assigning topics is the last step in the pipeline\n top_n_words: The number of words per topic to extract. Setting this\n too high can negatively impact topic embeddings as topics\n are typically best represented by at most 10 words.\n n_gram_range: The n-gram range for the CountVectorizer.\n vectorizer_model: Pass in your own CountVectorizer from scikit-learn\n ctfidf_model: Pass in your own c-TF-IDF model to update the representations\n representation_model: Pass in a model that fine-tunes the topic representations\n calculated through c-TF-IDF. Models from `bertopic.representation`\n are supported.\n\n Examples:\n In order to update the topic representation, you will need to first fit the topic\n model and extract topics from them. Based on these, you can update the representation:\n\n ```python\n topic_model.update_topics(docs, n_gram_range=(2, 3))\n ```\n\n You can also use a custom vectorizer to update the representation:\n\n ```python\n from sklearn.feature_extraction.text import CountVectorizer\n vectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words=\"english\")\n topic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n ```\n\n You can also use this function to change or map the topics to something else.\n You can update them as follows:\n\n ```python\n topic_model.update_topics(docs, my_updated_topics)\n ```\n \"\"\"\n check_documents_type(docs)\n check_is_fitted(self)\n if not n_gram_range:\n n_gram_range = self.n_gram_range\n\n if top_n_words > 100:\n logger.warning(\n \"Note that extracting more than 100 words from a sparse can slow down computation quite a bit.\"\n )\n self.top_n_words = top_n_words\n self.vectorizer_model = vectorizer_model or CountVectorizer(ngram_range=n_gram_range)\n self.ctfidf_model = ctfidf_model or ClassTfidfTransformer()\n self.representation_model = representation_model\n\n if topics is None:\n topics = self.topics_\n else:\n logger.warning(\n \"Using a custom list of topic assignments may lead to errors if \"\n \"topic reduction techniques are used afterwards. Make sure that \"\n \"manually assigning topics is the last step in the pipeline.\"\n \"Note that topic embeddings will also be created through weighted\"\n \"c-TF-IDF embeddings instead of centroid embeddings.\"\n )\n\n documents = pd.DataFrame({\"Document\": docs, \"Topic\": topics, \"ID\": range(len(docs)), \"Image\": images})\n documents_per_topic = documents.groupby([\"Topic\"], as_index=False).agg({\"Document\": \" \".join})\n\n # Update topic sizes and assignments\n self._update_topic_size(documents)\n\n # Extract words and update topic labels\n self.c_tf_idf_, words = self._c_tf_idf(documents_per_topic)\n self.topic_representations_ = self._extract_words_per_topic(words, documents)\n\n # Update topic vectors\n if set(topics) != set(self.topics_):\n # Remove outlier topic embedding if all that has changed is the outlier class\n same_position = all(\n [\n True if old_topic == new_topic else False\n for old_topic, new_topic in zip(self.topics_, topics)\n if old_topic != -1\n ]\n )\n if same_position and -1 not in topics and -1 in self.topics_:\n self.topic_embeddings_ = self.topic_embeddings_[1:]\n else:\n self._create_topic_vectors()\n\n def get_topics(self, full: bool = False) -> Mapping[str, Tuple[str, float]]:\n \"\"\"Return topics with top n words and their c-TF-IDF score.\n\n Arguments:\n full: If True, returns all different forms of topic representations\n for each topic, including aspects\n\n Returns:\n self.topic_representations_: The top n words per topic and the corresponding c-TF-IDF score\n\n Examples:\n ```python\n all_topics = topic_model.get_topics()\n ```\n \"\"\"\n check_is_fitted(self)\n\n if full:\n topic_representations = {\"Main\": self.topic_representations_}\n topic_representations.update(self.topic_aspects_)\n return topic_representations\n else:\n return self.topic_representations_\n\n def get_topic(self, topic: int, full: bool = False) -> Union[Mapping[str, Tuple[str, float]], bool]:\n \"\"\"Return top n words for a specific topic and their c-TF-IDF scores.\n\n Arguments:\n topic: A specific topic for which you want its representation\n full: If True, returns all different forms of topic representations\n for a topic, including aspects\n\n Returns:\n The top n words for a specific word and its respective c-TF-IDF scores\n\n Examples:\n ```python\n topic = topic_model.get_topic(12)\n ```\n \"\"\"\n check_is_fitted(self)\n if topic in self.topic_representations_:\n if full:\n representations = {\"Main\": self.topic_representations_[topic]}\n aspects = {aspect: representations[topic] for aspect, representations in self.topic_aspects_.items()}\n representations.update(aspects)\n return representations\n else:\n return self.topic_representations_[topic]\n else:\n return False\n\n def get_topic_info(self, topic: int | None = None) -> pd.DataFrame:\n \"\"\"Get information about each topic including its ID, frequency, and name.\n\n Arguments:\n topic: A specific topic for which you want the frequency\n\n Returns:\n info: The information relating to either a single topic or all topics\n\n Examples:\n ```python\n info_df = topic_model.get_topic_info()\n ```\n \"\"\"\n check_is_fitted(self)\n\n info = pd.DataFrame(self.topic_sizes_.items(), columns=[\"Topic\", \"Count\"]).sort_values(\"Topic\")\n info[\"Name\"] = info.Topic.map(self.topic_labels_)\n\n # Custom label\n if self.custom_labels_ is not None:\n if len(self.custom_labels_) == len(info):\n labels = {topic - self._outliers: label for topic, label in enumerate(self.custom_labels_)}\n info[\"CustomName\"] = info[\"Topic\"].map(labels)\n\n # Main Keywords\n values = {topic: list(next(zip(*values))) for topic, values in self.topic_representations_.items()}\n info[\"Representation\"] = info[\"Topic\"].map(values)\n\n # Extract all topic aspects\n if self.topic_aspects_:\n for aspect, values in self.topic_aspects_.items():\n if isinstance(list(values.values())[-1], list):\n if isinstance(list(values.values())[-1][0], tuple) or isinstance(\n list(values.values())[-1][0], list\n ):\n values = {topic: list(next(zip(*value))) for topic, value in values.items()}\n elif isinstance(list(values.values())[-1][0], str):\n values = {topic: \" \".join(value).strip() for topic, value in values.items()}\n info[aspect] = info[\"Topic\"].map(values)\n\n # Representative Docs / Images\n if self.representative_docs_ is not None:\n info[\"Representative_Docs\"] = info[\"Topic\"].map(self.representative_docs_)\n if self.representative_images_ is not None:\n info[\"Representative_Images\"] = info[\"Topic\"].map(self.representative_images_)\n\n # Select specific topic to return\n if topic is not None:\n info = info.loc[info.Topic == topic, :]\n\n return info.reset_index(drop=True)\n\n def get_topic_freq(self, topic: int | None = None) -> Union[pd.DataFrame, int]:\n \"\"\"Return the size of topics (descending order).\n\n Arguments:\n topic: A specific topic for which you want the frequency\n\n Returns:\n Either the frequency of a single topic or dataframe with\n the frequencies of all topics\n\n Examples:\n To extract the frequency of all topics:\n\n ```python\n frequency = topic_model.get_topic_freq()\n ```\n\n To get the frequency of a single topic:\n\n ```python\n frequency = topic_model.get_topic_freq(12)\n ```\n \"\"\"\n check_is_fitted(self)\n if isinstance(topic, int):\n return self.topic_sizes_[topic]\n else:\n return pd.DataFrame(self.topic_sizes_.items(), columns=[\"Topic\", \"Count\"]).sort_values(\n \"Count\", ascending=False\n )\n\n def get_document_info(\n self,\n docs: List[str],\n df: pd.DataFrame = None,\n metadata: Mapping[str, Any] | None = None,\n ) -> pd.DataFrame:\n \"\"\"Get information about the documents on which the topic was trained\n including the documents themselves, their respective topics, the name\n of each topic, the top n words of each topic, whether it is a\n representative document, and probability of the clustering if the cluster\n model supports it.\n\n There are also options to include other meta data, such as the topic\n distributions or the x and y coordinates of the reduced embeddings.\n\n Arguments:\n docs: The documents on which the topic model was trained.\n df: A dataframe containing the metadata and the documents on which\n the topic model was originally trained on.\n metadata: A dictionary with meta data for each document in the form\n of column name (key) and the respective values (value).\n\n Returns:\n document_info: A dataframe with several statistics regarding\n the documents on which the topic model was trained.\n\n Usage:\n\n To get the document info, you will only need to pass the documents on which\n the topic model was trained:\n\n ```python\n document_info = topic_model.get_document_info(docs)\n ```\n\n There are additionally options to include meta data, such as the topic\n distributions. Moreover, we can pass the original dataframe that contains\n the documents and extend it with the information retrieved from BERTopic:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n\n # The original data in a dataframe format to include the target variable\n data = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\n df = pd.DataFrame({\"Document\": data['data'], \"Class\": data['target']})\n\n # Add information about the percentage of the document that relates to the topic\n topic_distr, _ = topic_model.approximate_distribution(docs, batch_size=1000)\n distributions = [distr[topic] if topic != -1 else 0 for topic, distr in zip(topics, topic_distr)]\n\n # Create our documents dataframe using the original dataframe and meta data about\n # the topic distributions\n document_info = topic_model.get_document_info(docs, df=df,\n metadata={\"Topic_distribution\": distributions})\n \"\"\"\n check_documents_type(docs)\n if df is not None:\n document_info = df.copy()\n document_info[\"Document\"] = docs\n document_info[\"Topic\"] = self.topics_\n else:\n document_info = pd.DataFrame({\"Document\": docs, \"Topic\": self.topics_})\n\n # Add topic info through `.get_topic_info()`\n topic_info = self.get_topic_info().drop(\"Count\", axis=1)\n document_info = document_info.merge(topic_info, on=\"Topic\", how=\"left\")\n\n # Add top n words\n top_n_words = {topic: \" - \".join(next(zip(*self.get_topic(topic)))) for topic in set(self.topics_)}\n document_info[\"Top_n_words\"] = document_info.Topic.map(top_n_words)\n\n # Add flat probabilities\n if self.probabilities_ is not None:\n if len(self.probabilities_.shape) == 1:\n document_info[\"Probability\"] = self.probabilities_\n else:\n document_info[\"Probability\"] = [\n max(probs) if topic != -1 else 1 - sum(probs)\n for topic, probs in zip(self.topics_, self.probabilities_)\n ]\n\n # Add representative document labels\n repr_docs = [repr_doc for repr_docs in self.representative_docs_.values() for repr_doc in repr_docs]\n document_info[\"Representative_document\"] = False\n document_info.loc[document_info.Document.isin(repr_docs), \"Representative_document\"] = True\n\n # Add custom meta data provided by the user\n if metadata is not None:\n for column, values in metadata.items():\n document_info[column] = values\n return document_info\n\n def get_representative_docs(self, topic: int | None = None) -> List[str]:\n \"\"\"Extract the best representing documents per topic.\n\n Note:\n This does not extract all documents per topic as all documents\n are not saved within BERTopic. To get all documents, please\n run the following:\n\n ```python\n # When you used `.fit_transform`:\n df = pd.DataFrame({\"Document\": docs, \"Topic\": topic})\n\n # When you used `.fit`:\n df = pd.DataFrame({\"Document\": docs, \"Topic\": topic_model.topics_})\n ```\n\n Arguments:\n topic: A specific topic for which you want\n the representative documents\n\n Returns:\n Representative documents of the chosen topic\n\n Examples:\n To extract the representative docs of all topics:\n\n ```python\n representative_docs = topic_model.get_representative_docs()\n ```\n\n To get the representative docs of a single topic:\n\n ```python\n representative_docs = topic_model.get_representative_docs(12)\n ```\n \"\"\"\n check_is_fitted(self)\n if isinstance(topic, int):\n if self.representative_docs_.get(topic):\n return self.representative_docs_[topic]\n else:\n return None\n else:\n return self.representative_docs_\n\n @staticmethod\n def get_topic_tree(\n hier_topics: pd.DataFrame,\n max_distance: float | None = None,\n tight_layout: bool = False,\n ) -> str:\n \"\"\"Extract the topic tree such that it can be printed.\n\n Arguments:\n hier_topics: A dataframe containing the structure of the topic tree.\n This is the output of `topic_model.hierarchical_topics()`\n max_distance: The maximum distance between two topics. This value is\n based on the Distance column in `hier_topics`.\n tight_layout: Whether to use a tight layout (narrow width) for\n easier readability if you have hundreds of topics.\n\n Returns:\n A tree that has the following structure when printed:\n .\n .\n └─health_medical_disease_patients_hiv\n β”œβ”€patients_medical_disease_candida_health\n β”‚ β”œβ”€β– β”€β”€candida_yeast_infection_gonorrhea_infections ── Topic: 48\n β”‚ └─patients_disease_cancer_medical_doctor\n β”‚ β”œβ”€β– β”€β”€hiv_medical_cancer_patients_doctor ── Topic: 34\n β”‚ └─■──pain_drug_patients_disease_diet ── Topic: 26\n └─■──health_newsgroup_tobacco_vote_votes ── Topic: 9\n\n The blocks (β– ) indicate that the topic is one you can directly access\n from `topic_model.get_topic`. In other words, they are the original un-grouped topics.\n\n Examples:\n ```python\n # Train model\n from bertopic import BERTopic\n topic_model = BERTopic()\n topics, probs = topic_model.fit_transform(docs)\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n\n # Print topic tree\n tree = topic_model.get_topic_tree(hierarchical_topics)\n print(tree)\n ```\n \"\"\"\n width = 1 if tight_layout else 4\n if max_distance is None:\n max_distance = hier_topics.Distance.max() + 1\n\n max_original_topic = hier_topics.Parent_ID.astype(int).min() - 1\n\n # Extract mapping from ID to name\n topic_to_name = dict(zip(hier_topics.Child_Left_ID, hier_topics.Child_Left_Name))\n topic_to_name.update(dict(zip(hier_topics.Child_Right_ID, hier_topics.Child_Right_Name)))\n topic_to_name = {topic: name[:100] for topic, name in topic_to_name.items()}\n\n # Create tree\n tree = {\n str(row[1].Parent_ID): [\n str(row[1].Child_Left_ID),\n str(row[1].Child_Right_ID),\n ]\n for row in hier_topics.iterrows()\n }\n\n def get_tree(start, tree):\n \"\"\"Based on: https://stackoverflow.com/a/51920869/10532563.\"\"\"\n\n def _tree(to_print, start, parent, tree, grandpa=None, indent=\"\"):\n # Get distance between merged topics\n distance = hier_topics.loc[\n (hier_topics.Child_Left_ID == parent) | (hier_topics.Child_Right_ID == parent),\n \"Distance\",\n ]\n distance = distance.to_numpy()[0] if len(distance) > 0 else 10\n\n if parent != start:\n if grandpa is None:\n to_print += topic_to_name[parent]\n else:\n if int(parent) <= max_original_topic:\n # Do not append topic ID if they are not merged\n if distance < max_distance:\n to_print += \"■──\" + topic_to_name[parent] + f\" ── Topic: {parent}\" + \"\\n\"\n else:\n to_print += \"O \\n\"\n else:\n to_print += topic_to_name[parent] + \"\\n\"\n\n if parent not in tree:\n return to_print\n\n for child in tree[parent][:-1]:\n to_print += indent + \"β”œ\" + \"─\"\n to_print = _tree(to_print, start, child, tree, parent, indent + \"β”‚\" + \" \" * width)\n\n child = tree[parent][-1]\n to_print += indent + \"β””\" + \"─\"\n to_print = _tree(to_print, start, child, tree, parent, indent + \" \" * (width + 1))\n\n return to_print\n\n to_print = \".\" + \"\\n\"\n to_print = _tree(to_print, start, start, tree)\n return to_print\n\n start = str(hier_topics.Parent_ID.astype(int).max())\n return get_tree(start, tree)\n\n def set_topic_labels(self, topic_labels: Union[List[str], Mapping[int, str]]) -> None:\n \"\"\"Set custom topic labels in your fitted BERTopic model.\n\n Arguments:\n topic_labels: If a list of topic labels, it should contain the same number\n of labels as there are topics. This must be ordered\n from the topic with the lowest ID to the highest ID,\n including topic -1 if it exists.\n If a dictionary of `topic ID`: `topic_label`, it can have\n any number of topics as it will only map the topics found\n in the dictionary.\n\n Examples:\n First, we define our topic labels with `.generate_topic_labels` in which\n we can customize our topic labels:\n\n ```python\n topic_labels = topic_model.generate_topic_labels(nr_words=2,\n topic_prefix=True,\n word_length=10,\n separator=\", \")\n ```\n\n Then, we pass these `topic_labels` to our topic model which\n can be accessed at any time with `.custom_labels_`:\n\n ```python\n topic_model.set_topic_labels(topic_labels)\n topic_model.custom_labels_\n ```\n\n You might want to change only a few topic labels instead of all of them.\n To do so, you can pass a dictionary where the keys are the topic IDs and\n its keys the topic labels:\n\n ```python\n topic_model.set_topic_labels({0: \"Space\", 1: \"Sports\", 2: \"Medicine\"})\n topic_model.custom_labels_\n ```\n \"\"\"\n unique_topics = sorted(set(self.topics_))\n\n if isinstance(topic_labels, dict):\n if self.custom_labels_ is not None:\n original_labels = {topic: label for topic, label in zip(unique_topics, self.custom_labels_)}\n else:\n info = self.get_topic_info()\n original_labels = dict(zip(info.Topic, info.Name))\n custom_labels = [\n topic_labels.get(topic) if topic_labels.get(topic) else original_labels[topic]\n for topic in unique_topics\n ]\n\n elif isinstance(topic_labels, list):\n if len(topic_labels) == len(unique_topics):\n custom_labels = topic_labels\n else:\n raise ValueError(\n \"Make sure that `topic_labels` contains the same number of labels as there are topics.\"\n )\n\n self.custom_labels_ = custom_labels\n\n def generate_topic_labels(\n self,\n nr_words: int = 3,\n topic_prefix: bool = True,\n word_length: int | None = None,\n separator: str = \"_\",\n aspect: str | None = None,\n ) -> List[str]:\n \"\"\"Get labels for each topic in a user-defined format.\n\n Arguments:\n nr_words: Top `n` words per topic to use\n topic_prefix: Whether to use the topic ID as a prefix.\n If set to True, the topic ID will be separated\n using the `separator`\n word_length: The maximum length of each word in the topic label.\n Some words might be relatively long and setting this\n value helps to make sure that all labels have relatively\n similar lengths.\n separator: The string with which the words and topic prefix will be\n separated. Underscores are the default but a nice alternative\n is `\", \"`.\n aspect: The aspect from which to generate topic labels\n\n Returns:\n topic_labels: A list of topic labels sorted from the lowest topic ID to the highest.\n If the topic model was trained using HDBSCAN, the lowest topic ID is -1,\n otherwise it is 0.\n\n Examples:\n To create our custom topic labels, usage is rather straightforward:\n\n ```python\n topic_labels = topic_model.generate_topic_labels(nr_words=2, separator=\", \")\n ```\n \"\"\"\n unique_topics = sorted(set(self.topics_))\n\n topic_labels = []\n for topic in unique_topics:\n if aspect:\n words, _ = zip(*self.topic_aspects_[aspect][topic])\n else:\n words, _ = zip(*self.get_topic(topic))\n\n if word_length:\n words = [word[:word_length] for word in words][:nr_words]\n else:\n words = list(words)[:nr_words]\n\n if topic_prefix:\n topic_label = f\"{topic}{separator}\" + separator.join(words)\n else:\n topic_label = separator.join(words)\n\n topic_labels.append(topic_label)\n\n return topic_labels\n\n def merge_topics(\n self,\n docs: List[str],\n topics_to_merge: List[Union[Iterable[int], int]],\n images: List[str] | None = None,\n ) -> None:\n \"\"\"Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n topics_to_merge: Either a list of topics or a list of list of topics\n to merge. For example:\n [1, 2, 3] will merge topics 1, 2 and 3\n [[1, 2], [3, 4]] will merge topics 1 and 2, and\n separately merge topics 3 and 4.\n images: A list of paths to the images used when calling either\n `fit` or `fit_transform`.\n\n Examples:\n If you want to merge topics 1, 2, and 3:\n\n ```python\n topics_to_merge = [1, 2, 3]\n topic_model.merge_topics(docs, topics_to_merge)\n ```\n\n or if you want to merge topics 1 and 2, and separately\n merge topics 3 and 4:\n\n ```python\n topics_to_merge = [[1, 2],\n [3, 4]]\n topic_model.merge_topics(docs, topics_to_merge)\n ```\n \"\"\"\n check_is_fitted(self)\n check_documents_type(docs)\n documents = pd.DataFrame(\n {\n \"Document\": docs,\n \"Topic\": self.topics_,\n \"Image\": images,\n \"ID\": range(len(docs)),\n }\n )\n\n mapping = {topic: topic for topic in set(self.topics_)}\n if isinstance(topics_to_merge[0], int):\n for topic in sorted(topics_to_merge):\n mapping[topic] = topics_to_merge[0]\n elif isinstance(topics_to_merge[0], Iterable):\n for topic_group in sorted(topics_to_merge):\n for topic in topic_group:\n mapping[topic] = topic_group[0]\n else:\n raise ValueError(\"Make sure that `topics_to_merge` is eithera list of topics or a list of list of topics.\")\n\n # Track mappings and sizes of topics for merging topic embeddings\n mappings = defaultdict(list)\n for key, val in sorted(mapping.items()):\n mappings[val].append(key)\n mappings = {\n topic_to: {\n \"topics_from\": topics_from,\n \"topic_sizes\": [self.topic_sizes_[topic] for topic in topics_from],\n }\n for topic_to, topics_from in mappings.items()\n }\n\n # Update topics\n documents.Topic = documents.Topic.map(mapping)\n self.topic_mapper_.add_mappings(mapping, topic_model=self)\n documents = self._sort_mappings_by_frequency(documents)\n self._extract_topics(documents, mappings=mappings)\n self._update_topic_size(documents)\n self._save_representative_docs(documents)\n self.probabilities_ = self._map_probabilities(self.probabilities_)\n\n def delete_topics(\n self,\n topics_to_delete: List[int],\n ) -> None:\n \"\"\"Delete topics from the topic model.\n\n The deleted topics will be mapped to -1 (outlier topic). Core topic attributes\n like topic embeddings and c-TF-IDF will be automatically updated.\n\n Arguments:\n topics_to_delete: List of topics to delete\n \"\"\"\n check_is_fitted(self)\n\n topics_df = pd.DataFrame({\"Topic\": self.topics_})\n\n # Check if -1 exists in the current topics\n had_outliers = -1 in set(self.topics_)\n\n # If adding -1 for the first time, initialize its attributes\n if not had_outliers and any(topic in topics_to_delete for topic in self.topics_):\n # Initialize c-TF-IDF for -1 topic (zeros)\n outlier_row = np.zeros((1, self.c_tf_idf_.shape[1]))\n outlier_row = sp.csr_matrix(outlier_row)\n self.c_tf_idf_ = sp.vstack([outlier_row, self.c_tf_idf_])\n\n # Initialize topic embeddings for -1 topic (zeros)\n outlier_embedding = np.zeros((1, self.topic_embeddings_.shape[1]))\n self.topic_embeddings_ = np.vstack([outlier_embedding, self.topic_embeddings_])\n\n # Initialize topic representations for -1 topic: (\"\", 1e-05)\n self.topic_representations_[-1] = [(\"\", 1e-05)]\n\n # Initialize representative docs for -1 topic (empty list)\n self.representative_docs_[-1] = []\n\n # Initialize representative images for -1 topic if images are being used\n if self.representative_images_ is not None:\n outlier_image = np.zeros((1, self.representative_images_.shape[1]))\n self.representative_images_ = np.vstack([outlier_image, self.representative_images_])\n\n # Initialize custom labels for -1 topic if they exist\n if hasattr(self, \"custom_labels_\") and self.custom_labels_ is not None:\n self.custom_labels_[-1] = \"\"\n\n # Initialize ctfidf model diagonal for -1 topic (ones) if it exists\n if hasattr(self, \"ctfidf_model\") and self.ctfidf_model is not None:\n n_features = self.ctfidf_model._idf_diag.shape[1]\n outlier_diag = sp.csr_matrix(([1.0], ([0], [0])), shape=(1, n_features))\n self.ctfidf_model._idf_diag = sp.vstack([outlier_diag, self.ctfidf_model._idf_diag])\n\n # Initialize topic aspects for -1 topic (empty dict for each aspect) if they exist\n if hasattr(self, \"topic_aspects_\") and self.topic_aspects_ is not None:\n for aspect in self.topic_aspects_:\n self.topic_aspects_[aspect][-1] = {}\n\n # First map deleted topics to -1\n mapping = {topic: -1 if topic in topics_to_delete else topic for topic in set(self.topics_)}\n mapping[-1] = -1\n\n # Track mappings and sizes of topics for merging topic embeddings\n mappings = defaultdict(list)\n for key, val in sorted(mapping.items()):\n mappings[val].append(key)\n mappings = {\n topic_to: {\n \"topics_from\": topics_from,\n \"topic_sizes\": [self.topic_sizes_[topic] for topic in topics_from],\n }\n for topic_to, topics_from in mappings.items()\n }\n\n # remove deleted topics and update attributes\n topics_df.Topic = topics_df.Topic.map(mapping)\n self.topic_mapper_.add_mappings(mapping, topic_model=deepcopy(self))\n topics_df = self._sort_mappings_by_frequency(topics_df)\n self._update_topic_size(topics_df)\n self.probabilities_ = self._map_probabilities(self.probabilities_)\n\n final_mapping = self.topic_mapper_.get_mappings(original_topics=False)\n\n # Update dictionary-based attributes to remove deleted topics\n # Handle topic_aspects_ if it exists\n if hasattr(self, \"topic_aspects_\") and self.topic_aspects_ is not None:\n new_aspects = {\n aspect: {\n (final_mapping[old_topic] if old_topic != -1 else -1): content\n for old_topic, content in topics.items()\n if old_topic not in topics_to_delete\n }\n for aspect, topics in self.topic_aspects_.items()\n }\n self.topic_aspects_ = new_aspects\n\n # Update custom labels if they exist\n if hasattr(self, \"custom_labels_\") and self.custom_labels_ is not None:\n new_labels = {\n (final_mapping[old_topic] if old_topic != -1 else -1): label\n for old_topic, label in self.custom_labels_.items()\n if old_topic not in topics_to_delete\n }\n self.custom_labels_ = new_labels\n\n # Update topic representations\n new_representations = {\n (final_mapping[old_topic] if old_topic != -1 else -1): content\n for old_topic, content in self.topic_representations_.items()\n if old_topic not in topics_to_delete\n }\n self.topic_representations_ = new_representations\n\n # Update representative docs if they exist\n new_representative_docs = {\n (final_mapping[old_topic] if old_topic != -1 else -1): docs\n for old_topic, docs in self.representative_docs_.items()\n if old_topic not in topics_to_delete\n }\n self.representative_docs_ = new_representative_docs\n\n # Update representative images if they exist\n if self.representative_images_ is not None:\n # Create a mask for non-deleted topics\n mask = np.array([topic not in topics_to_delete for topic in range(len(self.representative_images_))])\n self.representative_images_ = self.representative_images_[mask] if mask.any() else None\n\n # Update array-based attributes using masks to remove deleted topics\n for attr in [\"topic_embeddings_\", \"c_tf_idf_\"]:\n matrix = getattr(self, attr)\n mask = np.array([topic not in topics_to_delete for topic in range(matrix.shape[0])])\n setattr(self, attr, matrix[mask])\n\n # Update ctfidf model to remove deleted topics if it exists\n if hasattr(self, \"ctfidf_model\") and self.ctfidf_model is not None:\n mask = np.array([topic not in topics_to_delete for topic in range(self.ctfidf_model._idf_diag.shape[0])])\n self.ctfidf_model._idf_diag = self.ctfidf_model._idf_diag[mask]\n\n def reduce_topics(\n self,\n docs: List[str],\n nr_topics: Union[int, str] = 20,\n images: List[str] | None = None,\n use_ctfidf: bool = False,\n ) -> None:\n \"\"\"Reduce the number of topics to a fixed number of topics\n or automatically.\n\n If nr_topics is an integer, then the number of topics is reduced\n to nr_topics using `AgglomerativeClustering` on the cosine distance matrix\n of the topic c-TF-IDF or semantic embeddings.\n\n If nr_topics is `\"auto\"`, then HDBSCAN is used to automatically\n reduce the number of topics by running it on the topic embeddings.\n\n The topics, their sizes, and representations are updated.\n\n Arguments:\n docs: The docs you used when calling either `fit` or `fit_transform`\n nr_topics: The number of topics you want reduced to\n images: A list of paths to the images used when calling either\n `fit` or `fit_transform`\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the\n embeddings from the embedding model are used.\n\n Updates:\n topics_ : Assigns topics to their merged representations.\n probabilities_ : Assigns probabilities to their merged representations.\n\n Examples:\n You can further reduce the topics by passing the documents with their\n topics and probabilities (if they were calculated):\n\n ```python\n topic_model.reduce_topics(docs, nr_topics=30)\n ```\n\n You can then access the updated topics and probabilities with:\n\n ```python\n topics = topic_model.topics_\n probabilities = topic_model.probabilities_\n ```\n \"\"\"\n check_is_fitted(self)\n check_documents_type(docs)\n\n self.nr_topics = nr_topics\n documents = pd.DataFrame(\n {\n \"Document\": docs,\n \"Topic\": self.topics_,\n \"Image\": images,\n \"ID\": range(len(docs)),\n }\n )\n\n # Reduce number of topics\n documents = self._reduce_topics(documents, use_ctfidf)\n self._merged_topics = None\n self._save_representative_docs(documents)\n self.probabilities_ = self._map_probabilities(self.probabilities_)\n\n return self\n\n def reduce_outliers(\n self,\n documents: List[str],\n topics: List[int],\n images: List[str] | None = None,\n strategy: str = \"distributions\",\n probabilities: np.ndarray = None,\n threshold: float = 0,\n embeddings: np.ndarray = None,\n distributions_params: Mapping[str, Any] = {},\n ) -> List[int]:\n \"\"\"Reduce outliers by merging them with their nearest topic according\n to one of several strategies.\n\n When using HDBSCAN, DBSCAN, or OPTICS, a number of outlier documents might be created\n that do not fall within any of the created topics. These are labeled as -1.\n This function allows the user to match outlier documents with their nearest topic\n using one of the following strategies using the `strategy` parameter:\n * \"probabilities\"\n This uses the soft-clustering as performed by HDBSCAN to find the\n best matching topic for each outlier document. To use this, make\n sure to calculate the `probabilities` beforehand by instantiating\n BERTopic with `calculate_probabilities=True`.\n * \"distributions\"\n Use the topic distributions, as calculated with `.approximate_distribution`\n to find the most frequent topic in each outlier document. You can use the\n `distributions_params` variable to tweak the parameters of\n `.approximate_distribution`.\n * \"c-tf-idf\"\n Calculate the c-TF-IDF representation for each outlier document and\n find the best matching c-TF-IDF topic representation using\n cosine similarity.\n * \"embeddings\"\n Using the embeddings of each outlier documents, find the best\n matching topic embedding using cosine similarity.\n\n Arguments:\n documents: A list of documents for which we reduce or remove the outliers.\n topics: The topics that correspond to the documents\n images: A list of paths to the images used when calling either\n `fit` or `fit_transform`\n strategy: The strategy used for reducing outliers.\n Options:\n * \"probabilities\"\n This uses the soft-clustering as performed by HDBSCAN\n to find the best matching topic for each outlier document.\n\n * \"distributions\"\n Use the topic distributions, as calculated with `.approximate_distribution`\n to find the most frequent topic in each outlier document.\n\n * \"c-tf-idf\"\n Calculate the c-TF-IDF representation for outlier documents and\n find the best matching c-TF-IDF topic representation.\n\n * \"embeddings\"\n Calculate the embeddings for outlier documents and\n find the best matching topic embedding.\n probabilities: Probabilities generated by HDBSCAN for each document when using the strategy `\"probabilities\"`.\n threshold: The threshold for assigning topics to outlier documents. This value\n represents the minimum probability when `strategy=\"probabilities\"`.\n For all other strategies, it represents the minimum similarity.\n embeddings: The pre-computed embeddings to be used when `strategy=\"embeddings\"`.\n If this is None, then it will compute the embeddings for the outlier documents.\n distributions_params: The parameters used in `.approximate_distribution` when using\n the strategy `\"distributions\"`.\n\n Returns:\n new_topics: The updated topics\n\n Usage:\n\n The default settings uses the `\"distributions\"` strategy:\n\n ```python\n new_topics = topic_model.reduce_outliers(docs, topics)\n ```\n\n When you use the `\"probabilities\"` strategy, make sure to also pass the probabilities\n as generated through HDBSCAN:\n\n ```python\n from bertopic import BERTopic\n topic_model = BERTopic(calculate_probabilities=True)\n topics, probs = topic_model.fit_transform(docs)\n\n new_topics = topic_model.reduce_outliers(docs, topics, probabilities=probs, strategy=\"probabilities\")\n ```\n \"\"\"\n if not self._outliers:\n raise ValueError(\"No outliers to reduce.\")\n\n if images is not None:\n strategy = \"embeddings\"\n\n # Check correct use of parameters\n if strategy.lower() == \"probabilities\" and probabilities is None:\n raise ValueError(\"Make sure to pass in `probabilities` in order to use the probabilities strategy\")\n\n # Reduce outliers by extracting most likely topics through the topic-term probability matrix\n if strategy.lower() == \"probabilities\":\n new_topics = [\n np.argmax(prob) if np.max(prob) >= threshold and topic == -1 else topic\n for topic, prob in zip(topics, probabilities)\n ]\n\n # Reduce outliers by extracting most frequent topics through calculating of Topic Distributions\n elif strategy.lower() == \"distributions\":\n outlier_ids = [index for index, topic in enumerate(topics) if topic == -1]\n outlier_docs = [documents[index] for index in outlier_ids]\n topic_distr, _ = self.approximate_distribution(\n outlier_docs, min_similarity=threshold, **distributions_params\n )\n outlier_topics = iter([np.argmax(prob) if sum(prob) > 0 else -1 for prob in topic_distr])\n new_topics = [topic if topic != -1 else next(outlier_topics) for topic in topics]\n\n # Reduce outliers by finding the most similar c-TF-IDF representations\n elif strategy.lower() == \"c-tf-idf\":\n outlier_ids = [index for index, topic in enumerate(topics) if topic == -1]\n outlier_docs = [documents[index] for index in outlier_ids]\n\n # Calculate c-TF-IDF of outlier documents with all topics\n bow_doc = self.vectorizer_model.transform(outlier_docs)\n c_tf_idf_doc = self.ctfidf_model.transform(bow_doc)\n similarity = cosine_similarity(c_tf_idf_doc, self.c_tf_idf_[self._outliers :])\n\n # Update topics\n similarity[similarity < threshold] = 0\n outlier_topics = iter([np.argmax(sim) if sum(sim) > 0 else -1 for sim in similarity])\n new_topics = [topic if topic != -1 else next(outlier_topics) for topic in topics]\n\n # Reduce outliers by finding the most similar topic embeddings\n elif strategy.lower() == \"embeddings\":\n if self.embedding_model is None and embeddings is None:\n raise ValueError(\n \"To use this strategy, you will need to pass a model to `embedding_model`\"\n \"when instantiating BERTopic.\"\n )\n outlier_ids = [index for index, topic in enumerate(topics) if topic == -1]\n if images is not None:\n outlier_docs = [images[index] for index in outlier_ids]\n else:\n outlier_docs = [documents[index] for index in outlier_ids]\n\n # Extract or calculate embeddings for outlier documents\n if embeddings is not None:\n outlier_embeddings = np.array([embeddings[index] for index in outlier_ids])\n elif images is not None:\n outlier_images = [images[index] for index in outlier_ids]\n outlier_embeddings = self.embedding_model.embed_images(outlier_images, verbose=self.verbose)\n else:\n outlier_embeddings = self.embedding_model.embed_documents(outlier_docs)\n similarity = cosine_similarity(outlier_embeddings, self.topic_embeddings_[self._outliers :])\n\n # Update topics\n similarity[similarity < threshold] = 0\n outlier_topics = iter([np.argmax(sim) if sum(sim) > 0 else -1 for sim in similarity])\n new_topics = [topic if topic != -1 else next(outlier_topics) for topic in topics]\n\n return new_topics\n\n def visualize_topics(\n self,\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n use_ctfidf: bool = False,\n custom_labels: bool = False,\n title: str = \"Intertopic Distance Map\",\n width: int = 650,\n height: int = 650,\n ) -> \"go.Figure\":\n \"\"\"Visualize topics, their sizes, and their corresponding words.\n\n This visualization is highly inspired by LDAvis, a great visualization\n technique typically reserved for LDA.\n\n Arguments:\n topics: A selection of topics to visualize\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`.\n top_n_topics: Only select the top n most frequent topics\n use_ctfidf: Whether to use c-TF-IDF representations instead of the embeddings from the embedding model.\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_topics()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics()\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_topics(\n self,\n topics=topics,\n top_n_topics=top_n_topics,\n use_ctfidf=use_ctfidf,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_documents(\n self,\n docs: List[str],\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n sample: float | None = None,\n hide_annotations: bool = False,\n hide_document_hover: bool = False,\n custom_labels: bool = False,\n title: str = \"Documents and Topics\",\n width: int = 1200,\n height: int = 750,\n ) -> \"go.Figure\":\n \"\"\"Visualize documents and their topics in 2D.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n docs: The documents you used when calling either `fit` or `fit_transform`\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n sample: The percentage of documents in each topic that you would like to keep.\n Value can be between 0 and 1. Setting this value to, for example,\n 0.1 (10% of documents in each topic) makes it easier to visualize\n millions of documents as a subset is chosen.\n hide_annotations: Hide the names of the traces on top of each cluster.\n hide_document_hover: Hide the content of the documents when hovering over\n specific points. Helps to speed up generation of visualization.\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_documents(docs)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic\n topic_model = BERTopic().fit(docs, embeddings)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_documents(docs, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n fig.write_html(\"path/to/file.html\")\n ```\n\n \n \"\"\"\n check_is_fitted(self)\n check_documents_type(docs)\n return plotting.visualize_documents(\n self,\n docs=docs,\n topics=topics,\n embeddings=embeddings,\n reduced_embeddings=reduced_embeddings,\n sample=sample,\n hide_annotations=hide_annotations,\n hide_document_hover=hide_document_hover,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_document_datamap(\n self,\n docs: List[str] | None = None,\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n custom_labels: Union[bool, str] = False,\n title: str = \"Documents and Topics\",\n sub_title: Union[str, None] = None,\n width: int = 1200,\n height: int = 750,\n interactive: bool = False,\n enable_search: bool = False,\n topic_prefix: bool = False,\n datamap_kwds: dict = {},\n int_datamap_kwds: dict = {},\n ) -> \"fig.Figure\":\n \"\"\"Visualize documents and their topics in 2D as a static plot for publication using\n DataMapPlot. This works best if there are between 5 and 60 topics. It is therefore best\n to use a sufficiently large `min_topic_size` or set `nr_topics` when building the model.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n docs: The documents you used when calling either `fit` or `fit_transform`.\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`. Documents not in these topics will be shown\n as noise points.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n sub_title: Sub-title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n interactive: Whether to create an interactive plot using DataMapPlot's `create_interactive_plot`.\n enable_search: Whether to enable search in the interactive plot. Only works if `interactive=True`.\n topic_prefix: Prefix to add to the topic number when displaying the topic name.\n datamap_kwds: Keyword args be passed on to DataMapPlot's `create_plot` function\n if you are not using the interactive version.\n See the DataMapPlot documentation for more details.\n int_datamap_kwds: Keyword args be passed on to DataMapPlot's `create_interactive_plot` function\n if you are using the interactive version.\n See the DataMapPlot documentation for more details.\n\n Returns:\n figure: A Matplotlib Figure object.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_document_datamap(docs)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic\n topic_model = BERTopic(min_topic_size=36).fit(docs, embeddings)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_document_datamap(docs, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\n fig.savefig(\"path/to/file.png\", bbox_inches=\"tight\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_document_datamap(\n self,\n docs,\n topics,\n embeddings,\n reduced_embeddings,\n custom_labels,\n title,\n sub_title,\n width,\n height,\n interactive,\n enable_search,\n topic_prefix,\n datamap_kwds,\n int_datamap_kwds,\n )\n\n def visualize_hierarchical_documents(\n self,\n docs: List[str],\n hierarchical_topics: pd.DataFrame,\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n sample: Union[float, int] | None = None,\n hide_annotations: bool = False,\n hide_document_hover: bool = True,\n nr_levels: int = 10,\n level_scale: str = \"linear\",\n custom_labels: bool = False,\n title: str = \"Hierarchical Documents and Topics\",\n width: int = 1200,\n height: int = 750,\n ) -> \"go.Figure\":\n \"\"\"Visualize documents and their topics in 2D at different levels of hierarchy.\n\n Arguments:\n docs: The documents you used when calling either `fit` or `fit_transform`\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n sample: The percentage of documents in each topic that you would like to keep.\n Value can be between 0 and 1. Setting this value to, for example,\n 0.1 (10% of documents in each topic) makes it easier to visualize\n millions of documents as a subset is chosen.\n hide_annotations: Hide the names of the traces on top of each cluster.\n hide_document_hover: Hide the content of the documents when hovering over\n specific points. Helps to speed up generation of visualizations.\n nr_levels: The number of levels to be visualized in the hierarchy. First, the distances\n in `hierarchical_topics.Distance` are split in `nr_levels` lists of distances with\n equal length. Then, for each list of distances, the merged topics, that have\n a distance less or equal to the maximum distance of the selected list of distances, are selected.\n NOTE: To get all possible merged steps, make sure that `nr_levels` is equal to\n the length of `hierarchical_topics`.\n level_scale: Whether to apply a linear or logarithmic ('log') scale levels of the distance\n vector. Linear scaling will perform an equal number of merges at each level\n while logarithmic scaling will perform more mergers in earlier levels to\n provide more resolution at higher levels (this can be used for when the number\n of topics is large).\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n NOTE: Custom labels are only generated for the original\n un-merged topics.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic and extract hierarchical topics\n topic_model = BERTopic().fit(docs, embeddings)\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n fig.write_html(\"path/to/file.html\")\n ```\n\n \n \"\"\"\n check_is_fitted(self)\n check_documents_type(docs)\n return plotting.visualize_hierarchical_documents(\n self,\n docs=docs,\n hierarchical_topics=hierarchical_topics,\n topics=topics,\n embeddings=embeddings,\n reduced_embeddings=reduced_embeddings,\n sample=sample,\n hide_annotations=hide_annotations,\n hide_document_hover=hide_document_hover,\n nr_levels=nr_levels,\n level_scale=level_scale,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_term_rank(\n self,\n topics: List[int] | None = None,\n log_scale: bool = False,\n custom_labels: bool = False,\n title: str = \"Term score decline per Topic\",\n width: int = 800,\n height: int = 500,\n ) -> \"go.Figure\":\n \"\"\"Visualize the ranks of all terms across all topics.\n\n Each topic is represented by a set of words. These words, however,\n do not all equally represent the topic. This visualization shows\n how many words are needed to represent a topic and at which point\n the beneficial effect of adding words starts to decline.\n\n Arguments:\n topics: A selection of topics to visualize. These will be colored\n red where all others will be colored black.\n log_scale: Whether to represent the ranking on a log scale\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the ranks of all words across\n all topics simply run:\n\n ```python\n topic_model.visualize_term_rank()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_term_rank()\n fig.write_html(\"path/to/file.html\")\n ```\n\n Reference:\n\n This visualization was heavily inspired by the\n \"Term Probability Decline\" visualization found in an\n analysis by the amazing [tmtoolkit](https://tmtoolkit.readthedocs.io/).\n Reference to that specific analysis can be found\n [here](https://wzbsocialsciencecenter.github.io/tm_corona/tm_analysis.html).\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_term_rank(\n self,\n topics=topics,\n log_scale=log_scale,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_topics_over_time(\n self,\n topics_over_time: pd.DataFrame,\n top_n_topics: int | None = None,\n topics: List[int] | None = None,\n normalize_frequency: bool = False,\n custom_labels: bool = False,\n title: str = \"Topics over Time\",\n width: int = 1250,\n height: int = 450,\n ) -> \"go.Figure\":\n \"\"\"Visualize topics over time.\n\n Arguments:\n topics_over_time: The topics you would like to be visualized with the\n corresponding topic representation\n top_n_topics: To visualize the most frequent topics instead of all\n topics: Select which topics you would like to be visualized\n normalize_frequency: Whether to normalize each topic's frequency individually\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n A plotly.graph_objects.Figure including all traces\n\n Examples:\n To visualize the topics over time, simply run:\n\n ```python\n topics_over_time = topic_model.topics_over_time(docs, timestamps)\n topic_model.visualize_topics_over_time(topics_over_time)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics_over_time(topics_over_time)\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_topics_over_time(\n self,\n topics_over_time=topics_over_time,\n top_n_topics=top_n_topics,\n topics=topics,\n normalize_frequency=normalize_frequency,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_topics_per_class(\n self,\n topics_per_class: pd.DataFrame,\n top_n_topics: int = 10,\n topics: List[int] | None = None,\n normalize_frequency: bool = False,\n custom_labels: bool = False,\n title: str = \"Topics per Class\",\n width: int = 1250,\n height: int = 900,\n ) -> \"go.Figure\":\n \"\"\"Visualize topics per class.\n\n Arguments:\n topics_per_class: The topics you would like to be visualized with the\n corresponding topic representation\n top_n_topics: To visualize the most frequent topics instead of all\n topics: Select which topics you would like to be visualized\n normalize_frequency: Whether to normalize each topic's frequency individually\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n A plotly.graph_objects.Figure including all traces\n\n Examples:\n To visualize the topics per class, simply run:\n\n ```python\n topics_per_class = topic_model.topics_per_class(docs, classes)\n topic_model.visualize_topics_per_class(topics_per_class)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics_per_class(topics_per_class)\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_topics_per_class(\n self,\n topics_per_class=topics_per_class,\n top_n_topics=top_n_topics,\n topics=topics,\n normalize_frequency=normalize_frequency,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_distribution(\n self,\n probabilities: np.ndarray,\n min_probability: float = 0.015,\n custom_labels: bool = False,\n title: str = \"Topic Probability Distribution\",\n width: int = 800,\n height: int = 600,\n ) -> \"go.Figure\":\n \"\"\"Visualize the distribution of topic probabilities.\n\n Arguments:\n probabilities: An array of probability scores\n min_probability: The minimum probability score to visualize.\n All others are ignored.\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n Make sure to fit the model before and only input the\n probabilities of a single document:\n\n ```python\n topic_model.visualize_distribution(topic_model.probabilities_[0])\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_distribution(topic_model.probabilities_[0])\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_distribution(\n self,\n probabilities=probabilities,\n min_probability=min_probability,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_approximate_distribution(\n self,\n document: str,\n topic_token_distribution: np.ndarray,\n normalize: bool = False,\n ):\n \"\"\"Visualize the topic distribution calculated by `.approximate_topic_distribution`\n on a token level. Thereby indicating the extent to which a certain word or phrase belongs\n to a specific topic. The assumption here is that a single word can belong to multiple\n similar topics and as such can give information about the broader set of topics within\n a single document.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n document: The document for which you want to visualize\n the approximated topic distribution.\n topic_token_distribution: The topic-token distribution of the document as\n extracted by `.approximate_topic_distribution`\n normalize: Whether to normalize, between 0 and 1 (summing up to 1), the\n topic distribution values.\n\n Returns:\n df: A stylized dataframe indicating the best fitting topics\n for each token.\n\n Examples:\n ```python\n # Calculate the topic distributions on a token level\n # Note that we need to have `calculate_token_level=True`\n topic_distr, topic_token_distr = topic_model.approximate_distribution(\n docs, calculate_token_level=True\n )\n\n # Visualize the approximated topic distributions\n df = topic_model.visualize_approximate_distribution(docs[0], topic_token_distr[0])\n df\n ```\n\n To revert this stylized dataframe back to a regular dataframe,\n you can run the following:\n\n ```python\n df.data.columns = [column.strip() for column in df.data.columns]\n df = df.data\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_approximate_distribution(\n self,\n document=document,\n topic_token_distribution=topic_token_distribution,\n normalize=normalize,\n )\n\n def visualize_hierarchy(\n self,\n orientation: str = \"left\",\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n use_ctfidf: bool = True,\n custom_labels: bool = False,\n title: str = \"Hierarchical Clustering\",\n width: int = 1000,\n height: int = 600,\n hierarchical_topics: pd.DataFrame = None,\n linkage_function: Callable[[csr_matrix], np.ndarray] | None = None,\n distance_function: Callable[[csr_matrix], csr_matrix] | None = None,\n color_threshold: int = 1,\n ) -> \"go.Figure\":\n \"\"\"Visualize a hierarchical structure of the topics.\n\n A ward linkage function is used to perform the\n hierarchical clustering based on the cosine distance\n matrix between c-TF-IDF or semantic embeddings of the topics.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n orientation: The orientation of the figure.\n Either 'left' or 'bottom'\n topics: A selection of topics to visualize\n top_n_topics: Only select the top n most frequent topics\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the\n embeddings from the embedding model are used.\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n NOTE: Custom labels are only generated for the original\n un-merged topics.\n title: Title of the plot.\n width: The width of the figure. Only works if orientation is set to 'left'\n height: The height of the figure. Only works if orientation is set to 'bottom'\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children.\n NOTE: The hierarchical topic names are only visualized\n if both `topics` and `top_n_topics` are not set.\n linkage_function: The linkage function to use. Default is:\n `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`\n NOTE: Make sure to use the same `linkage_function` as used\n in `topic_model.hierarchical_topics`.\n distance_function: The distance function to use on the c-TF-IDF matrix. Default is:\n `lambda x: 1 - cosine_similarity(x)`\n NOTE: Make sure to use the same `distance_function` as used\n in `topic_model.hierarchical_topics`.\n color_threshold: Value at which the separation of clusters will be made which\n will result in different colors for different clusters.\n A higher value will typically lead to less colored clusters.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the hierarchical structure of\n topics simply run:\n\n ```python\n topic_model.visualize_hierarchy()\n ```\n\n If you also want the labels of hierarchical topics visualized,\n run the following:\n\n ```python\n # Extract hierarchical topics and their representations\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n\n # Visualize these representations\n topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n ```\n\n If you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_hierarchy()\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_hierarchy(\n self,\n orientation=orientation,\n topics=topics,\n top_n_topics=top_n_topics,\n use_ctfidf=use_ctfidf,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n hierarchical_topics=hierarchical_topics,\n linkage_function=linkage_function,\n distance_function=distance_function,\n color_threshold=color_threshold,\n )\n\n def visualize_heatmap(\n self,\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n n_clusters: int | None = None,\n use_ctfidf: bool = False,\n custom_labels: bool = False,\n title: str = \"Similarity Matrix\",\n width: int = 800,\n height: int = 800,\n ) -> \"go.Figure\":\n \"\"\"Visualize a heatmap of the topic's similarity matrix.\n\n Based on the cosine similarity matrix between c-TF-IDFs or semantic embeddings of the topics,\n a heatmap is created showing the similarity between topics.\n\n Arguments:\n topics: A selection of topics to visualize.\n top_n_topics: Only select the top n most frequent topics.\n n_clusters: Create n clusters and order the similarity\n matrix by those clusters.\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the\n embeddings from the embedding model are used.\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the similarity matrix of\n topics simply run:\n\n ```python\n topic_model.visualize_heatmap()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_heatmap()\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_heatmap(\n self,\n topics=topics,\n top_n_topics=top_n_topics,\n n_clusters=n_clusters,\n use_ctfidf=use_ctfidf,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n )\n\n def visualize_barchart(\n self,\n topics: List[int] | None = None,\n top_n_topics: int = 8,\n n_words: int = 5,\n custom_labels: bool = False,\n title: str = \"Topic Word Scores\",\n width: int = 250,\n height: int = 250,\n autoscale: bool = False,\n ) -> \"go.Figure\":\n \"\"\"Visualize a barchart of selected topics.\n\n Arguments:\n topics: A selection of topics to visualize.\n top_n_topics: Only select the top n most frequent topics.\n n_words: Number of words to show in a topic\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n title: Title of the plot.\n width: The width of each figure.\n height: The height of each figure.\n autoscale: Whether to automatically calculate the height of the figures to fit the whole bar text\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the barchart of selected topics\n simply run:\n\n ```python\n topic_model.visualize_barchart()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_barchart()\n fig.write_html(\"path/to/file.html\")\n ```\n \"\"\"\n check_is_fitted(self)\n return plotting.visualize_barchart(\n self,\n topics=topics,\n top_n_topics=top_n_topics,\n n_words=n_words,\n custom_labels=custom_labels,\n title=title,\n width=width,\n height=height,\n autoscale=autoscale,\n )\n\n def save(\n self,\n path,\n serialization: Literal[\"safetensors\", \"pickle\", \"pytorch\"] = \"pickle\",\n save_embedding_model: Union[bool, str] = True,\n save_ctfidf: bool = False,\n ):\n \"\"\"Saves the model to the specified path or folder.\n\n When saving the model, make sure to also keep track of the versions\n of dependencies and Python used. Loading and saving the model should\n be done using the same dependencies and Python. Moreover, models\n saved in one version of BERTopic should not be loaded in other versions.\n\n Arguments:\n path: If `serialization` is 'safetensors' or `pytorch`, this is a directory.\n If `serialization` is `pickle`, then this is a file.\n serialization: If `pickle`, the entire model will be pickled. If `safetensors`\n or `pytorch` the model will be saved without the embedding,\n dimensionality reduction, and clustering algorithms.\n This is a very efficient format and typically advised.\n save_embedding_model: If serialization is `pickle`, then you can choose to skip\n saving the embedding model. If serialization is `safetensors`\n or `pytorch`, this variable can be used as a string pointing\n towards a huggingface model.\n save_ctfidf: Whether to save c-TF-IDF information if serialization is `safetensors`\n or `pytorch`\n\n Examples:\n To save the model in an efficient and safe format (safetensors) with c-TF-IDF information:\n\n ```python\n topic_model.save(\"model_dir\", serialization=\"safetensors\", save_ctfidf=True)\n ```\n\n If you wish to also add a pointer to the embedding model, which will be downloaded from\n HuggingFace upon loading:\n\n ```python\n embedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\n topic_model.save(\"model_dir\", serialization=\"safetensors\", save_embedding_model=embedding_model)\n ```\n\n or if you want save the full model with pickle:\n\n ```python\n topic_model.save(\"my_model\")\n ```\n\n NOTE: Pickle can run arbitrary code and is generally considered to be less safe than\n safetensors.\n \"\"\"\n if serialization == \"pickle\":\n logger.warning(\n \"When you use `pickle` to save/load a BERTopic model,\"\n \"please make sure that the environments in which you save\"\n \"and load the model are **exactly** the same. The version of BERTopic,\"\n \"its dependencies, and python need to remain the same.\"\n )\n\n with open(path, \"wb\") as file:\n # This prevents the vectorizer from being too large in size if `min_df` was\n # set to a value higher than 1\n self.vectorizer_model.stop_words_ = None\n\n if not save_embedding_model:\n embedding_model = self.embedding_model\n self.embedding_model = None\n joblib.dump(self, file)\n self.embedding_model = embedding_model\n else:\n joblib.dump(self, file)\n elif serialization == \"safetensors\" or serialization == \"pytorch\":\n # Directory\n save_directory = Path(path)\n save_directory.mkdir(exist_ok=True, parents=True)\n\n # Check embedding model\n if (\n save_embedding_model\n and hasattr(self.embedding_model, \"_hf_model\")\n and not isinstance(save_embedding_model, str)\n ):\n save_embedding_model = self.embedding_model._hf_model\n elif not save_embedding_model:\n logger.warning(\n \"You are saving a BERTopic model without explicitly defining an embedding model.\"\n \"If you are using a sentence-transformers model or a HuggingFace model supported\"\n \"by sentence-transformers, please save the model by using a pointer towards that model.\"\n \"For example, `save_embedding_model='sentence-transformers/all-mpnet-base-v2'`\"\n )\n\n # Minimal\n save_utils.save_hf(model=self, save_directory=save_directory, serialization=serialization)\n save_utils.save_topics(model=self, path=save_directory / \"topics.json\")\n save_utils.save_images(model=self, path=save_directory / \"images\")\n save_utils.save_config(\n model=self,\n path=save_directory / \"config.json\",\n embedding_model=save_embedding_model,\n )\n\n # Additional\n if save_ctfidf:\n if self.c_tf_idf_ is None:\n logger.warning(\n \"The c-TF-IDF matrix could not be saved as it was not found. \"\n \"This typically occurs when merging BERTopic models with `BERTopic.merge_models`.\"\n )\n else:\n save_utils.save_ctfidf(\n model=self,\n save_directory=save_directory,\n serialization=serialization,\n )\n save_utils.save_ctfidf_config(model=self, path=save_directory / \"ctfidf_config.json\")\n\n @classmethod\n def load(cls, path: str, embedding_model=None):\n \"\"\"Loads the model from the specified path or directory.\n\n Arguments:\n path: Either load a BERTopic model from a file (`.pickle`) or a folder containing\n `.safetensors` or `.bin` files.\n embedding_model: Additionally load in an embedding model if it was not saved\n in the BERTopic model file or directory.\n\n Examples:\n ```python\n BERTopic.load(\"model_dir\")\n ```\n\n or if you did not save the embedding model:\n\n ```python\n BERTopic.load(\"model_dir\", embedding_model=\"all-MiniLM-L6-v2\")\n ```\n \"\"\"\n file_or_dir = Path(path)\n\n # Load from Pickle\n if file_or_dir.is_file():\n with open(file_or_dir, \"rb\") as file:\n if embedding_model:\n topic_model = joblib.load(file)\n topic_model.embedding_model = select_backend(embedding_model, verbose=topic_model.verbose)\n else:\n topic_model = joblib.load(file)\n return topic_model\n\n # Load from directory or HF\n if file_or_dir.is_dir():\n topics, params, tensors, ctfidf_tensors, ctfidf_config, images = save_utils.load_local_files(file_or_dir)\n elif \"/\" in str(path):\n topics, params, tensors, ctfidf_tensors, ctfidf_config, images = save_utils.load_files_from_hf(path)\n else:\n raise ValueError(\"Make sure to either pass a valid directory or HF model.\")\n topic_model = _create_model_from_files(\n topics,\n params,\n tensors,\n ctfidf_tensors,\n ctfidf_config,\n images,\n warn_no_backend=(embedding_model is None),\n )\n\n # Replace embedding model if one is specifically chosen\n if embedding_model is not None:\n topic_model.embedding_model = select_backend(embedding_model, verbose=topic_model.verbose)\n\n return topic_model\n\n @classmethod\n def merge_models(cls, models, min_similarity: float = 0.7, embedding_model=None):\n \"\"\"Merge multiple pre-trained BERTopic models into a single model.\n\n The models are merged as if they were all saved using pytorch or\n safetensors, so a minimal version without c-TF-IDF.\n\n To do this, we choose the first model in the list of\n models as a baseline. Then, we check each model whether\n they contain topics that are not in the baseline.\n This check is based on the cosine similarity between\n topics embeddings. If topic embeddings between two models\n are similar, then the topic of the second model is re-assigned\n to the first. If they are dissimilar, the topic of the second\n model is assigned to the first.\n\n In essence, we simply check whether sufficiently \"new\"\n topics emerge and add them.\n\n Arguments:\n models: A list of fitted BERTopic models\n min_similarity: The minimum similarity for when topics are merged.\n embedding_model: Additionally load in an embedding model if necessary.\n\n Returns:\n A new BERTopic model that was created as if you were\n loading a model from the HuggingFace Hub without c-TF-IDF\n\n Examples:\n ```python\n from bertopic import BERTopic\n from sklearn.datasets import fetch_20newsgroups\n\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\n # Create three separate models\n topic_model_1 = BERTopic(min_topic_size=5).fit(docs[:4000])\n topic_model_2 = BERTopic(min_topic_size=5).fit(docs[4000:8000])\n topic_model_3 = BERTopic(min_topic_size=5).fit(docs[8000:])\n\n # Combine all models into one\n merged_model = BERTopic.merge_models([topic_model_1, topic_model_2, topic_model_3])\n ```\n \"\"\"\n\n def choose_backend():\n \"\"\"Choose the backend to use for saving the model.\"\"\"\n try:\n import torch # noqa: F401\n\n return \"pytorch\"\n except (ModuleNotFoundError, ImportError):\n try:\n import safetensors # noqa: F401\n\n return \"safetensors\"\n except (ModuleNotFoundError, ImportError):\n raise ImportError(\n \"Neither pytorch nor safetensors is installed. \"\n \"Please install at least one of these packages:\\n\"\n \" pip install torch\\n\"\n \" pip install safetensors\"\n )\n\n # Temporarily save model and push to HF\n with TemporaryDirectory() as tmpdir:\n # Save model weights and config.\n all_topics, all_params, all_tensors = [], [], []\n for index, model in enumerate(models):\n model.save(tmpdir, serialization=choose_backend())\n topics, params, tensors, _, _, _ = save_utils.load_local_files(Path(tmpdir))\n all_topics.append(topics)\n all_params.append(params)\n all_tensors.append(np.array(tensors[\"topic_embeddings\"]))\n\n # Create a base set of parameters\n if index == 0:\n merged_topics = topics\n merged_params = params\n merged_tensors = np.array(tensors[\"topic_embeddings\"])\n merged_topics[\"custom_labels\"] = None\n\n for tensors, selected_topics in zip(all_tensors[1:], all_topics[1:]):\n # Calculate similarity matrix\n sim_matrix = cosine_similarity(tensors, merged_tensors)\n sims = np.max(sim_matrix, axis=1)\n\n # Extract new topics\n new_topics = sorted(\n [index - selected_topics[\"_outliers\"] for index, sim in enumerate(sims) if sim < min_similarity]\n )\n max_topic = max(set(merged_topics[\"topics\"]))\n\n # Merge Topic Representations\n new_topics_dict = {}\n for new_topic in new_topics:\n if new_topic != -1:\n max_topic += 1\n new_topics_dict[new_topic] = max_topic\n merged_topics[\"topic_representations\"][str(max_topic)] = selected_topics[\"topic_representations\"][\n str(new_topic)\n ]\n merged_topics[\"topic_labels\"][str(max_topic)] = selected_topics[\"topic_labels\"][str(new_topic)]\n\n # Add new aspects\n if selected_topics[\"topic_aspects\"]:\n aspects_1 = set(merged_topics[\"topic_aspects\"].keys())\n aspects_2 = set(selected_topics[\"topic_aspects\"].keys())\n aspects_diff = aspects_2.difference(aspects_1)\n if aspects_diff:\n for aspect in aspects_diff:\n merged_topics[\"topic_aspects\"][aspect] = {}\n\n # If the original model does not have topic aspects but the to be added model does\n if not merged_topics.get(\"topic_aspects\"):\n merged_topics[\"topic_aspects\"] = selected_topics[\"topic_aspects\"]\n\n # If they both contain topic aspects, add to the existing set of aspects\n else:\n for aspect, values in selected_topics[\"topic_aspects\"].items():\n merged_topics[\"topic_aspects\"][aspect][str(max_topic)] = values[str(new_topic)]\n\n # Add new embeddings\n new_tensors = tensors[new_topic + selected_topics[\"_outliers\"]]\n merged_tensors = np.vstack([merged_tensors, new_tensors])\n\n # Topic Mapper\n merged_topics[\"topic_mapper\"] = TopicMapper(list(range(-1, max_topic + 1, 1))).mappings_\n\n # Find similar topics and re-assign those from the new models\n sims_idx = np.argmax(sim_matrix, axis=1)\n sims = np.max(sim_matrix, axis=1)\n to_merge = {\n a - selected_topics[\"_outliers\"]: b - merged_topics[\"_outliers\"]\n for a, (b, val) in enumerate(zip(sims_idx, sims))\n if val >= min_similarity\n }\n to_merge.update(new_topics_dict)\n to_merge[-1] = -1\n topics = [to_merge[topic] for topic in selected_topics[\"topics\"]]\n merged_topics[\"topics\"].extend(topics)\n merged_topics[\"topic_sizes\"] = dict(Counter(merged_topics[\"topics\"]))\n\n # Create a new model from the merged parameters\n merged_tensors = {\"topic_embeddings\": merged_tensors}\n merged_model = _create_model_from_files(\n merged_topics,\n merged_params,\n merged_tensors,\n None,\n None,\n None,\n warn_no_backend=False,\n )\n merged_model.embedding_model = models[0].embedding_model\n\n # Replace embedding model if one is specifically chosen\n verbose = any([model.verbose for model in models])\n if embedding_model is not None and type(merged_model.embedding_model) is BaseEmbedder:\n merged_model.embedding_model = select_backend(embedding_model, verbose=verbose)\n return merged_model\n\n def push_to_hf_hub(\n self,\n repo_id: str,\n commit_message: str = \"Add BERTopic model\",\n token: str | None = None,\n revision: str | None = None,\n private: bool = False,\n create_pr: bool = False,\n model_card: bool = True,\n serialization: str = \"safetensors\",\n save_embedding_model: Union[str, bool] = True,\n save_ctfidf: bool = False,\n ):\n \"\"\"Push your BERTopic model to a HuggingFace Hub.\n\n Whenever you want to upload files to the Hub, you need to log in to your HuggingFace account:\n\n * Log in to your HuggingFace account with the following command:\n ```bash\n huggingface-cli login\n\n # or using an environment variable\n huggingface-cli login --token $HUGGINGFACE_TOKEN\n ```\n * Alternatively, you can programmatically login using login() in a notebook or a script:\n ```python\n from huggingface_hub import login\n login()\n ```\n * Or you can give a token with the `token` variable\n\n Arguments:\n repo_id: The name of your HuggingFace repository\n commit_message: A commit message\n token: Token to add if not already logged in\n revision: Repository revision\n private: Whether to create a private repository\n create_pr: Whether to upload the model as a Pull Request\n model_card: Whether to automatically create a modelcard\n serialization: The type of serialization.\n Either `safetensors` or `pytorch`\n save_embedding_model: A pointer towards a HuggingFace model to be loaded in with\n SentenceTransformers. E.g.,\n `sentence-transformers/all-MiniLM-L6-v2`\n save_ctfidf: Whether to save c-TF-IDF information\n\n\n Examples:\n ```python\n topic_model.push_to_hf_hub(\n repo_id=\"ArXiv\",\n save_ctfidf=True,\n save_embedding_model=\"sentence-transformers/all-MiniLM-L6-v2\"\n )\n ```\n \"\"\"\n return save_utils.push_to_hf_hub(\n model=self,\n repo_id=repo_id,\n commit_message=commit_message,\n token=token,\n revision=revision,\n private=private,\n create_pr=create_pr,\n model_card=model_card,\n serialization=serialization,\n save_embedding_model=save_embedding_model,\n save_ctfidf=save_ctfidf,\n )\n\n def get_params(self, deep: bool = False) -> Mapping[str, Any]:\n \"\"\"Get parameters for this estimator.\n\n Adapted from:\n https://github.com/scikit-learn/scikit-learn/blob/b3ea3ed6a/sklearn/base.py#L178\n\n Arguments:\n deep: bool, default=True\n If True, will return the parameters for this estimator and\n contained subobjects that are estimators.\n\n Returns:\n out: Parameter names mapped to their values.\n \"\"\"\n out = dict()\n for key in self._get_param_names():\n value = getattr(self, key)\n if deep and hasattr(value, \"get_params\"):\n deep_items = value.get_params().items()\n out.update((key + \"__\" + k, val) for k, val in deep_items)\n out[key] = value\n return out\n\n def _extract_embeddings(\n self,\n documents: Union[List[str], str],\n images: List[str] | None = None,\n method: str = \"document\",\n verbose: bool | None = None,\n ) -> np.ndarray:\n \"\"\"Extract sentence/document embeddings through pre-trained embeddings\n For an overview of pre-trained models: https://www.sbert.net/docs/pretrained_models.html.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs\n images: A list of paths to the images to fit on or the images themselves\n method: Whether to extract document or word-embeddings, options are \"document\" and \"word\"\n verbose: Whether to show a progressbar demonstrating the time to extract embeddings\n\n Returns:\n embeddings: The extracted embeddings.\n \"\"\"\n if isinstance(documents, str):\n documents = [documents]\n\n if images is not None and hasattr(self.embedding_model, \"embed_images\"):\n embeddings = self.embedding_model.embed(documents=documents, images=images, verbose=verbose)\n elif method == \"word\":\n embeddings = self.embedding_model.embed_words(words=documents, verbose=verbose)\n elif method == \"document\":\n embeddings = self.embedding_model.embed_documents(documents, verbose=verbose)\n elif documents[0] is None and images is None:\n raise ValueError(\n \"Make sure to use an embedding model that can either embed documents\"\n \"or images depending on which you want to embed.\"\n )\n else:\n raise ValueError(\n \"Wrong method for extracting document/word embeddings. \"\n \"Either choose 'word' or 'document' as the method. \"\n )\n return embeddings\n\n def _images_to_text(self, documents: pd.DataFrame, embeddings: np.ndarray) -> pd.DataFrame:\n \"\"\"Convert images to text.\"\"\"\n logger.info(\"Images - Converting images to text. This might take a while.\")\n if isinstance(self.representation_model, dict):\n for tuner in self.representation_model.values():\n if getattr(tuner, \"image_to_text_model\", False):\n documents = tuner.image_to_text(documents, embeddings)\n elif isinstance(self.representation_model, list):\n for tuner in self.representation_model:\n if getattr(tuner, \"image_to_text_model\", False):\n documents = tuner.image_to_text(documents, embeddings)\n elif isinstance(self.representation_model, BaseRepresentation):\n if getattr(self.representation_model, \"image_to_text_model\", False):\n documents = self.representation_model.image_to_text(documents, embeddings)\n logger.info(\"Images - Completed \\u2713\")\n return documents\n\n def _map_predictions(self, predictions: List[int]) -> List[int]:\n \"\"\"Map predictions to the correct topics if topics were reduced.\"\"\"\n mappings = self.topic_mapper_.get_mappings(original_topics=True)\n mapped_predictions = [mappings[prediction] if prediction in mappings else -1 for prediction in predictions]\n return mapped_predictions\n\n def _reduce_dimensionality(\n self,\n embeddings: Union[np.ndarray, csr_matrix],\n y: Union[List[int], np.ndarray] = None,\n partial_fit: bool = False,\n ) -> np.ndarray:\n \"\"\"Reduce dimensionality of embeddings using UMAP and train a UMAP model.\n\n Arguments:\n embeddings: The extracted embeddings using the sentence transformer module.\n y: The target class for (semi)-supervised dimensionality reduction\n partial_fit: Whether to run `partial_fit` for online learning\n\n Returns:\n umap_embeddings: The reduced embeddings\n \"\"\"\n logger.info(\"Dimensionality - Fitting the dimensionality reduction algorithm\")\n # Partial fit\n if partial_fit:\n if hasattr(self.umap_model, \"partial_fit\"):\n self.umap_model = self.umap_model.partial_fit(embeddings)\n umap_embeddings = self.umap_model.transform(embeddings)\n elif self.topic_representations_ is None:\n if hasattr(self.umap_model, \"fit_transform\"):\n umap_embeddings = self.umap_model.fit_transform(embeddings)\n else:\n self.umap_model.fit(embeddings)\n umap_embeddings = self.umap_model.transform(embeddings)\n else:\n umap_embeddings = self.umap_model.transform(embeddings)\n\n # Regular fit\n else:\n try:\n # cuml umap needs y to be an numpy array\n y = np.array(y) if y is not None else None\n if hasattr(self.umap_model, \"fit_transform\"):\n umap_embeddings = self.umap_model.fit_transform(embeddings, y=y)\n else:\n self.umap_model.fit(embeddings, y=y)\n umap_embeddings = self.umap_model.transform(embeddings)\n except TypeError:\n if hasattr(self.umap_model, \"fit_transform\"):\n umap_embeddings = self.umap_model.fit_transform(embeddings, y=y)\n else:\n self.umap_model.fit(embeddings, y=y)\n umap_embeddings = self.umap_model.transform(embeddings)\n\n logger.info(\"Dimensionality - Completed \\u2713\")\n return np.nan_to_num(umap_embeddings)\n\n def _cluster_embeddings(\n self,\n umap_embeddings: np.ndarray,\n documents: pd.DataFrame,\n partial_fit: bool = False,\n y: np.ndarray = None,\n ) -> Tuple[pd.DataFrame, np.ndarray]:\n \"\"\"Cluster UMAP reduced embeddings with HDBSCAN.\n\n Arguments:\n umap_embeddings: The reduced sentence embeddings with UMAP\n documents: Dataframe with documents and their corresponding IDs\n partial_fit: Whether to run `partial_fit` for online learning\n y: Array of topics to use\n\n Returns:\n documents: Updated dataframe with documents and their corresponding IDs\n and newly added Topics\n probabilities: The distribution of probabilities\n \"\"\"\n logger.info(\"Cluster - Start clustering the reduced embeddings\")\n if partial_fit:\n self.hdbscan_model = self.hdbscan_model.partial_fit(umap_embeddings)\n labels = self.hdbscan_model.labels_\n documents[\"Topic\"] = labels\n self.topics_ = labels\n else:\n try:\n self.hdbscan_model.fit(umap_embeddings, y=y)\n except TypeError:\n self.hdbscan_model.fit(umap_embeddings)\n\n try:\n labels = self.hdbscan_model.labels_\n except AttributeError:\n labels = y\n documents[\"Topic\"] = labels\n self._update_topic_size(documents)\n\n # Extract probabilities\n probabilities = None\n if hasattr(self.hdbscan_model, \"probabilities_\"):\n probabilities = self.hdbscan_model.probabilities_\n\n if self.calculate_probabilities and is_supported_hdbscan(self.hdbscan_model):\n probabilities = hdbscan_delegator(self.hdbscan_model, \"all_points_membership_vectors\")\n\n if not partial_fit:\n self.topic_mapper_ = TopicMapper(self.topics_)\n logger.info(\"Cluster - Completed \\u2713\")\n return documents, probabilities\n\n def _zeroshot_topic_modeling(\n self, documents: pd.DataFrame, embeddings: np.ndarray\n ) -> Tuple[pd.DataFrame, np.array, pd.DataFrame, np.array]:\n \"\"\"Find documents that could be assigned to either one of the topics in self.zeroshot_topic_list.\n\n We transform the topics in `self.zeroshot_topic_list` to embeddings and\n compare them through cosine similarity with the document embeddings.\n If they pass the `self.zeroshot_min_similarity` threshold, they are assigned.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs\n embeddings: The document embeddings\n\n Returns:\n documents: The leftover documents that were not assigned to any topic\n embeddings: The leftover embeddings that were not assigned to any topic\n \"\"\"\n logger.info(\"Zeroshot Step 1 - Finding documents that could be assigned to either one of the zero-shot topics\")\n # Similarity between document and zero-shot topic embeddings\n zeroshot_embeddings = self._extract_embeddings(self.zeroshot_topic_list)\n cosine_similarities = cosine_similarity(embeddings, zeroshot_embeddings)\n assignment = np.argmax(cosine_similarities, 1)\n assignment_vals = np.max(cosine_similarities, 1)\n assigned_ids = [index for index, value in enumerate(assignment_vals) if value >= self.zeroshot_min_similarity]\n non_assigned_ids = [\n index for index, value in enumerate(assignment_vals) if value < self.zeroshot_min_similarity\n ]\n\n # Assign topics\n assigned_documents = documents.iloc[assigned_ids]\n assigned_documents[\"Topic\"] = [topic for topic in assignment[assigned_ids]]\n assigned_documents[\"Old_ID\"] = assigned_documents[\"ID\"].copy()\n assigned_documents[\"ID\"] = range(len(assigned_documents))\n assigned_embeddings = embeddings[assigned_ids]\n\n # Check that if a number of topics was specified, it exceeds the number of zeroshot topics matched\n num_zeroshot_topics = len(assigned_documents[\"Topic\"].unique())\n if self.nr_topics != \"auto\":\n if self.nr_topics and not self.nr_topics > num_zeroshot_topics:\n raise ValueError(\n f\"The set nr_topics ({self.nr_topics}) must exceed the number of matched zero-shot topics \"\n f\"({num_zeroshot_topics}). Consider raising nr_topics or raising the \"\n f\"zeroshot_min_similarity ({self.zeroshot_min_similarity}).\"\n )\n\n # Select non-assigned topics to be clustered\n documents = documents.iloc[non_assigned_ids]\n documents[\"Old_ID\"] = documents[\"ID\"].copy()\n documents[\"ID\"] = range(len(documents))\n embeddings = embeddings[non_assigned_ids]\n\n if len(documents) == 0:\n self.topics_ = assigned_documents[\"Topic\"].to_numpy().tolist()\n self.topic_mapper_ = TopicMapper(self.topics_)\n\n logger.info(\"Zeroshot Step 1 - Completed \\u2713\")\n return documents, embeddings, assigned_documents, assigned_embeddings\n\n def _is_zeroshot(self):\n \"\"\"Check whether zero-shot topic modeling is possible.\n\n * Embedding model is necessary to convert zero-shot topics to embeddings\n * Zero-shot topics should be defined\n \"\"\"\n if self.zeroshot_topic_list is not None and self.embedding_model is not None:\n return True\n return False\n\n def _combine_zeroshot_topics(\n self,\n documents: pd.DataFrame,\n embeddings: np.ndarray,\n assigned_documents: pd.DataFrame,\n assigned_embeddings: np.ndarray,\n ) -> Tuple[pd.DataFrame, np.ndarray]:\n \"\"\"Combine the zero-shot topics with the clustered topics.\n\n The zero-shot topics will be inserted between the outlier topic (that may or may not exist) and the rest of the\n topics from clustering. The rest of the topics from clustering will be given new IDs to correspond to topics\n after zero-shot topics.\n\n Documents and embeddings used in zero-shot topic modeling and clustering and re-merged.\n\n Arguments:\n documents: DataFrame with clustered documents and their corresponding IDs\n embeddings: The document embeddings for clustered documents\n assigned_documents: DataFrame with documents and their corresponding IDs\n that were assigned to a zero-shot topic\n assigned_embeddings: The document embeddings for documents that were assigned to a zero-shot topic\n\n Returns:\n documents: DataFrame with all the original documents with their topic assignments\n embeddings: np.ndarray of embeddings aligned with the documents\n \"\"\"\n logger.info(\"Zeroshot Step 2 - Combining topics from zero-shot topic modeling with topics from clustering...\")\n # Combine Zero-shot topics with topics from clustering\n zeroshot_topic_idx_to_topic_id = {\n zeroshot_topic_id: new_topic_id\n for new_topic_id, zeroshot_topic_id in enumerate(set(assigned_documents.Topic))\n }\n self._topic_id_to_zeroshot_topic_idx = {\n new_topic_id: zeroshot_topic_id\n for new_topic_id, zeroshot_topic_id in enumerate(set(assigned_documents.Topic))\n }\n assigned_documents.Topic = assigned_documents.Topic.map(zeroshot_topic_idx_to_topic_id)\n num_zeroshot_topics = len(zeroshot_topic_idx_to_topic_id)\n\n # Insert zeroshot topics between outlier cluster and other clusters\n documents.Topic = documents.Topic.apply(\n lambda topic_id: topic_id + num_zeroshot_topics if topic_id != -1 else topic_id\n )\n\n # Combine the clustered documents/embeddings with assigned documents/embeddings in the original order\n documents = pd.concat([documents, assigned_documents])\n embeddings = np.vstack([embeddings, assigned_embeddings])\n documents.ID = documents.Old_ID\n sorted_indices = documents.Old_ID.argsort()\n documents = documents.iloc[sorted_indices]\n embeddings = embeddings[sorted_indices]\n\n # Update topic sizes and topic mapper\n self._update_topic_size(documents)\n self.topic_mapper_ = TopicMapper(self.topics_)\n\n logger.info(\"Zeroshot Step 2 - Completed \\u2713\")\n return documents, embeddings\n\n def _guided_topic_modeling(self, embeddings: np.ndarray) -> Tuple[List[int], np.array]:\n \"\"\"Apply Guided Topic Modeling.\n\n We transform the seeded topics to embeddings using the\n same embedder as used for generating document embeddings.\n\n Then, we apply cosine similarity between the embeddings\n and set labels for documents that are more similar to\n one of the topics than the average document.\n\n If a document is more similar to the average document\n than any of the topics, it gets the -1 label and is\n thereby not included in UMAP.\n\n Arguments:\n embeddings: The document embeddings\n\n Returns:\n y: The labels for each seeded topic\n embeddings: Updated embeddings\n \"\"\"\n logger.info(\"Guided - Find embeddings highly related to seeded topics.\")\n # Create embeddings from the seeded topics\n seed_topic_list = [\" \".join(seed_topic) for seed_topic in self.seed_topic_list]\n seed_topic_embeddings = self._extract_embeddings(seed_topic_list, verbose=self.verbose)\n seed_topic_embeddings = np.vstack([seed_topic_embeddings, embeddings.mean(axis=0)])\n\n # Label documents that are most similar to one of the seeded topics\n sim_matrix = cosine_similarity(embeddings, seed_topic_embeddings)\n y = [np.argmax(sim_matrix[index]) for index in range(sim_matrix.shape[0])]\n y = [val if val != len(seed_topic_list) else -1 for val in y]\n\n # Average the document embeddings related to the seeded topics with the\n # embedding of the seeded topic to force the documents in a cluster\n for seed_topic in range(len(seed_topic_list)):\n indices = [index for index, topic in enumerate(y) if topic == seed_topic]\n embeddings[indices] = embeddings[indices] * 0.75 + seed_topic_embeddings[seed_topic] * 0.25\n logger.info(\"Guided - Completed \\u2713\")\n return y, embeddings\n\n def _extract_topics(\n self,\n documents: pd.DataFrame,\n embeddings: np.ndarray = None,\n mappings=None,\n verbose: bool = False,\n fine_tune_representation: bool = True,\n ):\n \"\"\"Extract topics from the clusters using a class-based TF-IDF.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs\n embeddings: The document embeddings\n mappings: The mappings from topic to word\n verbose: Whether to log the process of extracting topics\n fine_tune_representation: If True, the topic representation will be fine-tuned using representation models.\n If False, the topic representation will remain as the base c-TF-IDF representation.\n\n Returns:\n c_tf_idf: The resulting matrix giving a value (importance score) for each word per topic\n \"\"\"\n if verbose:\n action = \"Fine-tuning\" if fine_tune_representation else \"Extracting\"\n method = \"representation models\" if fine_tune_representation else \"c-TF-IDF for topic reduction\"\n logger.info(f\"Representation - {action} topics using {method}.\")\n\n documents_per_topic = documents.groupby([\"Topic\"], as_index=False).agg({\"Document\": \" \".join})\n self.c_tf_idf_, words = self._c_tf_idf(documents_per_topic)\n self.topic_representations_ = self._extract_words_per_topic(\n words,\n documents,\n fine_tune_representation=fine_tune_representation,\n calculate_aspects=fine_tune_representation,\n embeddings=embeddings,\n )\n self._create_topic_vectors(documents=documents, embeddings=embeddings, mappings=mappings)\n\n if verbose:\n logger.info(\"Representation - Completed \\u2713\")\n\n def _save_representative_docs(self, documents: pd.DataFrame):\n \"\"\"Save the 3 most representative docs per topic.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs\n\n Updates:\n self.representative_docs_: Populate each topic with 3 representative docs\n \"\"\"\n repr_docs, _, _, _ = self._extract_representative_docs(\n self.c_tf_idf_,\n documents,\n self.topic_representations_,\n nr_samples=500,\n nr_repr_docs=3,\n )\n self.representative_docs_ = repr_docs\n\n def _extract_representative_docs(\n self,\n c_tf_idf: csr_matrix,\n documents: pd.DataFrame,\n topics: Mapping[str, List[Tuple[str, float]]],\n nr_samples: int = 500,\n nr_repr_docs: int = 5,\n diversity: float | None = None,\n ) -> Union[List[str], List[List[int]]]:\n \"\"\"Approximate most representative documents per topic by sampling\n a subset of the documents in each topic and calculating which are\n most representative to their topic based on the cosine similarity between\n c-TF-IDF representations.\n\n Arguments:\n c_tf_idf: The topic c-TF-IDF representation\n documents: All input documents\n topics: The candidate topics as calculated with c-TF-IDF\n nr_samples: The number of candidate documents to extract per topic\n nr_repr_docs: The number of representative documents to extract per topic\n diversity: The diversity between the most representative documents.\n If None, no MMR is used. Otherwise, accepts values between 0 and 1.\n\n Returns:\n repr_docs_mappings: A dictionary from topic to representative documents\n representative_docs: A flat list of representative documents\n repr_doc_indices: Ordered indices of representative documents\n that belong to each topic\n repr_doc_ids: The indices of representative documents\n that belong to each topic\n \"\"\"\n # Sample documents per topic\n documents_per_topic = (\n documents.drop(\"Image\", axis=1, errors=\"ignore\")\n .groupby(\"Topic\")\n .sample(n=nr_samples, replace=True, random_state=42)\n .drop_duplicates()\n )\n\n # Find and extract documents that are most similar to the topic\n repr_docs = []\n repr_docs_indices = []\n repr_docs_mappings = {}\n repr_docs_ids = []\n labels = sorted(list(topics.keys()))\n for index, topic in enumerate(labels):\n # Slice data\n selection = documents_per_topic.loc[documents_per_topic.Topic == topic, :]\n selected_docs = selection[\"Document\"].to_numpy()\n selected_docs_ids = selection.index.tolist()\n\n # Calculate similarity\n nr_docs = nr_repr_docs if len(selected_docs) > nr_repr_docs else len(selected_docs)\n bow = self.vectorizer_model.transform(selected_docs)\n ctfidf = self.ctfidf_model.transform(bow)\n sim_matrix = cosine_similarity(ctfidf, c_tf_idf[index])\n\n # Use MMR to find representative but diverse documents\n if diversity:\n docs = mmr(\n c_tf_idf[index],\n ctfidf,\n selected_docs,\n top_n=nr_docs,\n diversity=diversity,\n )\n\n # Extract top n most representative documents\n else:\n indices = np.argpartition(sim_matrix.reshape(1, -1)[0], -nr_docs)[-nr_docs:]\n docs = [selected_docs[index] for index in indices]\n\n doc_ids = [selected_docs_ids[index] for index, doc in enumerate(selected_docs) if doc in docs]\n repr_docs_ids.append(doc_ids)\n repr_docs.extend(docs)\n repr_docs_indices.append([repr_docs_indices[-1][-1] + i + 1 if index != 0 else i for i in range(nr_docs)])\n repr_docs_mappings = {topic: repr_docs[i[0] : i[-1] + 1] for topic, i in zip(topics.keys(), repr_docs_indices)}\n\n return repr_docs_mappings, repr_docs, repr_docs_indices, repr_docs_ids\n\n def _create_topic_vectors(\n self,\n documents: pd.DataFrame = None,\n embeddings: np.ndarray = None,\n mappings=None,\n ):\n \"\"\"Creates embeddings per topics based on their topic representation.\n\n As a default, topic vectors (topic embeddings) are created by taking\n the average of all document embeddings within a topic. If topics are\n merged, then a weighted average of topic embeddings is taken based on\n the initial topic sizes.\n\n For the `.partial_fit` and `.update_topics` method, the average\n of all document embeddings is not taken since those are not known.\n Instead, the weighted average of the embeddings of the top n words\n is taken for each topic. The weighting is done based on the c-TF-IDF\n score. This will put more emphasis to words that represent a topic best.\n \"\"\"\n # Topic embeddings based on input embeddings\n if embeddings is not None and documents is not None:\n topic_embeddings = []\n topics = documents.sort_values(\"Topic\").Topic.unique()\n topic_ids = documents[\"Topic\"].to_numpy()\n doc_ids = documents[\"ID\"].to_numpy().astype(int)\n for topic in topics:\n mask = topic_ids == topic\n topic_embeddings.append(embeddings[doc_ids[mask]].mean(axis=0))\n self.topic_embeddings_ = np.array(topic_embeddings)\n\n # Topic embeddings when merging topics\n elif self.topic_embeddings_ is not None and mappings is not None:\n topic_embeddings_dict = {}\n for topic_to, topics_from in mappings.items():\n topic_ids = topics_from[\"topics_from\"]\n topic_sizes = topics_from[\"topic_sizes\"]\n if topic_ids:\n embds = np.array(self.topic_embeddings_)[np.array(topic_ids) + self._outliers]\n topic_embedding = np.average(embds, axis=0, weights=topic_sizes)\n topic_embeddings_dict[topic_to] = topic_embedding\n\n # Re-order topic embeddings\n topics_to_map = {\n topic_mapping[0]: topic_mapping[1] for topic_mapping in np.array(self.topic_mapper_.mappings_)[:, -2:]\n }\n topic_embeddings = {}\n for topic, embds in topic_embeddings_dict.items():\n topic_embeddings[topics_to_map[topic]] = embds\n unique_topics = sorted(list(topic_embeddings.keys()))\n self.topic_embeddings_ = np.array([topic_embeddings[topic] for topic in unique_topics])\n\n # Topic embeddings based on keyword representations\n elif self.embedding_model is not None and type(self.embedding_model) is not BaseEmbedder:\n topic_list = list(self.topic_representations_.keys())\n topic_list.sort()\n\n # Only extract top n words\n n = len(self.topic_representations_[topic_list[0]])\n if self.top_n_words < n:\n n = self.top_n_words\n\n # Extract embeddings for all words in all topics\n topic_words = [self.get_topic(topic) for topic in topic_list]\n topic_words = [word[0] for topic in topic_words for word in topic]\n word_embeddings = self._extract_embeddings(topic_words, method=\"word\", verbose=False)\n\n # Take the weighted average of word embeddings in a topic based on their c-TF-IDF value\n # The embeddings var is a single numpy matrix and therefore slicing is necessary to\n # access the words per topic\n topic_embeddings = []\n for i, topic in enumerate(topic_list):\n word_importance = [val[1] for val in self.get_topic(topic)]\n if sum(word_importance) == 0:\n word_importance = [1 for _ in range(len(self.get_topic(topic)))]\n topic_embedding = np.average(\n word_embeddings[i * n : n + (i * n)],\n weights=word_importance,\n axis=0,\n )\n topic_embeddings.append(topic_embedding)\n\n self.topic_embeddings_ = np.array(topic_embeddings)\n\n def _c_tf_idf(\n self,\n documents_per_topic: pd.DataFrame,\n fit: bool = True,\n partial_fit: bool = False,\n ) -> Tuple[csr_matrix, List[str]]:\n \"\"\"Calculate a class-based TF-IDF where m is the number of total documents.\n\n Arguments:\n documents_per_topic: The joined documents per topic such that each topic has a single\n string made out of multiple documents\n m: The total number of documents (unjoined)\n fit: Whether to fit a new vectorizer or use the fitted self.vectorizer_model\n partial_fit: Whether to run `partial_fit` for online learning\n\n Returns:\n tf_idf: The resulting matrix giving a value (importance score) for each word per topic\n words: The names of the words to which values were given\n \"\"\"\n documents = self._preprocess_text(documents_per_topic.Document.values)\n\n if partial_fit:\n X = self.vectorizer_model.partial_fit(documents).update_bow(documents)\n elif fit:\n X = self.vectorizer_model.fit_transform(documents)\n else:\n X = self.vectorizer_model.transform(documents)\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = self.vectorizer_model.get_feature_names_out()\n else:\n words = self.vectorizer_model.get_feature_names()\n\n multiplier = None\n if self.ctfidf_model.seed_words and self.seed_topic_list:\n seed_topic_list = [seed for seeds in self.seed_topic_list for seed in seeds]\n multiplier = np.array(\n [self.ctfidf_model.seed_multiplier if word in self.ctfidf_model.seed_words else 1 for word in words]\n )\n multiplier = np.array([1.2 if word in seed_topic_list else value for value, word in zip(multiplier, words)])\n elif self.ctfidf_model.seed_words:\n multiplier = np.array(\n [self.ctfidf_model.seed_multiplier if word in self.ctfidf_model.seed_words else 1 for word in words]\n )\n elif self.seed_topic_list:\n seed_topic_list = [seed for seeds in self.seed_topic_list for seed in seeds]\n multiplier = np.array([1.2 if word in seed_topic_list else 1 for word in words])\n\n if fit:\n self.ctfidf_model = self.ctfidf_model.fit(X, multiplier=multiplier)\n\n c_tf_idf = self.ctfidf_model.transform(X)\n\n return c_tf_idf, words\n\n def _update_topic_size(self, documents: pd.DataFrame):\n \"\"\"Calculate the topic sizes.\n\n Arguments:\n documents: Updated dataframe with documents and their corresponding IDs and newly added Topics\n \"\"\"\n self.topic_sizes_ = collections.Counter(documents.Topic.to_numpy().tolist())\n self.topics_ = documents.Topic.astype(int).tolist()\n\n def _extract_words_per_topic(\n self,\n words: List[str],\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix = None,\n fine_tune_representation: bool = True,\n calculate_aspects: bool = True,\n embeddings: np.ndarray = None,\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Based on tf_idf scores per topic, extract the top n words per topic.\n\n If the top words per topic need to be extracted, then only the `words` parameter\n needs to be passed. If the top words per topic in a specific timestamp, then it\n is important to pass the timestamp-based c-TF-IDF matrix and its corresponding\n labels.\n\n Arguments:\n words: List of all words (sorted according to tf_idf matrix position)\n documents: DataFrame with documents and their topic IDs\n c_tf_idf: A c-TF-IDF matrix from which to calculate the top words\n fine_tune_representation: If True, the topic representation will be fine-tuned using representation models.\n If False, the topic representation will remain as the base c-TF-IDF representation.\n calculate_aspects: Whether to calculate additional topic aspects\n embeddings: Pre-trained document embeddings. These can be used\n instead of an embedding model\n\n Returns:\n topics: The top words per topic\n \"\"\"\n if c_tf_idf is None:\n c_tf_idf = self.c_tf_idf_\n\n labels = sorted(list(documents.Topic.unique()))\n labels = [int(label) for label in labels]\n\n # Get at least the top 30 indices and values per row in a sparse c-TF-IDF matrix\n top_n_words = max(self.top_n_words, 30)\n indices = self._top_n_idx_sparse(c_tf_idf, top_n_words)\n scores = self._top_n_values_sparse(c_tf_idf, indices)\n sorted_indices = np.argsort(scores, 1)\n indices = np.take_along_axis(indices, sorted_indices, axis=1)\n scores = np.take_along_axis(scores, sorted_indices, axis=1)\n\n # Get top 30 words per topic based on c-TF-IDF score\n base_topics = {\n label: [\n (words[word_index], score) if word_index is not None and score > 0 else (\"\", 0.00001)\n for word_index, score in zip(indices[index][::-1], scores[index][::-1])\n ]\n for index, label in enumerate(labels)\n }\n\n # Fine-tune the topic representations\n topics = base_topics.copy()\n if not self.representation_model or not fine_tune_representation:\n # Default representation: c_tf_idf + top_n_words\n topics = {label: values[: self.top_n_words] for label, values in topics.items()}\n elif fine_tune_representation and isinstance(self.representation_model, list):\n for tuner in self.representation_model:\n topics = tuner.extract_topics(self, documents, c_tf_idf, topics)\n elif fine_tune_representation and isinstance(self.representation_model, KeyBERTInspired):\n topics = self.representation_model.extract_topics(self, documents, c_tf_idf, topics, embeddings)\n elif fine_tune_representation and isinstance(self.representation_model, BaseRepresentation):\n topics = self.representation_model.extract_topics(self, documents, c_tf_idf, topics)\n elif fine_tune_representation and isinstance(self.representation_model, dict):\n if self.representation_model.get(\"Main\"):\n main_model = self.representation_model[\"Main\"]\n if isinstance(main_model, BaseRepresentation):\n topics = main_model.extract_topics(self, documents, c_tf_idf, topics)\n elif isinstance(main_model, list):\n for tuner in main_model:\n topics = tuner.extract_topics(self, documents, c_tf_idf, topics)\n else:\n raise TypeError(f\"unsupported type {type(main_model).__name__} for representation_model['Main']\")\n else:\n # Default representation: c_tf_idf + top_n_words\n topics = {label: values[: self.top_n_words] for label, values in topics.items()}\n else:\n raise TypeError(f\"unsupported type {type(self.representation_model).__name__} for representation_model\")\n\n # Extract additional topic aspects\n if calculate_aspects and isinstance(self.representation_model, dict):\n for aspect, aspect_model in self.representation_model.items():\n if aspect != \"Main\":\n aspects = base_topics.copy()\n if not aspect_model:\n # Default representation: c_tf_idf + top_n_words\n aspects = {label: values[: self.top_n_words] for label, values in aspects.items()}\n if isinstance(aspect_model, list):\n for tuner in aspect_model:\n aspects = tuner.extract_topics(self, documents, c_tf_idf, aspects)\n elif isinstance(aspect_model, BaseRepresentation):\n aspects = aspect_model.extract_topics(self, documents, c_tf_idf, aspects)\n else:\n raise TypeError(\n f\"unsupported type {type(aspect_model).__name__} for representation_model[{aspect!r}]\"\n )\n self.topic_aspects_[aspect] = aspects\n\n return topics\n\n def _reduce_topics(self, documents: pd.DataFrame, use_ctfidf: bool = False) -> pd.DataFrame:\n \"\"\"Reduce topics to self.nr_topics.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs and Topics\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, semantic\n embeddings are used.\n\n Returns:\n documents: Updated dataframe with documents and the reduced number of Topics\n \"\"\"\n logger.info(\"Topic reduction - Reducing number of topics\")\n initial_nr_topics = len(self.get_topics())\n\n if isinstance(self.nr_topics, int):\n if self.nr_topics < initial_nr_topics:\n documents = self._reduce_to_n_topics(documents, use_ctfidf)\n else:\n logger.info(\n f\"Topic reduction - Number of topics ({self.nr_topics}) is equal or higher than the clustered topics({len(self.get_topics())}).\"\n )\n self._extract_topics(documents, verbose=self.verbose)\n return documents\n elif isinstance(self.nr_topics, str):\n documents = self._auto_reduce_topics(documents, use_ctfidf)\n else:\n raise ValueError(\"nr_topics needs to be an int or 'auto'! \")\n\n logger.info(\n f\"Topic reduction - Reduced number of topics from {initial_nr_topics} to {len(self.get_topic_freq())}\"\n )\n return documents\n\n def _reduce_to_n_topics(self, documents: pd.DataFrame, use_ctfidf: bool = False) -> pd.DataFrame:\n \"\"\"Reduce topics to self.nr_topics.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs and Topics\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, semantic\n embedding are used.\n\n Returns:\n documents: Updated dataframe with documents and the reduced number of Topics\n \"\"\"\n topics = documents.Topic.tolist().copy()\n\n # Create topic distance matrix\n topic_embeddings = select_topic_representation(\n self.c_tf_idf_, self.topic_embeddings_, use_ctfidf, output_ndarray=True\n )[0][self._outliers :]\n distance_matrix = 1 - cosine_similarity(topic_embeddings)\n np.fill_diagonal(distance_matrix, 0)\n\n # Cluster the topic embeddings using AgglomerativeClustering\n if version.parse(sklearn_version) >= version.parse(\"1.4.0\"):\n cluster = AgglomerativeClustering(self.nr_topics - self._outliers, metric=\"precomputed\", linkage=\"average\")\n else:\n cluster = AgglomerativeClustering(\n self.nr_topics - self._outliers,\n affinity=\"precomputed\",\n linkage=\"average\",\n )\n cluster.fit(distance_matrix)\n new_topics = [cluster.labels_[topic] if topic != -1 else -1 for topic in topics]\n\n # Track mappings and sizes of topics for merging topic embeddings\n mapped_topics = {from_topic: to_topic for from_topic, to_topic in zip(topics, new_topics)}\n basic_mappings = defaultdict(list)\n for key, val in sorted(mapped_topics.items()):\n basic_mappings[val].append(key)\n mappings = {\n topic_to: {\n \"topics_from\": topics_from,\n \"topic_sizes\": [self.topic_sizes_[topic] for topic in topics_from],\n }\n for topic_to, topics_from in basic_mappings.items()\n }\n\n # Map topics\n documents.Topic = new_topics\n self._update_topic_size(documents)\n self.topic_mapper_.add_mappings(mapped_topics, topic_model=self)\n\n # Update representations\n documents = self._sort_mappings_by_frequency(documents)\n self._extract_topics(documents, mappings=mappings, verbose=self.verbose)\n\n self._update_topic_size(documents)\n return documents\n\n def _auto_reduce_topics(self, documents: pd.DataFrame, use_ctfidf: bool = False) -> pd.DataFrame:\n \"\"\"Reduce the number of topics automatically using HDBSCAN.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs and Topics\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the\n embeddings from the embedding model are used.\n\n Returns:\n documents: Updated dataframe with documents and the reduced number of Topics\n \"\"\"\n topics = documents.Topic.tolist().copy()\n unique_topics = sorted(list(documents.Topic.unique()))[self._outliers :]\n\n # Find similar topics\n embeddings = select_topic_representation(\n self.c_tf_idf_, self.topic_embeddings_, use_ctfidf, output_ndarray=True\n )[0]\n norm_data = normalize(embeddings, norm=\"l2\")\n\n if HAS_HDBSCAN:\n predictions = HDBSCAN(\n min_cluster_size=2,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n ).fit_predict(norm_data[self._outliers :])\n else:\n predictions = SK_HDBSCAN(\n min_cluster_size=2, metric=\"euclidean\", cluster_selection_method=\"eom\", n_jobs=-1\n ).fit_predict(norm_data[self._outliers :])\n\n # Map clusters to their lowest topic_id\n cluster_to_lowest = {}\n for cluster, topic_id in zip(predictions, unique_topics):\n if cluster != -1: # Ignore unclustered items\n if cluster not in cluster_to_lowest:\n cluster_to_lowest[cluster] = topic_id\n else:\n cluster_to_lowest[cluster] = min(cluster_to_lowest[cluster], topic_id)\n\n # Map each topic_id to the lowest topic_id in its cluster\n mapped_topics = {}\n for cluster, topic_id in zip(predictions, unique_topics):\n if cluster == -1:\n mapped_topics[topic_id] = topic_id # No clustering, stays the same\n else:\n mapped_topics[topic_id] = cluster_to_lowest[cluster]\n\n documents.Topic = documents.Topic.map(mapped_topics).fillna(documents.Topic).astype(int)\n mapped_topics = {from_topic: to_topic for from_topic, to_topic in zip(topics, documents.Topic.tolist())}\n\n # Track mappings and sizes of topics for merging topic embeddings\n mappings = defaultdict(list)\n for key, val in sorted(mapped_topics.items()):\n mappings[val].append(key)\n mappings = {\n topic_to: {\n \"topics_from\": topics_from,\n \"topic_sizes\": [self.topic_sizes_[topic] for topic in topics_from],\n }\n for topic_to, topics_from in mappings.items()\n }\n\n # Update documents and topics\n self.topic_mapper_.add_mappings(mapped_topics, topic_model=self)\n documents = self._sort_mappings_by_frequency(documents)\n self._extract_topics(documents, mappings=mappings, verbose=self.verbose)\n self._update_topic_size(documents)\n return documents\n\n def _sort_mappings_by_frequency(self, documents: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Reorder mappings by their frequency.\n\n For example, if topic 88 was mapped to topic\n 5 and topic 5 turns out to be the largest topic,\n then topic 5 will be topic 0. The second largest\n will be topic 1, etc.\n\n If there are no mappings since no reduction of topics\n took place, then the topics will simply be ordered\n by their frequency and will get the topic ids based\n on that order.\n\n This means that -1 will remain the outlier class, and\n that the rest of the topics will be in descending order\n of ids and frequency.\n\n Arguments:\n documents: Dataframe with documents and their corresponding IDs and Topics\n\n Returns:\n documents: Updated dataframe with documents and the mapped\n and re-ordered topic ids\n \"\"\"\n # No need to sort if it's the first pass of zero-shot topic modeling\n nr_zeroshot = len(self._topic_id_to_zeroshot_topic_idx)\n if self._is_zeroshot and not self.nr_topics and nr_zeroshot > 0:\n return documents\n\n # Map topics based on frequency\n self._update_topic_size(documents)\n df = pd.DataFrame(self.topic_sizes_.items(), columns=[\"Old_Topic\", \"Size\"]).sort_values(\"Size\", ascending=False)\n df = df[df.Old_Topic != -1]\n sorted_topics = {**{-1: -1}, **dict(zip(df.Old_Topic, range(len(df))))}\n self.topic_mapper_.add_mappings(sorted_topics, topic_model=self)\n\n # Map documents\n documents.Topic = documents.Topic.map(sorted_topics).fillna(documents.Topic).astype(int)\n self._update_topic_size(documents)\n return documents\n\n def _map_probabilities(\n self, probabilities: Union[np.ndarray, None], original_topics: bool = False\n ) -> Union[np.ndarray, None]:\n \"\"\"Map the probabilities to the reduced topics.\n This is achieved by adding together the probabilities\n of all topics that are mapped to the same topic. Then,\n the topics that were mapped from are set to 0 as they\n were reduced.\n\n Arguments:\n probabilities: An array containing probabilities\n original_topics: Whether we want to map from the\n original topics to the most recent topics\n or from the second-most recent topics.\n\n Returns:\n mapped_probabilities: Updated probabilities\n \"\"\"\n mappings = self.topic_mapper_.get_mappings(original_topics)\n\n # Map array of probabilities (probability for assigned topic per document)\n if probabilities is not None:\n if len(probabilities.shape) == 2:\n mapped_probabilities = np.zeros(\n (\n probabilities.shape[0],\n len(set(mappings.values())) - self._outliers,\n )\n )\n for from_topic, to_topic in mappings.items():\n if to_topic != -1 and from_topic != -1:\n mapped_probabilities[:, to_topic] += probabilities[:, from_topic]\n\n return mapped_probabilities\n\n return probabilities\n\n def _preprocess_text(self, documents: np.ndarray) -> List[str]:\n r\"\"\"Basic preprocessing of text.\n\n Steps:\n * Replace \\n and \\t with whitespace\n * Only keep alpha-numerical characters\n \"\"\"\n cleaned_documents = [doc.replace(\"\\n\", \" \") for doc in documents]\n cleaned_documents = [doc.replace(\"\\t\", \" \") for doc in cleaned_documents]\n if self.language == \"english\":\n cleaned_documents = [re.sub(r\"[^A-Za-z0-9 ]+\", \"\", doc) for doc in cleaned_documents]\n cleaned_documents = [doc if doc != \"\" else \"emptydoc\" for doc in cleaned_documents]\n return cleaned_documents\n\n @staticmethod\n def _top_n_idx_sparse(matrix: csr_matrix, n: int) -> np.ndarray:\n \"\"\"Return indices of top n values in each row of a sparse matrix.\n\n Retrieved from:\n https://stackoverflow.com/questions/49207275/finding-the-top-n-values-in-a-row-of-a-scipy-sparse-matrix\n\n Arguments:\n matrix: The sparse matrix from which to get the top n indices per row\n n: The number of highest values to extract from each row\n\n Returns:\n indices: The top n indices per row\n \"\"\"\n indices = []\n for le, ri in zip(matrix.indptr[:-1], matrix.indptr[1:]):\n n_row_pick = min(n, ri - le)\n values = matrix.indices[le + np.argpartition(matrix.data[le:ri], -n_row_pick)[-n_row_pick:]]\n values = [values[index] if len(values) >= index + 1 else None for index in range(n)]\n indices.append(values)\n return np.array(indices)\n\n @staticmethod\n def _top_n_values_sparse(matrix: csr_matrix, indices: np.ndarray) -> np.ndarray:\n \"\"\"Return the top n values for each row in a sparse matrix.\n\n Arguments:\n matrix: The sparse matrix from which to get the top n indices per row\n indices: The top n indices per row\n\n Returns:\n top_values: The top n scores per row\n \"\"\"\n top_values = []\n for row, values in enumerate(indices):\n scores = np.array([matrix[row, value] if value is not None else 0 for value in values])\n top_values.append(scores)\n return np.array(top_values)\n\n @classmethod\n def _get_param_names(cls):\n \"\"\"Get parameter names for the estimator.\n\n Adapted from:\n https://github.com/scikit-learn/scikit-learn/blob/b3ea3ed6a/sklearn/base.py#L178\n \"\"\"\n init_signature = inspect.signature(cls.__init__)\n parameters = sorted(\n [p.name for p in init_signature.parameters.values() if p.name != \"self\" and p.kind != p.VAR_KEYWORD]\n )\n return parameters\n\n def __str__(self):\n \"\"\"Get a string representation of the current object.\n\n Returns:\n str: Human readable representation of the most important model parameters.\n The parameters that represent models are ignored due to their length.\n \"\"\"\n parameters = \"\"\n for parameter, value in self.get_params().items():\n value = str(value)\n if \"(\" in value and value[0] != \"(\":\n value = value.split(\"(\")[0] + \"(...)\"\n parameters += f\"{parameter}={value}, \"\n\n return f\"BERTopic({parameters[:-2]})\"\n\n\nclass TopicMapper:\n \"\"\"Keep track of Topic Mappings.\n\n The number of topics can be reduced\n by merging them together. This mapping\n needs to be tracked in BERTopic as new\n predictions need to be mapped to the new\n topics.\n\n These mappings are tracked in the `self.mappings_`\n attribute where each set of topic is stacked horizontally.\n For example, the most recent topics can be found in the\n last column. To get a mapping, simply take two columns\n of topics.\n\n In other words, it is represented as graph:\n Topic 1 --> Topic 11 --> Topic 4 --> etc.\n\n Attributes:\n self.mappings_ (np.ndarray) : A matrix indicating the mappings from one topic\n to another. The columns represent a collection of topics\n at any time. The last column represents the current state\n of topics and the first column represents the initial state\n of topics.\n \"\"\"\n\n def __init__(self, topics: List[int]):\n \"\"\"Initialization of Topic Mapper.\n\n Arguments:\n topics: A list of topics per document\n \"\"\"\n base_topics = np.array(sorted(set(topics)))\n topics = base_topics.copy().reshape(-1, 1)\n self.mappings_ = np.hstack([topics.copy(), topics.copy()]).tolist()\n\n def get_mappings(self, original_topics: bool = True) -> Mapping[int, int]:\n \"\"\"Get mappings from either the original topics or\n the second-most recent topics to the current topics.\n\n Arguments:\n original_topics: Whether we want to map from the\n original topics to the most recent topics\n or from the second-most recent topics.\n\n Returns:\n mappings: The mappings from old topics to new topics\n\n Examples:\n To get mappings, simply call:\n ```python\n mapper = TopicMapper(topics)\n mappings = mapper.get_mappings(original_topics=False)\n ```\n \"\"\"\n if original_topics:\n mappings = np.array(self.mappings_)[:, [0, -1]]\n mappings = dict(zip(mappings[:, 0], mappings[:, 1]))\n else:\n mappings = np.array(self.mappings_)[:, [-3, -1]]\n mappings = dict(zip(mappings[:, 0], mappings[:, 1]))\n return mappings\n\n def add_mappings(self, mappings: Mapping[int, int], topic_model: BERTopic):\n \"\"\"Add new column(s) of topic mappings.\n\n Arguments:\n mappings: The mappings to add\n topic_model: The topic model this TopicMapper belongs to\n \"\"\"\n for topics in self.mappings_:\n topic = topics[-1]\n if topic in mappings:\n topics.append(mappings[topic])\n else:\n topics.append(-1)\n\n # When zero-shot topic(s) are present in the topics to merge,\n # determine whether to take one of the zero-shot topic labels\n # or use a calculated representation.\n if topic_model._is_zeroshot() and len(topic_model._topic_id_to_zeroshot_topic_idx) > 0:\n new_topic_id_to_zeroshot_topic_idx = {}\n topics_to_map = {\n topic_mapping[0]: topic_mapping[1]\n for topic_mapping in np.array(topic_model.topic_mapper_.mappings_)[:, -2:]\n }\n\n # Map topic_to to topics_from\n mapping = defaultdict(list)\n for key, value in topics_to_map.items():\n mapping[value].append(key)\n\n for topic_to, topics_from in mapping.items():\n # which of the original topics are zero-shot\n zeroshot_topic_ids = [\n topic_id for topic_id in topics_from if topic_id in topic_model._topic_id_to_zeroshot_topic_idx\n ]\n if len(zeroshot_topic_ids) == 0:\n continue\n\n # If any of the original topics are zero-shot, take the best fitting zero-shot label\n # if the cosine similarity with the new topic exceeds the zero-shot threshold\n zeroshot_labels = [\n topic_model.zeroshot_topic_list[topic_model._topic_id_to_zeroshot_topic_idx[topic_id]]\n for topic_id in zeroshot_topic_ids\n ]\n zeroshot_embeddings = topic_model._extract_embeddings(zeroshot_labels)\n cosine_similarities = cosine_similarity(\n zeroshot_embeddings, [topic_model.topic_embeddings_[topic_to]]\n ).flatten()\n best_zeroshot_topic_idx = np.argmax(cosine_similarities)\n best_cosine_similarity = cosine_similarities[best_zeroshot_topic_idx]\n if best_cosine_similarity >= topic_model.zeroshot_min_similarity:\n # Using the topic ID from before mapping, get the idx into the zeroshot topic list\n new_topic_id_to_zeroshot_topic_idx[topic_to] = topic_model._topic_id_to_zeroshot_topic_idx[\n zeroshot_topic_ids[best_zeroshot_topic_idx]\n ]\n topic_model._topic_id_to_zeroshot_topic_idx = new_topic_id_to_zeroshot_topic_idx\n\n def add_new_topics(self, mappings: Mapping[int, int]):\n \"\"\"Add new row(s) of topic mappings.\n\n Arguments:\n mappings: The mappings to add\n \"\"\"\n length = len(self.mappings_[0])\n for key, value in mappings.items():\n to_append = [key] + ([None] * (length - 2)) + [value]\n self.mappings_.append(to_append)\n\n\ndef _create_model_from_files(\n topics: Mapping[str, Any],\n params: Mapping[str, Any],\n tensors: Mapping[str, np.array],\n ctfidf_tensors: Mapping[str, Any] | None = None,\n ctfidf_config: Mapping[str, Any] | None = None,\n images: Mapping[int, Any] | None = None,\n warn_no_backend: bool = True,\n):\n \"\"\"Create a BERTopic model from a variety of inputs.\n\n Arguments:\n topics: A dictionary containing topic metadata, including:\n - Topic representations, labels, sizes, custom labels, etc.\n params: BERTopic-specific hyperparams, including HF embedding_model ID\n if given.\n tensors: The topic embeddings\n ctfidf_tensors: The c-TF-IDF representations\n ctfidf_config: The config for CountVectorizer and c-TF-IDF\n images: The images per topic\n warn_no_backend: Whether to warn the user if no backend is given\n \"\"\"\n params[\"n_gram_range\"] = tuple(params[\"n_gram_range\"])\n\n if ctfidf_config is not None:\n ngram_range = ctfidf_config[\"vectorizer_model\"][\"params\"][\"ngram_range\"]\n ctfidf_config[\"vectorizer_model\"][\"params\"][\"ngram_range\"] = tuple(ngram_range)\n\n params[\"n_gram_range\"] = tuple(params[\"n_gram_range\"])\n\n # Select HF model through SentenceTransformers\n try:\n from sentence_transformers import SentenceTransformer\n\n embedding_model = select_backend(SentenceTransformer(params[\"embedding_model\"]))\n except: # noqa: E722\n embedding_model = BaseEmbedder()\n\n if warn_no_backend:\n logger.warning(\n \"You are loading a BERTopic model without explicitly defining an embedding model.\"\n \" If you want to also load in an embedding model, make sure to use\"\n \" `BERTopic.load(my_model, embedding_model=my_embedding_model)`.\"\n )\n\n if params.get(\"embedding_model\") is not None:\n del params[\"embedding_model\"]\n\n # Prepare our empty sub-models\n empty_dimensionality_model = BaseDimensionalityReduction()\n empty_cluster_model = BaseCluster()\n\n # Fit BERTopic without actually performing any clustering\n topic_model = BERTopic(\n embedding_model=embedding_model,\n umap_model=empty_dimensionality_model,\n hdbscan_model=empty_cluster_model,\n **params,\n )\n topic_model.topic_embeddings_ = tensors[\"topic_embeddings\"]\n topic_model.topic_representations_ = {int(key): val for key, val in topics[\"topic_representations\"].items()}\n topic_model.topics_ = topics[\"topics\"]\n topic_model.topic_sizes_ = {int(key): val for key, val in topics[\"topic_sizes\"].items()}\n topic_model.custom_labels_ = topics[\"custom_labels\"]\n\n if topics.get(\"topic_aspects\"):\n topic_aspects = {}\n for aspect, values in topics[\"topic_aspects\"].items():\n if aspect != \"Visual_Aspect\":\n topic_aspects[aspect] = {int(topic): value for topic, value in values.items()}\n topic_model.topic_aspects_ = topic_aspects\n\n if images is not None:\n topic_model.topic_aspects_[\"Visual_Aspect\"] = images\n\n # Topic Mapper\n topic_model.topic_mapper_ = TopicMapper([0])\n topic_model.topic_mapper_.mappings_ = topics[\"topic_mapper\"]\n\n if ctfidf_tensors is not None:\n topic_model.c_tf_idf_ = csr_matrix(\n (\n ctfidf_tensors[\"data\"],\n ctfidf_tensors[\"indices\"],\n ctfidf_tensors[\"indptr\"],\n ),\n shape=ctfidf_tensors[\"shape\"],\n )\n\n # CountVectorizer\n topic_model.vectorizer_model = CountVectorizer(**ctfidf_config[\"vectorizer_model\"][\"params\"])\n topic_model.vectorizer_model.vocabulary_ = ctfidf_config[\"vectorizer_model\"][\"vocab\"]\n\n # ClassTfidfTransformer\n topic_model.ctfidf_model.reduce_frequent_words = ctfidf_config[\"ctfidf_model\"][\"reduce_frequent_words\"]\n topic_model.ctfidf_model.bm25_weighting = ctfidf_config[\"ctfidf_model\"][\"bm25_weighting\"]\n idf = ctfidf_tensors[\"diag\"]\n topic_model.ctfidf_model._idf_diag = sp.diags(\n idf, offsets=0, shape=(len(idf), len(idf)), format=\"csr\", dtype=np.float64\n )\n return topic_model\n" }, { "path": "bertopic/_save_utils.py", "content": "import os\nimport json\nimport numpy as np\n\nfrom pathlib import Path\nfrom tempfile import TemporaryDirectory\n\n\n# HuggingFace Hub\ntry:\n from huggingface_hub import (\n create_repo,\n get_hf_file_metadata,\n hf_hub_download,\n hf_hub_url,\n repo_type_and_id_from_hf_id,\n upload_folder,\n )\n\n _has_hf_hub = True\nexcept ImportError:\n _has_hf_hub = False\n\n# Typing\nfrom typing import Union\n\n# Pytorch check\ntry:\n import torch\n\n _has_torch = True\nexcept ImportError:\n _has_torch = False\n\n# Image check\ntry:\n from PIL import Image\n\n _has_vision = True\nexcept ImportError:\n _has_vision = False\n\n\nTOPICS_NAME = \"topics.json\"\nCONFIG_NAME = \"config.json\"\n\nHF_WEIGHTS_NAME = \"topic_embeddings.bin\" # default pytorch pkl\nHF_SAFE_WEIGHTS_NAME = \"topic_embeddings.safetensors\" # safetensors version\n\nCTFIDF_WEIGHTS_NAME = \"ctfidf.bin\" # default pytorch pkl\nCTFIDF_SAFE_WEIGHTS_NAME = \"ctfidf.safetensors\" # safetensors version\nCTFIDF_CFG_NAME = \"ctfidf_config.json\"\n\nMODEL_CARD_TEMPLATE = \"\"\"\n---\ntags:\n- bertopic\nlibrary_name: bertopic\npipeline_tag: {PIPELINE_TAG}\n---\n\n# {MODEL_NAME}\n\nThis is a [BERTopic](https://github.com/MaartenGr/BERTopic) model.\nBERTopic is a flexible and modular topic modeling framework that allows for the generation of easily interpretable topics from large datasets.\n\n## Usage\n\nTo use this model, please install BERTopic:\n\n```\npip install -U bertopic\n```\n\nYou can use the model as follows:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic.load(\"{PATH}\")\n\ntopic_model.get_topic_info()\n```\n\n## Topic overview\n\n* Number of topics: {NR_TOPICS}\n* Number of training documents: {NR_DOCUMENTS}\n\n
\n Click here for an overview of all topics.\n\n {TOPICS}\n\n
\n\n## Training hyperparameters\n\n{HYPERPARAMS}\n\n## Framework versions\n\n{FRAMEWORKS}\n\"\"\"\n\n\ndef push_to_hf_hub(\n model,\n repo_id: str,\n commit_message: str = \"Add BERTopic model\",\n token: str | None = None,\n revision: str | None = None,\n private: bool = False,\n create_pr: bool = False,\n model_card: bool = True,\n serialization: str = \"safetensors\",\n save_embedding_model: Union[str, bool] = True,\n save_ctfidf: bool = False,\n):\n \"\"\"Push your BERTopic model to a HuggingFace Hub.\n\n Arguments:\n model: The BERTopic model to push\n repo_id: The name of your HuggingFace repository\n commit_message: A commit message\n token: Token to add if not already logged in\n revision: Repository revision\n private: Whether to create a private repository\n create_pr: Whether to upload the model as a Pull Request\n model_card: Whether to automatically create a modelcard\n serialization: The type of serialization.\n Either `safetensors` or `pytorch`\n save_embedding_model: A pointer towards a HuggingFace model to be loaded in with\n SentenceTransformers. E.g.,\n `sentence-transformers/all-MiniLM-L6-v2`\n save_ctfidf: Whether to save c-TF-IDF information\n \"\"\"\n if not _has_hf_hub:\n raise ValueError(\"Make sure you have the huggingface hub installed via `pip install --upgrade huggingface_hub`\")\n\n # Create repo if it doesn't exist yet and infer complete repo_id\n repo_url = create_repo(repo_id, token=token, private=private, exist_ok=True)\n _, repo_owner, repo_name = repo_type_and_id_from_hf_id(repo_url)\n repo_id = f\"{repo_owner}/{repo_name}\"\n\n # Temporarily save model and push to HF\n with TemporaryDirectory() as tmpdir:\n # Save model weights and config.\n model.save(\n tmpdir,\n serialization=serialization,\n save_embedding_model=save_embedding_model,\n save_ctfidf=save_ctfidf,\n )\n\n # Add README if it does not exist\n try:\n get_hf_file_metadata(hf_hub_url(repo_id=repo_id, filename=\"README.md\", revision=revision))\n except: # noqa: E722\n if model_card:\n readme_text = generate_readme(model, repo_id)\n readme_path = Path(tmpdir) / \"README.md\"\n readme_path.write_text(readme_text, encoding=\"utf8\")\n\n # Upload model\n return upload_folder(\n repo_id=repo_id,\n folder_path=tmpdir,\n revision=revision,\n create_pr=create_pr,\n commit_message=commit_message,\n )\n\n\ndef load_local_files(path):\n \"\"\"Load local BERTopic files.\"\"\"\n # Load json configs\n topics = load_cfg_from_json(path / TOPICS_NAME)\n params = load_cfg_from_json(path / CONFIG_NAME)\n\n # Load Topic Embeddings\n safetensor_path = path / HF_SAFE_WEIGHTS_NAME\n if safetensor_path.is_file():\n tensors = load_safetensors(safetensor_path)\n else:\n torch_path = path / HF_WEIGHTS_NAME\n if torch_path.is_file():\n tensors = torch.load(torch_path, map_location=\"cpu\")\n tensors = {k: v.numpy() for k, v in tensors.items()}\n\n # c-TF-IDF\n try:\n ctfidf_tensors = None\n safetensor_path = path / CTFIDF_SAFE_WEIGHTS_NAME\n if safetensor_path.is_file():\n ctfidf_tensors = load_safetensors(safetensor_path)\n else:\n torch_path = path / CTFIDF_WEIGHTS_NAME\n if torch_path.is_file():\n ctfidf_tensors = torch.load(torch_path, map_location=\"cpu\")\n ctfidf_tensors = {k: v.numpy() for k, v in ctfidf_tensors.items()}\n ctfidf_config = load_cfg_from_json(path / CTFIDF_CFG_NAME)\n except: # noqa: E722\n ctfidf_config, ctfidf_tensors = None, None\n\n # Load images\n images = None\n if _has_vision:\n try:\n Image.open(path / \"images/0.jpg\")\n _has_images = True\n except: # noqa: E722\n _has_images = False\n\n if _has_images:\n topic_list = list(topics[\"topic_representations\"].keys())\n images = {}\n for topic in topic_list:\n image = Image.open(path / f\"images/{topic}.jpg\")\n images[int(topic)] = image\n\n return topics, params, tensors, ctfidf_tensors, ctfidf_config, images\n\n\ndef load_files_from_hf(path):\n \"\"\"Load files from HuggingFace.\"\"\"\n path = str(path)\n\n # Configs\n topics = load_cfg_from_json(hf_hub_download(path, TOPICS_NAME, revision=None))\n params = load_cfg_from_json(hf_hub_download(path, CONFIG_NAME, revision=None))\n\n # Topic Embeddings\n try:\n tensors = hf_hub_download(path, HF_SAFE_WEIGHTS_NAME, revision=None)\n tensors = load_safetensors(tensors)\n except: # noqa: E722\n tensors = hf_hub_download(path, HF_WEIGHTS_NAME, revision=None)\n tensors = torch.load(tensors, map_location=\"cpu\")\n\n # c-TF-IDF\n try:\n ctfidf_config = load_cfg_from_json(hf_hub_download(path, CTFIDF_CFG_NAME, revision=None))\n try:\n ctfidf_tensors = hf_hub_download(path, CTFIDF_SAFE_WEIGHTS_NAME, revision=None)\n ctfidf_tensors = load_safetensors(ctfidf_tensors)\n except: # noqa: E722\n ctfidf_tensors = hf_hub_download(path, CTFIDF_WEIGHTS_NAME, revision=None)\n ctfidf_tensors = torch.load(ctfidf_tensors, map_location=\"cpu\")\n except: # noqa: E722\n ctfidf_config, ctfidf_tensors = None, None\n\n # Load images if they exist\n images = None\n if _has_vision:\n try:\n hf_hub_download(path, \"images/0.jpg\", revision=None)\n _has_images = True\n except: # noqa: E722\n _has_images = False\n\n if _has_images:\n topic_list = list(topics[\"topic_representations\"].keys())\n images = {}\n for topic in topic_list:\n image = Image.open(hf_hub_download(path, f\"images/{topic}.jpg\", revision=None))\n images[int(topic)] = image\n\n return topics, params, tensors, ctfidf_tensors, ctfidf_config, images\n\n\ndef generate_readme(model, repo_id: str):\n \"\"\"Generate README for HuggingFace model card.\"\"\"\n model_card = MODEL_CARD_TEMPLATE\n topic_table_head = \"| Topic ID | Topic Keywords | Topic Frequency | Label | \\n|----------|----------------|-----------------|-------| \\n\"\n\n # Get Statistics\n model_name = repo_id.split(\"/\")[-1]\n params = {param: value for param, value in model.get_params().items() if \"model\" not in param}\n params = \"\\n\".join([f\"* {param}: {value}\" for param, value in params.items()])\n topics = sorted(list(set(model.topics_)))\n nr_topics = str(len(set(model.topics_)))\n\n if model.topic_sizes_ is not None:\n nr_documents = str(sum(model.topic_sizes_.values()))\n else:\n nr_documents = \"\"\n\n # Topic information\n topic_keywords = [\" - \".join(next(zip(*model.get_topic(topic)))[:5]) for topic in topics]\n topic_freq = [model.get_topic_freq(topic) for topic in topics]\n topic_labels = model.custom_labels_ if model.custom_labels_ else [model.topic_labels_[topic] for topic in topics]\n topics = [\n f\"| {topic} | {topic_keywords[index]} | {topic_freq[topic]} | {topic_labels[index]} | \\n\"\n for index, topic in enumerate(topics)\n ]\n topics = topic_table_head + \"\".join(topics)\n frameworks = \"\\n\".join([f\"* {param}: {value}\" for param, value in get_package_versions().items()])\n\n # Fill Statistics into model card\n model_card = model_card.replace(\"{MODEL_NAME}\", model_name)\n model_card = model_card.replace(\"{PATH}\", repo_id)\n model_card = model_card.replace(\"{NR_TOPICS}\", nr_topics)\n model_card = model_card.replace(\"{TOPICS}\", topics.strip())\n model_card = model_card.replace(\"{NR_DOCUMENTS}\", nr_documents)\n model_card = model_card.replace(\"{HYPERPARAMS}\", params)\n model_card = model_card.replace(\"{FRAMEWORKS}\", frameworks)\n\n # Fill Pipeline tag\n has_visual_aspect = check_has_visual_aspect(model)\n if not has_visual_aspect:\n model_card = model_card.replace(\"{PIPELINE_TAG}\", \"text-classification\")\n else:\n model_card = model_card.replace(\"pipeline_tag: {PIPELINE_TAG}\\n\", \"\") # TODO add proper tag for this instance\n\n return model_card\n\n\ndef save_hf(model, save_directory, serialization: str):\n \"\"\"Save topic embeddings, either safely (using safetensors) or using legacy pytorch.\"\"\"\n tensors = np.array(model.topic_embeddings_, dtype=np.float32)\n\n if serialization == \"safetensors\":\n tensors = {\"topic_embeddings\": tensors}\n save_safetensors(save_directory / HF_SAFE_WEIGHTS_NAME, tensors)\n if serialization == \"pytorch\":\n assert _has_torch, \"`pip install pytorch` to save as bin\"\n tensors = {\"topic_embeddings\": torch.from_numpy(tensors)}\n torch.save(tensors, save_directory / HF_WEIGHTS_NAME)\n\n\ndef save_ctfidf(model, save_directory: str, serialization: str):\n \"\"\"Save c-TF-IDF sparse matrix.\"\"\"\n indptr = model.c_tf_idf_.indptr\n indices = model.c_tf_idf_.indices\n data = model.c_tf_idf_.data\n shape = np.array(model.c_tf_idf_.shape)\n diag = np.array(model.ctfidf_model._idf_diag.data)\n\n if serialization == \"safetensors\":\n tensors = {\n \"indptr\": indptr,\n \"indices\": indices,\n \"data\": data,\n \"shape\": shape,\n \"diag\": diag,\n }\n save_safetensors(save_directory / CTFIDF_SAFE_WEIGHTS_NAME, tensors)\n if serialization == \"pytorch\":\n assert _has_torch, \"`pip install pytorch` to save as .bin\"\n tensors = {\n \"indptr\": torch.from_numpy(indptr),\n \"indices\": torch.from_numpy(indices),\n \"data\": torch.from_numpy(data),\n \"shape\": torch.from_numpy(shape),\n \"diag\": torch.from_numpy(diag),\n }\n torch.save(tensors, save_directory / CTFIDF_WEIGHTS_NAME)\n\n\ndef save_ctfidf_config(model, path):\n \"\"\"Save parameters to recreate CountVectorizer and c-TF-IDF.\"\"\"\n config = {}\n\n # Recreate ClassTfidfTransformer\n config[\"ctfidf_model\"] = {\n \"bm25_weighting\": model.ctfidf_model.bm25_weighting,\n \"reduce_frequent_words\": model.ctfidf_model.reduce_frequent_words,\n }\n\n # Recreate CountVectorizer\n cv_params = model.vectorizer_model.get_params()\n del cv_params[\"tokenizer\"], cv_params[\"preprocessor\"], cv_params[\"dtype\"]\n if not isinstance(cv_params[\"analyzer\"], str):\n del cv_params[\"analyzer\"]\n\n config[\"vectorizer_model\"] = {\n \"params\": cv_params,\n \"vocab\": model.vectorizer_model.vocabulary_,\n }\n\n with path.open(\"w\") as f:\n json.dump(config, f, indent=2)\n\n\ndef save_config(model, path: str, embedding_model):\n \"\"\"Save BERTopic configuration.\"\"\"\n path = Path(path)\n params = model.get_params()\n config = {param: value for param, value in params.items() if \"model\" not in param}\n\n # Embedding model tag to be used in sentence-transformers\n if isinstance(embedding_model, str):\n config[\"embedding_model\"] = embedding_model\n\n with path.open(\"w\") as f:\n json.dump(config, f, indent=2)\n\n return config\n\n\ndef check_has_visual_aspect(model):\n \"\"\"Check if model has visual aspect.\"\"\"\n if _has_vision:\n for aspect, value in model.topic_aspects_.items():\n if isinstance(value[0], Image.Image):\n return True\n\n\ndef save_images(model, path: str):\n \"\"\"Save topic images.\"\"\"\n if _has_vision:\n visual_aspects = None\n for aspect, value in model.topic_aspects_.items():\n if isinstance(value[0], Image.Image):\n visual_aspects = model.topic_aspects_[aspect]\n break\n\n if visual_aspects is not None:\n path.mkdir(exist_ok=True, parents=True)\n for topic, image in visual_aspects.items():\n image.save(path / f\"{topic}.jpg\")\n\n\ndef save_topics(model, path: str):\n \"\"\"Save Topic-specific information.\"\"\"\n path = Path(path)\n\n if _has_vision:\n selected_topic_aspects = {}\n for aspect, value in model.topic_aspects_.items():\n if not isinstance(value[0], Image.Image):\n selected_topic_aspects[aspect] = value\n else:\n selected_topic_aspects[\"Visual_Aspect\"] = True\n else:\n selected_topic_aspects = model.topic_aspects_\n\n topics = {\n \"topic_representations\": model.topic_representations_,\n \"topics\": [int(topic) for topic in model.topics_],\n \"topic_sizes\": model.topic_sizes_,\n \"topic_mapper\": np.array(model.topic_mapper_.mappings_, dtype=int).tolist(),\n \"topic_labels\": model.topic_labels_,\n \"custom_labels\": model.custom_labels_,\n \"_outliers\": int(model._outliers),\n \"topic_aspects\": selected_topic_aspects,\n }\n\n with path.open(\"w\") as f:\n json.dump(topics, f, indent=2, cls=NumpyEncoder)\n\n\ndef load_cfg_from_json(json_file: Union[str, os.PathLike]):\n \"\"\"Load configuration from json.\"\"\"\n with open(json_file, \"r\", encoding=\"utf-8\") as reader:\n text = reader.read()\n return json.loads(text)\n\n\nclass NumpyEncoder(json.JSONEncoder):\n def default(self, obj):\n if isinstance(obj, np.integer):\n return int(obj)\n if isinstance(obj, np.floating):\n return float(obj)\n return super(NumpyEncoder, self).default(obj)\n\n\ndef get_package_versions():\n \"\"\"Get versions of main dependencies of BERTopic.\"\"\"\n try:\n import platform\n from numpy import __version__ as np_version\n from pandas import __version__ as pandas_version\n from sklearn import __version__ as sklearn_version\n from plotly import __version__ as plotly_version\n\n try:\n from importlib.metadata import version\n\n hdbscan_version = version(\"hdbscan\")\n except (ImportError, ModuleNotFoundError):\n hdbscan_version = None\n\n try:\n from umap import __version__ as umap_version\n except (ImportError, ModuleNotFoundError):\n umap_version = None\n\n try:\n from sentence_transformers import __version__ as sbert_version\n except (ImportError, ModuleNotFoundError):\n sbert_version = None\n\n try:\n from numba import __version__ as numba_version\n except (ImportError, ModuleNotFoundError):\n numba_version = None\n\n try:\n from transformers import __version__ as transformers_version\n except (ImportError, ModuleNotFoundError):\n transformers_version = None\n\n return {\n \"Numpy\": np_version,\n \"HDBSCAN\": hdbscan_version,\n \"UMAP\": umap_version,\n \"Pandas\": pandas_version,\n \"Scikit-Learn\": sklearn_version,\n \"Sentence-transformers\": sbert_version,\n \"Transformers\": transformers_version,\n \"Numba\": numba_version,\n \"Plotly\": plotly_version,\n \"Python\": platform.python_version(),\n }\n except Exception as e:\n return e\n\n\ndef load_safetensors(path):\n \"\"\"Load safetensors and check whether it is installed.\"\"\"\n try:\n import safetensors.numpy\n\n return safetensors.numpy.load_file(path)\n except ImportError:\n raise ValueError(\"`pip install safetensors` to load .safetensors\")\n\n\ndef save_safetensors(path, tensors):\n \"\"\"Save safetensors and check whether it is installed.\"\"\"\n try:\n import safetensors.numpy\n\n safetensors.numpy.save_file(tensors, path)\n except ImportError:\n raise ValueError(\"`pip install safetensors` to save as .safetensors\")\n" }, { "path": "bertopic/_utils.py", "content": "import numpy as np\nimport pandas as pd\nimport logging\nfrom collections.abc import Iterable\nfrom scipy.sparse import csr_matrix\nfrom scipy.spatial.distance import squareform\nfrom typing import Optional, Union, Tuple, Any\n\n\nclass MyLogger:\n def __init__(self):\n self.logger = logging.getLogger(\"BERTopic\")\n\n def configure(self, level):\n self.set_level(level)\n self._add_handler()\n self.logger.propagate = False\n\n def info(self, message):\n self.logger.info(f\"{message}\")\n\n def warning(self, message):\n self.logger.warning(f\"WARNING: {message}\")\n\n def set_level(self, level):\n levels = [\"DEBUG\", \"INFO\", \"WARNING\", \"ERROR\", \"CRITICAL\"]\n if level in levels:\n self.logger.setLevel(level)\n\n def _add_handler(self):\n sh = logging.StreamHandler()\n sh.setFormatter(logging.Formatter(\"%(asctime)s - %(name)s - %(message)s\"))\n self.logger.addHandler(sh)\n\n # Remove duplicate handlers\n if len(self.logger.handlers) > 1:\n self.logger.handlers = [self.logger.handlers[0]]\n\n\ndef check_documents_type(documents):\n \"\"\"Check whether the input documents are indeed a list of strings.\"\"\"\n if isinstance(documents, pd.DataFrame):\n raise TypeError(\"Make sure to supply a list of strings, not a dataframe.\")\n elif isinstance(documents, Iterable) and not isinstance(documents, str):\n if not any([isinstance(doc, str) for doc in documents]):\n raise TypeError(\"Make sure that the iterable only contains strings.\")\n else:\n raise TypeError(\"Make sure that the documents variable is an iterable containing strings only.\")\n\n\ndef check_embeddings_shape(embeddings, docs):\n \"\"\"Check if the embeddings have the correct shape.\"\"\"\n if embeddings is not None:\n if not any([isinstance(embeddings, np.ndarray), isinstance(embeddings, csr_matrix)]):\n raise ValueError(\"Make sure to input embeddings as a numpy array or scipy.sparse.csr.csr_matrix. \")\n else:\n if embeddings.shape[0] != len(docs):\n raise ValueError(\n \"Make sure that the embeddings are a numpy array with shape: \"\n \"(len(docs), vector_dim) where vector_dim is the dimensionality \"\n \"of the vector embeddings. \"\n )\n\n\ndef check_is_fitted(topic_model):\n \"\"\"Checks if the model was fitted by verifying the presence of self.matches.\n\n Arguments:\n topic_model: BERTopic instance for which the check is performed.\n\n Returns:\n None\n\n Raises:\n ValueError: If the matches were not found.\n \"\"\"\n msg = \"This %(name)s instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.\"\n\n if topic_model.topics_ is None:\n raise ValueError(msg % {\"name\": type(topic_model).__name__})\n\n\nclass NotInstalled:\n \"\"\"This object is used to notify the user that additional dependencies need to be\n installed in order to use the string matching model.\n \"\"\"\n\n def __init__(self, tool, dep, custom_msg=None):\n self.tool = tool\n self.dep = dep\n\n msg = f\"In order to use {self.tool} you will need to install via;\\n\\n\"\n if custom_msg is not None:\n msg += custom_msg\n else:\n msg += f\"pip install bertopic[{self.dep}]\\n\\n\"\n self.msg = msg\n\n def __getattr__(self, *args, **kwargs):\n raise ModuleNotFoundError(self.msg)\n\n def __call__(self, *args, **kwargs):\n raise ModuleNotFoundError(self.msg)\n\n\ndef validate_distance_matrix(X, n_samples):\n \"\"\"Validate the distance matrix and convert it to a condensed distance matrix\n if necessary.\n\n A valid distance matrix is either a square matrix of shape (n_samples, n_samples)\n with zeros on the diagonal and non-negative values or condensed distance matrix\n of shape (n_samples * (n_samples - 1) / 2,) containing the upper triangular of the\n distance matrix.\n\n Arguments:\n X: Distance matrix to validate.\n n_samples: Number of samples in the dataset.\n\n Returns:\n X: Validated distance matrix.\n\n Raises:\n ValueError: If the distance matrix is not valid.\n \"\"\"\n # Make sure it is the 1-D condensed distance matrix with zeros on the diagonal\n s = X.shape\n if len(s) == 1:\n # check it has correct size\n n = s[0]\n if n != (n_samples * (n_samples - 1) / 2):\n raise ValueError(\"The condensed distance matrix must have shape (n*(n-1)/2,).\")\n elif len(s) == 2:\n # check it has correct size\n if (s[0] != n_samples) or (s[1] != n_samples):\n raise ValueError(\"The distance matrix must be of shape (n, n) where n is the number of samples.\")\n # force zero diagonal and convert to condensed\n np.fill_diagonal(X, 0)\n X = squareform(X)\n else:\n raise ValueError(\n \"The distance matrix must be either a 1-D condensed \"\n \"distance matrix of shape (n*(n-1)/2,) or a \"\n \"2-D square distance matrix of shape (n, n).\"\n \"where n is the number of documents.\"\n \"Got a distance matrix of shape %s\" % str(s)\n )\n\n # Make sure its entries are non-negative\n if np.any(X < 0):\n raise ValueError(\"Distance matrix cannot contain negative values.\")\n\n return X\n\n\ndef get_unique_distances(dists: np.array, noise_max=1e-7) -> np.array:\n \"\"\"Check if the consecutive elements in the distance array are the same. If so, a small noise\n is added to one of the elements to make sure that the array does not contain duplicates.\n\n Arguments:\n dists: distance array sorted in the increasing order.\n noise_max: the maximal magnitude of noise to be added.\n\n Returns:\n Unique distances sorted in the preserved increasing order.\n \"\"\"\n dists_cp = dists.copy()\n\n for i in range(dists.shape[0] - 1):\n if dists[i] == dists[i + 1]:\n # returns the next unique distance or the current distance with the added noise\n next_unique_dist = next((d for d in dists[i + 1 :] if d != dists[i]), dists[i] + noise_max)\n\n # the noise can never be large then the difference between the next unique distance and the current one\n curr_max_noise = min(noise_max, next_unique_dist - dists_cp[i])\n dists_cp[i + 1] = np.random.uniform(low=dists_cp[i] + curr_max_noise / 2, high=dists_cp[i] + curr_max_noise)\n return dists_cp\n\n\ndef select_topic_representation(\n ctfidf_embeddings: Optional[Union[np.ndarray, csr_matrix]] = None,\n embeddings: Optional[Union[np.ndarray, csr_matrix]] = None,\n use_ctfidf: bool = True,\n output_ndarray: bool = False,\n) -> Tuple[np.ndarray, bool]:\n \"\"\"Select the topic representation.\n\n Arguments:\n ctfidf_embeddings: The c-TF-IDF embedding matrix\n embeddings: The topic embedding matrix\n use_ctfidf: Whether to use the c-TF-IDF representation. If False, topics embedding representation is used, if it\n exists. Default is True.\n output_ndarray: Whether to convert the selected representation into ndarray\n Raises\n ValueError:\n - If no topic representation was found\n - If c-TF-IDF embeddings are not a numpy array or a scipy.sparse.csr_matrix\n\n Returns:\n The selected topic representation and a boolean indicating whether it is c-TF-IDF.\n \"\"\"\n\n def to_ndarray(array: Union[np.ndarray, csr_matrix]) -> np.ndarray:\n if isinstance(array, csr_matrix):\n return array.toarray()\n return array\n\n logger = MyLogger()\n\n if use_ctfidf:\n if ctfidf_embeddings is None:\n logger.warning(\n \"No c-TF-IDF matrix was found despite it is supposed to be used (`use_ctfidf` is True). \"\n \"Defaulting to semantic embeddings.\"\n )\n repr_, ctfidf_used = embeddings, False\n else:\n repr_, ctfidf_used = ctfidf_embeddings, True\n else:\n if embeddings is None:\n logger.warning(\n \"No topic embeddings were found despite they are supposed to be used (`use_ctfidf` is False). \"\n \"Defaulting to c-TF-IDF representation.\"\n )\n repr_, ctfidf_used = ctfidf_embeddings, True\n else:\n repr_, ctfidf_used = embeddings, False\n\n return to_ndarray(repr_) if output_ndarray else repr_, ctfidf_used\n\n\nclass MockPlotlyModule:\n \"\"\"Mock module that raises an error when plotly functions are called.\"\"\"\n\n def __getattr__(self, name: str) -> Any:\n def mock_function(*args, **kwargs):\n raise ImportError(f\"Plotly is required to use '{name}'. Install it with uv pip install plotly\")\n\n return mock_function\n" }, { "path": "bertopic/backend/__init__.py", "content": "from ._base import BaseEmbedder\nfrom ._word_doc import WordDocEmbedder\nfrom ._utils import languages\nfrom bertopic._utils import NotInstalled\n\n# OpenAI Embeddings\ntry:\n from bertopic.backend._openai import OpenAIBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install openai` \\n\\n\"\n OpenAIBackend = NotInstalled(\"OpenAI\", \"OpenAI\", custom_msg=msg)\n\n# Cohere Embeddings\ntry:\n from bertopic.backend._cohere import CohereBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install cohere` \\n\\n\"\n CohereBackend = NotInstalled(\"Cohere\", \"Cohere\", custom_msg=msg)\n\n# Multimodal Embeddings\ntry:\n from bertopic.backend._multimodal import MultiModalBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install bertopic[vision]` \\n\\n\"\n MultiModalBackend = NotInstalled(\"Vision\", \"Vision\", custom_msg=msg)\n\n# Model2Vec Embeddings\ntry:\n from bertopic.backend._model2vec import Model2VecBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install model2vec` \\n\\n\"\n Model2VecBackend = NotInstalled(\"Model2Vec\", \"Model2Vec\", custom_msg=msg)\n\n# FasteEmbed Embeddings\ntry:\n from bertopic.backend._fastembed import FastEmbedBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install fastembed` \\n\\n\"\n FastEmbedBackend = NotInstalled(\"FastEmbed\", \"FastEmbed\", custom_msg=msg)\n\n\n# Langchain Embedddings\ntry:\n from bertopic.backend._langchain import LangChainBackend\nexcept ModuleNotFoundError:\n msg = \"`pip install langchain` \\n\\n\"\n LangChainBackend = NotInstalled(\"LangChain\", \"LangChain\", custom_msg=msg)\n\n\n__all__ = [\n \"BaseEmbedder\",\n \"CohereBackend\",\n \"FastEmbedBackend\",\n \"LangChainBackend\",\n \"Model2VecBackend\",\n \"MultiModalBackend\",\n \"OpenAIBackend\",\n \"WordDocEmbedder\",\n \"languages\",\n]\n" }, { "path": "bertopic/backend/_base.py", "content": "import numpy as np\nfrom typing import List\n\n\nclass BaseEmbedder:\n \"\"\"The Base Embedder used for creating embedding models.\n\n Arguments:\n embedding_model: The main embedding model to be used for extracting\n document and word embedding\n word_embedding_model: The embedding model used for extracting word\n embeddings only. If this model is selected,\n then the `embedding_model` is purely used for\n creating document embeddings.\n \"\"\"\n\n def __init__(self, embedding_model=None, word_embedding_model=None):\n self.embedding_model = embedding_model\n self.word_embedding_model = word_embedding_model\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n pass\n\n def embed_words(self, words: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n words: A list of words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Word embeddings with shape (n, m) with `n` words\n that each have an embeddings size of `m`\n\n \"\"\"\n return self.embed(words, verbose)\n\n def embed_documents(self, document: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n document: A list of documents to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document embeddings with shape (n, m) with `n` documents\n that each have an embeddings size of `m`\n \"\"\"\n return self.embed(document, verbose)\n" }, { "path": "bertopic/backend/_cohere.py", "content": "import time\nimport numpy as np\nfrom tqdm import tqdm\nfrom typing import Any, List, Mapping\nfrom bertopic.backend import BaseEmbedder\n\n\nclass CohereBackend(BaseEmbedder):\n \"\"\"Cohere Embedding Model.\n\n Arguments:\n client: A `cohere` client.\n embedding_model: A Cohere model. Default is \"large\".\n For an overview of models see:\n https://docs.cohere.ai/docs/generation-card\n delay_in_seconds: If a `batch_size` is given, use this set\n the delay in seconds between batches.\n batch_size: The size of each batch.\n embed_kwargs: Kwargs passed to `cohere.Client.embed`.\n Can be used to define additional parameters\n such as `input_type`\n\n Examples:\n ```python\n import cohere\n from bertopic.backend import CohereBackend\n\n client = cohere.Client(\"APIKEY\")\n cohere_model = CohereBackend(client)\n ```\n\n If you want to specify `input_type`:\n\n ```python\n cohere_model = CohereBackend(\n client,\n embedding_model=\"embed-english-v3.0\",\n embed_kwargs={\"input_type\": \"clustering\"}\n )\n ```\n \"\"\"\n\n def __init__(\n self,\n client,\n embedding_model: str = \"large\",\n delay_in_seconds: float | None = None,\n batch_size: int | None = None,\n embed_kwargs: Mapping[str, Any] = {},\n ):\n super().__init__()\n self.client = client\n self.embedding_model = embedding_model\n self.delay_in_seconds = delay_in_seconds\n self.batch_size = batch_size\n self.embed_kwargs = embed_kwargs\n\n if self.embed_kwargs.get(\"model\"):\n self.embedding_model = embed_kwargs.get(\"model\")\n else:\n self.embed_kwargs[\"model\"] = self.embedding_model\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Batch-wise embedding extraction\n if self.batch_size is not None:\n embeddings = []\n for batch in tqdm(self._chunks(documents), disable=not verbose):\n response = self.client.embed(texts=batch, **self.embed_kwargs)\n embeddings.extend(response.embeddings)\n\n # Delay subsequent calls\n if self.delay_in_seconds:\n time.sleep(self.delay_in_seconds)\n\n # Extract embeddings all at once\n else:\n response = self.client.embed(texts=documents, **self.embed_kwargs)\n embeddings = response.embeddings\n return np.array(embeddings)\n\n def _chunks(self, documents):\n for i in range(0, len(documents), self.batch_size):\n yield documents[i : i + self.batch_size]\n" }, { "path": "bertopic/backend/_fastembed.py", "content": "import numpy as np\nfrom typing import List\nfrom fastembed import TextEmbedding\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass FastEmbedBackend(BaseEmbedder):\n \"\"\"FastEmbed embedding model.\n\n The FastEmbed embedding model used for generating sentence embeddings.\n\n Arguments:\n embedding_model: A FastEmbed embedding model\n\n Examples:\n To create a model, you can load in a string pointing to a supported\n FastEmbed model:\n\n ```python\n from bertopic.backend import FastEmbedBackend\n\n sentence_model = FastEmbedBackend(\"BAAI/bge-small-en-v1.5\")\n ```\n \"\"\"\n\n def __init__(self, embedding_model: str = \"BAAI/bge-small-en-v1.5\"):\n super().__init__()\n\n supported_models = [m[\"model\"] for m in TextEmbedding.list_supported_models()]\n\n if isinstance(embedding_model, str) and embedding_model in supported_models:\n self.embedding_model = TextEmbedding(model_name=embedding_model)\n else:\n raise ValueError(\n \"Please select a correct FasteEmbed model: \\n\"\n \"the model must be a string and must be supported. \\n\"\n \"The supported TextEmbedding model list is here: https://qdrant.github.io/fastembed/examples/Supported_Models/\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n embeddings = np.array(list(self.embedding_model.embed(documents, show_progress_bar=verbose)))\n return embeddings\n" }, { "path": "bertopic/backend/_flair.py", "content": "import numpy as np\nfrom tqdm import tqdm\nfrom typing import Union, List\nfrom flair.data import Sentence\nfrom flair.embeddings import DocumentEmbeddings, TokenEmbeddings, DocumentPoolEmbeddings\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass FlairBackend(BaseEmbedder):\n \"\"\"Flair Embedding Model.\n\n The Flair embedding model used for generating document and\n word embeddings.\n\n Arguments:\n embedding_model: A Flair embedding model\n\n Examples:\n ```python\n from bertopic.backend import FlairBackend\n from flair.embeddings import WordEmbeddings, DocumentPoolEmbeddings\n\n # Create a Flair Embedding model\n glove_embedding = WordEmbeddings('crawl')\n document_glove_embeddings = DocumentPoolEmbeddings([glove_embedding])\n\n # Pass the Flair model to create a new backend\n flair_embedder = FlairBackend(document_glove_embeddings)\n ```\n \"\"\"\n\n def __init__(self, embedding_model: Union[TokenEmbeddings, DocumentEmbeddings]):\n super().__init__()\n\n # Flair word embeddings\n if isinstance(embedding_model, TokenEmbeddings):\n self.embedding_model = DocumentPoolEmbeddings([embedding_model])\n\n # Flair document embeddings + disable fine tune to prevent CUDA OOM\n # https://github.com/flairNLP/flair/issues/1719\n elif isinstance(embedding_model, DocumentEmbeddings):\n if \"fine_tune\" in embedding_model.__dict__:\n embedding_model.fine_tune = False\n self.embedding_model = embedding_model\n\n else:\n raise ValueError(\n \"Please select a correct Flair model by either using preparing a token or document \"\n \"embedding model: \\n\"\n \"`from flair.embeddings import TransformerDocumentEmbeddings` \\n\"\n \"`roberta = TransformerDocumentEmbeddings('roberta-base')`\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n embeddings = []\n for document in tqdm(documents, disable=not verbose):\n try:\n sentence = Sentence(document) if document else Sentence(\"an empty document\")\n self.embedding_model.embed(sentence)\n except RuntimeError:\n sentence = Sentence(\"an empty document\")\n self.embedding_model.embed(sentence)\n embedding = sentence.embedding.detach().cpu().numpy()\n embeddings.append(embedding)\n embeddings = np.asarray(embeddings)\n return embeddings\n" }, { "path": "bertopic/backend/_gensim.py", "content": "import numpy as np\nfrom tqdm import tqdm\nfrom typing import List\nfrom bertopic.backend import BaseEmbedder\nfrom gensim.models.keyedvectors import Word2VecKeyedVectors\n\n\nclass GensimBackend(BaseEmbedder):\n \"\"\"Gensim Embedding Model.\n\n The Gensim embedding model is typically used for word embeddings with\n GloVe, Word2Vec or FastText.\n\n Arguments:\n embedding_model: A Gensim embedding model\n\n Examples:\n ```python\n from bertopic.backend import GensimBackend\n import gensim.downloader as api\n\n ft = api.load('fasttext-wiki-news-subwords-300')\n ft_embedder = GensimBackend(ft)\n ```\n \"\"\"\n\n def __init__(self, embedding_model: Word2VecKeyedVectors):\n super().__init__()\n\n if isinstance(embedding_model, Word2VecKeyedVectors):\n self.embedding_model = embedding_model\n else:\n raise ValueError(\n \"Please select a correct Gensim model: \\n\"\n \"`import gensim.downloader as api` \\n\"\n \"`ft = api.load('fasttext-wiki-news-subwords-300')`\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n vector_shape = self.embedding_model.get_vector(next(iter(self.embedding_model.index_to_key))).shape[0]\n empty_vector = np.zeros(vector_shape)\n\n # Extract word embeddings and pool to document-level\n embeddings = []\n for doc in tqdm(documents, disable=not verbose, position=0, leave=True):\n embedding = [\n self.embedding_model.get_vector(word)\n for word in doc.split()\n if word in self.embedding_model.key_to_index\n ]\n\n if len(embedding) > 0:\n embeddings.append(np.mean(embedding, axis=0))\n else:\n embeddings.append(empty_vector)\n\n embeddings = np.array(embeddings)\n return embeddings\n" }, { "path": "bertopic/backend/_hftransformers.py", "content": "import numpy as np\n\nfrom tqdm import tqdm\nfrom typing import List\nfrom torch.utils.data import Dataset\nfrom sklearn.preprocessing import normalize\nfrom transformers.pipelines import Pipeline\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass HFTransformerBackend(BaseEmbedder):\n \"\"\"Hugging Face transformers model.\n\n This uses the `transformers.pipelines.pipeline` to define and create\n a feature generation pipeline from which embeddings can be extracted.\n\n Arguments:\n embedding_model: A Hugging Face feature extraction pipeline\n\n Examples:\n To use a Hugging Face transformers model, load in a pipeline and point\n to any model found on their model hub (https://huggingface.co/models):\n\n ```python\n from bertopic.backend import HFTransformerBackend\n from transformers.pipelines import pipeline\n\n hf_model = pipeline(\"feature-extraction\", model=\"distilbert-base-cased\")\n embedding_model = HFTransformerBackend(hf_model)\n ```\n \"\"\"\n\n def __init__(self, embedding_model: Pipeline):\n super().__init__()\n\n if isinstance(embedding_model, Pipeline):\n self.embedding_model = embedding_model\n else:\n raise ValueError(\n \"Please select a correct transformers pipeline. For example: \"\n \"pipeline('feature-extraction', model='distilbert-base-cased', device=0)\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n dataset = MyDataset(documents)\n\n embeddings = []\n for document, features in tqdm(\n zip(documents, self.embedding_model(dataset, truncation=True, padding=True)),\n total=len(dataset),\n disable=not verbose,\n ):\n embeddings.append(self._embed(document, features))\n\n return np.array(embeddings)\n\n def _embed(self, document: str, features: np.ndarray) -> np.ndarray:\n \"\"\"Mean pooling.\n\n Arguments:\n document: The document for which to extract the attention mask\n features: The embeddings for each token\n\n Adopted from:\n https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2#usage-huggingface-transformers\n \"\"\"\n token_embeddings = np.array(features)\n attention_mask = self.embedding_model.tokenizer(document, truncation=True, padding=True, return_tensors=\"np\")[\n \"attention_mask\"\n ]\n input_mask_expanded = np.broadcast_to(np.expand_dims(attention_mask, -1), token_embeddings.shape)\n sum_embeddings = np.sum(token_embeddings * input_mask_expanded, 1)\n sum_mask = np.clip(\n input_mask_expanded.sum(1),\n a_min=1e-9,\n a_max=input_mask_expanded.sum(1).max(),\n )\n embedding = normalize(sum_embeddings / sum_mask)[0]\n return embedding\n\n\nclass MyDataset(Dataset):\n \"\"\"Dataset to pass to `transformers.pipelines.pipeline`.\"\"\"\n\n def __init__(self, docs):\n self.docs = docs\n\n def __len__(self):\n return len(self.docs)\n\n def __getitem__(self, idx):\n return self.docs[idx]\n" }, { "path": "bertopic/backend/_langchain.py", "content": "from typing import List\n\nimport numpy as np\nfrom bertopic.backend import BaseEmbedder\nfrom langchain_core.embeddings import Embeddings\n\n\nclass LangChainBackend(BaseEmbedder):\n \"\"\"LangChain Embedding Model.\n\n This class uses the LangChain Embedding class to embed the documents.\n Argument:\n embedding_model: A LangChain Embedding Instance.\n\n Examples:\n ```python\n from langchain_community.embeddings import HuggingFaceInstructEmbeddings\n from bertopic.backend import LangChainBackend\n\n hf_embedding = HuggingFaceInstructEmbeddings()\n langchain_embedder = LangChainBackend(hf_embedding)\n ```\n \"\"\"\n\n def __init__(self, embedding_model: Embeddings):\n self.embedding_model = embedding_model\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Prepare documents, replacing empty strings with a single space\n prepared_documents = [\" \" if doc == \"\" else doc for doc in documents]\n response = self.embedding_model.embed_documents(prepared_documents)\n return np.array(response)\n" }, { "path": "bertopic/backend/_model2vec.py", "content": "import numpy as np\nfrom typing import List, Union\nfrom model2vec import StaticModel\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass Model2VecBackend(BaseEmbedder):\n \"\"\"Model2Vec embedding model.\n\n Arguments:\n embedding_model: Either a model2vec model or a\n string pointing to a model2vec model\n distill: Indicates whether to distill a sentence-transformers compatible model.\n The distillation will happen during fitting of the topic model.\n NOTE: Only works if `embedding_model` is a string.\n distill_kwargs: Keyword arguments to pass to the distillation process\n of `model2vec.distill.distill`\n distill_vectorizer: A CountVectorizer used for creating a custom vocabulary\n based on the same documents used for topic modeling.\n NOTE: If \"vocabulary\" is in `distill_kwargs`, this will be ignored.\n\n Examples:\n To create a model, you can load in a string pointing to a\n model2vec model:\n\n ```python\n from bertopic.backend import Model2VecBackend\n\n sentence_model = Model2VecBackend(\"minishlab/potion-base-8M\")\n ```\n\n or you can instantiate a model yourself:\n\n ```python\n from bertopic.backend import Model2VecBackend\n from model2vec import StaticModel\n\n embedding_model = StaticModel.from_pretrained(\"minishlab/potion-base-8M\")\n sentence_model = Model2VecBackend(embedding_model)\n ```\n\n If you want to distill a sentence-transformers model with the vocabulary of the documents,\n run the following:\n\n ```python\n from bertopic.backend import Model2VecBackend\n\n sentence_model = Model2VecBackend(\"sentence-transformers/all-MiniLM-L6-v2\", distill=True)\n ```\n \"\"\"\n\n def __init__(\n self,\n embedding_model: Union[str, StaticModel],\n distill: bool = False,\n distill_kwargs: dict = {},\n distill_vectorizer: str | None = None,\n ):\n super().__init__()\n\n self.distill = distill\n self.distill_kwargs = distill_kwargs\n self.distill_vectorizer = distill_vectorizer\n self._has_distilled = False\n\n # When we distill, we need a string pointing to a sentence-transformer model\n if self.distill:\n self._check_model2vec_installation()\n if not self.distill_vectorizer:\n self.distill_vectorizer = CountVectorizer()\n if isinstance(embedding_model, str):\n self.embedding_model = embedding_model\n else:\n raise ValueError(\"Please pass a string pointing to a sentence-transformer model when distilling.\")\n\n # If we don't distill, we can pass a model2vec model directly or load from a string\n elif isinstance(embedding_model, StaticModel):\n self.embedding_model = embedding_model\n elif isinstance(embedding_model, str):\n self.embedding_model = StaticModel.from_pretrained(embedding_model)\n else:\n raise ValueError(\n \"Please select a correct Model2Vec model: \\n\"\n \"`from model2vec import StaticModel` \\n\"\n \"`model = StaticModel.from_pretrained('minishlab/potion-base-8M')`\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Distill the model\n if self.distill and not self._has_distilled:\n from model2vec.distill import distill\n\n # Distill with the vocabulary of the documents\n if not self.distill_kwargs.get(\"vocabulary\"):\n X = self.distill_vectorizer.fit_transform(documents)\n word_counts = np.array(X.sum(axis=0)).flatten()\n words = self.distill_vectorizer.get_feature_names_out()\n vocabulary = [word for word, _ in sorted(zip(words, word_counts), key=lambda x: x[1], reverse=True)]\n self.distill_kwargs[\"vocabulary\"] = vocabulary\n\n # Distill the model\n self.embedding_model = distill(self.embedding_model, **self.distill_kwargs)\n\n # Distillation should happen only once and not for every embed call\n # The distillation should only happen the first time on the entire vocabulary\n self._has_distilled = True\n\n # Embed the documents\n embeddings = self.embedding_model.encode(documents, show_progress_bar=verbose)\n return embeddings\n\n def _check_model2vec_installation(self):\n try:\n from model2vec.distill import distill # noqa: F401\n except ImportError:\n raise ImportError(\"To distill a model using model2vec, you need to run `pip install model2vec[distill]`\")\n" }, { "path": "bertopic/backend/_multimodal.py", "content": "import numpy as np\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom typing import List, Union\nfrom sentence_transformers import SentenceTransformer\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass MultiModalBackend(BaseEmbedder):\n \"\"\"Multimodal backend using Sentence-transformers.\n\n The sentence-transformers embedding model used for\n generating word, document, and image embeddings.\n\n Arguments:\n embedding_model: A sentence-transformers embedding model that\n can either embed both images and text or only text.\n If it only embeds text, then `image_model` needs\n to be used to embed the images.\n image_model: A sentence-transformers embedding model that is used\n to embed only images.\n batch_size: The sizes of image batches to pass\n\n Examples:\n To create a model, you can load in a string pointing to a\n sentence-transformers model:\n\n ```python\n from bertopic.backend import MultiModalBackend\n\n sentence_model = MultiModalBackend(\"clip-ViT-B-32\")\n ```\n\n or you can instantiate a model yourself:\n ```python\n from bertopic.backend import MultiModalBackend\n from sentence_transformers import SentenceTransformer\n\n embedding_model = SentenceTransformer(\"clip-ViT-B-32\")\n sentence_model = MultiModalBackend(embedding_model)\n ```\n \"\"\"\n\n def __init__(\n self,\n embedding_model: Union[str, SentenceTransformer],\n image_model: Union[str, SentenceTransformer] = None,\n batch_size: int = 32,\n ):\n super().__init__()\n self.batch_size = batch_size\n\n # Text or Text+Image model\n if isinstance(embedding_model, SentenceTransformer):\n self.embedding_model = embedding_model\n elif isinstance(embedding_model, str):\n self.embedding_model = SentenceTransformer(embedding_model)\n else:\n raise ValueError(\n \"Please select a correct SentenceTransformers model: \\n\"\n \"`from sentence_transformers import SentenceTransformer` \\n\"\n \"`model = SentenceTransformer('clip-ViT-B-32')`\"\n )\n\n # Image Model\n self.image_model = None\n if image_model is not None:\n if isinstance(image_model, SentenceTransformer):\n self.image_model = image_model\n elif isinstance(image_model, str):\n self.image_model = SentenceTransformer(image_model)\n else:\n raise ValueError(\n \"Please select a correct SentenceTransformers model: \\n\"\n \"`from sentence_transformers import SentenceTransformer` \\n\"\n \"`model = SentenceTransformer('clip-ViT-B-32')`\"\n )\n\n try:\n self.tokenizer = self.embedding_model._first_module().processor.tokenizer\n except AttributeError:\n self.tokenizer = self.embedding_model.tokenizer\n except: # noqa: E722\n self.tokenizer = None\n\n def embed(self, documents: List[str], images: List[str] | None = None, verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words or images into an n-dimensional\n matrix of embeddings.\n\n Either documents, images, or both can be provided. If both are provided,\n then the embeddings are averaged.\n\n Arguments:\n documents: A list of documents or words to be embedded\n images: A list of image paths to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Embed documents\n doc_embeddings = None\n if documents[0] is not None:\n doc_embeddings = self.embed_documents(documents)\n\n # Embed images\n image_embeddings = None\n if isinstance(images, list):\n image_embeddings = self.embed_images(images, verbose)\n\n # Average embeddings\n averaged_embeddings = None\n if doc_embeddings is not None and image_embeddings is not None:\n averaged_embeddings = np.mean([doc_embeddings, image_embeddings], axis=0)\n\n if averaged_embeddings is not None:\n return averaged_embeddings\n elif doc_embeddings is not None:\n return doc_embeddings\n elif image_embeddings is not None:\n return image_embeddings\n\n def embed_documents(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n truncated_docs = [self._truncate_document(doc) for doc in documents]\n embeddings = self.embedding_model.encode(truncated_docs, show_progress_bar=verbose)\n return embeddings\n\n def embed_words(self, words: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n words: A list of words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n embeddings = self.embedding_model.encode(words, show_progress_bar=verbose)\n return embeddings\n\n def embed_images(self, images, verbose):\n if self.batch_size:\n nr_iterations = int(np.ceil(len(images) / self.batch_size))\n\n # Embed images per batch\n embeddings = []\n for i in tqdm(range(nr_iterations), disable=not verbose):\n start_index = i * self.batch_size\n end_index = (i * self.batch_size) + self.batch_size\n\n images_to_embed = [\n Image.open(image) if isinstance(image, str) else image for image in images[start_index:end_index]\n ]\n if self.image_model is not None:\n img_emb = self.image_model.encode(images_to_embed)\n else:\n img_emb = self.embedding_model.encode(images_to_embed, show_progress_bar=False)\n embeddings.extend(img_emb.tolist())\n\n # Close images\n if isinstance(images[0], str):\n for image in images_to_embed:\n image.close()\n embeddings = np.array(embeddings)\n else:\n images_to_embed = [Image.open(filepath) for filepath in images]\n if self.image_model is not None:\n embeddings = self.image_model.encode(images_to_embed)\n else:\n embeddings = self.embedding_model.encode(images_to_embed, show_progress_bar=False)\n return embeddings\n\n def _truncate_document(self, document):\n if self.tokenizer:\n tokens = self.tokenizer.encode(document)\n\n if len(tokens) > 77:\n # Skip the starting token, only include 75 tokens\n truncated_tokens = tokens[1:76]\n document = self.tokenizer.decode(truncated_tokens)\n\n # Recursive call here, because the encode(decode()) can have different result\n return self._truncate_document(document)\n\n return document\n" }, { "path": "bertopic/backend/_openai.py", "content": "import time\nimport openai\nimport numpy as np\nfrom tqdm import tqdm\nfrom typing import List, Mapping, Any\nfrom bertopic.backend import BaseEmbedder\n\n\nclass OpenAIBackend(BaseEmbedder):\n \"\"\"OpenAI Embedding Model.\n\n Arguments:\n client: A `openai.OpenAI` client.\n embedding_model: An OpenAI model. Default is\n For an overview of models see:\n https://platform.openai.com/docs/models/embeddings\n delay_in_seconds: If a `batch_size` is given, use this set\n the delay in seconds between batches.\n batch_size: The size of each batch.\n generator_kwargs: Kwargs passed to `openai.Embedding.create`.\n Can be used to define custom engines or\n deployment_ids.\n\n Examples:\n ```python\n import openai\n from bertopic.backend import OpenAIBackend\n\n client = openai.OpenAI(api_key=\"sk-...\")\n openai_embedder = OpenAIBackend(client, \"text-embedding-ada-002\")\n ```\n \"\"\"\n\n def __init__(\n self,\n client: openai.OpenAI,\n embedding_model: str = \"text-embedding-ada-002\",\n delay_in_seconds: float | None = None,\n batch_size: int | None = None,\n generator_kwargs: Mapping[str, Any] = {},\n ):\n super().__init__()\n self.client = client\n self.embedding_model = embedding_model\n self.delay_in_seconds = delay_in_seconds\n self.batch_size = batch_size\n self.generator_kwargs = generator_kwargs\n\n if self.generator_kwargs.get(\"model\"):\n self.embedding_model = generator_kwargs.get(\"model\")\n elif not self.generator_kwargs.get(\"engine\"):\n self.generator_kwargs[\"model\"] = self.embedding_model\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Prepare documents, replacing empty strings with a single space\n prepared_documents = [\" \" if doc == \"\" else doc for doc in documents]\n\n # Batch-wise embedding extraction\n if self.batch_size is not None:\n embeddings = []\n for batch in tqdm(self._chunks(prepared_documents), disable=not verbose):\n response = self.client.embeddings.create(input=batch, **self.generator_kwargs)\n embeddings.extend([r.embedding for r in response.data])\n\n # Delay subsequent calls\n if self.delay_in_seconds:\n time.sleep(self.delay_in_seconds)\n\n # Extract embeddings all at once\n else:\n response = self.client.embeddings.create(input=prepared_documents, **self.generator_kwargs)\n embeddings = [r.embedding for r in response.data]\n return np.array(embeddings)\n\n def _chunks(self, documents):\n for i in range(0, len(documents), self.batch_size):\n yield documents[i : i + self.batch_size]\n" }, { "path": "bertopic/backend/_sentencetransformers.py", "content": "import numpy as np\nfrom typing import List, Union\nfrom sentence_transformers import SentenceTransformer\nfrom sentence_transformers.models import StaticEmbedding\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass SentenceTransformerBackend(BaseEmbedder):\n \"\"\"Sentence-transformers embedding model.\n\n The sentence-transformers embedding model used for generating document and\n word embeddings.\n\n Arguments:\n embedding_model: A sentence-transformers embedding model\n model2vec: Indicates whether `embedding_model` is a model2vec model.\n NOTE: Only works if `embedding_model` is a string.\n Otherwise, you can pass the model2vec model directly to `embedding_model`.\n\n Examples:\n To create a model, you can load in a string pointing to a\n sentence-transformers model:\n\n ```python\n from bertopic.backend import SentenceTransformerBackend\n\n sentence_model = SentenceTransformerBackend(\"all-MiniLM-L6-v2\")\n ```\n\n or you can instantiate a model yourself:\n\n ```python\n from bertopic.backend import SentenceTransformerBackend\n from sentence_transformers import SentenceTransformer\n\n embedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n sentence_model = SentenceTransformerBackend(embedding_model)\n ```\n\n If you want to use a model2vec model without having to install model2vec,\n you can pass the model2vec model as a string:\n\n ```python\n from bertopic.backend import SentenceTransformerBackend\n from sentence_transformers import SentenceTransformer\n\n embedding_model = SentenceTransformer(\"minishlab/potion-base-8M\", model2vec=True)\n sentence_model = SentenceTransformerBackend(embedding_model)\n ```\n \"\"\"\n\n def __init__(self, embedding_model: Union[str, SentenceTransformer], model2vec: bool = False):\n super().__init__()\n\n self._hf_model = None\n if model2vec and isinstance(embedding_model, str):\n static_embedding = StaticEmbedding.from_model2vec(embedding_model)\n self.embedding_model = SentenceTransformer(modules=[static_embedding])\n elif isinstance(embedding_model, SentenceTransformer):\n self.embedding_model = embedding_model\n elif isinstance(embedding_model, str):\n self.embedding_model = SentenceTransformer(embedding_model)\n self._hf_model = embedding_model\n else:\n raise ValueError(\n \"Please select a correct SentenceTransformers model: \\n\"\n \"`from sentence_transformers import SentenceTransformer` \\n\"\n \"`model = SentenceTransformer('all-MiniLM-L6-v2')`\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n embeddings = self.embedding_model.encode(documents, show_progress_bar=verbose)\n return embeddings\n" }, { "path": "bertopic/backend/_sklearn.py", "content": "from bertopic.backend import BaseEmbedder\nfrom sklearn.utils.validation import check_is_fitted, NotFittedError\n\n\nclass SklearnEmbedder(BaseEmbedder):\n \"\"\"Scikit-Learn based embedding model.\n\n This component allows the usage of scikit-learn pipelines for generating document and\n word embeddings.\n\n Arguments:\n pipe: A scikit-learn pipeline that can `.transform()` text.\n\n Examples:\n Scikit-Learn is very flexible and it allows for many representations.\n A relatively simple pipeline is shown below.\n\n ```python\n from sklearn.pipeline import make_pipeline\n from sklearn.decomposition import TruncatedSVD\n from sklearn.feature_extraction.text import TfidfVectorizer\n\n from bertopic.backend import SklearnEmbedder\n\n pipe = make_pipeline(\n TfidfVectorizer(),\n TruncatedSVD(100)\n )\n\n sklearn_embedder = SklearnEmbedder(pipe)\n topic_model = BERTopic(embedding_model=sklearn_embedder)\n ```\n\n This pipeline first constructs a sparse representation based on TF/idf and then\n makes it dense by applying SVD. Alternatively, you might also construct something\n more elaborate. As long as you construct a scikit-learn compatible pipeline, you\n should be able to pass it to Bertopic.\n\n !!! Warning\n One caveat to be aware of is that scikit-learns base `Pipeline` class does not\n support the `.partial_fit()`-API. If you have a pipeline that theoretically should\n be able to support online learning then you might want to explore\n the [scikit-partial](https://github.com/koaning/scikit-partial) project.\n \"\"\"\n\n def __init__(self, pipe):\n super().__init__()\n self.pipe = pipe\n\n def embed(self, documents, verbose=False):\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: No-op variable that's kept around to keep the API consistent. If you want to get feedback on training times, you should use the sklearn API.\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n try:\n check_is_fitted(self.pipe)\n embeddings = self.pipe.transform(documents)\n except NotFittedError:\n embeddings = self.pipe.fit_transform(documents)\n\n return embeddings\n" }, { "path": "bertopic/backend/_spacy.py", "content": "import numpy as np\nfrom tqdm import tqdm\nfrom typing import List\nfrom bertopic.backend import BaseEmbedder\n\n\nclass SpacyBackend(BaseEmbedder):\n \"\"\"Spacy embedding model.\n\n The Spacy embedding model used for generating document and\n word embeddings.\n\n Arguments:\n embedding_model: A spacy embedding model\n\n Examples:\n To create a Spacy backend, you need to create an nlp object and\n pass it through this backend:\n\n ```python\n import spacy\n from bertopic.backend import SpacyBackend\n\n nlp = spacy.load(\"en_core_web_md\", exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])\n spacy_model = SpacyBackend(nlp)\n ```\n\n To load in a transformer model use the following:\n\n ```python\n import spacy\n from thinc.api import set_gpu_allocator, require_gpu\n from bertopic.backend import SpacyBackend\n\n nlp = spacy.load(\"en_core_web_trf\", exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])\n set_gpu_allocator(\"pytorch\")\n require_gpu(0)\n spacy_model = SpacyBackend(nlp)\n ```\n\n If you run into gpu/memory-issues, please use:\n\n ```python\n import spacy\n from bertopic.backend import SpacyBackend\n\n spacy.prefer_gpu()\n nlp = spacy.load(\"en_core_web_trf\", exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])\n spacy_model = SpacyBackend(nlp)\n ```\n \"\"\"\n\n def __init__(self, embedding_model):\n super().__init__()\n\n if \"spacy\" in str(type(embedding_model)):\n self.embedding_model = embedding_model\n else:\n raise ValueError(\n \"Please select a correct Spacy model by either using a string such as 'en_core_web_md' \"\n \"or create a nlp model using: `nlp = spacy.load('en_core_web_md')\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n # Handle empty documents, spaCy models automatically map\n # empty strings to the zero vector\n empty_document = \" \"\n\n # Extract embeddings\n embeddings = []\n for doc in tqdm(documents, position=0, leave=True, disable=not verbose):\n embedding = self.embedding_model(doc or empty_document)\n if embedding.has_vector:\n embedding = embedding.vector\n else:\n embedding = embedding._.trf_data.tensors[-1][0]\n\n if not isinstance(embedding, np.ndarray) and hasattr(embedding, \"get\"):\n # Convert cupy array to numpy array\n embedding = embedding.get()\n embeddings.append(embedding)\n\n return np.array(embeddings)\n" }, { "path": "bertopic/backend/_use.py", "content": "import numpy as np\nfrom tqdm import tqdm\nfrom typing import List\n\nfrom bertopic.backend import BaseEmbedder\n\n\nclass USEBackend(BaseEmbedder):\n \"\"\"Universal Sentence Encoder.\n\n USE encodes text into high-dimensional vectors that\n are used for semantic similarity in BERTopic.\n\n Arguments:\n embedding_model: An USE embedding model\n\n Examples:\n ```python\n import tensorflow_hub\n from bertopic.backend import USEBackend\n\n embedding_model = tensorflow_hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n use_embedder = USEBackend(embedding_model)\n ```\n \"\"\"\n\n def __init__(self, embedding_model):\n super().__init__()\n\n try:\n embedding_model([\"test sentence\"])\n self.embedding_model = embedding_model\n except TypeError:\n raise ValueError(\n \"Please select a correct USE model: \\n\"\n \"`import tensorflow_hub` \\n\"\n \"`embedding_model = tensorflow_hub.load(path_to_model)`\"\n )\n\n def embed(self, documents: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n documents/words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n documents: A list of documents or words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document/words embeddings with shape (n, m) with `n` documents/words\n that each have an embeddings size of `m`\n \"\"\"\n embeddings = np.array(\n [self.embedding_model([doc]).cpu().numpy()[0] for doc in tqdm(documents, disable=not verbose)]\n )\n return embeddings\n" }, { "path": "bertopic/backend/_utils.py", "content": "from ._base import BaseEmbedder\n\n# Imports for light-weight variant of BERTopic\nfrom bertopic.backend._sklearn import SklearnEmbedder\nfrom bertopic._utils import MyLogger\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.pipeline import Pipeline as ScikitPipeline\n\nlogger = MyLogger()\nlogger.configure(\"WARNING\")\n\nlanguages = [\n \"arabic\",\n \"bulgarian\",\n \"catalan\",\n \"czech\",\n \"danish\",\n \"german\",\n \"greek\",\n \"english\",\n \"spanish\",\n \"estonian\",\n \"persian\",\n \"finnish\",\n \"french\",\n \"canadian french\",\n \"galician\",\n \"gujarati\",\n \"hebrew\",\n \"hindi\",\n \"croatian\",\n \"hungarian\",\n \"armenian\",\n \"indonesian\",\n \"italian\",\n \"japanese\",\n \"georgian\",\n \"korean\",\n \"kurdish\",\n \"lithuanian\",\n \"latvian\",\n \"macedonian\",\n \"mongolian\",\n \"marathi\",\n \"malay\",\n \"burmese\",\n \"norwegian bokmal\",\n \"dutch\",\n \"polish\",\n \"portuguese\",\n \"brazilian portuguese\",\n \"romanian\",\n \"russian\",\n \"slovak\",\n \"slovenian\",\n \"albanian\",\n \"serbian\",\n \"swedish\",\n \"thai\",\n \"turkish\",\n \"ukrainian\",\n \"urdu\",\n \"vietnamese\",\n \"chinese (simplified)\",\n \"chinese (traditional)\",\n]\n\n\ndef select_backend(embedding_model, language: str | None = None, verbose: bool = False) -> BaseEmbedder:\n \"\"\"Select an embedding model based on language or a specific provided model.\n When selecting a language, we choose all-MiniLM-L6-v2 for English and\n paraphrase-multilingual-MiniLM-L12-v2 for all other languages as it support 100+ languages.\n If sentence-transformers is not installed, in the case of a lightweight installation,\n a scikit-learn backend is default.\n\n Returns:\n model: The selected model backend.\n \"\"\"\n logger.set_level(\"INFO\" if verbose else \"WARNING\")\n\n # BERTopic language backend\n if isinstance(embedding_model, BaseEmbedder):\n return embedding_model\n\n # Scikit-learn backend\n if isinstance(embedding_model, ScikitPipeline):\n return SklearnEmbedder(embedding_model)\n\n # Flair word embeddings\n if \"flair\" in str(type(embedding_model)):\n from bertopic.backend._flair import FlairBackend\n\n return FlairBackend(embedding_model)\n\n # Spacy embeddings\n if \"spacy\" in str(type(embedding_model)):\n from bertopic.backend._spacy import SpacyBackend\n\n return SpacyBackend(embedding_model)\n\n # Gensim embeddings\n if \"gensim\" in str(type(embedding_model)):\n from bertopic.backend._gensim import GensimBackend\n\n return GensimBackend(embedding_model)\n\n # USE embeddings\n if \"tensorflow\" and \"saved_model\" in str(type(embedding_model)):\n from bertopic.backend._use import USEBackend\n\n return USEBackend(embedding_model)\n\n # Sentence Transformer embeddings\n if \"sentence_transformers\" in str(type(embedding_model)) or isinstance(embedding_model, str):\n from ._sentencetransformers import SentenceTransformerBackend\n\n return SentenceTransformerBackend(embedding_model)\n\n # Hugging Face embeddings\n if \"transformers\" and \"pipeline\" in str(type(embedding_model)):\n from ._hftransformers import HFTransformerBackend\n\n return HFTransformerBackend(embedding_model)\n\n # Model2Vec embeddings\n if \"model2vec\" in str(type(embedding_model)):\n from ._model2vec import Model2VecBackend\n\n return Model2VecBackend(embedding_model)\n\n # FastEmbed word embeddings\n if \"fastembed\" in str(type(embedding_model)):\n from bertopic.backend._fastembed import FastEmbedBackend\n\n return FastEmbedBackend(embedding_model)\n\n # Select embedding model based on language\n if language:\n try:\n from ._sentencetransformers import SentenceTransformerBackend\n\n if language.lower() in [\"English\", \"english\", \"en\"]:\n return SentenceTransformerBackend(\"sentence-transformers/all-MiniLM-L6-v2\")\n elif language.lower() in languages or language == \"multilingual\":\n return SentenceTransformerBackend(\"sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2\")\n else:\n raise ValueError(\n f\"{language} is currently not supported. However, you can \"\n f\"create any embeddings yourself and pass it through fit_transform(docs, embeddings)\\n\"\n \"Else, please select a language from the following list:\\n\"\n f\"{languages}\"\n )\n\n # A ModuleNotFoundError might be a lightweight installation\n except ModuleNotFoundError as e:\n if e.name != \"sentence_transformers\":\n # Error occurred in a downstream module, probably not a lightweight install\n raise e\n # Whole sentence_transformers module is missing, probably a lightweight install\n if verbose:\n logger.info(\n \"Automatically selecting lightweight scikit-learn embedding backend as sentence-transformers appears to not be installed.\"\n )\n pipe = make_pipeline(TfidfVectorizer(), TruncatedSVD(100))\n return SklearnEmbedder(pipe)\n\n from ._sentencetransformers import SentenceTransformerBackend\n\n return SentenceTransformerBackend(\"sentence-transformers/all-MiniLM-L6-v2\")\n" }, { "path": "bertopic/backend/_word_doc.py", "content": "import numpy as np\nfrom typing import List\nfrom bertopic.backend._base import BaseEmbedder\nfrom bertopic.backend._utils import select_backend\n\n\nclass WordDocEmbedder(BaseEmbedder):\n \"\"\"Combine a document- and word-level embedder.\"\"\"\n\n def __init__(self, embedding_model, word_embedding_model):\n super().__init__()\n\n self.embedding_model = select_backend(embedding_model)\n self.word_embedding_model = select_backend(word_embedding_model)\n\n def embed_words(self, words: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n words: A list of words to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Word embeddings with shape (n, m) with `n` words\n that each have an embeddings size of `m`\n\n \"\"\"\n return self.word_embedding_model.embed(words, verbose)\n\n def embed_documents(self, document: List[str], verbose: bool = False) -> np.ndarray:\n \"\"\"Embed a list of n words into an n-dimensional\n matrix of embeddings.\n\n Arguments:\n document: A list of documents to be embedded\n verbose: Controls the verbosity of the process\n\n Returns:\n Document embeddings with shape (n, m) with `n` documents\n that each have an embeddings size of `m`\n \"\"\"\n return self.embedding_model.embed(document, verbose)\n" }, { "path": "bertopic/cluster/__init__.py", "content": "from ._base import BaseCluster\n\n__all__ = [\n \"BaseCluster\",\n]\n" }, { "path": "bertopic/cluster/_base.py", "content": "import numpy as np\n\n\nclass BaseCluster:\n \"\"\"The Base Cluster class.\n\n Using this class directly in BERTopic will make it skip\n over the cluster step. As a result, topics need to be passed\n to BERTopic in the form of its `y` parameter in order to create\n topic representations.\n\n Examples:\n This will skip over the cluster step in BERTopic:\n\n ```python\n from bertopic import BERTopic\n from bertopic.cluster import BaseCluster\n\n empty_cluster_model = BaseCluster()\n\n topic_model = BERTopic(hdbscan_model=empty_cluster_model)\n ```\n\n Then, this class can be used to perform manual topic modeling.\n That is, topic modeling on a topics that were already generated before\n without the need to learn them:\n\n ```python\n topic_model.fit(docs, y=y)\n ```\n \"\"\"\n\n def fit(self, X, y=None):\n if y is not None:\n self.labels_ = y\n else:\n self.labels_ = None\n return self\n\n def transform(self, X: np.ndarray) -> np.ndarray:\n return X\n" }, { "path": "bertopic/cluster/_utils.py", "content": "import numpy as np\n\n\ndef hdbscan_delegator(model, func: str, embeddings: np.ndarray = None):\n \"\"\"Function used to select the HDBSCAN-like model for generating\n predictions and probabilities.\n\n Arguments:\n model: The cluster model.\n func: The function to use. Options:\n - \"approximate_predict\"\n - \"all_points_membership_vectors\"\n - \"membership_vector\"\n embeddings: Input embeddings for \"approximate_predict\"\n and \"membership_vector\"\n \"\"\"\n try:\n import hdbscan\n except (ImportError, ModuleNotFoundError):\n hdbscan = type(\"hdbscan\", (), {\"HDBSCAN\": None})()\n\n # Approximate predict\n if func == \"approximate_predict\":\n if isinstance(model, hdbscan.HDBSCAN):\n predictions, probabilities = hdbscan.approximate_predict(model, embeddings)\n return predictions, probabilities\n\n str_type_model = str(type(model)).lower()\n if \"cuml\" in str_type_model and \"hdbscan\" in str_type_model:\n from cuml.cluster import hdbscan as cuml_hdbscan\n\n predictions, probabilities = cuml_hdbscan.approximate_predict(model, embeddings)\n return predictions, probabilities\n\n predictions = model.predict(embeddings)\n return predictions, None\n\n # All points membership\n if func == \"all_points_membership_vectors\":\n if isinstance(model, hdbscan.HDBSCAN):\n return hdbscan.all_points_membership_vectors(model)\n\n str_type_model = str(type(model)).lower()\n if \"cuml\" in str_type_model and \"hdbscan\" in str_type_model:\n from cuml.cluster import hdbscan as cuml_hdbscan\n\n return cuml_hdbscan.all_points_membership_vectors(model)\n\n return None\n\n # membership_vector\n if func == \"membership_vector\":\n if isinstance(model, hdbscan.HDBSCAN):\n probabilities = hdbscan.membership_vector(model, embeddings)\n return probabilities\n\n str_type_model = str(type(model)).lower()\n if \"cuml\" in str_type_model and \"hdbscan\" in str_type_model:\n from cuml.cluster import hdbscan as cuml_hdbscan\n\n probabilities = cuml_hdbscan.membership_vector(model, embeddings)\n return probabilities\n\n return None\n\n\ndef is_supported_hdbscan(model):\n \"\"\"Check whether the input model is a supported HDBSCAN-like model.\"\"\"\n try:\n import hdbscan\n except (ImportError, ModuleNotFoundError):\n hdbscan = type(\"hdbscan\", (), {\"HDBSCAN\": None})()\n\n if isinstance(model, hdbscan.HDBSCAN):\n return True\n\n str_type_model = str(type(model)).lower()\n if \"cuml\" in str_type_model and \"hdbscan\" in str_type_model:\n return True\n\n return False\n" }, { "path": "bertopic/dimensionality/__init__.py", "content": "from ._base import BaseDimensionalityReduction\n\n__all__ = [\n \"BaseDimensionalityReduction\",\n]\n" }, { "path": "bertopic/dimensionality/_base.py", "content": "import numpy as np\n\n\nclass BaseDimensionalityReduction:\n \"\"\"The Base Dimensionality Reduction class.\n\n You can use this to skip over the dimensionality reduction step in BERTopic.\n\n Examples:\n This will skip over the reduction step in BERTopic:\n\n ```python\n from bertopic import BERTopic\n from bertopic.dimensionality import BaseDimensionalityReduction\n\n empty_reduction_model = BaseDimensionalityReduction()\n\n topic_model = BERTopic(umap_model=empty_reduction_model)\n ```\n \"\"\"\n\n def fit(self, X: np.ndarray = None, y: np.ndarray = None):\n return self\n\n def transform(self, X: np.ndarray) -> np.ndarray:\n return X\n" }, { "path": "bertopic/plotting/__init__.py", "content": "from ._topics import visualize_topics\nfrom ._heatmap import visualize_heatmap\nfrom ._barchart import visualize_barchart\nfrom ._documents import visualize_documents\nfrom ._term_rank import visualize_term_rank\nfrom ._hierarchy import visualize_hierarchy\nfrom ._datamap import visualize_document_datamap\nfrom ._distribution import visualize_distribution\nfrom ._topics_over_time import visualize_topics_over_time\nfrom ._topics_per_class import visualize_topics_per_class\nfrom ._hierarchical_documents import visualize_hierarchical_documents\nfrom ._approximate_distribution import visualize_approximate_distribution\n\n\n__all__ = [\n \"visualize_approximate_distribution\",\n \"visualize_barchart\",\n \"visualize_distribution\",\n \"visualize_document_datamap\",\n \"visualize_documents\",\n \"visualize_heatmap\",\n \"visualize_hierarchical_documents\",\n \"visualize_hierarchy\",\n \"visualize_term_rank\",\n \"visualize_topics\",\n \"visualize_topics_over_time\",\n \"visualize_topics_per_class\",\n]\n" }, { "path": "bertopic/plotting/_approximate_distribution.py", "content": "import numpy as np\nimport pandas as pd\n\ntry:\n from pandas.io.formats.style import Styler # noqa: F401\n\n HAS_JINJA = True\nexcept (ModuleNotFoundError, ImportError):\n HAS_JINJA = False\n\n\ndef visualize_approximate_distribution(\n topic_model,\n document: str,\n topic_token_distribution: np.ndarray,\n normalize: bool = False,\n):\n \"\"\"Visualize the topic distribution calculated by `.approximate_topic_distribution`\n on a token level. Thereby indicating the extend to which a certain word or phrases belong\n to a specific topic. The assumption here is that a single word can belong to multiple\n similar topics and as such give information about the broader set of topics within\n a single document.\n\n Note:\n This function will return a stylized pandas dataframe if Jinja2 is installed. If not,\n it will only return a pandas dataframe without color highlighting. To install jinja:\n\n `pip install jinja2`\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n document: The document for which you want to visualize\n the approximated topic distribution.\n topic_token_distribution: The topic-token distribution of the document as\n extracted by `.approximate_topic_distribution`\n normalize: Whether to normalize, between 0 and 1 (summing to 1), the\n topic distribution values.\n\n Returns:\n df: A stylized dataframe indicating the best fitting topics\n for each token.\n\n Examples:\n ```python\n # Calculate the topic distributions on a token level\n # Note that we need to have `calculate_token_level=True`\n topic_distr, topic_token_distr = topic_model.approximate_distribution(\n docs, calculate_token_level=True\n )\n\n # Visualize the approximated topic distributions\n df = topic_model.visualize_approximate_distribution(docs[0], topic_token_distr[0])\n df\n ```\n\n To revert this stylized dataframe back to a regular dataframe,\n you can run the following:\n\n ```python\n df.data.columns = [column.strip() for column in df.data.columns]\n df = df.data\n ```\n \"\"\"\n # Tokenize document\n analyzer = topic_model.vectorizer_model.build_tokenizer()\n tokens = analyzer(document)\n\n if len(tokens) == 0:\n raise ValueError(\"Make sure that your document contains at least 1 token.\")\n\n # Prepare dataframe with results\n if normalize:\n df = pd.DataFrame(topic_token_distribution / topic_token_distribution.sum()).T\n else:\n df = pd.DataFrame(topic_token_distribution).T\n\n df.columns = [f\"{token}_{i}\" for i, token in enumerate(tokens)]\n df.columns = [f\"{token}{' ' * i}\" for i, token in enumerate(tokens)]\n df.index = list(topic_model.topic_labels_.values())[topic_model._outliers :]\n df = df.loc[(df.sum(axis=1) != 0), :]\n\n # Style the resulting dataframe\n def text_color(val):\n color = \"white\" if val == 0 else \"black\"\n return \"color: %s\" % color\n\n def highligh_color(data, color=\"white\"):\n attr = \"background-color: {}\".format(color)\n return pd.DataFrame(np.where(data == 0, attr, \"\"), index=data.index, columns=data.columns)\n\n if len(df) == 0:\n return df\n elif HAS_JINJA:\n df = (\n df.style.format(\"{:.3f}\")\n .background_gradient(cmap=\"Blues\", axis=None)\n .applymap(lambda x: text_color(x))\n .apply(highligh_color, axis=None)\n )\n return df\n" }, { "path": "bertopic/plotting/_barchart.py", "content": "import itertools\nimport numpy as np\nfrom typing import List, Union\n\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\n\n\ndef visualize_barchart(\n topic_model,\n topics: List[int] | None = None,\n top_n_topics: int = 8,\n n_words: int = 5,\n custom_labels: Union[bool, str] = False,\n title: str = \"Topic Word Scores\",\n width: int = 250,\n height: int = 250,\n autoscale: bool = False,\n) -> go.Figure:\n \"\"\"Visualize a barchart of selected topics.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics: A selection of topics to visualize.\n top_n_topics: Only select the top n most frequent topics.\n n_words: Number of words to show in a topic\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of each figure.\n height: The height of each figure.\n autoscale: Whether to automatically calculate the height of the figures to fit the whole bar text\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the barchart of selected topics\n simply run:\n\n ```python\n topic_model.visualize_barchart()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_barchart()\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n colors = itertools.cycle([\"#D55E00\", \"#0072B2\", \"#CC79A7\", \"#E69F00\", \"#56B4E9\", \"#009E73\", \"#F0E442\"])\n\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n topics = list(topics)\n elif top_n_topics is not None:\n topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n topics = sorted(freq_df.Topic.to_list()[0:6])\n\n # Initialize figure\n if isinstance(custom_labels, str):\n subplot_titles = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in topics]\n subplot_titles = [\"_\".join([label[0] for label in labels[:4]]) for labels in subplot_titles]\n subplot_titles = [label if len(label) < 30 else label[:27] + \"...\" for label in subplot_titles]\n elif topic_model.custom_labels_ is not None and custom_labels:\n subplot_titles = [topic_model.custom_labels_[topic + topic_model._outliers] for topic in topics]\n else:\n subplot_titles = [f\"Topic {topic}\" for topic in topics]\n columns = 4\n rows = int(np.ceil(len(topics) / columns))\n fig = make_subplots(\n rows=rows,\n cols=columns,\n shared_xaxes=False,\n horizontal_spacing=0.1,\n vertical_spacing=0.4 / rows if rows > 1 else 0,\n subplot_titles=subplot_titles,\n )\n\n # Add barchart for each topic\n row = 1\n column = 1\n for topic in topics:\n words = [word + \" \" for word, _ in topic_model.get_topic(topic)][:n_words][::-1]\n scores = [score for _, score in topic_model.get_topic(topic)][:n_words][::-1]\n\n fig.add_trace(\n go.Bar(x=scores, y=words, orientation=\"h\", marker_color=next(colors)),\n row=row,\n col=column,\n )\n\n if autoscale:\n if len(words) > 12:\n height = 250 + (len(words) - 12) * 11\n\n if len(words) > 9:\n fig.update_yaxes(tickfont=dict(size=(height - 140) // len(words)))\n\n if column == columns:\n column = 1\n row += 1\n else:\n column += 1\n\n # Stylize graph\n fig.update_layout(\n template=\"plotly_white\",\n showlegend=False,\n title={\n \"text\": f\"{title}\",\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width * 4,\n height=height * rows if rows > 1 else height * 1.3,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n )\n\n fig.update_xaxes(showgrid=True)\n fig.update_yaxes(showgrid=True)\n\n return fig\n" }, { "path": "bertopic/plotting/_datamap.py", "content": "import numpy as np\nimport pandas as pd\nfrom typing import List, Union\nfrom warnings import warn\n\ntry:\n import datamapplot\n from matplotlib.figure import Figure\nexcept ImportError:\n warn(\"Data map plotting is unavailable unless datamapplot is installed.\")\n\n # Create a dummy figure type for typing\n class Figure(object):\n pass\n\n\ndef visualize_document_datamap(\n topic_model,\n docs: List[str] | None = None,\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n custom_labels: Union[bool, str] = False,\n title: str = \"Documents and Topics\",\n sub_title: Union[str, None] = None,\n width: int = 1200,\n height: int = 750,\n interactive: bool = False,\n enable_search: bool = False,\n topic_prefix: bool = False,\n datamap_kwds: dict = {},\n int_datamap_kwds: dict = {},\n) -> Figure:\n \"\"\"Visualize documents and their topics in 2D as a static plot for publication using\n DataMapPlot.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n docs: The documents you used when calling either `fit` or `fit_transform`.\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`. Documents not in these topics will be shown\n as noise points.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n sub_title: Sub-title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n interactive: Whether to create an interactive plot using DataMapPlot's `create_interactive_plot`.\n enable_search: Whether to enable search in the interactive plot. Only works if `interactive=True`.\n topic_prefix: Prefix to add to the topic number when displaying the topic name.\n datamap_kwds: Keyword args be passed on to DataMapPlot's `create_plot` function\n if you are not using the interactive version.\n See the DataMapPlot documentation for more details.\n int_datamap_kwds: Keyword args be passed on to DataMapPlot's `create_interactive_plot` function\n if you are using the interactive version.\n See the DataMapPlot documentation for more details.\n\n Returns:\n figure: A Matplotlib Figure object.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_document_datamap(docs)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic\n topic_model = BERTopic().fit(docs, embeddings)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_document_datamap(docs, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\n fig.savefig(\"path/to/file.png\", bbox_inches=\"tight\")\n ```\n \"DataMapPlot\n \"\"\"\n topic_per_doc = topic_model.topics_\n\n df = pd.DataFrame({\"topic\": np.array(topic_per_doc)})\n df[\"doc\"] = docs\n df[\"topic\"] = topic_per_doc\n\n # Extract embeddings if not already done\n if embeddings is None and reduced_embeddings is None:\n embeddings_to_reduce = topic_model._extract_embeddings(df.doc.to_list(), method=\"document\")\n else:\n embeddings_to_reduce = embeddings\n\n # Reduce input embeddings\n if reduced_embeddings is None:\n try:\n from umap import UMAP\n\n umap_model = UMAP(n_neighbors=15, n_components=2, min_dist=0.15, metric=\"cosine\").fit(embeddings_to_reduce)\n embeddings_2d = umap_model.embedding_\n except (ImportError, ModuleNotFoundError):\n raise ModuleNotFoundError(\n \"UMAP is required if the embeddings are not yet reduced in dimensionality. Please install it using `pip install umap-learn`.\"\n )\n else:\n embeddings_2d = reduced_embeddings\n\n unique_topics = set(topic_per_doc)\n\n # Prepare text and names\n if isinstance(custom_labels, str):\n names = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in unique_topics]\n names = [\" \".join([label[0] for label in labels[:4]]) for labels in names]\n names = [label if len(label) < 30 else label[:27] + \"...\" for label in names]\n elif topic_model.custom_labels_ is not None and custom_labels:\n names = [topic_model.custom_labels_[topic + topic_model._outliers] for topic in unique_topics]\n else:\n if topic_prefix:\n names = [\n f\"Topic-{topic}: \" + \" \".join([word for word, value in topic_model.get_topic(topic)][:3])\n for topic in unique_topics\n ]\n else:\n names = [\" \".join([word for word, value in topic_model.get_topic(topic)][:3]) for topic in unique_topics]\n\n topic_name_mapping = {topic_num: topic_name for topic_num, topic_name in zip(unique_topics, names)}\n topic_name_mapping[-1] = \"Unlabelled\"\n\n # If a set of topics is chosen, set everything else to \"Unlabelled\"\n if topics is not None:\n selected_topics = set(topics)\n for topic_num in topic_name_mapping:\n if topic_num not in selected_topics:\n topic_name_mapping[topic_num] = \"Unlabelled\"\n\n # Map in topic names and plot\n named_topic_per_doc = pd.Series(topic_per_doc).map(topic_name_mapping).to_numpy()\n\n if interactive:\n figure = datamapplot.create_interactive_plot(\n embeddings_2d,\n named_topic_per_doc,\n hover_text=docs,\n enable_search=enable_search,\n width=width,\n height=height,\n **int_datamap_kwds,\n )\n else:\n figure, _ = datamapplot.create_plot(\n embeddings_2d,\n named_topic_per_doc,\n figsize=(width / 100, height / 100),\n dpi=100,\n title=title,\n sub_title=sub_title,\n **datamap_kwds,\n )\n\n return figure\n" }, { "path": "bertopic/plotting/_distribution.py", "content": "import numpy as np\nfrom typing import Union\nimport plotly.graph_objects as go\n\n\ndef visualize_distribution(\n topic_model,\n probabilities: np.ndarray,\n min_probability: float = 0.015,\n custom_labels: Union[bool, str] = False,\n title: str = \"Topic Probability Distribution\",\n width: int = 800,\n height: int = 600,\n) -> go.Figure:\n \"\"\"Visualize the distribution of topic probabilities.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n probabilities: An array of probability scores\n min_probability: The minimum probability score to visualize.\n All others are ignored.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n Make sure to fit the model before and only input the\n probabilities of a single document:\n\n ```python\n topic_model.visualize_distribution(probabilities[0])\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_distribution(probabilities[0])\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n if len(probabilities.shape) != 1:\n raise ValueError(\n \"This visualization cannot be used if you have set `calculate_probabilities` to False \"\n \"as it uses the topic probabilities of all topics. \"\n )\n if len(probabilities[probabilities > min_probability]) == 0:\n raise ValueError(\n \"There are no values where `min_probability` is higher than the \"\n \"probabilities that were supplied. Lower `min_probability` to prevent this error.\"\n )\n\n # Get values and indices equal or exceed the minimum probability\n labels_idx = np.argwhere(probabilities >= min_probability).flatten()\n vals = probabilities[labels_idx].tolist()\n\n # Create labels\n if isinstance(custom_labels, str):\n labels = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in labels_idx]\n labels = [\"_\".join([label[0] for label in l[:4]]) for l in labels] # noqa: E741\n labels = [label if len(label) < 30 else label[:27] + \"...\" for label in labels]\n elif topic_model.custom_labels_ is not None and custom_labels:\n labels = [topic_model.custom_labels_[idx + topic_model._outliers] for idx in labels_idx]\n else:\n labels = []\n for idx in labels_idx:\n words = topic_model.get_topic(idx)\n if words:\n label = [word[0] for word in words[:5]]\n label = f\"Topic {idx}: {'_'.join(label)}\"\n label = label[:40] + \"...\" if len(label) > 40 else label\n labels.append(label)\n else:\n vals.remove(probabilities[idx])\n\n # Create Figure\n fig = go.Figure(\n go.Bar(\n x=vals,\n y=labels,\n marker=dict(\n color=\"#C8D2D7\",\n line=dict(color=\"#6E8484\", width=1),\n ),\n orientation=\"h\",\n )\n )\n\n fig.update_layout(\n xaxis_title=\"Probability\",\n title={\n \"text\": f\"{title}\",\n \"y\": 0.95,\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n template=\"simple_white\",\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n )\n\n return fig\n" }, { "path": "bertopic/plotting/_documents.py", "content": "import numpy as np\nimport pandas as pd\nimport plotly.graph_objects as go\n\nfrom typing import List, Union\n\n\ndef visualize_documents(\n topic_model,\n docs: List[str],\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n sample: float | None = None,\n hide_annotations: bool = False,\n hide_document_hover: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Documents and Topics\",\n width: int = 1200,\n height: int = 750,\n):\n \"\"\"Visualize documents and their topics in 2D.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n docs: The documents you used when calling either `fit` or `fit_transform`\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n sample: The percentage of documents in each topic that you would like to keep.\n Value can be between 0 and 1. Setting this value to, for example,\n 0.1 (10% of documents in each topic) makes it easier to visualize\n millions of documents as a subset is chosen.\n hide_annotations: Hide the names of the traces on top of each cluster.\n hide_document_hover: Hide the content of the documents when hovering over\n specific points. Helps to speed up generation of visualization.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_documents(docs)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic\n topic_model = BERTopic().fit(docs, embeddings)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_documents(docs, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n fig.write_html(\"path/to/file.html\")\n ```\n\n \n \"\"\"\n topic_per_doc = topic_model.topics_\n\n # Sample the data to optimize for visualization and dimensionality reduction\n if sample is None or sample > 1:\n sample = 1\n\n indices = []\n for topic in set(topic_per_doc):\n s = np.where(np.array(topic_per_doc) == topic)[0]\n size = len(s) if len(s) < 100 else int(len(s) * sample)\n indices.extend(np.random.choice(s, size=size, replace=False))\n indices = np.array(indices)\n\n df = pd.DataFrame({\"topic\": np.array(topic_per_doc)[indices]})\n df[\"doc\"] = [docs[index] for index in indices]\n df[\"topic\"] = [topic_per_doc[index] for index in indices]\n\n # Extract embeddings if not already done\n if sample is None:\n if embeddings is None and reduced_embeddings is None:\n embeddings_to_reduce = topic_model._extract_embeddings(df.doc.to_list(), method=\"document\")\n else:\n embeddings_to_reduce = embeddings\n else:\n if embeddings is not None:\n embeddings_to_reduce = embeddings[indices]\n elif embeddings is None and reduced_embeddings is None:\n embeddings_to_reduce = topic_model._extract_embeddings(df.doc.to_list(), method=\"document\")\n\n # Reduce input embeddings\n if reduced_embeddings is None:\n try:\n from umap import UMAP\n\n umap_model = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric=\"cosine\").fit(embeddings_to_reduce)\n embeddings_2d = umap_model.embedding_\n except (ImportError, ModuleNotFoundError):\n raise ModuleNotFoundError(\n \"UMAP is required if the embeddings are not yet reduced in dimensionality. Please install it using `pip install umap-learn`.\"\n )\n elif sample is not None and reduced_embeddings is not None:\n embeddings_2d = reduced_embeddings[indices]\n elif sample is None and reduced_embeddings is not None:\n embeddings_2d = reduced_embeddings\n\n unique_topics = set(topic_per_doc)\n if topics is None:\n topics = unique_topics\n\n # Combine data\n df[\"x\"] = embeddings_2d[:, 0]\n df[\"y\"] = embeddings_2d[:, 1]\n\n # Prepare text and names\n if isinstance(custom_labels, str):\n names = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in unique_topics]\n names = [\"_\".join([label[0] for label in labels[:4]]) for labels in names]\n names = [label if len(label) < 30 else label[:27] + \"...\" for label in names]\n elif topic_model.custom_labels_ is not None and custom_labels:\n names = [topic_model.custom_labels_[topic + topic_model._outliers] for topic in unique_topics]\n else:\n names = [\n f\"{topic}_\" + \"_\".join([word for word, value in topic_model.get_topic(topic)][:3])\n for topic in unique_topics\n ]\n\n # Visualize\n fig = go.Figure()\n\n # Outliers and non-selected topics\n non_selected_topics = set(unique_topics).difference(topics)\n if len(non_selected_topics) == 0:\n non_selected_topics = [-1]\n\n selection = df.loc[df.topic.isin(non_selected_topics), :]\n selection[\"text\"] = \"\"\n selection.loc[len(selection), :] = [\n None,\n None,\n selection.x.mean(),\n selection.y.mean(),\n \"Other documents\",\n ]\n\n fig.add_trace(\n go.Scattergl(\n x=selection.x,\n y=selection.y,\n hovertext=selection.doc if not hide_document_hover else None,\n hoverinfo=\"text\",\n mode=\"markers+text\",\n name=\"other\",\n showlegend=False,\n marker=dict(color=\"#CFD8DC\", size=5, opacity=0.5),\n )\n )\n\n # Selected topics\n for name, topic in zip(names, unique_topics):\n if topic in topics and topic != -1:\n selection = df.loc[df.topic == topic, :]\n selection[\"text\"] = \"\"\n\n if not hide_annotations:\n selection.loc[len(selection), :] = [\n None,\n None,\n selection.x.mean(),\n selection.y.mean(),\n name,\n ]\n\n fig.add_trace(\n go.Scattergl(\n x=selection.x,\n y=selection.y,\n hovertext=selection.doc if not hide_document_hover else None,\n hoverinfo=\"text\",\n text=selection.text,\n mode=\"markers+text\",\n name=name,\n textfont=dict(\n size=12,\n ),\n marker=dict(size=5, opacity=0.5),\n )\n )\n\n # Add grid in a 'plus' shape\n x_range = (\n df.x.min() - abs((df.x.min()) * 0.15),\n df.x.max() + abs((df.x.max()) * 0.15),\n )\n y_range = (\n df.y.min() - abs((df.y.min()) * 0.15),\n df.y.max() + abs((df.y.max()) * 0.15),\n )\n fig.add_shape(\n type=\"line\",\n x0=sum(x_range) / 2,\n y0=y_range[0],\n x1=sum(x_range) / 2,\n y1=y_range[1],\n line=dict(color=\"#CFD8DC\", width=2),\n )\n fig.add_shape(\n type=\"line\",\n x0=x_range[0],\n y0=sum(y_range) / 2,\n x1=x_range[1],\n y1=sum(y_range) / 2,\n line=dict(color=\"#9E9E9E\", width=2),\n )\n fig.add_annotation(x=x_range[0], y=sum(y_range) / 2, text=\"D1\", showarrow=False, yshift=10)\n fig.add_annotation(y=y_range[1], x=sum(x_range) / 2, text=\"D2\", showarrow=False, xshift=10)\n\n # Stylize layout\n fig.update_layout(\n template=\"simple_white\",\n title={\n \"text\": f\"{title}\",\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width,\n height=height,\n )\n\n fig.update_xaxes(visible=False)\n fig.update_yaxes(visible=False)\n return fig\n" }, { "path": "bertopic/plotting/_heatmap.py", "content": "import numpy as np\nfrom typing import List, Union\nfrom scipy.cluster.hierarchy import fcluster, linkage\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom bertopic._utils import select_topic_representation\n\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n\ndef visualize_heatmap(\n topic_model,\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n n_clusters: int | None = None,\n use_ctfidf: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Similarity Matrix\",\n width: int = 800,\n height: int = 800,\n) -> go.Figure:\n \"\"\"Visualize a heatmap of the topic's similarity matrix.\n\n Based on the cosine similarity matrix between topic embeddings (either c-TF-IDF or the embeddings from the embedding\n model), a heatmap is created showing the similarity between topics.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics: A selection of topics to visualize.\n top_n_topics: Only select the top n most frequent topics.\n n_clusters: Create n clusters and order the similarity\n matrix by those clusters.\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the embeddings\n from the embedding model are used.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the similarity matrix of\n topics simply run:\n\n ```python\n topic_model.visualize_heatmap()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_heatmap()\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n embeddings = select_topic_representation(topic_model.c_tf_idf_, topic_model.topic_embeddings_, use_ctfidf)[0][\n topic_model._outliers :\n ]\n\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n topics = list(topics)\n elif top_n_topics is not None:\n topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n topics = sorted(freq_df.Topic.to_list())\n\n # Order heatmap by similar clusters of topics\n sorted_topics = topics\n if n_clusters:\n if n_clusters >= len(set(topics)):\n raise ValueError(\"Make sure to set `n_clusters` lower than the total number of unique topics.\")\n\n distance_matrix = cosine_similarity(embeddings[topics])\n Z = linkage(distance_matrix, \"ward\")\n clusters = fcluster(Z, t=n_clusters, criterion=\"maxclust\")\n\n # Extract new order of topics\n mapping = {cluster: [] for cluster in clusters}\n for topic, cluster in zip(topics, clusters):\n mapping[cluster].append(topic)\n mapping = [cluster for cluster in mapping.values()]\n sorted_topics = [topic for cluster in mapping for topic in cluster]\n\n # Select embeddings\n indices = np.array([topics.index(topic) for topic in sorted_topics])\n embeddings = embeddings[indices]\n distance_matrix = cosine_similarity(embeddings)\n\n # Create labels\n if isinstance(custom_labels, str):\n new_labels = [\n [[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in sorted_topics\n ]\n new_labels = [\"_\".join([label[0] for label in labels[:4]]) for labels in new_labels]\n new_labels = [label if len(label) < 30 else label[:27] + \"...\" for label in new_labels]\n elif topic_model.custom_labels_ is not None and custom_labels:\n new_labels = [topic_model.custom_labels_[topic + topic_model._outliers] for topic in sorted_topics]\n else:\n new_labels = [[[str(topic), None], *topic_model.get_topic(topic)] for topic in sorted_topics]\n new_labels = [\"_\".join([label[0] for label in labels[:4]]) for labels in new_labels]\n new_labels = [label if len(label) < 30 else label[:27] + \"...\" for label in new_labels]\n\n fig = px.imshow(\n distance_matrix,\n labels=dict(color=\"Similarity Score\"),\n x=new_labels,\n y=new_labels,\n color_continuous_scale=\"GnBu\",\n )\n\n fig.update_layout(\n title={\n \"text\": f\"{title}\",\n \"y\": 0.95,\n \"x\": 0.55,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n )\n fig.update_layout(showlegend=True)\n fig.update_layout(legend_title_text=\"Trend\")\n\n return fig\n" }, { "path": "bertopic/plotting/_hierarchical_documents.py", "content": "import numpy as np\nimport pandas as pd\nimport plotly.graph_objects as go\nimport math\n\nfrom typing import List, Union\n\n\ndef visualize_hierarchical_documents(\n topic_model,\n docs: List[str],\n hierarchical_topics: pd.DataFrame,\n topics: List[int] | None = None,\n embeddings: np.ndarray = None,\n reduced_embeddings: np.ndarray = None,\n sample: Union[float, int] | None = None,\n hide_annotations: bool = False,\n hide_document_hover: bool = True,\n nr_levels: int = 10,\n level_scale: str = \"linear\",\n custom_labels: Union[bool, str] = False,\n title: str = \"Hierarchical Documents and Topics\",\n width: int = 1200,\n height: int = 750,\n) -> go.Figure:\n \"\"\"Visualize documents and their topics in 2D at different levels of hierarchy.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n docs: The documents you used when calling either `fit` or `fit_transform`\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children\n topics: A selection of topics to visualize.\n Not to be confused with the topics that you get from `.fit_transform`.\n For example, if you want to visualize only topics 1 through 5:\n `topics = [1, 2, 3, 4, 5]`.\n embeddings: The embeddings of all documents in `docs`.\n reduced_embeddings: The 2D reduced embeddings of all documents in `docs`.\n sample: The percentage of documents in each topic that you would like to keep.\n Value can be between 0 and 1. Setting this value to, for example,\n 0.1 (10% of documents in each topic) makes it easier to visualize\n millions of documents as a subset is chosen.\n hide_annotations: Hide the names of the traces on top of each cluster.\n hide_document_hover: Hide the content of the documents when hovering over\n specific points. Helps to speed up generation of visualizations.\n nr_levels: The number of levels to be visualized in the hierarchy. First, the distances\n in `hierarchical_topics.Distance` are split in `nr_levels` lists of distances.\n Then, for each list of distances, the merged topics are selected that have a\n distance less or equal to the maximum distance of the selected list of distances.\n NOTE: To get all possible merged steps, make sure that `nr_levels` is equal to\n the length of `hierarchical_topics`.\n level_scale: Whether to apply a linear or logarithmic (log) scale levels of the distance\n vector. Linear scaling will perform an equal number of merges at each level\n while logarithmic scaling will perform more mergers in earlier levels to\n provide more resolution at higher levels (this can be used for when the number\n of topics is large).\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n NOTE: Custom labels are only generated for the original\n un-merged topics.\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics)\n ```\n\n Do note that this re-calculates the embeddings and reduces them to 2D.\n The advised and preferred pipeline for using this function is as follows:\n\n ```python\n from sklearn.datasets import fetch_20newsgroups\n from sentence_transformers import SentenceTransformer\n from bertopic import BERTopic\n from umap import UMAP\n\n # Prepare embeddings\n docs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n sentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n embeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n # Train BERTopic and extract hierarchical topics\n topic_model = BERTopic().fit(docs, embeddings)\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n\n # Reduce dimensionality of embeddings, this step is optional\n # reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n # Run the visualization with the original embeddings\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)\n\n # Or, if you have reduced the original embeddings already:\n topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n fig.write_html(\"path/to/file.html\")\n ```\n\n Note:\n This visualization was inspired by the scatter plot representation of Doc2Map:\n https://github.com/louisgeisler/Doc2Map\n\n \n \"\"\"\n topic_per_doc = topic_model.topics_\n\n # Sample the data to optimize for visualization and dimensionality reduction\n if sample is None or sample > 1:\n sample = 1\n\n indices = []\n for topic in set(topic_per_doc):\n s = np.where(np.array(topic_per_doc) == topic)[0]\n size = len(s) if len(s) < 100 else int(len(s) * sample)\n indices.extend(np.random.choice(s, size=size, replace=False))\n indices = np.array(indices)\n\n df = pd.DataFrame({\"topic\": np.array(topic_per_doc)[indices]})\n df[\"doc\"] = [docs[index] for index in indices]\n df[\"topic\"] = [topic_per_doc[index] for index in indices]\n\n # Extract embeddings if not already done\n if sample is None:\n if embeddings is None and reduced_embeddings is None:\n embeddings_to_reduce = topic_model._extract_embeddings(df.doc.to_list(), method=\"document\")\n else:\n embeddings_to_reduce = embeddings\n else:\n if embeddings is not None:\n embeddings_to_reduce = embeddings[indices]\n elif embeddings is None and reduced_embeddings is None:\n embeddings_to_reduce = topic_model._extract_embeddings(df.doc.to_list(), method=\"document\")\n\n # Reduce input embeddings\n if reduced_embeddings is None:\n try:\n from umap import UMAP\n\n umap_model = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric=\"cosine\").fit(embeddings_to_reduce)\n embeddings_2d = umap_model.embedding_\n except (ImportError, ModuleNotFoundError):\n raise ModuleNotFoundError(\n \"UMAP is required if the embeddings are not yet reduced in dimensionality. Please install it using `pip install umap-learn`.\"\n )\n elif sample is not None and reduced_embeddings is not None:\n embeddings_2d = reduced_embeddings[indices]\n elif sample is None and reduced_embeddings is not None:\n embeddings_2d = reduced_embeddings\n\n # Combine data\n df[\"x\"] = embeddings_2d[:, 0]\n df[\"y\"] = embeddings_2d[:, 1]\n\n # Create topic list for each level, levels are created by calculating the distance\n distances = hierarchical_topics.Distance.to_list()\n if level_scale == \"log\" or level_scale == \"logarithmic\":\n log_indices = (\n np.round(\n np.logspace(\n start=math.log(1, 10),\n stop=math.log(len(distances) - 1, 10),\n num=nr_levels,\n )\n )\n .astype(int)\n .tolist()\n )\n log_indices.reverse()\n max_distances = [distances[i] for i in log_indices]\n elif level_scale == \"lin\" or level_scale == \"linear\":\n max_distances = [\n distances[indices[-1]] for indices in np.array_split(range(len(hierarchical_topics)), nr_levels)\n ][::-1]\n else:\n raise ValueError(\"level_scale needs to be one of 'log' or 'linear'\")\n\n for index, max_distance in enumerate(max_distances):\n # Get topics below `max_distance`\n mapping = {topic: topic for topic in df.topic.unique()}\n selection = hierarchical_topics.loc[hierarchical_topics.Distance <= max_distance, :]\n selection.Parent_ID = selection.Parent_ID.astype(int)\n selection = selection.sort_values(\"Parent_ID\")\n\n for row in selection.iterrows():\n for topic in row[1].Topics:\n mapping[topic] = row[1].Parent_ID\n\n # Make sure the mappings are mapped 1:1\n mappings = [True for _ in mapping]\n while any(mappings):\n for i, (key, value) in enumerate(mapping.items()):\n if value in mapping.keys() and key != value:\n mapping[key] = mapping[value]\n else:\n mappings[i] = False\n\n # Create new column\n df[f\"level_{index + 1}\"] = df.topic.map(mapping)\n df[f\"level_{index + 1}\"] = df[f\"level_{index + 1}\"].astype(int)\n\n # Prepare topic names of original and merged topics\n trace_names = []\n topic_names = {}\n for topic in range(hierarchical_topics.Parent_ID.astype(int).max()):\n if topic < hierarchical_topics.Parent_ID.astype(int).min():\n if topic_model.get_topic(topic):\n if isinstance(custom_labels, str):\n trace_name = f\"{topic}_\" + \"_\".join(\n next(zip(*topic_model.topic_aspects_[custom_labels][topic]))[:3]\n )\n elif topic_model.custom_labels_ is not None and custom_labels:\n trace_name = topic_model.custom_labels_[topic + topic_model._outliers]\n else:\n trace_name = f\"{topic}_\" + \"_\".join([word[:20] for word, _ in topic_model.get_topic(topic)][:3])\n topic_names[topic] = {\n \"trace_name\": trace_name[:40],\n \"plot_text\": trace_name[:40],\n }\n trace_names.append(trace_name)\n else:\n trace_name = (\n f\"{topic}_\"\n + hierarchical_topics.loc[hierarchical_topics.Parent_ID == str(topic), \"Parent_Name\"].to_numpy()[0]\n )\n plot_text = \"_\".join([name[:20] for name in trace_name.split(\"_\")[:3]])\n topic_names[topic] = {\n \"trace_name\": trace_name[:40],\n \"plot_text\": plot_text[:40],\n }\n trace_names.append(trace_name)\n\n # Prepare traces\n all_traces = []\n for level in range(len(max_distances)):\n traces = []\n\n # Outliers\n if topic_model._outliers:\n traces.append(\n go.Scattergl(\n x=df.loc[(df[f\"level_{level + 1}\"] == -1), \"x\"],\n y=df.loc[df[f\"level_{level + 1}\"] == -1, \"y\"],\n mode=\"markers+text\",\n name=\"other\",\n hoverinfo=\"text\",\n hovertext=df.loc[(df[f\"level_{level + 1}\"] == -1), \"doc\"] if not hide_document_hover else None,\n showlegend=False,\n marker=dict(color=\"#CFD8DC\", size=5, opacity=0.5),\n )\n )\n\n # Selected topics\n if topics:\n selection = df.loc[(df.topic.isin(topics)), :]\n unique_topics = sorted([int(topic) for topic in selection[f\"level_{level + 1}\"].unique()])\n else:\n unique_topics = sorted([int(topic) for topic in df[f\"level_{level + 1}\"].unique()])\n\n for topic in unique_topics:\n if topic != -1:\n if topics:\n selection = df.loc[(df[f\"level_{level + 1}\"] == topic) & (df.topic.isin(topics)), :]\n else:\n selection = df.loc[df[f\"level_{level + 1}\"] == topic, :]\n\n if not hide_annotations:\n selection.loc[len(selection), :] = None\n selection[\"text\"] = \"\"\n selection.loc[len(selection) - 1, \"x\"] = selection.x.mean()\n selection.loc[len(selection) - 1, \"y\"] = selection.y.mean()\n selection.loc[len(selection) - 1, \"text\"] = topic_names[int(topic)][\"plot_text\"]\n\n traces.append(\n go.Scattergl(\n x=selection.x,\n y=selection.y,\n text=selection.text if not hide_annotations else None,\n hovertext=selection.doc if not hide_document_hover else None,\n hoverinfo=\"text\",\n name=topic_names[int(topic)][\"trace_name\"],\n mode=\"markers+text\",\n marker=dict(size=5, opacity=0.5),\n )\n )\n\n all_traces.append(traces)\n\n # Track and count traces\n nr_traces_per_set = [len(traces) for traces in all_traces]\n trace_indices = [(0, nr_traces_per_set[0])]\n for index, nr_traces in enumerate(nr_traces_per_set[1:]):\n start = trace_indices[index][1]\n end = nr_traces + start\n trace_indices.append((start, end))\n\n # Visualization\n fig = go.Figure()\n for traces in all_traces:\n for trace in traces:\n fig.add_trace(trace)\n\n for index in range(len(fig.data)):\n if index >= nr_traces_per_set[0]:\n fig.data[index].visible = False\n\n # Create and add slider\n steps = []\n for index, indices in enumerate(trace_indices):\n step = dict(\n method=\"update\",\n label=str(index),\n args=[{\"visible\": [False] * len(fig.data)}],\n )\n for index in range(indices[1] - indices[0]):\n step[\"args\"][0][\"visible\"][index + indices[0]] = True\n steps.append(step)\n\n sliders = [dict(currentvalue={\"prefix\": \"Level: \"}, pad={\"t\": 20}, steps=steps)]\n\n # Add grid in a 'plus' shape\n x_range = (\n df.x.min() - abs((df.x.min()) * 0.15),\n df.x.max() + abs((df.x.max()) * 0.15),\n )\n y_range = (\n df.y.min() - abs((df.y.min()) * 0.15),\n df.y.max() + abs((df.y.max()) * 0.15),\n )\n fig.add_shape(\n type=\"line\",\n x0=sum(x_range) / 2,\n y0=y_range[0],\n x1=sum(x_range) / 2,\n y1=y_range[1],\n line=dict(color=\"#CFD8DC\", width=2),\n )\n fig.add_shape(\n type=\"line\",\n x0=x_range[0],\n y0=sum(y_range) / 2,\n x1=x_range[1],\n y1=sum(y_range) / 2,\n line=dict(color=\"#9E9E9E\", width=2),\n )\n fig.add_annotation(x=x_range[0], y=sum(y_range) / 2, text=\"D1\", showarrow=False, yshift=10)\n fig.add_annotation(y=y_range[1], x=sum(x_range) / 2, text=\"D2\", showarrow=False, xshift=10)\n\n # Stylize layout\n fig.update_layout(\n sliders=sliders,\n template=\"simple_white\",\n title={\n \"text\": f\"{title}\",\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width,\n height=height,\n )\n\n fig.update_xaxes(visible=False)\n fig.update_yaxes(visible=False)\n return fig\n" }, { "path": "bertopic/plotting/_hierarchy.py", "content": "import numpy as np\nimport pandas as pd\nfrom typing import Callable, List, Union\nfrom scipy.sparse import csr_matrix\nfrom scipy.cluster import hierarchy as sch\nfrom sklearn.metrics.pairwise import cosine_similarity\n\nfrom bertopic._utils import select_topic_representation\n\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\n\nfrom bertopic._utils import validate_distance_matrix\n\n\ndef visualize_hierarchy(\n topic_model,\n orientation: str = \"left\",\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n use_ctfidf: bool = True,\n custom_labels: Union[bool, str] = False,\n title: str = \"Hierarchical Clustering\",\n width: int = 1000,\n height: int = 600,\n hierarchical_topics: pd.DataFrame = None,\n linkage_function: Callable[[csr_matrix], np.ndarray] | None = None,\n distance_function: Callable[[csr_matrix], csr_matrix] | None = None,\n color_threshold: int = 1,\n) -> go.Figure:\n \"\"\"Visualize a hierarchical structure of the topics.\n\n A ward linkage function is used to perform the\n hierarchical clustering based on the cosine distance\n matrix between topic embeddings (either c-TF-IDF or the embeddings from the embedding model).\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n orientation: The orientation of the figure.\n Either 'left' or 'bottom'\n topics: A selection of topics to visualize\n top_n_topics: Only select the top n most frequent topics\n use_ctfidf: Whether to calculate distances between topics based on c-TF-IDF embeddings. If False, the embeddings\n from the embedding model are used.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n NOTE: Custom labels are only generated for the original\n un-merged topics.\n title: Title of the plot.\n width: The width of the figure. Only works if orientation is set to 'left'\n height: The height of the figure. Only works if orientation is set to 'bottom'\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children.\n NOTE: The hierarchical topic names are only visualized\n if both `topics` and `top_n_topics` are not set.\n linkage_function: The linkage function to use. Default is:\n `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`\n NOTE: Make sure to use the same `linkage_function` as used\n in `topic_model.hierarchical_topics`.\n distance_function: The distance function to use on the c-TF-IDF matrix. Default is:\n `lambda x: 1 - cosine_similarity(x)`.\n You can pass any function that returns either a square matrix of\n shape (n_samples, n_samples) with zeros on the diagonal and\n non-negative values or condensed distance matrix of shape\n (n_samples * (n_samples - 1) / 2,) containing the upper\n triangular of the distance matrix.\n NOTE: Make sure to use the same `distance_function` as used\n in `topic_model.hierarchical_topics`.\n color_threshold: Value at which the separation of clusters will be made which\n will result in different colors for different clusters.\n A higher value will typically lead in less colored clusters.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the hierarchical structure of\n topics simply run:\n\n ```python\n topic_model.visualize_hierarchy()\n ```\n\n If you also want the labels visualized of hierarchical topics,\n run the following:\n\n ```python\n # Extract hierarchical topics and their representations\n hierarchical_topics = topic_model.hierarchical_topics(docs)\n\n # Visualize these representations\n topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n ```\n\n If you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_hierarchy()\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n if distance_function is None:\n distance_function = lambda x: 1 - cosine_similarity(x)\n\n if linkage_function is None:\n linkage_function = lambda x: sch.linkage(x, \"ward\", optimal_ordering=True)\n\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n topics = list(topics)\n elif top_n_topics is not None:\n topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n topics = sorted(freq_df.Topic.to_list())\n\n # Select embeddings\n all_topics = sorted(list(topic_model.get_topics().keys()))\n indices = np.array([all_topics.index(topic) for topic in topics])\n\n # Select topic embeddings\n embeddings = select_topic_representation(topic_model.c_tf_idf_, topic_model.topic_embeddings_, use_ctfidf)[0][\n indices\n ]\n\n # Annotations\n if hierarchical_topics is not None and len(topics) == len(freq_df.Topic.to_list()):\n annotations = _get_annotations(\n topic_model=topic_model,\n hierarchical_topics=hierarchical_topics,\n embeddings=embeddings,\n distance_function=distance_function,\n linkage_function=linkage_function,\n orientation=orientation,\n custom_labels=custom_labels,\n )\n else:\n annotations = None\n\n # wrap distance function to validate input and return a condensed distance matrix\n distance_function_viz = lambda x: validate_distance_matrix(distance_function(x), embeddings.shape[0])\n # Create dendogram\n fig = ff.create_dendrogram(\n embeddings,\n orientation=orientation,\n distfun=distance_function_viz,\n linkagefun=linkage_function,\n hovertext=annotations,\n color_threshold=color_threshold,\n )\n\n # Create nicer labels\n axis = \"yaxis\" if orientation == \"left\" else \"xaxis\"\n if isinstance(custom_labels, str):\n new_labels = [\n [[str(x), None]] + topic_model.topic_aspects_[custom_labels][x] for x in fig.layout[axis][\"ticktext\"]\n ]\n new_labels = [\"_\".join([label[0] for label in labels[:4]]) for labels in new_labels]\n new_labels = [label if len(label) < 30 else label[:27] + \"...\" for label in new_labels]\n elif topic_model.custom_labels_ is not None and custom_labels:\n new_labels = [\n topic_model.custom_labels_[topics[int(x)] + topic_model._outliers] for x in fig.layout[axis][\"ticktext\"]\n ]\n else:\n new_labels = [\n [[str(topics[int(x)]), None], *topic_model.get_topic(topics[int(x)])] for x in fig.layout[axis][\"ticktext\"]\n ]\n new_labels = [\"_\".join([label[0] for label in labels[:4]]) for labels in new_labels]\n new_labels = [label if len(label) < 30 else label[:27] + \"...\" for label in new_labels]\n\n # Stylize layout\n fig.update_layout(\n plot_bgcolor=\"#ECEFF1\",\n template=\"plotly_white\",\n title={\n \"text\": f\"{title}\",\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n )\n\n # Stylize orientation\n if orientation == \"left\":\n fig.update_layout(\n height=200 + (15 * len(topics)),\n width=width,\n yaxis=dict(tickmode=\"array\", ticktext=new_labels),\n )\n\n # Fix empty space on the bottom of the graph\n y_max = max([trace[\"y\"].max() + 5 for trace in fig[\"data\"]])\n y_min = min([trace[\"y\"].min() - 5 for trace in fig[\"data\"]])\n fig.update_layout(yaxis=dict(range=[y_min, y_max]))\n\n else:\n fig.update_layout(\n width=200 + (15 * len(topics)),\n height=height,\n xaxis=dict(tickmode=\"array\", ticktext=new_labels),\n )\n\n if hierarchical_topics is not None:\n for index in [0, 3]:\n axis = \"x\" if orientation == \"left\" else \"y\"\n xs = [data[\"x\"][index] for data in fig.data if (data[\"text\"] and data[axis][index] > 0)]\n ys = [data[\"y\"][index] for data in fig.data if (data[\"text\"] and data[axis][index] > 0)]\n hovertext = [data[\"text\"][index] for data in fig.data if (data[\"text\"] and data[axis][index] > 0)]\n\n fig.add_trace(\n go.Scatter(\n x=xs,\n y=ys,\n marker_color=\"black\",\n hovertext=hovertext,\n hoverinfo=\"text\",\n mode=\"markers\",\n showlegend=False,\n )\n )\n return fig\n\n\ndef _get_annotations(\n topic_model,\n hierarchical_topics: pd.DataFrame,\n embeddings: csr_matrix,\n linkage_function: Callable[[csr_matrix], np.ndarray],\n distance_function: Callable[[csr_matrix], csr_matrix],\n orientation: str,\n custom_labels: bool = False,\n) -> List[List[str]]:\n \"\"\"Get annotations by replicating linkage function calculation in scipy.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n hierarchical_topics: A dataframe that contains a hierarchy of topics\n represented by their parents and their children.\n NOTE: The hierarchical topic names are only visualized\n if both `topics` and `top_n_topics` are not set.\n embeddings: The c-TF-IDF matrix on which to model the hierarchy\n linkage_function: The linkage function to use. Default is:\n `lambda x: sch.linkage(x, 'ward', optimal_ordering=True)`\n NOTE: Make sure to use the same `linkage_function` as used\n in `topic_model.hierarchical_topics`.\n distance_function: The distance function to use on the c-TF-IDF matrix. Default is:\n `lambda x: 1 - cosine_similarity(x)`.\n You can pass any function that returns either a square matrix of\n shape (n_samples, n_samples) with zeros on the diagonal and\n non-negative values or condensed distance matrix of shape\n (n_samples * (n_samples - 1) / 2,) containing the upper\n triangular of the distance matrix.\n NOTE: Make sure to use the same `distance_function` as used\n in `topic_model.hierarchical_topics`.\n orientation: The orientation of the figure.\n Either 'left' or 'bottom'\n custom_labels: Whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n NOTE: Custom labels are only generated for the original\n un-merged topics.\n\n Returns:\n text_annotations: Annotations to be used within Plotly's `ff.create_dendogram`\n \"\"\"\n df = hierarchical_topics.loc[hierarchical_topics.Parent_Name != \"Top\", :]\n\n # Calculate distance\n X = distance_function(embeddings)\n X = validate_distance_matrix(X, embeddings.shape[0])\n\n # Calculate linkage and generate dendrogram\n Z = linkage_function(X)\n P = sch.dendrogram(Z, orientation=orientation, no_plot=True)\n\n # store topic no.(leaves) corresponding to the x-ticks in dendrogram\n x_ticks = np.arange(5, len(P[\"leaves\"]) * 10 + 5, 10)\n x_topic = dict(zip(P[\"leaves\"], x_ticks))\n\n topic_vals = dict()\n for key, val in x_topic.items():\n topic_vals[val] = [key]\n\n parent_topic = dict(zip(df.Parent_ID, df.Topics))\n\n # loop through every trace (scatter plot) in dendrogram\n text_annotations = []\n for index, trace in enumerate(P[\"icoord\"]):\n fst_topic = topic_vals[trace[0]]\n scnd_topic = topic_vals[trace[2]]\n\n if len(fst_topic) == 1:\n if isinstance(custom_labels, str):\n fst_name = f\"{fst_topic[0]}_\" + \"_\".join(\n next(zip(*topic_model.topic_aspects_[custom_labels][fst_topic[0]]))[:3]\n )\n elif topic_model.custom_labels_ is not None and custom_labels:\n fst_name = topic_model.custom_labels_[fst_topic[0] + topic_model._outliers]\n else:\n fst_name = \"_\".join([word for word, _ in topic_model.get_topic(fst_topic[0])][:5])\n else:\n for key, value in parent_topic.items():\n if set(value) == set(fst_topic):\n fst_name = df.loc[df.Parent_ID == key, \"Parent_Name\"].to_numpy()[0]\n\n if len(scnd_topic) == 1:\n if isinstance(custom_labels, str):\n scnd_name = f\"{scnd_topic[0]}_\" + \"_\".join(\n next(zip(*topic_model.topic_aspects_[custom_labels][scnd_topic[0]]))[:3]\n )\n elif topic_model.custom_labels_ is not None and custom_labels:\n scnd_name = topic_model.custom_labels_[scnd_topic[0] + topic_model._outliers]\n else:\n scnd_name = \"_\".join([word for word, _ in topic_model.get_topic(scnd_topic[0])][:5])\n else:\n for key, value in parent_topic.items():\n if set(value) == set(scnd_topic):\n scnd_name = df.loc[df.Parent_ID == key, \"Parent_Name\"].to_numpy()[0]\n\n text_annotations.append([fst_name, \"\", \"\", scnd_name])\n\n center = (trace[0] + trace[2]) / 2\n topic_vals[center] = fst_topic + scnd_topic\n\n return text_annotations\n" }, { "path": "bertopic/plotting/_term_rank.py", "content": "import numpy as np\nfrom typing import List, Union\nimport plotly.graph_objects as go\n\n\ndef visualize_term_rank(\n topic_model,\n topics: List[int] | None = None,\n log_scale: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Term score decline per Topic\",\n width: int = 800,\n height: int = 500,\n) -> go.Figure:\n \"\"\"Visualize the ranks of all terms across all topics.\n\n Each topic is represented by a set of words. These words, however,\n do not all equally represent the topic. This visualization shows\n how many words are needed to represent a topic and at which point\n the beneficial effect of adding words starts to decline.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics: A selection of topics to visualize. These will be colored\n red where all others will be colored black.\n log_scale: Whether to represent the ranking on a log scale\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n fig: A plotly figure\n\n Examples:\n To visualize the ranks of all words across\n all topics simply run:\n\n ```python\n topic_model.visualize_term_rank()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_term_rank()\n fig.write_html(\"path/to/file.html\")\n ```\n\n \n\n \n\n Reference:\n\n This visualization was heavily inspired by the\n \"Term Probability Decline\" visualization found in an\n analysis by the amazing [tmtoolkit](https://tmtoolkit.readthedocs.io/).\n Reference to that specific analysis can be found\n [here](https://wzbsocialsciencecenter.github.io/tm_corona/tm_analysis.html).\n \"\"\"\n topics = [] if topics is None else topics\n\n topic_ids = topic_model.get_topic_info().Topic.unique().tolist()\n topic_words = [topic_model.get_topic(topic) for topic in topic_ids]\n\n values = np.array([[value[1] for value in values] for values in topic_words])\n indices = np.array([[value + 1 for value in range(len(values))] for values in topic_words])\n\n # Create figure\n lines = []\n for topic, x, y in zip(topic_ids, indices, values):\n if not any(y > 1.5):\n # labels\n if isinstance(custom_labels, str):\n label = f\"{topic}_\" + \"_\".join(next(zip(*topic_model.topic_aspects_[custom_labels][topic]))[:3])\n elif topic_model.custom_labels_ is not None and custom_labels:\n label = topic_model.custom_labels_[topic + topic_model._outliers]\n else:\n label = f\"Topic {topic}:\" + \"_\".join([word[0] for word in topic_model.get_topic(topic)])\n label = label[:50]\n\n # line parameters\n color = \"red\" if topic in topics else \"black\"\n opacity = 1 if topic in topics else 0.1\n if any(y == 0):\n y[y == 0] = min(values[values > 0])\n y = np.log10(y, out=y, where=y > 0) if log_scale else y\n\n line = go.Scatter(\n x=x,\n y=y,\n name=\"\",\n hovertext=label,\n mode=\"lines+lines\",\n opacity=opacity,\n line=dict(color=color, width=1.5),\n )\n lines.append(line)\n\n fig = go.Figure(data=lines)\n\n # Stylize layout\n fig.update_xaxes(range=[0, len(indices[0])], tick0=1, dtick=2)\n fig.update_layout(\n showlegend=False,\n template=\"plotly_white\",\n title={\n \"text\": f\"{title}\",\n \"y\": 0.9,\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n )\n\n fig.update_xaxes(title_text=\"Term Rank\")\n if log_scale:\n fig.update_yaxes(title_text=\"c-TF-IDF score (log scale)\")\n else:\n fig.update_yaxes(title_text=\"c-TF-IDF score\")\n\n return fig\n" }, { "path": "bertopic/plotting/_topics.py", "content": "import numpy as np\nimport pandas as pd\n\ntry:\n from umap import UMAP\n\n HAS_UMAP = True\nexcept (ImportError, ModuleNotFoundError):\n HAS_UMAP = False\n\nfrom typing import List, Union\nfrom sklearn.preprocessing import MinMaxScaler\nfrom bertopic._utils import select_topic_representation\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n\ndef visualize_topics(\n topic_model,\n topics: List[int] | None = None,\n top_n_topics: int | None = None,\n use_ctfidf: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Intertopic Distance Map\",\n width: int = 650,\n height: int = 650,\n) -> go.Figure:\n \"\"\"Visualize topics, their sizes, and their corresponding words.\n\n This visualization is highly inspired by LDAvis, a great visualization\n technique typically reserved for LDA.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics: A selection of topics to visualize\n top_n_topics: Only select the top n most frequent topics\n use_ctfidf: Whether to use c-TF-IDF representations instead of the embeddings from the embedding model.\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Examples:\n To visualize the topics simply run:\n\n ```python\n topic_model.visualize_topics()\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics()\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n topics = list(topics)\n elif top_n_topics is not None:\n topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n topics = sorted(freq_df.Topic.to_list())\n\n # Extract topic words and their frequencies\n topic_list = sorted(topics)\n frequencies = [topic_model.topic_sizes_[topic] for topic in topic_list]\n if isinstance(custom_labels, str):\n words = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in topic_list]\n words = [\"_\".join([label[0] for label in labels[:4]]) for labels in words]\n words = [label if len(label) < 30 else label[:27] + \"...\" for label in words]\n elif custom_labels and topic_model.custom_labels_ is not None:\n words = [topic_model.custom_labels_[topic + topic_model._outliers] for topic in topic_list]\n else:\n words = [\" | \".join([word[0] for word in topic_model.get_topic(topic)[:5]]) for topic in topic_list]\n\n # Embed c-TF-IDF into 2D\n all_topics = sorted(list(topic_model.get_topics().keys()))\n indices = np.array([all_topics.index(topic) for topic in topics])\n\n embeddings, c_tfidf_used = select_topic_representation(\n topic_model.c_tf_idf_,\n topic_model.topic_embeddings_,\n use_ctfidf=use_ctfidf,\n output_ndarray=True,\n )\n embeddings = embeddings[indices]\n\n if HAS_UMAP:\n if c_tfidf_used:\n embeddings = MinMaxScaler().fit_transform(embeddings)\n embeddings = UMAP(\n n_neighbors=2, n_components=2, metric=\"hellinger\", init=\"random\", random_state=42\n ).fit_transform(embeddings)\n else:\n embeddings = UMAP(\n n_neighbors=2, n_components=2, metric=\"cosine\", init=\"random\", random_state=42\n ).fit_transform(embeddings)\n else:\n raise ModuleNotFoundError(\n \"UMAP is required to reduce the embeddings.. Please install it using `pip install umap-learn`.\"\n )\n\n # Visualize with plotly\n df = pd.DataFrame(\n {\n \"x\": embeddings[:, 0],\n \"y\": embeddings[:, 1],\n \"Topic\": topic_list,\n \"Words\": words,\n \"Size\": frequencies,\n }\n )\n return _plotly_topic_visualization(df, topic_list, title, width, height)\n\n\ndef _plotly_topic_visualization(df: pd.DataFrame, topic_list: List[str], title: str, width: int, height: int):\n \"\"\"Create plotly-based visualization of topics with a slider for topic selection.\"\"\"\n\n def get_color(topic_selected):\n if topic_selected == -1:\n marker_color = [\"#B0BEC5\" for _ in topic_list]\n else:\n marker_color = [\"red\" if topic == topic_selected else \"#B0BEC5\" for topic in topic_list]\n return [{\"marker.color\": [marker_color]}]\n\n # Prepare figure range\n x_range = (\n df.x.min() - abs((df.x.min()) * 0.15),\n df.x.max() + abs((df.x.max()) * 0.15),\n )\n y_range = (\n df.y.min() - abs((df.y.min()) * 0.15),\n df.y.max() + abs((df.y.max()) * 0.15),\n )\n\n # Plot topics\n fig = px.scatter(\n df,\n x=\"x\",\n y=\"y\",\n size=\"Size\",\n size_max=40,\n template=\"simple_white\",\n labels={\"x\": \"\", \"y\": \"\"},\n hover_data={\"Topic\": True, \"Words\": True, \"Size\": True, \"x\": False, \"y\": False},\n )\n fig.update_traces(marker=dict(color=\"#B0BEC5\", line=dict(width=2, color=\"DarkSlateGrey\")))\n\n # Update hover order\n fig.update_traces(\n hovertemplate=\"
\".join(\n [\n \"Topic %{customdata[0]}\",\n \"%{customdata[1]}\",\n \"Size: %{customdata[2]}\",\n ]\n )\n )\n\n # Create a slider for topic selection\n steps = [dict(label=f\"Topic {topic}\", method=\"update\", args=get_color(topic)) for topic in topic_list]\n sliders = [dict(active=0, pad={\"t\": 50}, steps=steps)]\n\n # Stylize layout\n fig.update_layout(\n title={\n \"text\": f\"{title}\",\n \"y\": 0.95,\n \"x\": 0.5,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n xaxis={\"visible\": False},\n yaxis={\"visible\": False},\n sliders=sliders,\n )\n\n # Update axes ranges\n fig.update_xaxes(range=x_range)\n fig.update_yaxes(range=y_range)\n\n # Add grid in a 'plus' shape\n fig.add_shape(\n type=\"line\",\n x0=sum(x_range) / 2,\n y0=y_range[0],\n x1=sum(x_range) / 2,\n y1=y_range[1],\n line=dict(color=\"#CFD8DC\", width=2),\n )\n fig.add_shape(\n type=\"line\",\n x0=x_range[0],\n y0=sum(y_range) / 2,\n x1=x_range[1],\n y1=sum(y_range) / 2,\n line=dict(color=\"#9E9E9E\", width=2),\n )\n fig.add_annotation(x=x_range[0], y=sum(y_range) / 2, text=\"D1\", showarrow=False, yshift=10)\n fig.add_annotation(y=y_range[1], x=sum(x_range) / 2, text=\"D2\", showarrow=False, xshift=10)\n fig.data = fig.data[::-1]\n\n return fig\n" }, { "path": "bertopic/plotting/_topics_over_time.py", "content": "import pandas as pd\nfrom typing import List, Union\nimport plotly.graph_objects as go\nfrom sklearn.preprocessing import normalize\n\n\ndef visualize_topics_over_time(\n topic_model,\n topics_over_time: pd.DataFrame,\n top_n_topics: int | None = None,\n topics: List[int] | None = None,\n normalize_frequency: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Topics over Time\",\n width: int = 1250,\n height: int = 450,\n) -> go.Figure:\n \"\"\"Visualize topics over time.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics_over_time: The topics you would like to be visualized with the\n corresponding topic representation\n top_n_topics: To visualize the most frequent topics instead of all\n topics: Select which topics you would like to be visualized\n normalize_frequency: Whether to normalize each topic's frequency individually\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n A plotly.graph_objects.Figure including all traces\n\n Examples:\n To visualize the topics over time, simply run:\n\n ```python\n topics_over_time = topic_model.topics_over_time(docs, timestamps)\n topic_model.visualize_topics_over_time(topics_over_time)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics_over_time(topics_over_time)\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n colors = [\n \"#E69F00\",\n \"#56B4E9\",\n \"#009E73\",\n \"#F0E442\",\n \"#D55E00\",\n \"#0072B2\",\n \"#CC79A7\",\n ]\n\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n selected_topics = list(topics)\n elif top_n_topics is not None:\n selected_topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n selected_topics = sorted(freq_df.Topic.to_list())\n\n # Prepare data\n if isinstance(custom_labels, str):\n topic_names = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in topics]\n topic_names = [\"_\".join([label[0] for label in labels[:4]]) for labels in topic_names]\n topic_names = [label if len(label) < 30 else label[:27] + \"...\" for label in topic_names]\n topic_names = {key: topic_names[index] for index, key in enumerate(topic_model.topic_labels_.keys())}\n elif topic_model.custom_labels_ is not None and custom_labels:\n topic_names = {\n key: topic_model.custom_labels_[key + topic_model._outliers] for key, _ in topic_model.topic_labels_.items()\n }\n else:\n topic_names = {\n key: value[:40] + \"...\" if len(value) > 40 else value for key, value in topic_model.topic_labels_.items()\n }\n topics_over_time[\"Name\"] = topics_over_time.Topic.map(topic_names)\n data = topics_over_time.loc[topics_over_time.Topic.isin(selected_topics), :].sort_values([\"Topic\", \"Timestamp\"])\n\n # Add traces\n fig = go.Figure()\n for index, topic in enumerate(data.Topic.unique()):\n trace_data = data.loc[data.Topic == topic, :]\n topic_name = trace_data.Name.to_numpy()[0]\n words = trace_data.Words.to_numpy()\n if normalize_frequency:\n y = normalize(trace_data.Frequency.to_numpy().reshape(1, -1))[0]\n else:\n y = trace_data.Frequency\n fig.add_trace(\n go.Scatter(\n x=trace_data.Timestamp,\n y=y,\n mode=\"lines\",\n marker_color=colors[index % 7],\n hoverinfo=\"text\",\n name=topic_name,\n hovertext=[f\"Topic {topic}
Words: {word}\" for word in words],\n )\n )\n\n # Styling of the visualization\n fig.update_xaxes(showgrid=True)\n fig.update_yaxes(showgrid=True)\n fig.update_layout(\n yaxis_title=\"Normalized Frequency\" if normalize_frequency else \"Frequency\",\n title={\n \"text\": f\"{title}\",\n \"y\": 0.95,\n \"x\": 0.40,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n template=\"simple_white\",\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n legend=dict(\n title=\"Global Topic Representation\",\n ),\n )\n return fig\n" }, { "path": "bertopic/plotting/_topics_per_class.py", "content": "import pandas as pd\nfrom typing import List, Union\nimport plotly.graph_objects as go\nfrom sklearn.preprocessing import normalize\n\n\ndef visualize_topics_per_class(\n topic_model,\n topics_per_class: pd.DataFrame,\n top_n_topics: int = 10,\n topics: List[int] | None = None,\n normalize_frequency: bool = False,\n custom_labels: Union[bool, str] = False,\n title: str = \"Topics per Class\",\n width: int = 1250,\n height: int = 900,\n) -> go.Figure:\n \"\"\"Visualize topics per class.\n\n Arguments:\n topic_model: A fitted BERTopic instance.\n topics_per_class: The topics you would like to be visualized with the\n corresponding topic representation\n top_n_topics: To visualize the most frequent topics instead of all\n topics: Select which topics you would like to be visualized\n normalize_frequency: Whether to normalize each topic's frequency individually\n custom_labels: If bool, whether to use custom topic labels that were defined using\n `topic_model.set_topic_labels`.\n If `str`, it uses labels from other aspects, e.g., \"Aspect1\".\n title: Title of the plot.\n width: The width of the figure.\n height: The height of the figure.\n\n Returns:\n A plotly.graph_objects.Figure including all traces\n\n Examples:\n To visualize the topics per class, simply run:\n\n ```python\n topics_per_class = topic_model.topics_per_class(docs, classes)\n topic_model.visualize_topics_per_class(topics_per_class)\n ```\n\n Or if you want to save the resulting figure:\n\n ```python\n fig = topic_model.visualize_topics_per_class(topics_per_class)\n fig.write_html(\"path/to/file.html\")\n ```\n \n \"\"\"\n colors = [\n \"#E69F00\",\n \"#56B4E9\",\n \"#009E73\",\n \"#F0E442\",\n \"#D55E00\",\n \"#0072B2\",\n \"#CC79A7\",\n ]\n\n # Select topics based on top_n and topics args\n freq_df = topic_model.get_topic_freq()\n freq_df = freq_df.loc[freq_df.Topic != -1, :]\n if topics is not None:\n selected_topics = list(topics)\n elif top_n_topics is not None:\n selected_topics = sorted(freq_df.Topic.to_list()[:top_n_topics])\n else:\n selected_topics = sorted(freq_df.Topic.to_list())\n\n # Prepare data\n if isinstance(custom_labels, str):\n topic_names = [[[str(topic), None]] + topic_model.topic_aspects_[custom_labels][topic] for topic in topics]\n topic_names = [\"_\".join([label[0] for label in labels[:4]]) for labels in topic_names]\n topic_names = [label if len(label) < 30 else label[:27] + \"...\" for label in topic_names]\n topic_names = {key: topic_names[index] for index, key in enumerate(topic_model.topic_labels_.keys())}\n elif topic_model.custom_labels_ is not None and custom_labels:\n topic_names = {\n key: topic_model.custom_labels_[key + topic_model._outliers] for key, _ in topic_model.topic_labels_.items()\n }\n else:\n topic_names = {\n key: value[:40] + \"...\" if len(value) > 40 else value for key, value in topic_model.topic_labels_.items()\n }\n topics_per_class[\"Name\"] = topics_per_class.Topic.map(topic_names)\n data = topics_per_class.loc[topics_per_class.Topic.isin(selected_topics), :]\n\n # Add traces\n fig = go.Figure()\n for index, topic in enumerate(selected_topics):\n if index == 0:\n visible = True\n else:\n visible = \"legendonly\"\n trace_data = data.loc[data.Topic == topic, :]\n topic_name = trace_data.Name.to_numpy()[0]\n words = trace_data.Words.to_numpy()\n if normalize_frequency:\n x = normalize(trace_data.Frequency.to_numpy().reshape(1, -1))[0]\n else:\n x = trace_data.Frequency\n fig.add_trace(\n go.Bar(\n y=trace_data.Class,\n x=x,\n visible=visible,\n marker_color=colors[index % 7],\n hoverinfo=\"text\",\n name=topic_name,\n orientation=\"h\",\n hovertext=[f\"Topic {topic}
Words: {word}\" for word in words],\n )\n )\n\n # Styling of the visualization\n fig.update_xaxes(showgrid=True)\n fig.update_yaxes(showgrid=True)\n fig.update_layout(\n xaxis_title=\"Normalized Frequency\" if normalize_frequency else \"Frequency\",\n yaxis_title=\"Class\",\n title={\n \"text\": f\"{title}\",\n \"y\": 0.95,\n \"x\": 0.40,\n \"xanchor\": \"center\",\n \"yanchor\": \"top\",\n \"font\": dict(size=22, color=\"Black\"),\n },\n template=\"simple_white\",\n width=width,\n height=height,\n hoverlabel=dict(bgcolor=\"white\", font_size=16, font_family=\"Rockwell\"),\n legend=dict(\n title=\"Global Topic Representation\",\n ),\n )\n return fig\n" }, { "path": "bertopic/representation/__init__.py", "content": "from bertopic._utils import NotInstalled\nfrom bertopic.representation._cohere import Cohere\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._keybert import KeyBERTInspired\nfrom bertopic.representation._mmr import MaximalMarginalRelevance\n\n\n# Llama CPP Generator\ntry:\n from bertopic.representation._llamacpp import LlamaCPP\nexcept ModuleNotFoundError:\n msg = \"`pip install llama-cpp-python` \\n\\n\"\n LlamaCPP = NotInstalled(\"llama.cpp\", \"llama-cpp-python\", custom_msg=msg)\n\n# Text Generation using transformers\ntry:\n from bertopic.representation._textgeneration import TextGeneration\nexcept ModuleNotFoundError:\n msg = \"`pip install bertopic` without `--no-deps` \\n\\n\"\n TextGeneration = NotInstalled(\"TextGeneration\", \"transformers\", custom_msg=msg)\n\n# Zero-shot classification using transformers\ntry:\n from bertopic.representation._zeroshot import ZeroShotClassification\nexcept ModuleNotFoundError:\n msg = \"`pip install bertopic` without `--no-deps` \\n\\n\"\n ZeroShotClassification = NotInstalled(\"ZeroShotClassification\", \"transformers\", custom_msg=msg)\n\n# OpenAI Generator\ntry:\n from bertopic.representation._openai import OpenAI\nexcept ModuleNotFoundError:\n msg = \"`pip install openai` \\n\\n\"\n OpenAI = NotInstalled(\"OpenAI\", \"openai\", custom_msg=msg)\n\n# LiteLLM Generator\ntry:\n from bertopic.representation._litellm import LiteLLM\nexcept ModuleNotFoundError:\n msg = \"`pip install litellm` \\n\\n\"\n LiteLLM = NotInstalled(\"LiteLLM\", \"litellm\", custom_msg=msg)\n\n# LangChain Generator\ntry:\n from bertopic.representation._langchain import LangChain\nexcept ModuleNotFoundError:\n msg = \"`pip install langchain` \\n\\n\"\n LangChain = NotInstalled(\"langchain\", \"langchain\", custom_msg=msg)\n\n# POS using Spacy\ntry:\n from bertopic.representation._pos import PartOfSpeech\nexcept ModuleNotFoundError:\n PartOfSpeech = NotInstalled(\"Part of Speech with Spacy\", \"spacy\")\n\n# Multimodal\ntry:\n from bertopic.representation._visual import VisualRepresentation\nexcept ModuleNotFoundError:\n VisualRepresentation = NotInstalled(\"a visual representation model\", \"vision\")\n\n\n__all__ = [\n \"BaseRepresentation\",\n \"Cohere\",\n \"KeyBERTInspired\",\n \"LangChain\",\n \"LiteLLM\",\n \"LlamaCPP\",\n \"MaximalMarginalRelevance\",\n \"OpenAI\",\n \"PartOfSpeech\",\n \"TextGeneration\",\n \"VisualRepresentation\",\n \"ZeroShotClassification\",\n]\n" }, { "path": "bertopic/representation/_base.py", "content": "import pandas as pd\nfrom scipy.sparse import csr_matrix\nfrom sklearn.base import BaseEstimator\nfrom typing import Mapping, List, Tuple\n\n\nclass BaseRepresentation(BaseEstimator):\n \"\"\"The base representation model for fine-tuning topic representations.\"\"\"\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Each representation model that inherits this class will have\n its arguments (topic_model, documents, c_tf_idf, topics)\n automatically passed. Therefore, the representation model\n will only have access to the information about topics related\n to those arguments.\n\n Arguments:\n topic_model: The BERTopic model that is fitted until topic\n representations are calculated.\n documents: A dataframe with columns \"Document\" and \"Topic\"\n that contains all documents with each corresponding\n topic.\n c_tf_idf: A c-TF-IDF representation that is typically\n identical to `topic_model.c_tf_idf_` except for\n dynamic, class-based, and hierarchical topic modeling\n where it is calculated on a subset of the documents.\n topics: A dictionary with topic (key) and tuple of word and\n weight (value) as calculated by c-TF-IDF. This is the\n default topics that are returned if no representation\n model is used.\n \"\"\"\n return topic_model.topic_representations_\n" }, { "path": "bertopic/representation/_cohere.py", "content": "import time\nimport pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Union, Callable\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import truncate_document, validate_truncate_document_parameters\n\n\nDEFAULT_PROMPT = \"\"\"\nThis is a list of texts where each collection of texts describe a topic. After each collection of texts, the name of the topic they represent is mentioned as a short-highly-descriptive title\n---\nTopic:\nSample texts from this topic:\n- Traditional diets in most cultures were primarily plant-based with a little meat on top, but with the rise of industrial style meat production and factory farming, meat has become a staple food.\n- Meat, but especially beef, is the word food in terms of emissions.\n- Eating meat doesn't make you a bad person, not eating meat doesn't make you a good one.\n\nKeywords: meat beef eat eating emissions steak food health processed chicken\nTopic name: Environmental impacts of eating meat\n---\nTopic:\nSample texts from this topic:\n- I have ordered the product weeks ago but it still has not arrived!\n- The website mentions that it only takes a couple of days to deliver but I still have not received mine.\n- I got a message stating that I received the monitor but that is not true!\n- It took a month longer to deliver than was advised...\n\nKeywords: deliver weeks product shipping long delivery received arrived arrive week\nTopic name: Shipping and delivery issues\n---\nTopic:\nSample texts from this topic:\n[DOCUMENTS]\nKeywords: [KEYWORDS]\nTopic name:\"\"\"\n\nDEFAULT_SYSTEM_PROMPT = \"You are an assistant that extracts high-level topics from texts.\"\n\n\nclass Cohere(BaseRepresentation):\n \"\"\"Use the Cohere API to generate topic labels based on their\n generative model.\n\n Find more about their models here:\n https://docs.cohere.ai/docs\n\n Arguments:\n client: A `cohere.Client`\n model: Model to use within Cohere, defaults to `\"xlarge\"`.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt\n to decide where the keywords and documents need to be\n inserted.\n system_prompt: The system prompt to be used in the model. If no system prompt is given,\n `self.default_system_prompt_` is used instead.\n delay_in_seconds: The delay in seconds between consecutive prompts\n in order to prevent RateLimitErrors.\n nr_docs: The number of documents to pass to OpenAI if a prompt\n with the `[\"DOCUMENTS\"]` tag is used.\n diversity: The diversity of documents to pass to OpenAI.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n\n Usage:\n\n To use this, you will need to install cohere first:\n\n `pip install cohere`\n\n Then, get yourself an API key and use Cohere's API as follows:\n\n ```python\n import cohere\n from bertopic.representation import Cohere\n from bertopic import BERTopic\n\n # Create your representation model\n co = cohere.Client(my_api_key)\n representation_model = Cohere(co)\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n\n You can also use a custom prompt:\n\n ```python\n prompt = \"I have the following documents: [DOCUMENTS]. What topic do they contain?\"\n representation_model = Cohere(co, prompt=prompt)\n ```\n \"\"\"\n\n def __init__(\n self,\n client,\n model: str = \"command-r\",\n prompt: str | None = None,\n system_prompt: str | None = None,\n delay_in_seconds: float | None = None,\n nr_docs: int = 4,\n diversity: float | None = None,\n doc_length: int | None = None,\n tokenizer: Union[str, Callable] | None = None,\n ):\n self.client = client\n self.model = model\n self.prompt = prompt if prompt is not None else DEFAULT_PROMPT\n self.system_prompt = system_prompt if system_prompt is not None else DEFAULT_SYSTEM_PROMPT\n self.default_prompt_ = DEFAULT_PROMPT\n self.default_system_prompt_ = DEFAULT_SYSTEM_PROMPT\n self.delay_in_seconds = delay_in_seconds\n self.nr_docs = nr_docs\n self.diversity = diversity\n self.doc_length = doc_length\n self.tokenizer = tokenizer\n validate_truncate_document_parameters(self.tokenizer, self.doc_length)\n\n self.prompts_ = []\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: Not used\n documents: Not used\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top 4 representative documents per topic\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity\n )\n\n # Generate using Cohere's Language Model\n updated_topics = {}\n for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):\n truncated_docs = [truncate_document(topic_model, self.doc_length, self.tokenizer, doc) for doc in docs]\n prompt = self._create_prompt(truncated_docs, topic, topics)\n self.prompts_.append(prompt)\n\n # Delay\n if self.delay_in_seconds:\n time.sleep(self.delay_in_seconds)\n\n request = self.client.chat(\n model=self.model,\n preamble=self.system_prompt,\n message=prompt,\n max_tokens=50,\n stop_sequences=[\"\\n\"],\n )\n label = request.text.strip()\n updated_topics[topic] = [(label, 1)] + [(\"\", 0) for _ in range(9)]\n\n return updated_topics\n\n def _create_prompt(self, docs, topic, topics):\n keywords = next(zip(*topics[topic]))\n\n # Use the Default Chat Prompt\n if self.prompt == DEFAULT_PROMPT:\n prompt = self.prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n prompt = self._replace_documents(prompt, docs)\n\n # Use a custom prompt that leverages keywords, documents or both using\n # custom tags, namely [KEYWORDS] and [DOCUMENTS] respectively\n else:\n prompt = self.prompt\n if \"[KEYWORDS]\" in prompt:\n prompt = prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n if \"[DOCUMENTS]\" in prompt:\n prompt = self._replace_documents(prompt, docs)\n\n return prompt\n\n @staticmethod\n def _replace_documents(prompt, docs):\n to_replace = \"\"\n for doc in docs:\n to_replace += f\"- {doc}\\n\"\n prompt = prompt.replace(\"[DOCUMENTS]\", to_replace)\n return prompt\n" }, { "path": "bertopic/representation/_keybert.py", "content": "import numpy as np\nimport pandas as pd\n\nfrom packaging import version\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Union\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom bertopic.representation._base import BaseRepresentation\nfrom sklearn import __version__ as sklearn_version\n\n\nclass KeyBERTInspired(BaseRepresentation):\n def __init__(\n self,\n top_n_words: int = 10,\n nr_repr_docs: int = 5,\n nr_samples: int = 500,\n nr_candidate_words: int = 100,\n random_state: int = 42,\n ):\n \"\"\"Use a KeyBERT-like model to fine-tune the topic representations.\n\n The algorithm follows KeyBERT but does some optimization in\n order to speed up inference.\n\n The steps are as follows. First, we extract the top n representative\n documents per topic. To extract the representative documents, we\n randomly sample a number of candidate documents per cluster\n which is controlled by the `nr_samples` parameter. Then,\n the top n representative documents are extracted by calculating\n the c-TF-IDF representation for the candidate documents and finding,\n through cosine similarity, which are closest to the topic c-TF-IDF representation.\n Next, the top n words per topic are extracted based on their\n c-TF-IDF representation, which is controlled by the `nr_repr_docs`\n parameter.\n\n Then, we extract the embeddings for words and representative documents\n and create topic embeddings by averaging the representative documents.\n Finally, the most similar words to each topic are extracted by\n calculating the cosine similarity between word and topic embeddings.\n\n Arguments:\n top_n_words: The top n words to extract per topic.\n nr_repr_docs: The number of representative documents to extract per cluster.\n nr_samples: The number of candidate documents to extract per cluster.\n nr_candidate_words: The number of candidate words per cluster.\n random_state: The random state for randomly sampling candidate documents.\n\n Usage:\n\n ```python\n from bertopic.representation import KeyBERTInspired\n from bertopic import BERTopic\n\n # Create your representation model\n representation_model = KeyBERTInspired()\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n \"\"\"\n self.top_n_words = top_n_words\n self.nr_repr_docs = nr_repr_docs\n self.nr_samples = nr_samples\n self.nr_candidate_words = nr_candidate_words\n self.random_state = random_state\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n embeddings: np.ndarray = None,\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: The topic c-TF-IDF representation\n topics: The candidate topics as calculated with c-TF-IDF\n embeddings: Pre-trained document embeddings. These can be used\n instead of an embedding model\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # We extract the top n representative documents per class\n _, representative_docs, repr_doc_indices, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, self.nr_samples, self.nr_repr_docs\n )\n\n # If document embeddings are precomputed, extract the embeddings of the representative documents based on repr_doc_indices\n repr_embeddings = None\n if embeddings is not None:\n repr_embeddings = [embeddings[index] for index in np.concatenate(repr_doc_indices)]\n\n # We extract the top n words per class\n topics = self._extract_candidate_words(topic_model, c_tf_idf, topics)\n\n # We calculate the similarity between word and document embeddings and create\n # topic embeddings from the representative document embeddings\n sim_matrix, words = self._extract_embeddings(\n topic_model, topics, representative_docs, repr_doc_indices, repr_embeddings\n )\n # Find the best matching words based on the similarity matrix for each topic\n updated_topics = self._extract_top_words(words, topics, sim_matrix)\n\n return updated_topics\n\n def _extract_candidate_words(\n self,\n topic_model,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"For each topic, extract candidate words based on the c-TF-IDF\n representation.\n\n Arguments:\n topic_model: A BERTopic model\n c_tf_idf: The topic c-TF-IDF representation\n topics: The top words per topic\n\n Returns:\n topics: The `self.top_n_words` per topic\n \"\"\"\n labels = [int(label) for label in sorted(list(topics.keys()))]\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = topic_model.vectorizer_model.get_feature_names_out()\n else:\n words = topic_model.vectorizer_model.get_feature_names()\n\n indices = topic_model._top_n_idx_sparse(c_tf_idf, self.nr_candidate_words)\n scores = topic_model._top_n_values_sparse(c_tf_idf, indices)\n sorted_indices = np.argsort(scores, 1)\n indices = np.take_along_axis(indices, sorted_indices, axis=1)\n scores = np.take_along_axis(scores, sorted_indices, axis=1)\n\n # Get top 30 words per topic based on c-TF-IDF score\n topics = {\n label: [\n (words[word_index], score) if word_index is not None and score > 0 else (\"\", 0.00001)\n for word_index, score in zip(indices[index][::-1], scores[index][::-1])\n ]\n for index, label in enumerate(labels)\n }\n topics = {label: next(zip(*values[: self.nr_candidate_words])) for label, values in topics.items()}\n\n return topics\n\n def _extract_embeddings(\n self,\n topic_model,\n topics: Mapping[str, List[Tuple[str, float]]],\n representative_docs: List[str],\n repr_doc_indices: List[List[int]],\n repr_embeddings: np.ndarray = None,\n ) -> Union[np.ndarray, List[str]]:\n \"\"\"Extract the representative document embeddings and create topic embeddings.\n Then extract word embeddings and calculate the cosine similarity between topic\n embeddings and the word embeddings. Topic embeddings are the average of\n representative document embeddings.\n\n Arguments:\n topic_model: A BERTopic model\n topics: The top words per topic\n representative_docs: A flat list of representative documents\n repr_doc_indices: The indices of representative documents\n that belong to each topic\n repr_embeddings: Embeddings of respective representative_docs\n\n Returns:\n sim: The similarity matrix between word and topic embeddings\n vocab: The complete vocabulary of input documents\n \"\"\"\n # Calculate representative document embeddings if there are no precomputed embeddings.\n if repr_embeddings is None:\n repr_embeddings = topic_model._extract_embeddings(representative_docs, method=\"document\", verbose=False)\n\n topic_embeddings = [np.mean(repr_embeddings[i[0] : i[-1] + 1], axis=0) for i in repr_doc_indices]\n\n # Calculate word embeddings and extract best matching with updated topic_embeddings\n vocab = list(set([word for words in topics.values() for word in words]))\n word_embeddings = topic_model._extract_embeddings(vocab, method=\"document\", verbose=False)\n sim = cosine_similarity(topic_embeddings, word_embeddings)\n\n return sim, vocab\n\n def _extract_top_words(\n self,\n vocab: List[str],\n topics: Mapping[str, List[Tuple[str, float]]],\n sim: np.ndarray,\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract the top n words per topic based on the\n similarity matrix between topics and words.\n\n Arguments:\n vocab: The complete vocabulary of input documents\n labels: All topic labels\n topics: The top words per topic\n sim: The similarity matrix between word and topic embeddings\n\n Returns:\n updated_topics: The updated topic representations\n \"\"\"\n labels = [int(label) for label in sorted(list(topics.keys()))]\n updated_topics = {}\n for i, topic in enumerate(labels):\n indices = [vocab.index(word) for word in topics[topic]]\n values = sim[:, indices][i]\n word_indices = [indices[index] for index in np.argsort(values)[-self.top_n_words :]]\n updated_topics[topic] = [\n (vocab[index], val) for val, index in zip(np.sort(values)[-self.top_n_words :], word_indices)\n ][::-1]\n\n return updated_topics\n" }, { "path": "bertopic/representation/_langchain.py", "content": "import pandas as pd\nfrom langchain.docstore.document import Document\nfrom scipy.sparse import csr_matrix\nfrom typing import Callable, Mapping, List, Tuple, Union\n\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import truncate_document, validate_truncate_document_parameters\n\nDEFAULT_PROMPT = \"What are these documents about? Please give a single label.\"\n\n\nclass LangChain(BaseRepresentation):\n \"\"\"Using chains in langchain to generate topic labels.\n\n The classic example uses `langchain.chains.question_answering.load_qa_chain`.\n This returns a chain that takes a list of documents and a question as input.\n\n You can also use Runnables such as those composed using the LangChain Expression Language.\n\n Arguments:\n chain: The langchain chain or Runnable with a `batch` method.\n Input keys must be `input_documents` and `question`.\n Output key must be `output_text`.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` in the prompt\n to decide where the keywords need to be\n inserted. Keywords won't be included unless\n indicated. Unlike other representation models,\n Langchain does not use the `\"[DOCUMENTS]\"` tag\n to insert documents into the prompt. The load_qa_chain function\n formats the representative documents within the prompt.\n nr_docs: The number of documents to pass to LangChain\n diversity: The diversity of documents to pass to LangChain.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`. They are decoded with\n whitespaces.\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n chain_config: The configuration for the langchain chain. Can be used to set options\n like max_concurrency to avoid rate limiting errors.\n Usage:\n\n To use this, you will need to install the langchain package first.\n Additionally, you will need an underlying LLM to support langchain,\n like openai:\n\n `pip install langchain`\n `pip install openai`\n\n Then, you can create your chain as follows:\n\n ```python\n from langchain.chains.question_answering import load_qa_chain\n from langchain.llms import OpenAI\n chain = load_qa_chain(OpenAI(temperature=0, openai_api_key=my_openai_api_key), chain_type=\"stuff\")\n ```\n\n Finally, you can pass the chain to BERTopic as follows:\n\n ```python\n from bertopic.representation import LangChain\n\n # Create your representation model\n representation_model = LangChain(chain)\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n\n You can also use a custom prompt:\n\n ```python\n prompt = \"What are these documents about? Please give a single label.\"\n representation_model = LangChain(chain, prompt=prompt)\n ```\n\n You can also use a Runnable instead of a chain.\n The example below uses the LangChain Expression Language:\n\n ```python\n from bertopic.representation import LangChain\n from langchain.chains.question_answering import load_qa_chain\n from langchain.chat_models import ChatAnthropic\n from langchain.schema.document import Document\n from langchain.schema.runnable import RunnablePassthrough\n from langchain_experimental.data_anonymizer.presidio import PresidioReversibleAnonymizer\n\n prompt = ...\n llm = ...\n\n # We will construct a special privacy-preserving chain using Microsoft Presidio\n\n pii_handler = PresidioReversibleAnonymizer(analyzed_fields=[\"PERSON\"])\n\n chain = (\n {\n \"input_documents\": (\n lambda inp: [\n Document(\n page_content=pii_handler.anonymize(\n d.page_content,\n language=\"en\",\n ),\n )\n for d in inp[\"input_documents\"]\n ]\n ),\n \"question\": RunnablePassthrough(),\n }\n | load_qa_chain(representation_llm, chain_type=\"stuff\")\n | (lambda output: {\"output_text\": pii_handler.deanonymize(output[\"output_text\"])})\n )\n\n representation_model = LangChain(chain, prompt=representation_prompt)\n ```\n \"\"\"\n\n def __init__(\n self,\n chain,\n prompt: str | None = None,\n nr_docs: int = 4,\n diversity: float | None = None,\n doc_length: int | None = None,\n tokenizer: Union[str, Callable] | None = None,\n chain_config=None,\n ):\n self.chain = chain\n self.prompt = prompt if prompt is not None else DEFAULT_PROMPT\n self.default_prompt_ = DEFAULT_PROMPT\n self.chain_config = chain_config\n self.nr_docs = nr_docs\n self.diversity = diversity\n self.doc_length = doc_length\n self.tokenizer = tokenizer\n validate_truncate_document_parameters(self.tokenizer, self.doc_length)\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, int]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: The topic c-TF-IDF representation\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top 4 representative documents per topic\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf=c_tf_idf,\n documents=documents,\n topics=topics,\n nr_samples=500,\n nr_repr_docs=self.nr_docs,\n diversity=self.diversity,\n )\n\n # Generate label using langchain's batch functionality\n chain_docs: List[List[Document]] = [\n [\n Document(page_content=truncate_document(topic_model, self.doc_length, self.tokenizer, doc))\n for doc in docs\n ]\n for docs in repr_docs_mappings.values()\n ]\n\n # `self.chain` must take `input_documents` and `question` as input keys\n # Use a custom prompt that leverages keywords, using the tag: [KEYWORDS]\n if \"[KEYWORDS]\" in self.prompt:\n prompts = []\n for topic in topics:\n keywords = next(zip(*topics[topic]))\n prompt = self.prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n prompts.append(prompt)\n\n inputs = [{\"input_documents\": docs, \"question\": prompt} for docs, prompt in zip(chain_docs, prompts)]\n\n else:\n inputs = [{\"input_documents\": docs, \"question\": self.prompt} for docs in chain_docs]\n\n # `self.chain` must return a dict with an `output_text` key\n # same output key as the `StuffDocumentsChain` returned by `load_qa_chain`\n outputs = self.chain.batch(inputs=inputs, config=self.chain_config)\n labels = [output[\"output_text\"].strip() for output in outputs]\n\n updated_topics = {\n topic: [(label, 1)] + [(\"\", 0) for _ in range(9)] for topic, label in zip(repr_docs_mappings.keys(), labels)\n }\n\n return updated_topics\n" }, { "path": "bertopic/representation/_litellm.py", "content": "import time\nfrom litellm import completion\nimport pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Any\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import retry_with_exponential_backoff\n\n\nDEFAULT_PROMPT = \"\"\"\nI have a topic that contains the following documents:\n[DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS]\nBased on the information above, extract a short topic label in the following format:\ntopic: \n\"\"\"\n\n\nclass LiteLLM(BaseRepresentation):\n \"\"\"Using the LiteLLM API to generate topic labels.\n\n For an overview of models see:\n https://docs.litellm.ai/docs/providers\n\n Arguments:\n model: Model to use. Defaults to OpenAI's \"gpt-3.5-turbo\".\n generator_kwargs: Kwargs passed to `litellm.completion`.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt\n to decide where the keywords and documents need to be\n inserted.\n delay_in_seconds: The delay in seconds between consecutive prompts\n in order to prevent RateLimitErrors.\n exponential_backoff: Retry requests with a random exponential backoff.\n A short sleep is used when a rate limit error is hit,\n then the requests is retried. Increase the sleep length\n if errors are hit until 10 unsuccesfull requests.\n If True, overrides `delay_in_seconds`.\n nr_docs: The number of documents to pass to LiteLLM if a prompt\n with the `[\"DOCUMENTS\"]` tag is used.\n diversity: The diversity of documents to pass to LiteLLM.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n\n Usage:\n\n To use this, you will need to install the litellm package first:\n\n `pip install litellm`\n\n Then, get yourself an API key of any provider (for instance OpenAI) and use it as follows:\n\n ```python\n import os\n from bertopic.representation import LiteLLM\n from bertopic import BERTopic\n\n # set ENV variables\n os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\"\n\n # Create your representation model\n representation_model = LiteLLM(model=\"gpt-3.5-turbo\")\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n\n You can also use a custom prompt:\n\n ```python\n prompt = \"I have the following documents: [DOCUMENTS] \\nThese documents are about the following topic: '\"\n representation_model = LiteLLM(model=\"gpt\", prompt=prompt)\n ```\n \"\"\" # noqa: D301\n\n def __init__(\n self,\n model: str = \"gpt-3.5-turbo\",\n prompt: str | None = None,\n generator_kwargs: Mapping[str, Any] = {},\n delay_in_seconds: float | None = None,\n exponential_backoff: bool = False,\n nr_docs: int = 4,\n diversity: float | None = None,\n ):\n self.model = model\n self.prompt = prompt if prompt else DEFAULT_PROMPT\n self.default_prompt_ = DEFAULT_PROMPT\n self.delay_in_seconds = delay_in_seconds\n self.exponential_backoff = exponential_backoff\n self.nr_docs = nr_docs\n self.diversity = diversity\n\n self.generator_kwargs = generator_kwargs\n if self.generator_kwargs.get(\"model\"):\n self.model = generator_kwargs.get(\"model\")\n if self.generator_kwargs.get(\"prompt\"):\n del self.generator_kwargs[\"prompt\"]\n\n def extract_topics(\n self, topic_model, documents: pd.DataFrame, c_tf_idf: csr_matrix, topics: Mapping[str, List[Tuple[str, float]]]\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: The topic c-TF-IDF representation\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top n representative documents per topic\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity\n )\n\n # Generate using a (Large) Language Model\n updated_topics = {}\n for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):\n prompt = self._create_prompt(docs, topic, topics)\n\n # Delay\n if self.delay_in_seconds:\n time.sleep(self.delay_in_seconds)\n\n messages = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": prompt},\n ]\n kwargs = {\"model\": self.model, \"messages\": messages, **self.generator_kwargs}\n if self.exponential_backoff:\n response = chat_completions_with_backoff(**kwargs)\n else:\n response = completion(**kwargs)\n label = response[\"choices\"][0][\"message\"][\"content\"].strip().replace(\"topic: \", \"\")\n\n updated_topics[topic] = [(label, 1)]\n\n return updated_topics\n\n def _create_prompt(self, docs, topic, topics):\n keywords = next(zip(*topics[topic]))\n\n # Use the Default Chat Prompt\n if self.prompt == DEFAULT_PROMPT:\n prompt = self.prompt.replace(\"[KEYWORDS]\", \" \".join(keywords))\n prompt = self._replace_documents(prompt, docs)\n\n # Use a custom prompt that leverages keywords, documents or both using\n # custom tags, namely [KEYWORDS] and [DOCUMENTS] respectively\n else:\n prompt = self.prompt\n if \"[KEYWORDS]\" in prompt:\n prompt = prompt.replace(\"[KEYWORDS]\", \" \".join(keywords))\n if \"[DOCUMENTS]\" in prompt:\n prompt = self._replace_documents(prompt, docs)\n\n return prompt\n\n @staticmethod\n def _replace_documents(prompt, docs):\n to_replace = \"\"\n for doc in docs:\n to_replace += f\"- {doc[:255]}\\n\"\n prompt = prompt.replace(\"[DOCUMENTS]\", to_replace)\n return prompt\n\n\ndef chat_completions_with_backoff(**kwargs):\n return retry_with_exponential_backoff(\n completion,\n )(**kwargs)\n" }, { "path": "bertopic/representation/_llamacpp.py", "content": "import pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom llama_cpp import Llama\nfrom typing import Mapping, List, Tuple, Any, Union, Callable\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import truncate_document, validate_truncate_document_parameters\n\n\nDEFAULT_PROMPT = \"\"\"\nThis is a list of texts where each collection of texts describe a topic. After each collection of texts, the name of the topic they represent is mentioned as a short-highly-descriptive title\n---\nTopic:\nSample texts from this topic:\n- Traditional diets in most cultures were primarily plant-based with a little meat on top, but with the rise of industrial style meat production and factory farming, meat has become a staple food.\n- Meat, but especially beef, is the word food in terms of emissions.\n- Eating meat doesn't make you a bad person, not eating meat doesn't make you a good one.\n\nKeywords: meat beef eat eating emissions steak food health processed chicken\nTopic name: Environmental impacts of eating meat\n---\nTopic:\nSample texts from this topic:\n- I have ordered the product weeks ago but it still has not arrived!\n- The website mentions that it only takes a couple of days to deliver but I still have not received mine.\n- I got a message stating that I received the monitor but that is not true!\n- It took a month longer to deliver than was advised...\n\nKeywords: deliver weeks product shipping long delivery received arrived arrive week\nTopic name: Shipping and delivery issues\n---\nTopic:\nSample texts from this topic:\n[DOCUMENTS]\nKeywords: [KEYWORDS]\nTopic name:\"\"\"\n\nDEFAULT_SYSTEM_PROMPT = \"You are an assistant that extracts high-level topics from texts.\"\n\n\nclass LlamaCPP(BaseRepresentation):\n \"\"\"A llama.cpp implementation to use as a representation model.\n\n Arguments:\n model: Either a string pointing towards a local LLM or a\n `llama_cpp.Llama` object.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt\n to decide where the keywords and documents need to be\n inserted.\n system_prompt: The system prompt to be used in the model. If no system prompt is given,\n `self.default_system_prompt_` is used instead.\n pipeline_kwargs: Kwargs that you can pass to the `llama_cpp.Llama`\n when it is called such as `max_tokens` to be generated.\n nr_docs: The number of documents to pass to OpenAI if a prompt\n with the `[\"DOCUMENTS\"]` tag is used.\n diversity: The diversity of documents to pass to OpenAI.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n\n Usage:\n\n To use a llama.cpp, first download the LLM:\n\n ```bash\n wget https://huggingface.co/TheBloke/zephyr-7B-alpha-GGUF/resolve/main/zephyr-7b-alpha.Q4_K_M.gguf\n ```\n\n Then, we can now use the model the model with BERTopic in just a couple of lines:\n\n ```python\n from bertopic import BERTopic\n from bertopic.representation import LlamaCPP\n\n # Use llama.cpp to load in a 4-bit quantized version of Zephyr 7B Alpha\n representation_model = LlamaCPP(\"zephyr-7b-alpha.Q4_K_M.gguf\")\n\n # Create our BERTopic model\n topic_model = BERTopic(representation_model=representation_model, verbose=True)\n ```\n\n If you want to have more control over the LLMs parameters, you can run it like so:\n\n ```python\n from bertopic import BERTopic\n from bertopic.representation import LlamaCPP\n from llama_cpp import Llama\n\n # Use llama.cpp to load in a 4-bit quantized version of Zephyr 7B Alpha\n llm = Llama(model_path=\"zephyr-7b-alpha.Q4_K_M.gguf\", n_gpu_layers=-1, n_ctx=4096, stop=\"Q:\")\n representation_model = LlamaCPP(llm)\n\n # Create our BERTopic model\n topic_model = BERTopic(representation_model=representation_model, verbose=True)\n ```\n \"\"\"\n\n def __init__(\n self,\n model: Union[str, Llama],\n prompt: str | None = None,\n system_prompt: str | None = None,\n pipeline_kwargs: Mapping[str, Any] = {},\n nr_docs: int = 4,\n diversity: float | None = None,\n doc_length: int | None = None,\n tokenizer: Union[str, Callable] | None = None,\n ):\n if isinstance(model, str):\n self.model = Llama(model_path=model, n_gpu_layers=-1, stop=\"\\n\", chat_format=\"ChatML\")\n elif isinstance(model, Llama):\n self.model = model\n else:\n raise ValueError(\n \"Make sure that the model that you\"\n \"pass is either a string referring to a\"\n \"local LLM or a ` llama_cpp.Llama` object.\"\n )\n self.prompt = prompt if prompt is not None else DEFAULT_PROMPT\n self.system_prompt = system_prompt if system_prompt is not None else DEFAULT_SYSTEM_PROMPT\n self.default_prompt_ = DEFAULT_PROMPT\n self.default_system_prompt_ = DEFAULT_SYSTEM_PROMPT\n self.pipeline_kwargs = pipeline_kwargs\n self.nr_docs = nr_docs\n self.diversity = diversity\n self.doc_length = doc_length\n self.tokenizer = tokenizer\n validate_truncate_document_parameters(self.tokenizer, self.doc_length)\n\n self.prompts_ = []\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topic representations and return a single label.\n\n Arguments:\n topic_model: A BERTopic model\n documents: Not used\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top 4 representative documents per topic\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity\n )\n\n updated_topics = {}\n for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):\n # Prepare prompt\n truncated_docs = [truncate_document(topic_model, self.doc_length, self.tokenizer, doc) for doc in docs]\n prompt = self._create_prompt(truncated_docs, topic, topics)\n self.prompts_.append(prompt)\n\n # Extract result from generator and use that as label\n # topic_description = self.model(prompt, **self.pipeline_kwargs)[\"choices\"]\n topic_description = self.model.create_chat_completion(\n messages=[{\"role\": \"system\", \"content\": self.system_prompt}, {\"role\": \"user\", \"content\": prompt}],\n **self.pipeline_kwargs,\n )\n label = topic_description[\"choices\"][0][\"message\"][\"content\"].strip()\n updated_topics[topic] = [(label, 1)] + [(\"\", 0) for _ in range(9)]\n\n return updated_topics\n\n def _create_prompt(self, docs, topic, topics):\n keywords = next(zip(*topics[topic]))\n\n # Use the Default Chat Prompt\n if self.prompt == DEFAULT_PROMPT:\n prompt = self.prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n prompt = self._replace_documents(prompt, docs)\n\n # Use a custom prompt that leverages keywords, documents or both using\n # custom tags, namely [KEYWORDS] and [DOCUMENTS] respectively\n else:\n prompt = self.prompt\n if \"[KEYWORDS]\" in prompt:\n prompt = prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n if \"[DOCUMENTS]\" in prompt:\n prompt = self._replace_documents(prompt, docs)\n\n return prompt\n\n @staticmethod\n def _replace_documents(prompt, docs):\n to_replace = \"\"\n for doc in docs:\n to_replace += f\"- {doc}\\n\"\n prompt = prompt.replace(\"[DOCUMENTS]\", to_replace)\n return prompt\n" }, { "path": "bertopic/representation/_mmr.py", "content": "import warnings\nimport numpy as np\nimport pandas as pd\nfrom typing import List, Mapping, Tuple\nfrom scipy.sparse import csr_matrix\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom bertopic.representation._base import BaseRepresentation\n\n\nclass MaximalMarginalRelevance(BaseRepresentation):\n \"\"\"Calculate Maximal Marginal Relevance (MMR)\n between candidate keywords and the document.\n\n MMR considers the similarity of keywords/keyphrases with the\n document, along with the similarity of already selected\n keywords and keyphrases. This results in a selection of keywords\n that maximize their within diversity with respect to the document.\n\n Arguments:\n diversity: How diverse the select keywords/keyphrases are.\n Values range between 0 and 1 with 0 being not diverse at all\n and 1 being most diverse.\n top_n_words: The number of keywords/keyhprases to return\n\n Usage:\n\n ```python\n from bertopic.representation import MaximalMarginalRelevance\n from bertopic import BERTopic\n\n # Create your representation model\n representation_model = MaximalMarginalRelevance(diversity=0.3)\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n \"\"\"\n\n def __init__(self, diversity: float = 0.1, top_n_words: int = 10):\n self.diversity = diversity\n self.top_n_words = top_n_words\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topic representations.\n\n Arguments:\n topic_model: The BERTopic model\n documents: Not used\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n if topic_model.embedding_model is None:\n warnings.warn(\n \"MaximalMarginalRelevance can only be used BERTopic was instantiated\"\n \"with the `embedding_model` parameter.\"\n )\n return topics\n\n updated_topics = {}\n for topic, topic_words in topics.items():\n words = [word[0] for word in topic_words]\n word_embeddings = topic_model._extract_embeddings(words, method=\"word\", verbose=False)\n topic_embedding = topic_model._extract_embeddings(\" \".join(words), method=\"word\", verbose=False).reshape(\n 1, -1\n )\n topic_words = mmr(\n topic_embedding,\n word_embeddings,\n words,\n self.diversity,\n self.top_n_words,\n )\n updated_topics[topic] = [(word, value) for word, value in topics[topic] if word in topic_words]\n return updated_topics\n\n\ndef mmr(\n doc_embedding: np.ndarray,\n word_embeddings: np.ndarray,\n words: List[str],\n diversity: float = 0.1,\n top_n: int = 10,\n) -> List[str]:\n \"\"\"Maximal Marginal Relevance.\n\n Arguments:\n doc_embedding: The document embeddings\n word_embeddings: The embeddings of the selected candidate keywords/phrases\n words: The selected candidate keywords/keyphrases\n diversity: The diversity of the selected embeddings.\n Values between 0 and 1.\n top_n: The top n items to return\n\n Returns:\n List[str]: The selected keywords/keyphrases\n \"\"\"\n # Extract similarity within words, and between words and the document\n word_doc_similarity = cosine_similarity(word_embeddings, doc_embedding)\n word_similarity = cosine_similarity(word_embeddings)\n\n # Initialize candidates and already choose best keyword/keyphras\n keywords_idx = [np.argmax(word_doc_similarity)]\n candidates_idx = [i for i in range(len(words)) if i != keywords_idx[0]]\n\n for _ in range(top_n - 1):\n # Extract similarities within candidates and\n # between candidates and selected keywords/phrases\n candidate_similarities = word_doc_similarity[candidates_idx, :]\n target_similarities = np.max(word_similarity[candidates_idx][:, keywords_idx], axis=1)\n\n # Calculate MMR\n mmr = (1 - diversity) * candidate_similarities - diversity * target_similarities.reshape(-1, 1)\n mmr_idx = candidates_idx[np.argmax(mmr)]\n\n # Update keywords & candidates\n keywords_idx.append(mmr_idx)\n candidates_idx.remove(mmr_idx)\n\n return [words[idx] for idx in keywords_idx]\n" }, { "path": "bertopic/representation/_openai.py", "content": "import time\nimport openai\nimport pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Any, Union, Callable\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import (\n retry_with_exponential_backoff,\n truncate_document,\n validate_truncate_document_parameters,\n)\n\n\nDEFAULT_CHAT_PROMPT = \"\"\"You will extract a short topic label from given documents and keywords.\nHere are two examples of topics you created before:\n\n# Example 1\nSample texts from this topic:\n- Traditional diets in most cultures were primarily plant-based with a little meat on top, but with the rise of industrial style meat production and factory farming, meat has become a staple food.\n- Meat, but especially beef, is the worst food in terms of emissions.\n- Eating meat doesn't make you a bad person, not eating meat doesn't make you a good one.\n\nKeywords: meat beef eat eating emissions steak food health processed chicken\ntopic: Environmental impacts of eating meat\n\n# Example 2\nSample texts from this topic:\n- I have ordered the product weeks ago but it still has not arrived!\n- The website mentions that it only takes a couple of days to deliver but I still have not received mine.\n- I got a message stating that I received the monitor but that is not true!\n- It took a month longer to deliver than was advised...\n\nKeywords: deliver weeks product shipping long delivery received arrived arrive week\ntopic: Shipping and delivery issues\n\n# Your task\nSample texts from this topic:\n[DOCUMENTS]\n\nKeywords: [KEYWORDS]\n\nBased on the information above, extract a short topic label (three words at most) in the following format:\ntopic: \n\"\"\"\n\nDEFAULT_SYSTEM_PROMPT = \"You are an assistant that extracts high-level topics from texts.\"\n\n\nclass OpenAI(BaseRepresentation):\n r\"\"\"Using the OpenAI API to generate topic labels based\n on one of their Completion of ChatCompletion models.\n\n For an overview see:\n https://platform.openai.com/docs/models\n\n Arguments:\n client: A `openai.OpenAI` client\n model: Model to use within OpenAI, defaults to `\"gpt-4o-mini\"`.\n generator_kwargs: Kwargs passed to `openai.Completion.create`\n for fine-tuning the output.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt\n to decide where the keywords and documents need to be\n inserted.\n system_prompt: The system prompt to be used in the model. If no system prompt is given,\n `self.default_system_prompt_` is used instead.\n delay_in_seconds: The delay in seconds between consecutive prompts\n in order to prevent RateLimitErrors.\n exponential_backoff: Retry requests with a random exponential backoff.\n A short sleep is used when a rate limit error is hit,\n then the requests is retried. Increase the sleep length\n if errors are hit until 10 unsuccessful requests.\n If True, overrides `delay_in_seconds`.\n nr_docs: The number of documents to pass to OpenAI if a prompt\n with the `[\"DOCUMENTS\"]` tag is used.\n diversity: The diversity of documents to pass to OpenAI.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n\n Usage:\n\n To use this, you will need to install the openai package first:\n\n `pip install openai`\n\n Then, get yourself an API key and use OpenAI's API as follows:\n\n ```python\n import openai\n from bertopic.representation import OpenAI\n from bertopic import BERTopic\n\n # Create your representation model\n client = openai.OpenAI(api_key=MY_API_KEY)\n representation_model = OpenAI(client, delay_in_seconds=5)\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n\n You can also use a custom prompt:\n\n ```python\n prompt = \"I have the following documents: [DOCUMENTS] \\nThese documents are about the following topic: '\"\n representation_model = OpenAI(client, prompt=prompt, delay_in_seconds=5)\n ```\n\n To choose a model:\n\n ```python\n representation_model = OpenAI(client, model=\"gpt-4o-mini\", delay_in_seconds=10)\n ```\n \"\"\"\n\n def __init__(\n self,\n client,\n model: str = \"gpt-4o-mini\",\n prompt: str | None = None,\n system_prompt: str | None = None,\n generator_kwargs: Mapping[str, Any] = {},\n delay_in_seconds: float | None = None,\n exponential_backoff: bool = False,\n nr_docs: int = 4,\n diversity: float | None = None,\n doc_length: int | None = None,\n tokenizer: Union[str, Callable] | None = None,\n **kwargs,\n ):\n self.client = client\n self.model = model\n\n if prompt is None:\n self.prompt = DEFAULT_CHAT_PROMPT\n else:\n self.prompt = prompt\n\n if system_prompt is None:\n self.system_prompt = DEFAULT_SYSTEM_PROMPT\n else:\n self.system_prompt = system_prompt\n\n self.default_prompt_ = DEFAULT_CHAT_PROMPT\n self.default_system_prompt_ = DEFAULT_SYSTEM_PROMPT\n self.delay_in_seconds = delay_in_seconds\n self.exponential_backoff = exponential_backoff\n self.nr_docs = nr_docs\n self.diversity = diversity\n self.doc_length = doc_length\n self.tokenizer = tokenizer\n validate_truncate_document_parameters(self.tokenizer, self.doc_length)\n\n self.prompts_ = []\n\n self.generator_kwargs = generator_kwargs\n if self.generator_kwargs.get(\"model\"):\n self.model = generator_kwargs.get(\"model\")\n del self.generator_kwargs[\"model\"]\n if self.generator_kwargs.get(\"prompt\"):\n del self.generator_kwargs[\"prompt\"]\n if not self.generator_kwargs.get(\"stop\"):\n self.generator_kwargs[\"stop\"] = \"\\n\"\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: The topic c-TF-IDF representation\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top n representative documents per topic\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity\n )\n\n # Generate using OpenAI's Language Model\n updated_topics = {}\n for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):\n truncated_docs = [truncate_document(topic_model, self.doc_length, self.tokenizer, doc) for doc in docs]\n prompt = self._create_prompt(truncated_docs, topic, topics)\n self.prompts_.append(prompt)\n\n # Delay\n if self.delay_in_seconds:\n time.sleep(self.delay_in_seconds)\n\n messages = [\n {\"role\": \"system\", \"content\": self.system_prompt},\n {\"role\": \"user\", \"content\": prompt},\n ]\n kwargs = {\n \"model\": self.model,\n \"messages\": messages,\n **self.generator_kwargs,\n }\n if self.exponential_backoff:\n response = chat_completions_with_backoff(self.client, **kwargs)\n else:\n response = self.client.chat.completions.create(**kwargs)\n\n # Check whether content was actually generated\n # Addresses #1570 for potential issues with OpenAI's content filter\n # Addresses #2176 for potential issues when openAI returns a None type object\n if response and hasattr(response.choices[0].message, \"content\"):\n label = response.choices[0].message.content.strip().replace(\"topic: \", \"\")\n else:\n label = \"No label returned\"\n\n updated_topics[topic] = [(label, 1)]\n\n return updated_topics\n\n def _create_prompt(self, docs, topic, topics):\n keywords = next(zip(*topics[topic]))\n\n # Use the Default Chat Prompt\n if self.prompt == DEFAULT_CHAT_PROMPT:\n prompt = self.prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n prompt = self._replace_documents(prompt, docs)\n\n # Use a custom prompt that leverages keywords, documents or both using\n # custom tags, namely [KEYWORDS] and [DOCUMENTS] respectively\n else:\n prompt = self.prompt\n if \"[KEYWORDS]\" in prompt:\n prompt = prompt.replace(\"[KEYWORDS]\", \", \".join(keywords))\n if \"[DOCUMENTS]\" in prompt:\n prompt = self._replace_documents(prompt, docs)\n\n return prompt\n\n @staticmethod\n def _replace_documents(prompt, docs):\n to_replace = \"\"\n for doc in docs:\n to_replace += f\"- {doc}\\n\"\n prompt = prompt.replace(\"[DOCUMENTS]\", to_replace)\n return prompt\n\n\ndef chat_completions_with_backoff(client, **kwargs):\n return retry_with_exponential_backoff(\n client.chat.completions.create,\n errors=(openai.RateLimitError,),\n )(**kwargs)\n" }, { "path": "bertopic/representation/_pos.py", "content": "import numpy as np\nimport pandas as pd\n\nimport spacy\nfrom spacy.matcher import Matcher\nfrom spacy.language import Language\n\nfrom packaging import version\nfrom scipy.sparse import csr_matrix\nfrom typing import List, Mapping, Tuple, Union\nfrom sklearn import __version__ as sklearn_version\nfrom bertopic.representation._base import BaseRepresentation\n\n\nclass PartOfSpeech(BaseRepresentation):\n \"\"\"Extract Topic Keywords based on their Part-of-Speech.\n\n DEFAULT_PATTERNS = [\n [{'POS': 'ADJ'}, {'POS': 'NOUN'}],\n [{'POS': 'NOUN'}],\n [{'POS': 'ADJ'}]\n ]\n\n From candidate topics, as extracted with c-TF-IDF,\n find documents that contain keywords found in the\n candidate topics. These candidate documents then\n serve as the representative set of documents from\n which the Spacy model can extract a set of candidate\n keywords for each topic.\n\n These candidate keywords are first judged by whether\n they fall within the DEFAULT_PATTERNS or the user-defined\n pattern. Then, the resulting keywords are sorted by\n their respective c-TF-IDF values.\n\n Arguments:\n model: The Spacy model to use\n top_n_words: The top n words to extract\n pos_patterns: Patterns for Spacy to use.\n See https://spacy.io/usage/rule-based-matching\n\n Usage:\n\n ```python\n from bertopic.representation import PartOfSpeech\n from bertopic import BERTopic\n\n # Create your representation model\n representation_model = PartOfSpeech(\"en_core_web_sm\")\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n\n You can define custom POS patterns to be extracted:\n\n ```python\n pos_patterns = [\n [{'POS': 'ADJ'}, {'POS': 'NOUN'}],\n [{'POS': 'NOUN'}], [{'POS': 'ADJ'}]\n ]\n representation_model = PartOfSpeech(\"en_core_web_sm\", pos_patterns=pos_patterns)\n ```\n \"\"\"\n\n def __init__(\n self,\n model: Union[str, Language] = \"en_core_web_sm\",\n top_n_words: int = 10,\n pos_patterns: List[str] | None = None,\n ):\n if isinstance(model, str):\n self.model = spacy.load(model)\n elif isinstance(model, Language):\n self.model = model\n else:\n raise ValueError(\n \"Make sure that the Spacy model that you\"\n \"pass is either a string referring to a\"\n \"Spacy model or a Spacy nlp object.\"\n )\n\n self.top_n_words = top_n_words\n\n if pos_patterns is None:\n self.pos_patterns = [\n [{\"POS\": \"ADJ\"}, {\"POS\": \"NOUN\"}],\n [{\"POS\": \"NOUN\"}],\n [{\"POS\": \"ADJ\"}],\n ]\n else:\n self.pos_patterns = pos_patterns\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n matcher = Matcher(self.model.vocab)\n matcher.add(\"Pattern\", self.pos_patterns)\n\n candidate_topics = {}\n for topic, values in topics.items():\n keywords = next(zip(*values))\n\n # Extract candidate documents\n candidate_documents = []\n for keyword in keywords:\n selection = documents.loc[documents.Topic == topic, :]\n selection = selection.loc[selection.Document.str.contains(keyword, regex=False), \"Document\"]\n if len(selection) > 0:\n for document in selection[:2]:\n candidate_documents.append(document)\n candidate_documents = list(set(candidate_documents))\n\n # Extract keywords\n docs_pipeline = self.model.pipe(candidate_documents)\n updated_keywords = []\n for doc in docs_pipeline:\n matches = matcher(doc)\n for _, start, end in matches:\n updated_keywords.append(doc[start:end].text)\n candidate_topics[topic] = list(set(updated_keywords))\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = list(topic_model.vectorizer_model.get_feature_names_out())\n else:\n words = list(topic_model.vectorizer_model.get_feature_names())\n\n # Match updated keywords with c-TF-IDF values\n words_lookup = dict(zip(words, range(len(words))))\n updated_topics = {topic: [] for topic in topics.keys()}\n\n for topic, candidate_keywords in candidate_topics.items():\n word_indices = np.sort(\n [words_lookup.get(keyword) for keyword in candidate_keywords if keyword in words_lookup]\n )\n vals = topic_model.c_tf_idf_[:, word_indices][topic + topic_model._outliers]\n indices = np.argsort(np.array(vals.todense().reshape(1, -1))[0])[-self.top_n_words :][::-1]\n vals = np.sort(np.array(vals.todense().reshape(1, -1))[0])[-self.top_n_words :][::-1]\n topic_words = [(words[word_indices[index]], val) for index, val in zip(indices, vals)]\n updated_topics[topic] = topic_words\n if len(updated_topics[topic]) < self.top_n_words:\n updated_topics[topic] += [(\"\", 0) for _ in range(self.top_n_words - len(updated_topics[topic]))]\n\n return updated_topics\n" }, { "path": "bertopic/representation/_textgeneration.py", "content": "import pandas as pd\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom transformers import pipeline, set_seed\nfrom transformers.pipelines.base import Pipeline\nfrom typing import Mapping, List, Tuple, Any, Union, Callable\nfrom bertopic.representation._base import BaseRepresentation\nfrom bertopic.representation._utils import truncate_document, validate_truncate_document_parameters\n\n\nDEFAULT_PROMPT = \"\"\"\nI have a topic described by the following keywords: [KEYWORDS].\nThe name of this topic is:\n\"\"\"\n\n\nclass TextGeneration(BaseRepresentation):\n \"\"\"Text2Text or text generation with transformers.\n\n Arguments:\n model: A transformers pipeline that should be initialized as \"text-generation\"\n for gpt-like models or \"text2text-generation\" for T5-like models.\n For example, `pipeline('text-generation', model='gpt2')`. If a string\n is passed, \"text-generation\" will be selected by default.\n prompt: The prompt to be used in the model. If no prompt is given,\n `self.default_prompt_` is used instead.\n NOTE: Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt\n to decide where the keywords and documents need to be\n inserted.\n pipeline_kwargs: Kwargs that you can pass to the transformers.pipeline\n when it is called.\n random_state: A random state to be passed to `transformers.set_seed`\n nr_docs: The number of documents to pass to OpenAI if a prompt\n with the `[\"DOCUMENTS\"]` tag is used.\n diversity: The diversity of documents to pass to OpenAI.\n Accepts values between 0 and 1. A higher\n values results in passing more diverse documents\n whereas lower values passes more similar documents.\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n\n Usage:\n\n To use a gpt-like model:\n\n ```python\n from bertopic.representation import TextGeneration\n from bertopic import BERTopic\n\n # Create your representation model\n generator = pipeline('text-generation', model='gpt2')\n representation_model = TextGeneration(generator)\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTo pic(representation_model=representation_model)\n ```\n\n You can use a custom prompt and decide where the keywords should\n be inserted by using the `[KEYWORDS]` or documents with thte `[DOCUMENTS]` tag:\n\n ```python\n from bertopic.representation import TextGeneration\n\n prompt = \"I have a topic described by the following keywords: [KEYWORDS]. Based on the previous keywords, what is this topic about?\"\"\n\n # Create your representation model\n generator = pipeline('text2text-generation', model='google/flan-t5-base')\n representation_model = TextGeneration(generator)\n ```\n \"\"\"\n\n def __init__(\n self,\n model: Union[str, pipeline],\n prompt: str | None = None,\n pipeline_kwargs: Mapping[str, Any] = {},\n random_state: int = 42,\n nr_docs: int = 4,\n diversity: float | None = None,\n doc_length: int | None = None,\n tokenizer: Union[str, Callable] | None = None,\n ):\n self.random_state = random_state\n set_seed(random_state)\n if isinstance(model, str):\n self.model = pipeline(\"text-generation\", model=model)\n elif isinstance(model, Pipeline):\n self.model = model\n else:\n raise ValueError(\n \"Make sure that the HF model that you\"\n \"pass is either a string referring to a\"\n \"HF model or a `transformers.pipeline` object.\"\n )\n self.prompt = prompt if prompt is not None else DEFAULT_PROMPT\n self.default_prompt_ = DEFAULT_PROMPT\n self.pipeline_kwargs = pipeline_kwargs\n self.nr_docs = nr_docs\n self.diversity = diversity\n self.doc_length = doc_length\n self.tokenizer = tokenizer\n validate_truncate_document_parameters(self.tokenizer, self.doc_length)\n\n self.prompts_ = []\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topic representations and return a single label.\n\n Arguments:\n topic_model: A BERTopic model\n documents: Not used\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Extract the top 4 representative documents per topic\n if self.prompt != DEFAULT_PROMPT and \"[DOCUMENTS]\" in self.prompt:\n repr_docs_mappings, _, _, _ = topic_model._extract_representative_docs(\n c_tf_idf, documents, topics, 500, self.nr_docs, self.diversity\n )\n else:\n repr_docs_mappings = {topic: None for topic in topics.keys()}\n\n updated_topics = {}\n for topic, docs in tqdm(repr_docs_mappings.items(), disable=not topic_model.verbose):\n # Prepare prompt\n truncated_docs = (\n [truncate_document(topic_model, self.doc_length, self.tokenizer, doc) for doc in docs]\n if docs is not None\n else docs\n )\n prompt = self._create_prompt(truncated_docs, topic, topics)\n self.prompts_.append(prompt)\n\n # Extract result from generator and use that as label\n topic_description = self.model(prompt, **self.pipeline_kwargs)\n topic_description = [\n (description[\"generated_text\"].replace(prompt, \"\"), 1) for description in topic_description\n ]\n\n if len(topic_description) < 10:\n topic_description += [(\"\", 0) for _ in range(10 - len(topic_description))]\n\n updated_topics[topic] = topic_description\n\n return updated_topics\n\n def _create_prompt(self, docs, topic, topics):\n keywords = \", \".join(next(zip(*topics[topic])))\n\n # Use the default prompt and replace keywords\n if self.prompt == DEFAULT_PROMPT:\n prompt = self.prompt.replace(\"[KEYWORDS]\", keywords)\n\n # Use a prompt that leverages either keywords or documents in\n # a custom location\n else:\n prompt = self.prompt\n if \"[KEYWORDS]\" in prompt:\n prompt = prompt.replace(\"[KEYWORDS]\", keywords)\n if \"[DOCUMENTS]\" in prompt:\n to_replace = \"\"\n for doc in docs:\n to_replace += f\"- {doc}\\n\"\n prompt = prompt.replace(\"[DOCUMENTS]\", to_replace)\n\n return prompt\n" }, { "path": "bertopic/representation/_utils.py", "content": "import random\nimport time\nfrom typing import Union\n\n\ndef truncate_document(topic_model, doc_length: Union[int, None], tokenizer: Union[str, callable], document: str) -> str:\n \"\"\"Truncate a document to a certain length.\n\n If you want to add a custom tokenizer, then it will need to have a `decode` and\n `encode` method. An example would be the following custom tokenizer:\n\n ```python\n class Tokenizer:\n 'A custom tokenizer that splits on commas'\n def encode(self, doc):\n return doc.split(\",\")\n\n def decode(self, doc_chunks):\n return \",\".join(doc_chunks)\n ```\n\n You can use this tokenizer by passing it to the `tokenizer` parameter.\n\n Arguments:\n topic_model: A BERTopic model\n doc_length: The maximum length of each document. If a document is longer,\n it will be truncated. If None, the entire document is passed.\n tokenizer: The tokenizer used to calculate to split the document into segments\n used to count the length of a document.\n * If tokenizer is 'char', then the document is split up\n into characters which are counted to adhere to `doc_length`\n * If tokenizer is 'whitespace', the document is split up\n into words separated by whitespaces. These words are counted\n and truncated depending on `doc_length`\n * If tokenizer is 'vectorizer', then the internal CountVectorizer\n is used to tokenize the document. These tokens are counted\n and truncated depending on `doc_length`. They are decoded with\n whitespaces.\n * If tokenizer is a callable, then that callable is used to tokenize\n the document. These tokens are counted and truncated depending\n on `doc_length`\n document: A single document\n\n Returns:\n truncated_document: A truncated document\n \"\"\"\n if doc_length is not None:\n if tokenizer == \"char\":\n truncated_document = document[:doc_length]\n elif tokenizer == \"whitespace\":\n truncated_document = \" \".join(document.split()[:doc_length])\n elif tokenizer == \"vectorizer\":\n tokenizer = topic_model.vectorizer_model.build_tokenizer()\n truncated_document = \" \".join(tokenizer(document)[:doc_length])\n elif hasattr(tokenizer, \"encode\") and hasattr(tokenizer, \"decode\"):\n encoded_document = tokenizer.encode(document)\n truncated_document = tokenizer.decode(encoded_document[:doc_length])\n return truncated_document\n return document\n\n\ndef validate_truncate_document_parameters(tokenizer, doc_length) -> Union[None, ValueError]:\n \"\"\"Validates parameters that are used in the function `truncate_document`.\"\"\"\n if tokenizer is None and doc_length is not None:\n raise ValueError(\n \"Please select from one of the valid options for the `tokenizer` parameter: \\n\"\n \"{'char', 'whitespace', 'vectorizer'} \\n\"\n \"If `tokenizer` is of type callable ensure it has methods to encode and decode a document \\n\"\n )\n elif tokenizer is not None and doc_length is None:\n raise ValueError(\"If `tokenizer` is provided, `doc_length` of type int must be provided as well.\")\n\n\ndef retry_with_exponential_backoff(\n func,\n initial_delay: float = 1,\n exponential_base: float = 2,\n jitter: bool = True,\n max_retries: int = 10,\n errors: tuple | None = None,\n):\n \"\"\"Retry a function with exponential backoff.\"\"\"\n\n def wrapper(*args, **kwargs):\n # Initialize variables\n num_retries = 0\n delay = initial_delay\n\n # Loop until a successful response or max_retries is hit or an exception is raised\n while True:\n try:\n return func(*args, **kwargs)\n\n # Retry on specific errors\n except errors:\n # Increment retries\n num_retries += 1\n\n # Check if max retries has been reached\n if num_retries > max_retries:\n raise Exception(f\"Maximum number of retries ({max_retries}) exceeded.\")\n\n # Increment the delay\n delay *= exponential_base * (1 + jitter * random.random())\n\n # Sleep for the delay\n time.sleep(delay)\n\n # Raise exceptions for any errors not specified\n except Exception as e:\n raise e\n\n return wrapper\n" }, { "path": "bertopic/representation/_visual.py", "content": "import numpy as np\nimport pandas as pd\n\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Union\nfrom transformers.pipelines import Pipeline, pipeline\n\nfrom bertopic.representation._mmr import mmr\nfrom bertopic.representation._base import BaseRepresentation\n\n\nclass VisualRepresentation(BaseRepresentation):\n \"\"\"From a collection of representative documents, extract\n images to represent topics. These topics are represented by a\n collage of images.\n\n Arguments:\n nr_repr_images: Number of representative images to extract\n nr_samples: The number of candidate documents to extract per cluster.\n image_height: The height of the resulting collage\n image_square: Whether to resize each image in the collage\n to a square. This can be visually more appealing\n if all input images are all almost squares.\n image_to_text_model: The model to caption images.\n batch_size: The number of images to pass to the\n `image_to_text_model`.\n\n Usage:\n\n ```python\n from bertopic.representation import VisualRepresentation\n from bertopic import BERTopic\n\n # The visual representation is typically not a core representation\n # and is advised to pass to BERTopic as an additional aspect.\n # Aspects can be labeled with dictionaries as shown below:\n representation_model = {\n \"Visual_Aspect\": VisualRepresentation()\n }\n\n # Use the representation model in BERTopic as a separate aspect\n topic_model = BERTopic(representation_model=representation_model)\n ```\n \"\"\"\n\n def __init__(\n self,\n nr_repr_images: int = 9,\n nr_samples: int = 500,\n image_height: Tuple[int, int] = 600,\n image_squares: bool = False,\n image_to_text_model: Union[str, Pipeline] = None,\n batch_size: int = 32,\n ):\n self.nr_repr_images = nr_repr_images\n self.nr_samples = nr_samples\n self.image_height = image_height\n self.image_squares = image_squares\n\n # Text-to-image model\n if isinstance(image_to_text_model, Pipeline):\n self.image_to_text_model = image_to_text_model\n elif isinstance(image_to_text_model, str):\n self.image_to_text_model = pipeline(\"image-to-text\", model=image_to_text_model)\n elif image_to_text_model is None:\n self.image_to_text_model = None\n else:\n raise ValueError(\n \"Please select a correct transformers pipeline. For example:\"\n \"pipeline('image-to-text', model='nlpconnect/vit-gpt2-image-captioning')\"\n )\n self.batch_size = batch_size\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: A BERTopic model\n documents: All input documents\n c_tf_idf: The topic c-TF-IDF representation\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n representative_images: Representative images per topic\n \"\"\"\n # Extract image ids of most representative documents\n images = documents[\"Image\"].to_numpy().tolist()\n (_, _, _, repr_docs_ids) = topic_model._extract_representative_docs(\n c_tf_idf,\n documents,\n topics,\n nr_samples=self.nr_samples,\n nr_repr_docs=self.nr_repr_images,\n )\n unique_topics = sorted(list(topics.keys()))\n\n # Combine representative images into a single representation\n representative_images = {}\n for topic in tqdm(unique_topics):\n # Get and order represetnative images\n sliced_examplars = repr_docs_ids[topic + topic_model._outliers]\n sliced_examplars = [sliced_examplars[i : i + 3] for i in range(0, len(sliced_examplars), 3)]\n images_to_combine = [\n [\n Image.open(images[index]) if isinstance(images[index], str) else images[index]\n for index in sub_indices\n ]\n for sub_indices in sliced_examplars\n ]\n\n # Concatenate representative images\n representative_image = get_concat_tile_resize(images_to_combine, self.image_height, self.image_squares)\n representative_images[topic] = representative_image\n\n # Make sure to properly close images\n if isinstance(images[0], str):\n for image_list in images_to_combine:\n for image in image_list:\n image.close()\n\n return representative_images\n\n def _convert_image_to_text(self, images: List[str], verbose: bool = False) -> List[str]:\n \"\"\"Convert a list of images to captions.\n\n Arguments:\n images: A list of images or words to be converted to text.\n verbose: Controls the verbosity of the process\n\n Returns:\n List of captions\n \"\"\"\n # Batch-wise image conversion\n if self.batch_size is not None:\n documents = []\n for batch in tqdm(self._chunks(images), disable=not verbose):\n outputs = self.image_to_text_model(batch)\n captions = [output[0][\"generated_text\"] for output in outputs]\n documents.extend(captions)\n\n # Convert images to text\n else:\n outputs = self.image_to_text_model(images)\n documents = [output[0][\"generated_text\"] for output in outputs]\n\n return documents\n\n def image_to_text(self, documents: pd.DataFrame, embeddings: np.ndarray) -> pd.DataFrame:\n \"\"\"Convert images to text.\"\"\"\n # Create image topic embeddings\n topics = documents.Topic.to_numpy().tolist()\n images = documents.Image.to_numpy().tolist()\n df = pd.DataFrame(np.hstack([np.array(topics).reshape(-1, 1), embeddings]))\n image_topic_embeddings = df.groupby(0).mean().to_numpy()\n\n # Extract image centroids\n image_centroids = {}\n unique_topics = sorted(list(set(topics)))\n for topic, topic_embedding in zip(unique_topics, image_topic_embeddings):\n indices = np.array([index for index, t in enumerate(topics) if t == topic])\n top_n = min([self.nr_repr_images, len(indices)])\n indices = mmr(\n topic_embedding.reshape(1, -1),\n embeddings[indices],\n indices,\n top_n=top_n,\n diversity=0.1,\n )\n image_centroids[topic] = indices\n\n # Extract documents\n documents = pd.DataFrame(columns=[\"Document\", \"ID\", \"Topic\", \"Image\"])\n current_id = 0\n for topic, image_ids in tqdm(image_centroids.items()):\n selected_images = [\n Image.open(images[index]) if isinstance(images[index], str) else images[index] for index in image_ids\n ]\n text = self._convert_image_to_text(selected_images)\n\n for doc, image_id in zip(text, image_ids):\n documents.loc[len(documents), :] = [\n doc,\n current_id,\n topic,\n images[image_id],\n ]\n current_id += 1\n\n # Properly close images\n if isinstance(images[image_ids[0]], str):\n for image in selected_images:\n image.close()\n\n return documents\n\n def _chunks(self, images):\n for i in range(0, len(images), self.batch_size):\n yield images[i : i + self.batch_size]\n\n\ndef get_concat_h_multi_resize(im_list):\n \"\"\"Code adapted from: https://note.nkmk.me/en/python-pillow-concat-images/.\"\"\"\n min_height = min(im.height for im in im_list)\n min_height = max(im.height for im in im_list)\n im_list_resize = []\n for im in im_list:\n im.resize((int(im.width * min_height / im.height), min_height), resample=0)\n im_list_resize.append(im)\n\n total_width = sum(im.width for im in im_list_resize)\n dst = Image.new(\"RGB\", (total_width, min_height), (255, 255, 255))\n pos_x = 0\n for im in im_list_resize:\n dst.paste(im, (pos_x, 0))\n pos_x += im.width\n return dst\n\n\ndef get_concat_v_multi_resize(im_list):\n \"\"\"Code adapted from: https://note.nkmk.me/en/python-pillow-concat-images/.\"\"\"\n min_width = min(im.width for im in im_list)\n min_width = max(im.width for im in im_list)\n im_list_resize = [im.resize((min_width, int(im.height * min_width / im.width)), resample=0) for im in im_list]\n total_height = sum(im.height for im in im_list_resize)\n dst = Image.new(\"RGB\", (min_width, total_height), (255, 255, 255))\n pos_y = 0\n for im in im_list_resize:\n dst.paste(im, (0, pos_y))\n pos_y += im.height\n return dst\n\n\ndef get_concat_tile_resize(im_list_2d, image_height=600, image_squares=False):\n \"\"\"Code adapted from: https://note.nkmk.me/en/python-pillow-concat-images/.\"\"\"\n images = [[image.copy() for image in images] for images in im_list_2d]\n\n # Create\n if image_squares:\n width = int(image_height / 3)\n height = int(image_height / 3)\n images = [[image.resize((width, height)) for image in images] for images in im_list_2d]\n\n # Resize images based on minimum size\n else:\n min_width = min([min([img.width for img in imgs]) for imgs in im_list_2d])\n min_height = min([min([img.height for img in imgs]) for imgs in im_list_2d])\n for i, imgs in enumerate(images):\n for j, img in enumerate(imgs):\n if img.height > img.width:\n images[i][j] = img.resize(\n (int(img.width * min_height / img.height), min_height),\n resample=0,\n )\n elif img.width > img.height:\n images[i][j] = img.resize((min_width, int(img.height * min_width / img.width)), resample=0)\n else:\n images[i][j] = img.resize((min_width, min_width))\n\n # Resize grid image\n images = [get_concat_h_multi_resize(im_list_h) for im_list_h in images]\n img = get_concat_v_multi_resize(images)\n height_percentage = image_height / float(img.size[1])\n adjusted_width = int((float(img.size[0]) * float(height_percentage)))\n img = img.resize((adjusted_width, image_height), Image.Resampling.LANCZOS)\n\n return img\n" }, { "path": "bertopic/representation/_zeroshot.py", "content": "import pandas as pd\nfrom transformers import pipeline\nfrom transformers.pipelines.base import Pipeline\nfrom scipy.sparse import csr_matrix\nfrom typing import Mapping, List, Tuple, Any\nfrom bertopic.representation._base import BaseRepresentation\n\n\nclass ZeroShotClassification(BaseRepresentation):\n \"\"\"Zero-shot Classification on topic keywords with candidate labels.\n\n Arguments:\n candidate_topics: A list of labels to assign to the topics if they\n exceed `min_prob`\n model: A transformers pipeline that should be initialized as\n \"zero-shot-classification\". For example,\n `pipeline(\"zero-shot-classification\", model=\"facebook/bart-large-mnli\")`\n pipeline_kwargs: Kwargs that you can pass to the transformers.pipeline\n when it is called. NOTE: Use `{\"multi_label\": True}`\n to extract multiple labels for each topic.\n min_prob: The minimum probability to assign a candidate label to a topic\n\n Usage:\n\n ```python\n from bertopic.representation import ZeroShotClassification\n from bertopic import BERTopic\n\n # Create your representation model\n candidate_topics = [\"space and nasa\", \"bicycles\", \"sports\"]\n representation_model = ZeroShotClassification(candidate_topics, model=\"facebook/bart-large-mnli\")\n\n # Use the representation model in BERTopic on top of the default pipeline\n topic_model = BERTopic(representation_model=representation_model)\n ```\n \"\"\"\n\n def __init__(\n self,\n candidate_topics: List[str],\n model: str = \"facebook/bart-large-mnli\",\n pipeline_kwargs: Mapping[str, Any] = {},\n min_prob: float = 0.8,\n ):\n self.candidate_topics = candidate_topics\n if isinstance(model, str):\n self.model = pipeline(\"zero-shot-classification\", model=model)\n elif isinstance(model, Pipeline):\n self.model = model\n else:\n raise ValueError(\n \"Make sure that the HF model that you\"\n \"pass is either a string referring to a\"\n \"HF model or a `transformers.pipeline` object.\"\n )\n self.pipeline_kwargs = pipeline_kwargs\n self.min_prob = min_prob\n\n def extract_topics(\n self,\n topic_model,\n documents: pd.DataFrame,\n c_tf_idf: csr_matrix,\n topics: Mapping[str, List[Tuple[str, float]]],\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\"Extract topics.\n\n Arguments:\n topic_model: Not used\n documents: Not used\n c_tf_idf: Not used\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n # Classify topics\n topic_descriptions = [\" \".join(next(zip(*topics[topic]))) for topic in topics.keys()]\n classifications = self.model(topic_descriptions, self.candidate_topics, **self.pipeline_kwargs)\n\n # Extract labels\n updated_topics = {}\n for topic, classification in zip(topics.keys(), classifications):\n topic_description = topics[topic]\n\n # Multi-label assignment\n if self.pipeline_kwargs.get(\"multi_label\"):\n topic_description = []\n for label, score in zip(classification[\"labels\"], classification[\"scores\"]):\n if score > self.min_prob:\n topic_description.append((label, score))\n\n # Single label assignment\n elif classification[\"scores\"][0] > self.min_prob:\n topic_description = [(classification[\"labels\"][0], classification[\"scores\"][0])]\n\n # Make sure that 10 items are returned\n if len(topic_description) == 0:\n topic_description = topics[topic]\n elif len(topic_description) < 10:\n topic_description += [(\"\", 0) for _ in range(10 - len(topic_description))]\n updated_topics[topic] = topic_description\n\n return updated_topics\n" }, { "path": "bertopic/vectorizers/__init__.py", "content": "from ._ctfidf import ClassTfidfTransformer\nfrom ._online_cv import OnlineCountVectorizer\n\n__all__ = [\"ClassTfidfTransformer\", \"OnlineCountVectorizer\"]\n" }, { "path": "bertopic/vectorizers/_ctfidf.py", "content": "from typing import List\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.preprocessing import normalize\nfrom sklearn.utils import check_array\nimport numpy as np\nimport scipy.sparse as sp\n\n\nclass ClassTfidfTransformer(TfidfTransformer):\n \"\"\"A Class-based TF-IDF procedure using scikit-learns TfidfTransformer as a base.\n\n ![](../algorithm/c-TF-IDF.svg)\n\n c-TF-IDF can best be explained as a TF-IDF formula adopted for multiple classes\n by joining all documents per class. Thus, each class is converted to a single document\n instead of set of documents. The frequency of each word **x** is extracted\n for each class **c** and is **l1** normalized. This constitutes the term frequency.\n\n Then, the term frequency is multiplied with IDF which is the logarithm of 1 plus\n the average number of words per class **A** divided by the frequency of word **x**\n across all classes.\n\n Arguments:\n bm25_weighting: Uses BM25-inspired idf-weighting procedure instead of the procedure\n as defined in the c-TF-IDF formula. It uses the following weighting scheme:\n `log(1+((avg_nr_samples - df + 0.5) / (df+0.5)))`\n reduce_frequent_words: Takes the square root of the bag-of-words after normalizing the matrix.\n Helps to reduce the impact of words that appear too frequently.\n seed_words: Specific words that will have their idf value increased by\n the value of `seed_multiplier`.\n NOTE: This will only increase the value of words that have an exact match.\n seed_multiplier: The value with which the idf values of the words in `seed_words`\n are multiplied.\n\n Examples:\n ```python\n transformer = ClassTfidfTransformer()\n ```\n \"\"\"\n\n def __init__(\n self,\n bm25_weighting: bool = False,\n reduce_frequent_words: bool = False,\n seed_words: List[str] | None = None,\n seed_multiplier: float = 2,\n ):\n self.bm25_weighting = bm25_weighting\n self.reduce_frequent_words = reduce_frequent_words\n self.seed_words = seed_words\n self.seed_multiplier = seed_multiplier\n super(ClassTfidfTransformer, self).__init__()\n\n def fit(self, X: sp.csr_matrix, multiplier: np.ndarray = None):\n \"\"\"Learn the idf vector (global term weights).\n\n Arguments:\n X: A matrix of term/token counts.\n multiplier: A multiplier for increasing/decreasing certain IDF scores\n \"\"\"\n X = check_array(X, accept_sparse=(\"csr\", \"csc\"))\n if not sp.issparse(X):\n X = sp.csr_matrix(X)\n dtype = np.float64\n\n if self.use_idf:\n _, n_features = X.shape\n\n # Calculate the frequency of words across all classes\n df = np.squeeze(np.asarray(X.sum(axis=0)))\n\n # Calculate the average number of samples as regularization\n avg_nr_samples = int(X.sum(axis=1).mean())\n\n # BM25-inspired weighting procedure\n if self.bm25_weighting:\n idf = np.log(1 + ((avg_nr_samples - df + 0.5) / (df + 0.5)))\n\n # Divide the average number of samples by the word frequency\n # +1 is added to force values to be positive\n else:\n idf = np.log((avg_nr_samples / df) + 1)\n\n # Multiplier to increase/decrease certain idf scores\n if multiplier is not None:\n idf = idf * multiplier\n\n self._idf_diag = sp.diags(\n idf,\n offsets=0,\n shape=(n_features, n_features),\n format=\"csr\",\n dtype=dtype,\n )\n\n return self\n\n def transform(self, X: sp.csr_matrix):\n \"\"\"Transform a count-based matrix to c-TF-IDF.\n\n Arguments:\n X (sparse matrix): A matrix of term/token counts.\n\n Returns:\n X (sparse matrix): A c-TF-IDF matrix\n \"\"\"\n if self.use_idf:\n X = normalize(X, axis=1, norm=\"l1\", copy=False)\n\n if self.reduce_frequent_words:\n X.data = np.sqrt(X.data)\n\n X = X * self._idf_diag\n\n return X\n" }, { "path": "bertopic/vectorizers/_online_cv.py", "content": "import numpy as np\nfrom itertools import chain\nfrom typing import List\n\nfrom scipy import sparse\nfrom scipy.sparse import csr_matrix\n\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n\nclass OnlineCountVectorizer(CountVectorizer):\n \"\"\"An online variant of the CountVectorizer with updating vocabulary.\n\n At each `.partial_fit`, its vocabulary is updated based on any OOV words\n it might find. Then, `.update_bow` can be used to track and update\n the Bag-of-Words representation. These functions are separated such that\n the vectorizer can be used in iteration without updating the Bag-of-Words\n representation can might speed up the fitting process. However, the\n `.update_bow` function is used in BERTopic to track changes in the\n topic representations and allow for decay.\n\n This class inherits its parameters and attributes from:\n `sklearn.feature_extraction.text.CountVectorizer`\n\n Arguments:\n decay: A value between [0, 1] to weight the percentage of frequencies\n the previous bag-of-words should be decreased. For example,\n a value of `.1` will decrease the frequencies in the bag-of-words\n matrix with 10% at each iteration.\n delete_min_df: Delete words at each iteration from its vocabulary\n that are below a minimum frequency.\n This will keep the resulting bag-of-words matrix small\n such that it does not explode in size with increasing\n vocabulary. If `decay` is None then this equals `min_df`.\n **kwargs: Set of parameters inherited from:\n `sklearn.feature_extraction.text.CountVectorizer`\n In practice, this means that you can still use parameters\n from the original CountVectorizer, like `stop_words` and\n `ngram_range`.\n\n Attributes:\n X_ (scipy.sparse.csr_matrix) : The Bag-of-Words representation\n\n Examples:\n ```python\n from bertopic.vectorizers import OnlineCountVectorizer\n vectorizer = OnlineCountVectorizer(stop_words=\"english\")\n\n for index, doc in enumerate(my_docs):\n vectorizer.partial_fit(doc)\n\n # Update and clean the bow every 100 iterations:\n if index % 100 == 0:\n X = vectorizer.update_bow()\n ```\n\n To use the model in BERTopic:\n\n ```python\n from bertopic import BERTopic\n from bertopic.vectorizers import OnlineCountVectorizer\n\n vectorizer_model = OnlineCountVectorizer(stop_words=\"english\")\n topic_model = BERTopic(vectorizer_model=vectorizer_model)\n ```\n\n References:\n Adapted from: https://github.com/idoshlomo/online_vectorizers\n \"\"\"\n\n def __init__(self, decay: float | None = None, delete_min_df: float | None = None, **kwargs):\n self.decay = decay\n self.delete_min_df = delete_min_df\n super(OnlineCountVectorizer, self).__init__(**kwargs)\n\n def partial_fit(self, raw_documents: List[str]) -> None:\n \"\"\"Perform a partial fit and update vocabulary with OOV tokens.\n\n Arguments:\n raw_documents: A list of documents\n \"\"\"\n if not hasattr(self, \"vocabulary_\"):\n return self.fit(raw_documents)\n\n analyzer = self.build_analyzer()\n analyzed_documents = [analyzer(doc) for doc in raw_documents]\n new_tokens = set(chain.from_iterable(analyzed_documents))\n oov_tokens = new_tokens.difference(set(self.vocabulary_.keys()))\n\n if oov_tokens:\n max_index = max(self.vocabulary_.values())\n oov_vocabulary = dict(\n zip(\n oov_tokens,\n list(range(max_index + 1, max_index + 1 + len(oov_tokens), 1)),\n )\n )\n self.vocabulary_.update(oov_vocabulary)\n\n return self\n\n def update_bow(self, raw_documents: List[str]) -> csr_matrix:\n \"\"\"Create or update the bag-of-words matrix.\n\n Update the bag-of-words matrix by adding the newly transformed\n documents. This may add empty columns if new words are found and/or\n add empty rows if new topics are found.\n\n During this process, the previous bag-of-words matrix might be\n decayed if `self.decay` has been set during init. Similarly, words\n that do not exceed `self.delete_min_df` are removed from its\n vocabulary and bag-of-words matrix.\n\n Arguments:\n raw_documents: A list of documents\n\n Returns:\n X_: Bag-of-words matrix\n \"\"\"\n if hasattr(self, \"X_\"):\n X = self.transform(raw_documents)\n\n # Add empty columns if new words are found\n columns = csr_matrix((self.X_.shape[0], X.shape[1] - self.X_.shape[1]), dtype=int)\n self.X_ = sparse.hstack([self.X_, columns])\n\n # Add empty rows if new topics are found\n rows = csr_matrix((X.shape[0] - self.X_.shape[0], self.X_.shape[1]), dtype=int)\n self.X_ = sparse.vstack([self.X_, rows])\n\n # Decay of BoW matrix\n if self.decay is not None:\n self.X_ = self.X_ * (1 - self.decay)\n\n self.X_ += X\n else:\n self.X_ = self.transform(raw_documents)\n\n if self.delete_min_df is not None:\n self._clean_bow()\n\n return self.X_\n\n def _clean_bow(self) -> None:\n \"\"\"Remove words that do not exceed `self.delete_min_df`.\"\"\"\n # Only keep words with a minimum frequency\n indices = np.where(self.X_.sum(0) >= self.delete_min_df)[1]\n indices_dict = {index: index for index in indices}\n self.X_ = self.X_[:, indices]\n\n # Update vocabulary with new words\n new_vocab = {}\n vocabulary_dict = {v: k for k, v in self.vocabulary_.items()}\n for i, index in enumerate(indices):\n if indices_dict.get(index) is not None:\n new_vocab[vocabulary_dict[index]] = i\n\n self.vocabulary_ = new_vocab\n" }, { "path": "docs/algorithm/algorithm.md", "content": "---\nhide:\n - navigation\n---\n\n# The Algorithm\n\nBelow, you will find different types of overviews of each step in BERTopic's main algorithm. Each successive overview will be more in-depth than the previous overview. This approach aims to make the underlying algorithm as intuitive as possible for a wide range of users.\n\n## **Visual Overview**\n\nBERTopic can be viewed as a sequence of steps to create its topic representations. There are five steps to this process:\n\n\n\nAlthough these steps are the default, there is some modularity to BERTopic. Each step in this process was carefully selected such that they are all somewhat independent from one another. For example, the tokenization step is not directly influenced by the embedding model that was used to convert the documents which allow us to be creative in how we perform the tokenization step.\n\nThis effect is especially strong in the clustering step. Models like HDBSCAN assume that clusters can have different shapes and forms. As a result, using a centroid-based technique to model the topic representations would not be beneficial since the centroid is not always representative of these types of clusters. A bag-of-words representation, however, makes very few assumptions concerning the shape and form of a cluster.\n\nAs a result, BERTopic is quite modular and can maintain its quality of topic generation throughout a variety of sub-models. In other words, BERTopic essentially allows you to **build your own topic model**:\n\n\n\nThere is extensive documentation on how to use each step in this pipeline:\n\n1. [Embeddings](../getting_started/embeddings/embeddings.html)\n2. [Dimensionality Reduction](../getting_started/dim_reduction/dim_reduction.html)\n3. [Clustering](../getting_started/clustering/clustering.html)\n4. [Tokenizer](../getting_started/vectorizers/vectorizers.html)\n5. [Weighting Scheme](../getting_started/ctfidf/ctfidf.html)\n6. [Representation Tuning](../getting_started/representation/representation.html)\n * [Large Language Models (LLM)](../getting_started/representation/llm.html)\n\n## **Code Overview**\nAfter going through the visual overview, this code overview demonstrates the algorithm using BERTopic. An advantage of using BERTopic is each major step in its algorithm can be explicitly defined, thereby making the process not only transparent but also more intuitive.\n\n\n```python\nfrom umap import UMAP\nfrom hdbscan import HDBSCAN\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom bertopic import BERTopic\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\n\n# Step 1 - Extract embeddings\nembedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\n# Step 2 - Reduce dimensionality\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine')\n\n# Step 3 - Cluster reduced embeddings\nhdbscan_model = HDBSCAN(min_cluster_size=15, metric='euclidean', cluster_selection_method='eom', prediction_data=True)\n\n# Step 4 - Tokenize topics\nvectorizer_model = CountVectorizer(stop_words=\"english\")\n\n# Step 5 - Create topic representation\nctfidf_model = ClassTfidfTransformer()\n\n# Step 6 - (Optional) Fine-tune topic representations with\n# a `bertopic.representation` model\nrepresentation_model = KeyBERTInspired()\n\n# All steps together\ntopic_model = BERTopic(\n embedding_model=embedding_model, # Step 1 - Extract embeddings\n umap_model=umap_model, # Step 2 - Reduce dimensionality\n hdbscan_model=hdbscan_model, # Step 3 - Cluster reduced embeddings\n vectorizer_model=vectorizer_model, # Step 4 - Tokenize topics\n ctfidf_model=ctfidf_model, # Step 5 - Extract topic words\n representation_model=representation_model # Step 6 - (Optional) Fine-tune topic representations\n)\n```\n\n\n## **Detailed Overview**\nThis overview describes each step in more detail such that you can get an intuitive feeling as to what models might fit best at each step in your use case.\n\n### **1. Embed documents**\nWe start by converting our documents to numerical representations. Although there are many methods for doing so the default in BERTopic is [sentence-transformers](https://github.com/UKPLab/sentence-transformers). These models are often optimized for semantic similarity which helps tremendously in our clustering task. Moreover, they are great for creating either document- or sentence-embeddings.\n
\nIn BERTopic, you can choose any sentence-transformers model but two models are set as defaults:\n\n* `\"all-MiniLM-L6-v2\"`\n* `\"paraphrase-multilingual-MiniLM-L12-v2\"`\n\nThe first is an English language model trained specifically for semantic similarity tasks which works quite\nwell for most use cases. The second model is very similar to the first with one major difference being that the\n`multilingual` models work for 50+ languages. This model is quite a bit larger than the first and is only selected if\nyou select any language other than English.\n\n!!! tip Embedding models\n\n Although BERTopic uses sentence-transformers models as a default, you can choose\n any embedding model that fits your use case. Follow the guide [here](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html) for selecting\n and customizing your model.\n\n\n### **2. Dimensionality reduction**\nAfter having created our numerical representations of the documents we have to reduce the dimensionality of these representations. Cluster models typically have difficulty handling high dimensional data due to the curse of dimensionality. There are great approaches that can reduce dimensionality, such as PCA, but as a default [UMAP](https://github.com/lmcinnes/umap) is selected in BERTopic. It is a technique that can keep some of a dataset's local and global structure when reducing its dimensionality. This structure is important to keep as it contains the information necessary to create clusters of semantically similar documents.\n\n!!! tip Dimensionality reduction models\n\n Although BERTopic uses UMAP as a default, you can choose\n any dimensionality reduction model that fits your use case. Follow the guide [here](https://maartengr.github.io/BERTopic/getting_started/dim_reduction/dim_reduction.html) for selecting\n and customizing your model.\n\n### **3. Cluster Documents**\nAfter having reduced our embeddings, we can start clustering our data. For that, we leverage a density-based clustering technique, HDBSCAN. It can find clusters of different shapes and has the nice feature of identifying outliers where possible. As a result, we do not force documents into a cluster where they might not belong. This will improve the resulting topic representation as there is less noise to draw from.\n\n!!! tip Cluster models\n\n Although BERTopic uses HDBSCAN as a default, you can choose\n any cluster model that fits your use case. Follow the guide [here](https://maartengr.github.io/BERTopic/getting_started/clustering/clustering.html) for selecting\n and customizing your model.\n\n### **4. Bag-of-words**\nBefore we can start creating the topic representation we first need to select a technique that allows for modularity in BERTopic's algorithm. When we use HDBSCAN as a cluster model, we may assume that our clusters have different degrees of density and different shapes. This means that a centroid-based topic representation technique might not be the best-fitting model. In other words, we want a topic representation technique that makes little to no assumption on the expected structure of the clusters.\n
\nTo do this, we first combine all documents in a cluster into a single document. That, very long, document then represents the cluster. Then, we can count how often each word appears in each cluster. This generates something called a bag-of-words representation in which the frequency of each word in each cluster can be found. This bag-of-words representation is therefore on a cluster level and not on a document level. This distinction is important as we are interested in words on a topic level (i.e., cluster level). By using a bag-of-words representation, no assumption is made concerning the structure of the clusters. Moreover, the bag-of-words representation is L1-normalized to account for clusters that have different sizes.\n\n!!! tip Bag-of-words and tokenization\n\n There are many ways you can tune or change the bag-of-words step. This step allows for processing the documents however you want without affecting the first step, embedding the documents. You can follow the guide [here](https://maartengr.github.io/BERTopic/getting_started/vectorizers/vectorizers.html) for more information about tokenization options in BERTopic.\n\n### **5. Topic representation**\nFrom the generated bag-of-words representation, we want to know what makes one cluster different from another. Which words are typical for cluster 1 and not so much for all other clusters? To solve this, we need to modify TF-IDF such that it considers topics (i.e., clusters) instead of documents.\n
\nWhen you apply TF-IDF as usual on a set of documents, what you are doing is comparing the importance of\nwords between documents. Now, what if, we instead treat all documents in a single category (e.g., a cluster) as a single document and then apply TF-IDF? The result would be importance scores for words within a cluster. The more important words are within a cluster, the more it is representative of that topic. In other words, if we extract the most important words per cluster, we get descriptions of **topics**! This model is called **class-based TF-IDF**:\n

\n\n\n\n
\nEach cluster is converted to a single document instead of a set of documents. Then, we extract the frequency of word `x` in class `c`, where `c` refers to the cluster we created before. This results in our class-based `tf` representation. This representation is L1-normalized to account for the differences in topic sizes.\n

\nThen, we take the logarithm of one plus the average number of words per class `A` divided by the frequency of word `x` across all classes. We add plus one within the logarithm to force values to be positive. This results in our class-based `idf` representation. Like with the classic TF-IDF, we then multiply `tf` with `idf` to get the importance score per word in each class. In other words, the classical TF-IDF procedure is **not** used here but a modified version of the algorithm that allows for a much better representation.\n\n!!! tip c-TF-IDF parameters\n\n In the `ClassTfidfTransformer`, there are a few parameters that might be worth exploring, including an option to perform additional BM-25 weighting. You can find more information about that [here](https://maartengr.github.io/BERTopic/getting_started/ctfidf/ctfidf.html).\n\n### **6. (Optional) Fine-tune Topic representation**\nAfter having generated the c-TF-IDF representations, we have a set of words that describe a collection of documents. c-TF-IDF\nis a method that can quickly generate accurate topic representations. However, with the fast developments in NLP-world, new\nand exciting methods are released weekly. In order to keep up with what is happening, there is the possibility to further fine-tune\nthese c-TF-IDF topics using GPT, T5, KeyBERT, Spacy, and other techniques. Many are implemented in BERTopic for you to use and play around with.\n\nMore specifically, we can consider the c-TF-IDF generated topics to be candidate topics. They each contain a set of keywords and\nrepresentative documents that we can use to further fine-tune the topic representations. Having a set of representative documents\nfor each topic is huge advantage as it allows for fine-tuning on a reduced number of documents. This reduces computation for\nlarge models as they only need to operate on that small set of representative documents for each topic. As a result,\nlarge language models like GPT and T5 becomes feasible in production settings and typically take less wall time than the dimensionality reduction\nand clustering steps.\n\nThe following models are implemented in `bertopic.representation`:\n\n* `MaximalMarginalRelevance`\n* `PartOfSpeech`\n* `KeyBERTInspired`\n* `ZeroShotClassification`\n* `TextGeneration` (HuggingFace)\n* `Cohere`\n* `OpenAI`\n* `LangChain`\n* `LiteLLM`\n* `LlamaCPP`\n\n!!! tip Models\n There are roughly two sets of models. **First** are the non-generative set of models that you can find [here](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html). These include models that focus on enhancing the keywords in the topic representations. **Second** are the generative models that attempt to label or summarize the topics instead. You can find an overview of [implemented LLMs here](https://maartengr.github.io/BERTopic/getting_started/representation/llm).\n" }, { "path": "docs/api/backends.md", "content": "# `Backends`\n\n::: bertopic.backend\n" }, { "path": "docs/api/bertopic.md", "content": "# `BERTopic`\n\n::: bertopic._bertopic.BERTopic\n" }, { "path": "docs/api/cluster copy.md", "content": "# `BaseCluster`\n\n::: bertopic.cluster._base.BaseCluster\n" }, { "path": "docs/api/cluster.md", "content": "# `BaseCluster`\n\n::: bertopic.cluster._base.BaseCluster\n" }, { "path": "docs/api/ctfidf.md", "content": "# `c-TF-IDF`\n\n::: bertopic.vectorizers.ClassTfidfTransformer\n" }, { "path": "docs/api/dimensionality.md", "content": "# `BaseDimensionalityReduction`\n\n::: bertopic.dimensionality._base.BaseDimensionalityReduction\n" }, { "path": "docs/api/plotting/barchart.md", "content": "# `Barchart`\n\n::: bertopic.plotting._barchart.visualize_barchart\n" }, { "path": "docs/api/plotting/distribution.md", "content": "# `Distribution`\n\n::: bertopic.plotting._distribution.visualize_distribution\n" }, { "path": "docs/api/plotting/document_datamap.md", "content": "# `Documents with DataMapPlot`\n\n::: bertopic.plotting._datamap.visualize_document_datamap\n" }, { "path": "docs/api/plotting/documents.md", "content": "# `Documents`\n\n::: bertopic.plotting._documents.visualize_documents\n" }, { "path": "docs/api/plotting/dtm.md", "content": "# `DTM`\n\n::: bertopic.plotting._topics_over_time.visualize_topics_over_time\n" }, { "path": "docs/api/plotting/heatmap.md", "content": "# `Heatmap`\n\n::: bertopic.plotting._heatmap.visualize_heatmap\n" }, { "path": "docs/api/plotting/hierarchical_documents.md", "content": "# `Hierarchical Documents`\n\n::: bertopic.plotting._hierarchical_documents.visualize_hierarchical_documents\n" }, { "path": "docs/api/plotting/hierarchy.md", "content": "# `Hierarchy`\n\n::: bertopic.plotting._hierarchy.visualize_hierarchy\n" }, { "path": "docs/api/plotting/term.md", "content": "# `Term Score Decline`\n\n::: bertopic.plotting._term_rank.visualize_term_rank\n" }, { "path": "docs/api/plotting/topics.md", "content": "# `Topics`\n\n::: bertopic.plotting._topics.visualize_topics\n" }, { "path": "docs/api/plotting/topics_per_class.md", "content": "# `Topics per Class`\n\n::: bertopic.plotting._topics_per_class.visualize_topics_per_class\n" }, { "path": "docs/api/plotting.md", "content": "# `Plotting`\n\n::: bertopic.plotting\n" }, { "path": "docs/api/representations.md", "content": "# `Representations`\n\n::: bertopic.representation\n" }, { "path": "docs/api/vectorizers.md", "content": "# `Vectorizers`\n\n::: bertopic.vectorizers._online_cv.OnlineCountVectorizer\n" }, { "path": "docs/changelog.md", "content": "---\nhide:\n - navigation\n---\n\n# Changelog\n\n\n## **Version 0.17.4**\n*Release date: 3 December, 2025*\n\n

Highlights:

\n\n* Add `.delete_topics` by [@shuanglovesdata](https://github.com/shuanglovesdata) in [#2322](https://github.com/MaartenGr/BERTopic/pull/2322)\n* Allow execution without plotly by [@luismavs](https://github.com/luismavs) in [#2401](https://github.com/MaartenGr/BERTopic/pull/2401)\n* Add tqdm to _litellm.py [@NFrnk](https://github.com/NFrnk) in [#2408](https://github.com/MaartenGr/BERTopic/pull/2408)\n* Drop support for python 3.9 by [@afuetterer](https://github.com/afuetterer) in [#2419](https://github.com/MaartenGr/BERTopic/pull/2419)\n* Make UMAP's init default to random on visualize_topics for reproducible visualization by [@makramab](https://github.com/makramab) in [#2412](https://github.com/MaartenGr/BERTopic/pull/2412)\n\n

cuML:

\n\nPreparing for MEGA!-scale BERTopic with [Multi-GPU UMAP](https://github.com/rapidsai/cuml/issues/3330#issuecomment-3376951158) and the following necessary updates:\n\n* Update installation instructions for cuML with BERTopic by [@csadorf](https://github.com/csadorf) in [#2446](https://github.com/MaartenGr/BERTopic/pull/2446)\n* Speed up `._create_topic_vectors` by replacing DataFrame .loc with NumPy masking [@jinsolp](https://github.com/jinsolp) in [#2406](https://github.com/MaartenGr/BERTopic/pull/2406)\n* Modify _reduce_dimensionality to use fit_transform by [@betatim](https://github.com/betatim) in [#2416](https://github.com/MaartenGr/BERTopic/pull/2416)\n\n

Fixes:

\n* Fix incorrect label in zero-shot svg in documentation by [@huisman](https://github.com/huisman) in [#2448](https://github.com/MaartenGr/BERTopic/pull/2448)\n* Enable ruff rule RUF by [@afuetterer](https://github.com/afuetterer) in [#2457](https://github.com/MaartenGr/BERTopic/pull/2457)\n* CI: bump github actions versions by [@afuetterer](https://github.com/afuetterer) in [#2427](https://github.com/MaartenGr/BERTopic/pull/2427)\n* CI: prefer action-pre-commit-uv for lint job by [@afuetterer](https://github.com/afuetterer) in [#2434](https://github.com/MaartenGr/BERTopic/pull/2434)\n* CI: switch to uv based project installation by [@afuetterer](https://github.com/afuetterer) in [#2445](https://github.com/MaartenGr/BERTopic/pull/2445)\n* Chore: update pre-commit hooks by [@afuetterer](https://github.com/afuetterer) in [#2414](https://github.com/MaartenGr/BERTopic/pull/2414) and [#2443](https://github.com/MaartenGr/BERTopic/pull/2443)\n* Chore: remove obsolete `version_info` check by [@afuetterer](https://github.com/afuetterer) in [#2444](https://github.com/MaartenGr/BERTopic/pull/2444)\n\n\n## **Version 0.17.3**\n*Release date: 8 July, 2025*\n\n* Fixed issues with `uv add bertopic` by adding `llvmlite` as an explicit dependency. Although the fixes in `v0.17.2` and `v0.17.2.post1` actually worked locally, this wasn't the case after pushing the package to pypi.org unfortunately.\n\n\n## **Version 0.17.1**\n*Release date: 8 July, 2025*\n\n

Highlights:

\n* Added FastEmbed backend by [@nickprock](https://github.com/nickprock) in [#2213](https://github.com/MaartenGr/BERTopic/pull/2312)\n* Added LangChain backend by [@regaltsui](https://github.com/regaltsui) in [#2303](https://github.com/MaartenGr/BERTopic/pull/2303)\n* Pass precomputed embeddings to KeyBERTInspired.extract_topics [@saikumaru](https://github.com/saikumaru) in [#2368](https://github.com/MaartenGr/BERTopic/pull/2368)\n\n

Fixes:

\n* Merge models without pytorch (using safetensors) by [@MaartenGr](https://github.com/MaartenGr) in [#2329](https://github.com/MaartenGr/BERTopic/pull/2329)\n* Fix installation issue with uv by [@MaartenGr](https://github.com/MaartenGr) in [#2328](https://github.com/MaartenGr/BERTopic/pull/2328)\n* Fix incorrect comparison in update_topics by [@uply23333](https://github.com/uply23333) in [#2336](https://github.com/MaartenGr/BERTopic/pull/2336)\n* Add missing comma under Exploration subsection by [@angelonazzaro](https://github.com/angelonazzaro) in [#2374](https://github.com/MaartenGr/BERTopic/pull/2374)\n* Fix typo in Lightweight installation under tips_and_tricks by [@angelonazzaro](https://github.com/angelonazzaro) in [#2375](https://github.com/MaartenGr/BERTopic/pull/2375)\n* Fix IndexError in zero-shot topic modeling by [@MaartenGr](https://github.com/MaartenGr) in [#2267](https://github.com/MaartenGr/BERTopic/pull/2267)\n\n## **Version 0.17.0**\n*Release date: 19 March, 2025*\n\n

Highlights:

\n* Light-weight installation without UMAP and HDBSCAN by [@MaartenGr](https://github.com/MaartenGr) in [#2289](https://github.com/MaartenGr/BERTopic/pull/2289)\n* Add Model2Vec as an embedding backend by [@MaartenGr](https://github.com/MaartenGr) in [#2245](https://github.com/MaartenGr/BERTopic/pull/2245)\n* Add LiteLLM as a representation model by [@MaartenGr](https://github.com/MaartenGr) in [#2213](https://github.com/MaartenGr/BERTopic/pull/2213)\n* Interactive DataMapPlot and deprecate non-chat OpenAI models by [@MaartenGr](https://github.com/MaartenGr) in [#2287](https://github.com/MaartenGr/BERTopic/pull/2287)\n\n

Fixes:

\n\n* Lightweight installation: use safetensors without torch by [@hedgeho](https://github.com/hedgeho) in [#2306](https://github.com/MaartenGr/BERTopic/pull/2306)\n* Fix missing links by [@MaartenGr](https://github.com/MaartenGr) in [#2305](https://github.com/MaartenGr/BERTopic/pull/2305)\n* Set up pre-commit hooks by [@afuetterer](https://github.com/afuetterer) in [#2283](https://github.com/MaartenGr/BERTopic/pull/2283)\n* Fix handling OpenAI returning None objects by [@jeaninejuliettes](https://github.com/jeaninejuliettes) in [#2280](https://github.com/MaartenGr/BERTopic/pull/2280)\n* Add support for python 3.13 by [@afuetterer](https://github.com/afuetterer) in [#2173](https://github.com/MaartenGr/BERTopic/pull/2173)\n* Added system prompts by [@Leo-LiHao](https://github.com/Leo-LiHao) in [#2145](https://github.com/MaartenGr/BERTopic/pull/2145)\n* More documentation for topic reduction by [@MaartenGr](https://github.com/MaartenGr) in [#2260](https://github.com/MaartenGr/BERTopic/pull/2260)\n* Drop support for python 3.8 by [@afuetterer](https://github.com/afuetterer) in [#2243](https://github.com/MaartenGr/BERTopic/pull/2243)\n* Fixed online topic modeling on GPU by [@SSivakumar12](https://github.com/SSivakumar12) in [#2181](https://github.com/MaartenGr/BERTopic/pull/2181)\n* Fixed hierarchical cluster visualization by [@PipaFlores](https://github.com/PipaFlores) in [#2191](https://github.com/MaartenGr/BERTopic/pull/2191)\n* Remove duplicated phrase by [@AndreaFrancis](https://github.com/AndreaFrancis) in [#2197](https://github.com/MaartenGr/BERTopic/pull/2197)\n\n## **Version 0.16.4**\n*Release date: 9 October, 2024*\n\n

Fixes:

\n\n* Fix ValueError in Guided Topic Modeling by [@RTChou](https://github.com/RTChou) in [#2115](https://github.com/MaartenGr/BERTopic/pull/2115)\n* Fix saving BERTopic when c-TF-IDF is None by [@sete39](https://github.com/sete39) in [#2112](https://github.com/MaartenGr/BERTopic/pull/2112)\n* Fix `KeyError: 'topics_from'` in [#2101](https://github.com/MaartenGr/BERTopic/pull/2101)\n* Fix issues related Zero-shot Topic Modeling by [@ianrandman](https://github.com/ianrandman) in [#2105](https://github.com/MaartenGr/BERTopic/pull/2105)\n* Fix regex matching being used in PartOfSpeech representation model by [@woranov](https://github.com/woranov) in [#2138](https://github.com/MaartenGr/BERTopic/pull/2138)\n* Update typo by [@saikumaru](https://github.com/saikumaru) in [#2162](https://github.com/MaartenGr/BERTopic/pull/2162)\n\n\n## **Version 0.16.3**\n*Release date: 22 July, 2024*\n\n

Highlights:

\n\n* Simplify zero-shot topic modeling by [@ianrandman](https://github.com/ianrandman) in [#2060](https://github.com/MaartenGr/BERTopic/pull/2060)\n* Option to choose between c-TF-IDF and Topic Embeddings in many functions by [@azikoss](https://github.com/azikoss) in [#1894](https://github.com/MaartenGr/BERTopic/pull/1894)\n * Use the `use_ctfidf` parameter in the following function to choose between c-TF-IDF and topic embeddings:\n * `hierarchical_topics`, `reduce_topics`, `visualize_hierarchy`, `visualize_heatmap`, `visualize_topics`\n* Linting with Ruff by [@afuetterer](https://github.com/afuetterer) in [#2033](https://github.com/MaartenGr/BERTopic/pull/2033)\n* Switch from setup.py to pyproject.toml by [@afuetterer](https://github.com/afuetterer) in [#1978](https://github.com/MaartenGr/BERTopic/pull/1978)\n* In multi-aspect context, allow Main model to be chained by [@ddicato](https://github.com/ddicato) in [#2002](https://github.com/MaartenGr/BERTopic/pull/2002)\n\n

Fixes:

\n\n* Added templates for [issues](https://github.com/MaartenGr/BERTopic/tree/master/.github/ISSUE_TEMPLATE) and [pull requests](https://github.com/MaartenGr/BERTopic/blob/master/.github/PULL_REQUEST_TEMPLATE.md)\n* Update River documentation example by [@Proteusiq](https://github.com/Proteusiq) in [#2004](https://github.com/MaartenGr/BERTopic/pull/2004)\n* Fix PartOfSpeech reproducibility by [@Greenpp](https://github.com/Greenpp) in [#1996](https://github.com/MaartenGr/BERTopic/pull/1996)\n* Fix PartOfSpeech ignoring first word by [@Greenpp](https://github.com/Greenpp) in [#2024](https://github.com/MaartenGr/BERTopic/pull/2024)\n* Make sklearn embedding backend auto-select more cautious by [@freddyheppell](https://github.com/freddyheppell) in [#1984](https://github.com/MaartenGr/BERTopic/pull/1984)\n* Fix typos by [@afuetterer](https://github.com/afuetterer) in [#1974](https://github.com/MaartenGr/BERTopic/pull/1974)\n* Fix hierarchical_topics(...) when the distances between three clusters are the same by [@azikoss](https://github.com/azikoss) in [#1929](https://github.com/MaartenGr/BERTopic/pull/1929)\n* Fixes to chain strategy example in outlier_reduction.md by [@reuning](https://github.com/reuning) in [#2065](https://github.com/MaartenGr/BERTopic/pull/2065)\n* Remove obsolete flake8 config and update line length by [@afuetterer](https://github.com/afuetterer) in [#22066](https://github.com/MaartenGr/BERTopic/pull/2066)\n\n\n## **Version 0.16.2**\n*Release date: 12 May, 2024*\n\n

Fixes:

\n\n* Fix issue with zeroshot topic modeling missing outlier [#1957](https://github.com/MaartenGr/BERTopic/issues/1957)\n* Bump github actions versions by [@afuetterer](https://github.com/afuetterer) in [#1941](https://github.com/MaartenGr/BERTopic/pull/1941)\n* Drop support for python 3.7 by [@afuetterer](https://github.com/afuetterer) in [#1949](https://github.com/MaartenGr/BERTopic/pull/1949)\n* Add testing python 3.10+ in Github actions by [@afuetterer](https://github.com/afuetterer) in [#1968](https://github.com/MaartenGr/BERTopic/pull/1968)\n* Speed up fitting CountVectorizer by [@dannywhuang](https://github.com/dannywhuang) in [#1938](https://github.com/MaartenGr/BERTopic/pull/1938)\n* Fix `transform` when using cuML HDBSCAN by [@beckernick](https://github.com/beckernick) in [#1960](https://github.com/MaartenGr/BERTopic/pull/1960)\n* Fix wrong link in algorithm documentation by [@naeyn](https://github.com/naeyn) in [#1970](https://github.com/MaartenGr/BERTopic/pull/1970)\n\n\n## **Version 0.16.1**\n*Release date: 21 April, 2024*\n\n

Highlights:

\n\n* Add Quantized [LLM Tutorial](https://colab.research.google.com/drive/1DdSHvVPJA3rmNfBWjCo2P1E9686xfxFx?usp=sharing)\n* Add optional [datamapplot](https://github.com/TutteInstitute/datamapplot) visualization using `topic_model.visualize_document_datamap` by [@lmcinnes](https://github.com/lmcinnes) in [#1750](https://github.com/MaartenGr/BERTopic/pull/1750)\n* Migrated OpenAIBackend to openai>=1 by [@peguerosdc](https://github.com/peguerosdc) in [#1724](https://github.com/MaartenGr/BERTopic/pull/1724)\n* Add automatic height scaling and font resize by [@ir2718](https://github.com/ir2718) in [#1863](https://github.com/MaartenGr/BERTopic/pull/1863)\n* Use `[KEYWORDS]` tags with the LangChain representation model by [@mcantimmy](https://github.com/mcantimmy) in [#1871](https://github.com/MaartenGr/BERTopic/pull/1871)\n\n\n

Fixes:

\n\n* Fixed issue with `.merge_models` seemingly skipping topic [#1898](https://github.com/MaartenGr/BERTopic/issues/1898)\n* Fixed Cohere client.embed TypeError [#1904](https://github.com/MaartenGr/BERTopic/issues/1904)\n* Fixed `AttributeError: 'TextGeneration' object has no attribute 'random_state'` [#1870](https://github.com/MaartenGr/BERTopic/issues/1870)\n* Fixed topic embeddings not properly updated if all outliers were removed [#1838](https://github.com/MaartenGr/BERTopic/issues/1838)\n* Fixed issue with representation models not properly merging [#1762](https://github.com/MaartenGr/BERTopic/issues/1762)\n* Fixed Embeddings not ordered correctly when using `.merge_models` [#1804](https://github.com/MaartenGr/BERTopic/issues/1804)\n* Fixed Outlier topic not in the 0th position when using zero-shot topic modeling causing prediction issues (amongst others) [#1804](https://github.com/MaartenGr/BERTopic/issues/1804)\n* Fixed Incorrect label in ZeroShot doc SVG [#1732](https://github.com/MaartenGr/BERTopic/issues/1732)\n* Fixed MultiModalBackend throws error with clip-ViT-B-32-multilingual-v1 [#1670](https://github.com/MaartenGr/BERTopic/issues/1670)\n* Fixed AuthenticationError while using OpenAI() [#1678](https://github.com/MaartenGr/BERTopic/issues/1678)\n\n* Update FAQ on Apple Silicon by [@benz0li](https://github.com/benz0li) in [#1901](https://github.com/MaartenGr/BERTopic/pull/1901)\n* Add documentation DataMapPlot + FAQ for running on Apple Silicon by [@dkapitan](https://github.com/dkapitan) in [#1854](https://github.com/MaartenGr/BERTopic/pull/1854)\n* Remove commas from pip install reference in readme by [@luisoala](https://github.com/luisoala) in [#1850](https://github.com/MaartenGr/BERTopic/pull/1850)\n* Spelling corrections by [@joouha](https://github.com/joouha) in [#1801](https://github.com/MaartenGr/BERTopic/pull/1801)\n* Replacing the deprecated `text-ada-001` model with the latest `text-embedding-3-small` from OpenAI by [@atmb4u](https://github.com/atmb4u) in [#1800](https://github.com/MaartenGr/BERTopic/pull/1800)\n* Prevent invalid empty input error when retrieving embeddings with openai backend by [@liaoelton](https://github.com/liaoelton) in [#1827](https://github.com/MaartenGr/BERTopic/pull/1827)\n* Remove spurious warning about missing embedding model by [@sliedes](https://github.com/sliedes) in [#1774](https://github.com/MaartenGr/BERTopic/pull/1774)\n* Fix type hint in ClassTfidfTransformer constructor [@snape](https://github.com/snape) in [#1803](https://github.com/MaartenGr/BERTopic/pull/1803)\n* Fix typo and simplify wording in OnlineCountVectorizer docstring by [@chrisji](https://github.com/chrisji) in [#1802](https://github.com/MaartenGr/BERTopic/pull/1802)\n* Fixed warning when saving a topic model without an embedding model by [@zilch42](https://github.com/zilch42) in [#1740](https://github.com/MaartenGr/BERTopic/pull/1740)\n* Fix bug in `TextGeneration` by [@manveersadhal](https://github.com/manveersadhal) in [#1726](https://github.com/MaartenGr/BERTopic/pull/1726)\n* Fix an incorrect link to usecases.md by [@nicholsonjf](https://github.com/nicholsonjf) in [#1731](https://github.com/MaartenGr/BERTopic/pull/1731)\n* Prevent `model` argument being passed twice when using `generator_kwargs` in OpenAI by [@ninavandiermen](https://github.com/ninavandiermen) in [#1733](https://github.com/MaartenGr/BERTopic/pull/1733)\n* Several fixes to the docstrings by [@arpadikuma](https://github.com/arpadikuma) in [#1719](https://github.com/MaartenGr/BERTopic/pull/1719)\n* Remove unused `cluster_df` variable in `hierarchical_topics` by [@shadiakiki1986](https://github.com/shadiakiki1986) in [#1701](https://github.com/MaartenGr/BERTopic/pull/1701)\n* Removed redundant quotation mark by [@LawrenceFulton](https://github.com/LawrenceFulton) in [#1695](https://github.com/MaartenGr/BERTopic/pull/1695)\n* Fix typo in merge models docs by [@zilch42](https://github.com/zilch42) in [#1660](https://github.com/MaartenGr/BERTopic/pull/1660)\n\n\n## **Version 0.16.0**\n*Release date: 26 November, 2023*\n\n

Highlights:

\n\n* Merge pre-trained BERTopic models with [**`.merge_models`**](https://maartengr.github.io/BERTopic/getting_started/merge/merge.html)\n * Combine models with different representations together!\n * Use this for *incremental/online topic modeling* to detect new incoming topics\n * First step towards *federated learning* with BERTopic\n* [**Zero-shot**](https://maartengr.github.io/BERTopic/getting_started/zeroshot/zeroshot.html) Topic Modeling\n * Use a predefined list of topics to assign documents\n * If needed, allows for further exploration of undefined topics\n* [**Seed (domain-specific) words**](https://maartengr.github.io/BERTopic/getting_started/seed_words/seed_words.html) with `ClassTfidfTransformer`\n * Make sure selected words are more likely to end up in the representation without influencing the clustering process\n* Added params to [**truncate documents**](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#truncating-documents) to length when using LLMs\n* Added [**LlamaCPP**](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#llamacpp) as a representation model\n* LangChain: Support for **LCEL Runnables** by [@joshuasundance-swca](https://github.com/joshuasundance-swca) in [#1586](https://github.com/MaartenGr/BERTopic/pull/1586)\n* Added `topics` parameter to `.topics_over_time` to select a subset of documents and topics\n* Documentation:\n * [Best practices Guide](https://maartengr.github.io/BERTopic/getting_started/best_practices/best_practices.html)\n * [Llama 2 Tutorial](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#llama-2)\n * [Zephyr Tutorial](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#zephyr-mistral-7b)\n * Improved [embeddings guidance](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html#sentence-transformers) (MTEB)\n * Improved logging throughout the package\n* Added support for **Cohere's Embed v3**:\n```python\ncohere_model = CohereBackend(\n client,\n embedding_model=\"embed-english-v3.0\",\n embed_kwargs={\"input_type\": \"clustering\"}\n)\n```\n\n

Fixes:

\n\n* Fixed n-gram Keywords need delimiting in OpenAI() [#1546](https://github.com/MaartenGr/BERTopic/issues/1546)\n* Fixed OpenAI v1.0 issues [#1629](https://github.com/MaartenGr/BERTopic/issues/1629)\n* Improved documentation/logging to address [#1589](https://github.com/MaartenGr/BERTopic/issues/1589), [#1591](https://github.com/MaartenGr/BERTopic/issues/1591)\n* Fixed engine support for Azure OpenAI embeddings [#1577](https://github.com/MaartenGr/BERTopic/issues/1487)\n* Fixed OpenAI Representation: KeyError: 'content' [#1570](https://github.com/MaartenGr/BERTopic/issues/1570)\n* Fixed Loading topic model with multiple topic aspects changes their format [#1487](https://github.com/MaartenGr/BERTopic/issues/1487)\n* Fix expired link in algorithm.md by [@burugaria7](https://github.com/burugaria7) in [#1396](https://github.com/MaartenGr/BERTopic/pull/1396)\n* Fix guided topic modeling in cuML's UMAP by [@stevetracvc](https://github.com/stevetracvc) in [#1326](https://github.com/MaartenGr/BERTopic/pull/1326)\n* OpenAI: Allow retrying on Service Unavailable errors by [@agamble](https://github.com/agamble) in [#1407](https://github.com/MaartenGr/BERTopic/pull/1407)\n* Fixed parameter naming for HDBSCAN in best practices by [@rnckp](https://github.com/rnckp) in [#1408](https://github.com/MaartenGr/BERTopic/pull/1408)\n* Fixed typo in tips_and_tricks.md by [@aronnoordhoek](https://github.com/aronnoordhoek) in [#1446](https://github.com/MaartenGr/BERTopic/pull/1446)\n* Fix typos in documentation by [@bobchien](https://github.com/bobchien) in [#1481](https://github.com/MaartenGr/BERTopic/pull/1481)\n* Fix IndexError when all outliers are removed by reduce_outliers by [@Aratako](https://github.com/Aratako) in [#1466](https://github.com/MaartenGr/BERTopic/pull/1466)\n* Fix TypeError on reduce_outliers \"probabilities\" by [@ananaphasia](https://github.com/ananaphasia) in [#1501](https://github.com/MaartenGr/BERTopic/pull/1501)\n* Add new line to fix markdown bullet point formatting by [@saeedesmaili](https://github.com/saeedesmaili) in [#1519](https://github.com/MaartenGr/BERTopic/pull/1519)\n* Update typo in topicrepresentation.md by [@oliviercaron](https://github.com/oliviercaron) in [#1537](https://github.com/MaartenGr/BERTopic/pull/1537)\n* Fix typo in FAQ by [@sandijou](https://github.com/sandijou) in [#1542](https://github.com/MaartenGr/BERTopic/pull/1542)\n* Fixed typos in best practices documentation by [@poomkusa](https://github.com/poomkusa) in [#1557](https://github.com/MaartenGr/BERTopic/pull/1557)\n* Correct TopicMapper doc example by [@chrisji](https://github.com/chrisji) in [#1637](https://github.com/MaartenGr/BERTopic/pull/1637)\n* Fix typing in hierarchical_topics by [@dschwalm](https://github.com/dschwalm) in [#1364](https://github.com/MaartenGr/BERTopic/pull/1364)\n* Fixed typing issue with threshold parameter in reduce_outliers by [@dschwalm](https://github.com/dschwalm) in [#1380](https://github.com/MaartenGr/BERTopic/pull/1380)\n* Fix several typos by [@mertyyanik](https://github.com/mertyyanik) in [#1307](https://github.com/MaartenGr/BERTopic/pull/1307)\n(#1307)\n* Fix inconsistent naming by [@rolanderdei](https://github.com/rolanderdei) in [#1073](https://github.com/MaartenGr/BERTopic/pull/1073)\n\n

Merge Pre-trained BERTopic Models

\n\nThe new `.merge_models` feature allows for any number of fitted BERTopic models to be merged. Doing so allows for a number of use cases:\n\n* **Incremental topic modeling** -- Continuously merge models together to detect whether new topics have appeared\n* **Federated Learning** - Train BERTopic models on different clients and combine them on a central server\n* **Minimal compute** - We can essentially batch the training process into multiple instances to reduce compute\n* **Different datasets** - When you have different datasets that you want to train separately on, for example with different languages, you can train each model separately and join them after training\n\nTo demonstrate merging different topic models with BERTopic, we use the ArXiv paper abstracts to see which topics they generally contain.\n\nFirst, we train three separate models on different parts of the data:\n\n```python\nfrom umap import UMAP\nfrom bertopic import BERTopic\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\n\n# Extract abstracts to train on and corresponding titles\nabstracts_1 = dataset[\"abstract\"][:5_000]\nabstracts_2 = dataset[\"abstract\"][5_000:10_000]\nabstracts_3 = dataset[\"abstract\"][10_000:15_000]\n\n# Create topic models\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\ntopic_model_1 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_1)\ntopic_model_2 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_2)\ntopic_model_3 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_3)\n```\n\nThen, we can combine all three models into one with `.merge_models`:\n\n```python\n# Combine all models into one\nmerged_model = BERTopic.merge_models([topic_model_1, topic_model_2, topic_model_3])\n```\n\n

Zero-shot Topic Modeling

\nZeroshot Topic Modeling is a technique that allows you to find pre-defined topics in large amounts of documents. This method allows you to not only find those specific topics but also create new topics for documents that would not fit with your predefined topics.\nThis allows for extensive flexibility as there are three scenario's to explore.\n\n* No zeroshot topics were detected. This means that none of the documents would fit with the predefined topics and a regular BERTopic would be run.\n* Only zeroshot topics were detected. Here, we would not need to find additional topics since all original documents were assigned to one of the predefined topics.\n* Both zeroshot topics and clustered topics were detected. This means that some documents would fit with the predefined topics where others would not. For the latter, new topics were found.\n\n![zeroshot](https://github.com/MaartenGr/BERTopic/assets/25746895/9cce6ee3-445f-440a-b93b-f8008578c839)\n\nIn order to use zero-shot BERTopic, we create a list of topics that we want to assign to our documents. However,\nthere may be several other topics that we know should be in the documents. The dataset that we use is small subset of ArXiv papers.\nWe know the data and believe there to be at least the following topics: *clustering*, *topic modeling*, and *large language models*.\nHowever, we are not sure whether other topics exist and want to explore those.\n\nUsing this feature is straightforward:\n\n```python\nfrom datasets import load_dataset\n\nfrom bertopic import BERTopic\nfrom bertopic.representation import KeyBERTInspired\n\n# We select a subsample of 5000 abstracts from ArXiv\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\ndocs = dataset[\"abstract\"][:5_000]\n\n# We define a number of topics that we know are in the documents\nzeroshot_topic_list = [\"Clustering\", \"Topic Modeling\", \"Large Language Models\"]\n\n# We fit our model using the zero-shot topics\n# and we define a minimum similarity. For each document,\n# if the similarity does not exceed that value, it will be used\n# for clustering instead.\ntopic_model = BERTopic(\n embedding_model=\"thenlper/gte-small\",\n min_topic_size=15,\n zeroshot_topic_list=zeroshot_topic_list,\n zeroshot_min_similarity=.85,\n representation_model=KeyBERTInspired()\n)\ntopics, _ = topic_model.fit_transform(docs)\n```\n\nWhen we run `topic_model.get_topic_info()` you will see something like this:\n\n![zeroshot_output](https://github.com/MaartenGr/BERTopic/assets/25746895/1801e0a9-cda7-4d74-929f-e975fa67404b)\n\n

Seed (Domain-specific) Words

\n\n\nWhen performing Topic Modeling, you are often faced with data that you are familiar with to a certain extend or that speaks a very specific language. In those cases, topic modeling techniques might have difficulties capturing and representing the semantic nature of domain specific abbreviations, slang, short form, acronyms, etc. For example, the *\"TNM\"* classification is a method for identifying the stage of most cancers. The word *\"TNM\"* is an abbreviation and might not be correctly captured in generic embedding models.\n\nTo make sure that certain domain specific words are weighted higher and are more often used in topic representations, you can set any number of `seed_words` in the `bertopic.vectorizer.ClassTfidfTransformer`. To do so, let's take a look at an example. We have a dataset of article abstracts and want to perform some topic modeling. Since we might be familiar with the data, there are certain words that we know should be generally important. Let's assume that we have in-depth knowledge about reinforcement learning and know that words like \"agent\" and \"robot\" should be important in such a topic were it to be found. Using the `ClassTfidfTransformer`, we can define those `seed_words` and also choose by how much their values are multiplied.\n\nThe full example is then as follows:\n\n```python\nfrom umap import UMAP\nfrom datasets import load_dataset\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\n# Let's take a subset of ArXiv abstracts as the training data\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\nabstracts = dataset[\"abstract\"][:5_000]\n\n# For illustration purposes, we make sure the output is fixed when running this code multiple times\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\n\n# We can choose any number of seed words for which we want their representation\n# to be strengthen. We increase the importance of these words as we want them to be more\n# likely to end up in the topic representations.\nctfidf_model = ClassTfidfTransformer(\n seed_words=[\"agent\", \"robot\", \"behavior\", \"policies\", \"environment\"],\n seed_multiplier=2\n)\n\n# We run the topic model with the seeded words\ntopic_model = BERTopic(\n umap_model=umap_model,\n min_topic_size=15,\n ctfidf_model=ctfidf_model,\n).fit(abstracts)\n```\n\n

Truncate Documents in LLMs

\n\nWhen using LLMs with BERTopic, we can truncate the input documents in `[DOCUMENTS]` in order to reduce the number of tokens that we have in our input prompt. To do so, all text generation modules have two parameters that we can tweak:\n\n* `doc_length` - The maximum length of each document. If a document is longer, it will be truncated. If None, the entire document is passed.\n* `tokenizer` - The tokenizer used to calculate to split the document into segments used to count the length of a document.\n * Options include `'char'`, `'whitespace'`, `'vectorizer'`, and a callable\n\nThis means that the definition of `doc_length` changes depending on what constitutes a token in the `tokenizer` parameter. If a token is a character, then `doc_length` refers to max length in characters. If a token is a word, then `doc_length` refers to the max length in words.\n\nLet's illustrate this with an example. In the code below, we will use [`tiktoken`](https://github.com/openai/tiktoken) to count the number of tokens in each document and limit them to 100 tokens. All documents that have more than 100 tokens will be truncated.\n\nWe use `bertopic.representation.OpenAI` to represent our topics with nicely written labels. We specify that documents that we put in the prompt cannot exceed 100 tokens each. Since we will put 4 documents in the prompt, they will total roughly 400 tokens:\n\n```python\nimport openai\nimport tiktoken\nfrom bertopic.representation import OpenAI\nfrom bertopic import BERTopic\n\n# Tokenizer\ntokenizer= tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n\n# Create your representation model\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(\n client,\n model=\"gpt-3.5-turbo\",\n delay_in_seconds=2,\n chat=True,\n nr_docs=4,\n doc_length=100,\n tokenizer=tokenizer\n)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n\n## **Version 0.15.0**\n*Release date: 29 May, 2023*\n\n

Highlights:

\n\n* [**Multimodal**](https://maartengr.github.io/BERTopic/getting_started/multimodal/multimodal.html) Topic Modeling\n * Train your topic modeling on text, images, or images and text!\n * Use the `bertopic.backend.MultiModalBackend` to embed images, text, both or even caption images!\n* [**Multi-Aspect**](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) Topic Modeling\n * Create multiple topic representations simultaneously\n* Improved [**Serialization**](https://maartengr.github.io/BERTopic/getting_started/serialization/serialization.html) options\n * Push your model to the HuggingFace Hub with `.push_to_hf_hub`\n * Safer, smaller and more flexible serialization options with `safetensors`\n * Thanks to a great collaboration with HuggingFace and the authors of [BERTransfer](https://github.com/opinionscience/BERTransfer)!\n* Added new embedding models\n * OpenAI: `bertopic.backend.OpenAIBackend`\n * Cohere: `bertopic.backend.CohereBackend`\n* Added example of [summarizing topics](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html#summarization) with OpenAI's GPT-models\n* Added `nr_docs` and `diversity` parameters to OpenAI and Cohere representation models\n* Use `custom_labels=\"Aspect1\"` to use the aspect labels for visualizations instead\n* Added cuML support for probability calculation in `.transform`\n* Updated **topic embeddings**\n * Centroids by default and c-TF-IDF weighted embeddings for `partial_fit` and `.update_topics`\n* Added `exponential_backoff` parameter to `OpenAI` model\n\n

Fixes:

\n\n* Fixed custom prompt not working in `TextGeneration`\n* Fixed [#1142](https://github.com/MaartenGr/BERTopic/pull/1142)\n* Add additional logic to handle cupy arrays by [@metasyn](https://github.com/metasyn) in [#1179](https://github.com/MaartenGr/BERTopic/pull/1179)\n* Fix hierarchy viz and handle any form of distance matrix by [@elashrry](https://github.com/elashrry) in [#1173](https://github.com/MaartenGr/BERTopic/pull/1173)\n* Updated languages list by [@sam9111](https://github.com/sam9111) in [#1099](https://github.com/MaartenGr/BERTopic/pull/1099)\n* Added level_scale argument to visualize_hierarchical_documents by [@zilch42](https://github.com/zilch42) in [#1106](https://github.com/MaartenGr/BERTopic/pull/1106)\n* Fix inconsistent naming by [@rolanderdei](https://github.com/rolanderdei) in [#1073](https://github.com/MaartenGr/BERTopic/pull/1073)\n\n

Multimodal Topic Modeling

\n\nWith v0.15, we can now perform multimodal topic modeling in BERTopic! The most basic example of multimodal topic modeling in BERTopic is when you have images that accompany your documents. This means that it is expected that each document has an image and vice versa. Instagram pictures, for example, almost always have some descriptions to them.\n\n
\n ![Image title](getting_started/multimodal/images_and_text.svg)\n
\n
\n\nIn this example, we are going to use images from `flickr` that each have a caption associated to it:\n\n```python\n# NOTE: This requires the `datasets` package which you can\n# install with `pip install datasets`\nfrom datasets import load_dataset\n\nds = load_dataset(\"maderix/flickr_bw_rgb\")\nimages = ds[\"train\"][\"image\"]\ndocs = ds[\"train\"][\"caption\"]\n```\n\nThe `docs` variable contains the captions for each image in `images`. We can now use these variables to run our multimodal example:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import VisualRepresentation\n\n# Additional ways of representing a topic\nvisual_model = VisualRepresentation()\n\n# Make sure to add the `visual_model` to a dictionary\nrepresentation_model = {\n \"Visual_Aspect\": visual_model,\n}\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\nWe can now access our image representations for each topic with `topic_model.topic_aspects_[\"Visual_Aspect\"]`.\nIf you want an overview of the topic images together with their textual representations in jupyter, you can run the following:\n\n```python\nimport base64\nfrom io import BytesIO\nfrom IPython.display import HTML\n\ndef image_base64(im):\n if isinstance(im, str):\n im = get_thumbnail(im)\n with BytesIO() as buffer:\n im.save(buffer, 'jpeg')\n return base64.b64encode(buffer.getvalue()).decode()\n\n\ndef image_formatter(im):\n return f''\n\n# Extract dataframe\ndf = topic_model.get_topic_info().drop(\"Representative_Docs\", 1).drop(\"Name\", 1)\n\n# Visualize the images\nHTML(df.to_html(formatters={'Visual_Aspect': image_formatter}, escape=False))\n```\n\n![images_and_text](https://github.com/MaartenGr/BERTopic/assets/25746895/3a741e2b-5810-4865-9664-0c6bb24ca3f9)\n\n\n

Multi-aspect Topic Modeling

\n\nIn this new release, we introduce `multi-aspect topic modeling`! During the `.fit` or `.fit_transform` stages, you can now get multiple representations of a single topic. In practice, it works by generating and storing all kinds of different topic representations (see image below).\n\n
\n ![Image title](getting_started/multiaspect/multiaspect.svg)\n
\n
\n\nThe approach is rather straightforward. We might want to represent our topics using a `PartOfSpeech` representation model but we might also want to try out `KeyBERTInspired` and compare those representation models. We can do this as follows:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic.representation import PartOfSpeech\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Documents to train on\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\n# The main representation of a topic\nmain_representation = KeyBERTInspired()\n\n# Additional ways of representing a topic\naspect_model1 = PartOfSpeech(\"en_core_web_sm\")\naspect_model2 = [KeyBERTInspired(top_n_words=30), MaximalMarginalRelevance(diversity=.5)]\n\n# Add all models together to be run in a single `fit`\nrepresentation_model = {\n \"Main\": main_representation,\n \"Aspect1\": aspect_model1,\n \"Aspect2\": aspect_model2\n}\ntopic_model = BERTopic(representation_model=representation_model).fit(docs)\n```\n\nAs show above, to perform multi-aspect topic modeling, we make sure that `representation_model` is a dictionary where each representation model pipeline is defined.\nThe main pipeline, that is used in most visualization options, is defined with the `\"Main\"` key. All other aspects can be defined however you want. In the example above, the two additional aspects that we are interested in are defined as `\"Aspect1\"` and `\"Aspect2\"`.\n\nAfter we have fitted our model, we can access all representations with `topic_model.get_topic_info()`:\n\n\n
\n\nAs you can see, there are a number of different representations for our topics that we can inspect. All aspects are found in `topic_model.topic_aspects_`.\n\n\n

Serialization

\n\nSaving, loading, and sharing a BERTopic model can be done in several ways. With this new release, it is now advised to go with `.safetensors` as that allows for a small, safe, and fast method for saving your BERTopic model. However, other formats, such as `.pickle` and pytorch `.bin` are also possible.\n\nThe methods are used as follows:\n\n```python\ntopic_model = BERTopic().fit(my_docs)\n\n# Method 1 - safetensors\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"safetensors\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 2 - pytorch\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"pytorch\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 3 - pickle\ntopic_model.save(\"my_model\", serialization=\"pickle\")\n```\n\nSaving the topic modeling with `.safetensors` or `pytorch` has a number of advantages:\n\n* `.safetensors` is a relatively **safe format**\n* The resulting model can be **very small** (often < 20MB>) since no sub-models need to be saved\n* Although version control is important, there is a bit more **flexibility** with respect to specific versions of packages\n* More easily used in **production**\n* **Share** models with the HuggingFace Hub\n\n

\n\n

\n\nThe above image, a model trained on 100,000 documents, demonstrates the differences in sizes comparing `safetensors`, `pytorch`, and `pickle`. The difference in sizes can mostly be explained due to the efficient saving procedure and that the clustering and dimensionality reductions are not saved in safetensors/pytorch since inference can be done based on the topic embeddings.\n\n\n\n\n

HuggingFace Hub

\n\nWhen you have created a BERTopic model, you can easily share it with other through the HuggingFace Hub. First, you need to log in to your HuggingFace account:\n\n```python\nfrom huggingface_hub import login\nlogin()\n```\n\nWhen you have logged in to your HuggingFace account, you can save and upload the model as follows:\n\n```python\nfrom bertopic import BERTopic\n\n# Train model\ntopic_model = BERTopic().fit(my_docs)\n\n# Push to HuggingFace Hub\ntopic_model.push_to_hf_hub(\n repo_id=\"MaartenGr/BERTopic_ArXiv\",\n save_ctfidf=True\n)\n\n# Load from HuggingFace\nloaded_model = BERTopic.load(\"MaartenGr/BERTopic_ArXiv\")\n```\n\n## **Version 0.14.1**\n*Release date: 2 March, 2023*\n\n

Highlights:

\n\n* Use [**ChatGPT**](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html#chatgpt) to create topic representations!:\n* Added `delay_in_seconds` parameter to OpenAI and Cohere representation models for throttling the API\n * Setting this between 5 and 10 allows for trial users now to use more easily without hitting RateLimitErrors\n* Fixed missing `title` param to visualization methods\n* Fixed probabilities not correctly aligning ([#1024](https://github.com/MaartenGr/BERTopic/issues/1024))\n* Fix typo in textgenerator [@dkopljar27](https://github.com/dkopljar27) in [#1002](https://github.com/MaartenGr/BERTopic/pull/1002)\n\n

ChatGPT

\n\nWithin OpenAI's API, the ChatGPT models use a different API structure compared to the GPT-3 models.\nIn order to use ChatGPT with BERTopic, we need to define the model and make sure to set `chat=True`:\n\n```python\nimport openai\nfrom bertopic import BERTopic\nfrom bertopic.representation import OpenAI\n\n# Create your representation model\nopenai.api_key = MY_API_KEY\nrepresentation_model = OpenAI(model=\"gpt-3.5-turbo\", delay_in_seconds=10, chat=True)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nPrompting with ChatGPT is very satisfying and can be customized in BERTopic by using certain tags.\nThere are currently two tags, namely `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"`.\nThese tags indicate where in the prompt they are to be replaced with a topics keywords and top 4 most representative documents respectively.\nFor example, if we have the following prompt:\n\n```python\nprompt = \"\"\"\nI have topic that contains the following documents: \\n[DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS]\n\nBased on the information above, extract a short topic label in the following format:\ntopic: \n\"\"\"\n```\n\nthen that will be rendered as follows and passed to OpenAI's API:\n\n```python\n\"\"\"\nI have a topic that contains the following documents:\n- Our videos are also made possible by your support on patreon.co.\n- If you want to help us make more videos, you can do so on patreon.com or get one of our posters from our shop.\n- If you want to help us make more videos, you can do so there.\n- And if you want to support us in our endeavor to survive in the world of online video, and make more videos, you can do so on patreon.com.\n\nThe topic is described by the following keywords: videos video you our support want this us channel patreon make on we if facebook to patreoncom can for and more watch\n\nBased on the information above, extract a short topic label in the following format:\ntopic: \n\"\"\"\n```\n\n!!! note\n Whenever you create a custom prompt, it is important to add\n ```\n Based on the information above, extract a short topic label in the following format:\n topic: \n ```\n at the end of your prompt as BERTopic extracts everything that comes after `topic: `. Having\n said that, if `topic: ` is not in the output, then it will simply extract the entire response, so\n feel free to experiment with the prompts.\n\n\n## **Version 0.14.0**\n*Release date: 14 February, 2023*\n\n

Highlights:

\n\n* Fine-tune [topic representations](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) with `bertopic.representation`\n * Diverse range of models, including KeyBERT, MMR, POS, Transformers, OpenAI, and more!'\n * Create your own prompts for text generation models, like GPT3:\n * Use `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"` in the prompt to decide where the keywords and set of representative documents need to be inserted.\n * Chain models to perform fine-grained fine-tuning\n * Create and customize your represention model\n* Improved the topic reduction technique when using `nr_topics=int`\n* Added `title` parameters for all graphs ([#800](https://github.com/MaartenGr/BERTopic/issues/800))\n\n\n

Fixes:

\n\n* Improve documentation ([#837](https://github.com/MaartenGr/BERTopic/issues/837), [#769](https://github.com/MaartenGr/BERTopic/issues/769), [#954](https://github.com/MaartenGr/BERTopic/issues/954), [#912](https://github.com/MaartenGr/BERTopic/issues/912), [#911](https://github.com/MaartenGr/BERTopic/issues/911))\n* Bump pyyaml ([#903](https://github.com/MaartenGr/BERTopic/issues/903))\n* Fix large number of representative docs ([#965](https://github.com/MaartenGr/BERTopic/issues/965))\n* Prevent stochastisch behavior in `.visualize_topics` ([#952](https://github.com/MaartenGr/BERTopic/issues/952))\n* Add custom labels parameter to `.visualize_topics` ([#976](https://github.com/MaartenGr/BERTopic/issues/976))\n* Fix cuML HDBSCAN type checks by [@FelSiq](https://github.com/FelSiq) in [#981](https://github.com/MaartenGr/BERTopic/pull/981)\n\n

API Changes:

\n* The `diversity` parameter was removed in favor of `bertopic.representation.MaximalMarginalRelevance`\n* The `representation_model` parameter was added to `bertopic.BERTopic`\n\n
\n\n

Representation Models

\n\nFine-tune the c-TF-IDF representation with a variety of models. Whether that is through a KeyBERT-Inspired model or GPT-3, the choice is up to you!\n\n\n\n
\n\n\n

KeyBERTInspired

\n\nThe algorithm follows some principles of [KeyBERT](https://github.com/MaartenGr/KeyBERT) but does some optimization in order to speed up inference. Usage is straightforward:\n\n![keybertinspired](https://user-images.githubusercontent.com/25746895/216336376-d2c4e5d6-6cf7-435c-904c-fc195aae7dcd.svg)\n\n```python\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = KeyBERTInspired()\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![keybert](https://user-images.githubusercontent.com/25746895/218417161-bfd5980e-43c7-498a-904a-b6018ba58d45.svg)\n\n

PartOfSpeech

\n\nOur candidate topics, as extracted with c-TF-IDF, do not take into account a keyword's part of speech as extracting noun-phrases from all documents can be computationally quite expensive. Instead, we can leverage c-TF-IDF to perform part of speech on a subset of keywords and documents that best represent a topic.\n\n![partofspeech](https://user-images.githubusercontent.com/25746895/216336534-48ff400e-72e1-4c50-9030-414576bac01e.svg)\n\n\n```python\nfrom bertopic.representation import PartOfSpeech\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = PartOfSpeech(\"en_core_web_sm\")\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![pos](https://user-images.githubusercontent.com/25746895/218417198-41c19b5c-251f-43c1-bfe2-0a480731565a.svg)\n\n\n

MaximalMarginalRelevance

\n\nWhen we calculate the weights of keywords, we typically do not consider whether we already have similar keywords in our topic. Words like \"car\" and \"cars\"\nessentially represent the same information and often redundant. We can use `MaximalMarginalRelevance` to improve diversity of our candidate topics:\n\n![mmr](https://user-images.githubusercontent.com/25746895/216336697-558f1409-8da3-4076-a21b-d87eec583ac7.svg)\n\n\n```python\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = MaximalMarginalRelevance(diversity=0.3)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![mmr (1)](https://user-images.githubusercontent.com/25746895/218417234-88b145e2-7293-43c0-888c-36abe469a48a.svg)\n\n

Zero-Shot Classification

\n\nTo perform zero-shot classification, we feed the model with the keywords as generated through c-TF-IDF and a set of candidate labels. If, for a certain topic, we find a similar enough label, then it is assigned. If not, then we keep the original c-TF-IDF keywords.\n\nWe use it in BERTopic as follows:\n\n```python\nfrom bertopic.representation import ZeroShotClassification\nfrom bertopic import BERTopic\n\n# Create your representation model\ncandidate_topics = [\"space and nasa\", \"bicycles\", \"sports\"]\nrepresentation_model = ZeroShotClassification(candidate_topics, model=\"facebook/bart-large-mnli\")\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![zero](https://user-images.githubusercontent.com/25746895/218417276-dcef3519-acba-4792-8601-45dc7ed39488.svg)\n\n

Text Generation: πŸ€— Transformers

\n\nNearly every week, there are new and improved models released on the πŸ€— [Model Hub](https://huggingface.co/models) that, with some creativity, allow for\nfurther fine-tuning of our c-TF-IDF based topics. These models range from text generation to zero-classification. In BERTopic, wrappers around these\nmethods are created as a way to support whatever might be released in the future.\n\nUsing a GPT-like model from the huggingface hub is rather straightforward:\n\n```python\nfrom bertopic.representation import TextGeneration\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = TextGeneration('gpt2')\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![hf](https://user-images.githubusercontent.com/25746895/218417310-2b0eabc7-296d-499d-888b-0ab48a65a2fb.svg)\n\n\n

Text Generation: Cohere

\n\nInstead of using a language model from πŸ€— transformers, we can use external APIs instead that\ndo the work for you. Here, we can use [Cohere](https://docs.cohere.ai/) to extract our topic labels from the candidate documents and keywords.\nTo use this, you will need to install cohere first:\n\n```bash\npip install cohere\n```\n\nThen, get yourself an API key and use Cohere's API as follows:\n\n```python\nimport cohere\nfrom bertopic.representation import Cohere\nfrom bertopic import BERTopic\n\n# Create your representation model\nco = cohere.Client(my_api_key)\nrepresentation_model = Cohere(co)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![cohere](https://user-images.githubusercontent.com/25746895/218417337-294cb52a-93c9-4fd5-b981-29b40e4f0c1e.svg)\n\n\n

Text Generation: OpenAI

\n\nInstead of using a language model from πŸ€— transformers, we can use external APIs instead that\ndo the work for you. Here, we can use [OpenAI](https://openai.com/api/) to extract our topic labels from the candidate documents and keywords.\nTo use this, you will need to install openai first:\n\n```\npip install openai\n```\n\nThen, get yourself an API key and use OpenAI's API as follows:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\nfrom bertopic import BERTopic\n\n# Create your representation model\nopenai.api_key = MY_API_KEY\nrepresentation_model = OpenAI()\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n![openai](https://user-images.githubusercontent.com/25746895/218417357-cf8c0fab-4450-43d3-b4fd-219ed276d870.svg)\n\n\n

Text Generation: LangChain

\n\n[Langchain](https://github.com/hwchase17/langchain) is a package that helps users with chaining large language models.\nIn BERTopic, we can leverage this package in order to more efficiently combine external knowledge. Here, this\nexternal knowledge are the most representative documents in each topic.\n\nTo use langchain, you will need to install the langchain package first. Additionally, you will need an underlying LLM to support langchain,\nlike openai:\n\n```bash\npip install langchain, openai\n```\n\nThen, you can create your chain as follows:\n\n```python\nfrom langchain.chains.question_answering import load_qa_chain\nfrom langchain.llms import OpenAI\nchain = load_qa_chain(OpenAI(temperature=0, openai_api_key=MY_API_KEY), chain_type=\"stuff\")\n```\n\nFinally, you can pass the chain to BERTopic as follows:\n\n```python\nfrom bertopic.representation import LangChain\n\n# Create your representation model\nrepresentation_model = LangChain(chain)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n\n## **Version 0.13.0**\n*Release date: 4 January, 2023*\n\n

Highlights:

\n\n* Calculate [topic distributions](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html) with `.approximate_distribution` regardless of the cluster model used\n * Generates topic distributions on a document- and token-levels\n * Can be used for any document regardless of its size!\n* [Fully supervised BERTopic](https://maartengr.github.io/BERTopic/getting_started/supervised/supervised.html)\n * You can now use a classification model for the clustering step instead to create a fully supervised topic model\n* [Manual topic modeling](https://maartengr.github.io/BERTopic/getting_started/manual/manual.html)\n * Generate topic representations from labels directly\n * Allows for skipping the embedding and clustering steps in order to go directly to the topic representation step\n* [Reduce outliers](https://maartengr.github.io/BERTopic/getting_started/outlier_reduction/outlier_reduction.html) with 4 different strategies using `.reduce_outliers`\n* Install BERTopic without `SentenceTransformers` for a [lightweight package](https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html#lightweight-installation):\n * `pip install --no-deps bertopic`\n * `pip install --upgrade numpy hdbscan umap-learn pandas scikit-learn tqdm plotly pyyaml`\n* Get meta data of trained documents such as topics and probabilities using `.get_document_info(docs)`\n* Added more support for cuML's HDBSCAN\n * Calculate and predict probabilities during `fit_transform` and `transform` respectively\n * This should give a major speed-up when setting `calculate_probabilities=True`\n* More images to the documentation and a lot of changes/updates/clarifications\n* Get representative documents for non-HDBSCAN models by comparing document and topic c-TF-IDF representations\n* Sklearn Pipeline [Embedder](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html#scikit-learn-embeddings) by [@koaning](https://github.com/koaning) in [#791](https://github.com/MaartenGr/BERTopic/pull/791)\n\n

Fixes:

\n\n* Improve `.partial_fit` documentation ([#837](https://github.com/MaartenGr/BERTopic/issues/837))\n* Fixed scipy linkage usage ([#807](https://github.com/MaartenGr/BERTopic/issues/807))\n* Fixed shifted heatmap ([#782](https://github.com/MaartenGr/BERTopic/issues/782))\n* Fixed SpaCy backend ([#744](https://github.com/MaartenGr/BERTopic/issues/744))\n* Fixed representative docs with small clusters (<3) ([#703](https://github.com/MaartenGr/BERTopic/issues/703))\n* Typo fixed by [@timpal0l](https://github.com/timpal0l) in [#734](https://github.com/MaartenGr/BERTopic/pull/734)\n* Typo fixed by [@srulikbd](https://github.com/timpal0l) in [#842](https://github.com/MaartenGr/BERTopic/pull/842)\n* Correcting iframe urls by [@Mustapha-AJEGHRIR](https://github.com/Mustapha-AJEGHRIR) in [#798](https://github.com/MaartenGr/BERTopic/pull/798)\n* Refactor embedding methods by [@zachschillaci27](https://github.com/zachschillaci27) in [#855](https://github.com/MaartenGr/BERTopic/pull/855)\n* Added diversity parameter to update_topics() function by [@anubhabdaserrr](https://github.com/anubhabdaserrr) in [#887](https://github.com/MaartenGr/BERTopic/pull/887)\n\n

Documentation

\n\nPersonally, I believe that documentation can be seen as a feature and is an often underestimated aspect of open-source. So I went a bit overboardπŸ˜…... and created an animation about the three pillars of BERTopic using Manim. There are many other visualizations added, one of each variation of BERTopic, and many smaller changes.\n\n\n\n

Topic Distributions

\n\nThe difficulty with a cluster-based topic modeling technique is that it does not directly consider that documents may contain multiple topics. With the new release, we can now model the distributions of topics! We even consider that a single word might be related to multiple topics. If a document is a mixture of topics, what is preventing a single word to be the same?\n\nTo do so, we approximate the distribution of topics in a document by calculating and summing the similarities of tokensets (achieved by applying a sliding window) with the topics:\n\n```python\n# After fitting your model run the following for either your trained documents or even unseen documents\ntopic_distr, _ = topic_model.approximate_distribution(docs)\n```\n\nTo calculate and visualize the topic distributions in a document on a token-level, we can run the following:\n\n```python\n# We need to calculate the topic distributions on a token level\ntopic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)\n\n# Create a visualization using a styled dataframe if Jinja2 is installed\ndf = topic_model.visualize_approximate_distribution(docs[0], topic_token_distr[0]); df\n```\n\n

Supervised Topic Modeling

\n\nBERTopic now supports fully-supervised classification! Instead of using a clustering algorithm, like HDBSCAN, we can replace it with a classifier, like Logistic Regression:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.dimensionality import BaseDimensionalityReduction\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.linear_model import LogisticRegression\n\n# Get labeled data\ndata= fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data['data']\ny = data['target']\n\n# Allows us to skip over the dimensionality reduction step\nempty_dimensionality_model = BaseDimensionalityReduction()\n\n# Create a classifier to be used instead of the cluster model\nclf= LogisticRegression()\n\n# Create a fully supervised BERTopic instance\ntopic_model= BERTopic(\n umap_model=empty_dimensionality_model,\n hdbscan_model=clf\n)\ntopics, probs = topic_model.fit_transform(docs, y=y)\n```\n\n

Manual Topic Modeling

\n\nWhen you already have a bunch of labels and simply want to extract topic representations from them, you might not need to actually learn how those can predicted. We can bypass the `embeddings -> dimensionality reduction -> clustering` steps and go straight to the c-TF-IDF representation of our labels:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.backend import BaseEmbedder\nfrom bertopic.cluster import BaseCluster\nfrom bertopic.dimensionality import BaseDimensionalityReduction\n\n# Prepare our empty sub-models and reduce frequent words while we are at it.\nempty_embedding_model = BaseEmbedder()\nempty_dimensionality_model = BaseDimensionalityReduction()\nempty_cluster_model = BaseCluster()\n\n# Fit BERTopic without actually performing any clustering\ntopic_model= BERTopic(\n embedding_model=empty_embedding_model,\n umap_model=empty_dimensionality_model,\n hdbscan_model=empty_cluster_model,\n)\ntopics, probs = topic_model.fit_transform(docs, y=y)\n```\n\n

Outlier Reduction

\n\nOutlier reduction is an frequently-discussed topic in BERTopic as its default cluster model, HDBSCAN, has a tendency to generate many outliers. This often helps in the topic representation steps, as we do not consider documents that are less relevant, but you might want to still assign those outliers to actual topics. In the modular philosophy of BERTopic, keeping training times in mind, it is now possible to perform outlier reduction **after** having trained your topic model. This allows for ease of iteration and prevents having to train BERTopic many times to find the parameters you are searching for. There are 4 different strategies that you can use, so make sure to check out the [documentation](https://maartengr.github.io/BERTopic/getting_started/outlier_reduction/outlier_reduction.html)!\n\nUsing it is rather straightforward:\n\n```python\nnew_topics = topic_model.reduce_outliers(docs, topics)\n```\n\n

Lightweight BERTopic

\n\nThe default embedding model in BERTopic is one of the amazing sentence-transformers models, namely `\"all-MiniLM-L6-v2\"`. Although this model performs well out of the box, it typically needs a GPU to transform the documents into embeddings in a reasonable time. Moreover, the installation requires `pytorch` which often results in a rather large environment, memory-wise.\n\nFortunately, it is possible to install BERTopic without `sentence-transformers` and use it as a lightweight solution instead. The installation can be done as follows:\n\n```bash\npip install --no-deps bertopic\npip install --upgrade numpy hdbscan umap-learn pandas scikit-learn tqdm plotly pyyaml\n```\n\nThen, we can use BERTopic without `sentence-transformers` as follows using a CPU-based embedding technique:\n\n```python\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\npipe = make_pipeline(\n TfidfVectorizer(),\n TruncatedSVD(100)\n)\n\ntopic_model = BERTopic(embedding_model=pipe)\n```\n\nAs a result, the entire package and resulting model can be run quickly on the CPU and no GPU is necessary!\n\n

Document Information

\n\nGet information about the documents on which the topic was trained including the documents themselves, their respective topics, the name of each topic, the top n words of each topic, whether it is a representative document, and the probability of the clustering if the cluster model supports it. There are also options to include other metadata, such as the topic distributions or the x and y coordinates of the reduced embeddings that you can learn more about here.\n\nTo get the document info, you will only need to pass the documents on which the topic model was trained:\n\n\n```python\n>>> topic_model.get_document_info(docs)\n\nDocument Topic\tName\t Top_n_words Probability ...\nI am sure some bashers of Pens...\t 0\t 0_game_team_games_season\tgame - team - games...\t 0.200010 ...\nMy brother is in the market for... -1 -1_can_your_will_any\t can - your - will...\t 0.420668 ...\nFinally you said what you dream...\t -1 -1_can_your_will_any\t can - your - will... 0.807259 ...\nThink! It is the SCSI card doing...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.071746 ...\n1) I have an old Jasmine drive...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.038983 ...\n```\n\n\n\n## **Version 0.12.0**\n*Release date: 5 September, 2022*\n\n**Highlights**:\n\n* Perform [online/incremental topic modeling](https://maartengr.github.io/BERTopic/getting_started/online/online.html) with `.partial_fit`\n* Expose [c-TF-IDF model](https://maartengr.github.io/BERTopic/getting_started/ctfidf/ctfidf.html) for customization with `bertopic.vectorizers.ClassTfidfTransformer`\n * The parameters `bm25_weighting` and `reduce_frequent_words` were added to potentially improve representations:\n* Expose attributes for easier access to internal data\n* Major changes to the [Algorithm](https://maartengr.github.io/BERTopic/algorithm/algorithm.html) page of the documentation, which now contains three overviews of the algorithm:\n * [Visualize Overview](https://maartengr.github.io/BERTopic/algorithm/algorithm.html#visual-overview)\n * [Code Overview](https://maartengr.github.io/BERTopic/algorithm/algorithm.html#code-overview)\n * [Detailed Overview](https://maartengr.github.io/BERTopic/algorithm/algorithm.html#detailed-overview)\n* Added an [example](https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html#keybert-bertopic) of combining BERTopic with KeyBERT\n* Added many tests with the intention of making development a bit more stable\n\n**Fixes**:\n\n* Fixed iteratively merging topics ([#632](https://github.com/MaartenGr/BERTopic/issues/632) and ([#648](https://github.com/MaartenGr/BERTopic/issues/648))\n* Fixed 0th topic not showing up in visualizations ([#667](https://github.com/MaartenGr/BERTopic/issues/667))\n* Fixed lowercasing not being optional ([#682](https://github.com/MaartenGr/BERTopic/issues/682))\n* Fixed spelling ([#664](https://github.com/MaartenGr/BERTopic/issues/664) and ([#673](https://github.com/MaartenGr/BERTopic/issues/673))\n* Fixed 0th topic not shown in `.get_topic_info` by [@oxymor0n](https://github.com/oxymor0n) in [#660](https://github.com/MaartenGr/BERTopic/pull/660)\n* Fixed spelling by [@domenicrosati](https://github.com/domenicrosati) in [#674](https://github.com/MaartenGr/BERTopic/pull/674)\n* Add custom labels and title options to barchart [@leloykun](https://github.com/leloykun) in [#694](https://github.com/MaartenGr/BERTopic/pull/694)\n\n**Online/incremental topic modeling**:\n\nOnline topic modeling (sometimes called \"incremental topic modeling\") is the ability to learn incrementally from a mini-batch of instances. Essentially, it is a way to update your topic model with data on which it was not trained on before. In Scikit-Learn, this technique is often modeled through a `.partial_fit` function, which is also used in BERTopic.\n\nAt a minimum, the cluster model needs to support a `.partial_fit` function in order to use this feature. The default HDBSCAN model will not work as it does not support online updating.\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.decomposition import IncrementalPCA\nfrom bertopic.vectorizers import OnlineCountVectorizer\nfrom bertopic import BERTopic\n\n# Prepare documents\nall_docs = fetch_20newsgroups(subset=subset, remove=('headers', 'footers', 'quotes'))[\"data\"]\ndoc_chunks = [all_docs[i:i+1000] for i in range(0, len(all_docs), 1000)]\n\n# Prepare sub-models that support online learning\numap_model = IncrementalPCA(n_components=5)\ncluster_model = MiniBatchKMeans(n_clusters=50, random_state=0)\nvectorizer_model = OnlineCountVectorizer(stop_words=\"english\", decay=.01)\n\ntopic_model = BERTopic(umap_model=umap_model,\n hdbscan_model=cluster_model,\n vectorizer_model=vectorizer_model)\n\n# Incrementally fit the topic model by training on 1000 documents at a time\nfor docs in doc_chunks:\n topic_model.partial_fit(docs)\n```\n\nOnly the topics for the most recent batch of documents are tracked. If you want to be using online topic modeling, not for a streaming setting but merely for low-memory use cases, then it is advised to also update the `.topics_` attribute as variations such as hierarchical topic modeling will not work afterward:\n\n```python\n# Incrementally fit the topic model by training on 1000 documents at a time and track the topics in each iteration\ntopics = []\nfor docs in doc_chunks:\n topic_model.partial_fit(docs)\n topics.extend(topic_model.topics_)\n\ntopic_model.topics_ = topics\n```\n\n**c-TF-IDF**:\n\nExplicitly define, use, and adjust the `ClassTfidfTransformer` with new parameters, `bm25_weighting` and `reduce_frequent_words`, to potentially improve the topic representation:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer(bm25_weighting=True)\ntopic_model = BERTopic(ctfidf_model=ctfidf_model)\n```\n\n**Attributes**:\n\nAfter having fitted your BERTopic instance, you can use the following attributes to have quick access to certain information, such as the topic assignment for each document in `topic_model.topics_`.\n\n| Attribute | Type | Description |\n|--------------------|----|---------------------------------------------------------------------------------------------|\n| topics_ | List[int] | The topics that are generated for each document after training or updating the topic model. The most recent topics are tracked. |\n| probabilities_ | List[float] | The probability of the assigned topic per document. These are only calculated if a HDBSCAN model is used for the clustering step. When `calculate_probabilities=True`, then it is the probabilities of all topics per document. |\n| topic_sizes_ | Mapping[int, int] | The size of each topic. |\n| topic_mapper_ | TopicMapper | A class for tracking topics and their mappings anytime they are merged, reduced, added, or removed. |\n| topic_representations_ | Mapping[int, Tuple[int, float]] | The top *n* terms per topic and their respective c-TF-IDF values. |\n| c_tf_idf_ | csr_matrix | The topic-term matrix as calculated through c-TF-IDF. To access its respective words, run `.vectorizer_model.get_feature_names()` or `.vectorizer_model.get_feature_names_out()` |\n| topic_labels_ | Mapping[int, str] | The default labels for each topic. |\n| custom_labels_ | List[str] | Custom labels for each topic as generated through `.set_topic_labels`. |\n| topic_embeddings_ | np.ndarray | The embeddings for each topic. It is calculated by taking the weighted average of word embeddings in a topic based on their c-TF-IDF values. |\n| representative_docs_ | Mapping[int, str] | The representative documents for each topic if HDBSCAN is used. |\n\n\n## **Version 0.11.0**\n*Release date: 11 July, 2022*\n\n**Highlights**:\n\n* Perform [hierarchical topic modeling](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html) with `.hierarchical_topics`\n\n```python\nhierarchical_topics = topic_model.hierarchical_topics(docs, topics)\n```\n\n* Visualize [hierarchical topic representations](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html#visualizations) with `.visualize_hierarchy`\n\n```python\ntopic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n```\n\n* Extract a [text-based hierarchical topic representation](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html#visualizations) with `.get_topic_tree`\n\n```python\ntree = topic_model.get_topic_tree(hierarchical_topics)\n```\n\n* Visualize [2D documents](https://maartengr.github.io/BERTopic/getting_started/visualization/visualization.html#visualize-documents) with `.visualize_documents()`\n\n```python\n# Use input embeddings\ntopic_model.visualize_documents(docs, embeddings=embeddings)\n\n# or use 2D reduced embeddings through a method of your own (e.g., PCA, t-SNE, UMAP, etc.)\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n```\n\n* Visualize [2D hierarchical documents](https://maartengr.github.io/BERTopic/getting_started/visualization/visualization.html#visualize-hierarchical-documents) with `.visualize_hierarchical_documents()`\n\n```python\n# Run the visualization with the original embeddings\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)\n\n# Or, if you have reduced the original embeddings already which speed things up quite a bit:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n```\n\n* Create [custom labels](https://maartengr.github.io/BERTopic/getting_started/topicrepresentation/topicrepresentation.html#custom-labels) to the topics throughout most visualizations\n\n```python\n# Generate topic labels\ntopic_labels = topic_model.generate_topic_labels(nr_words=3, topic_prefix=False, word_length=10, separator=\", \")\n\n# Set them internally in BERTopic\ntopic_model.set_topic_labels(topics_labels)\n```\n\n* Manually [merge topics](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html#merge-topics) with `.merge_topics()`\n\n```python\n# Merge topics 1, 2, and 3\ntopics_to_merge = [1, 2, 3]\ntopic_model.merge_topics(docs, topics, topics_to_merge)\n\n# Merge topics 1 and 2, and separately merge topics 3 and 4\ntopics_to_merge = [[1, 2], [3, 4]]\ntopic_model.merge_topics(docs, topics, topics_to_merge)\n```\n\n* Added example for finding similar topics between two models in the [tips & tricks](https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html) page\n* Add multi-modal example in the [tips & tricks](https://maartengr.github.io/BERTopic/getting_started/tips_and_tricks/tips_and_tricks.html) page\n* Added native [Hugging Face transformers](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html#hugging-face-transformers) support\n\n**Fixes**:\n\n* Fix support for k-Means in `.visualize_heatmap` ([#532](https://github.com/MaartenGr/BERTopic/issues/532))\n* Fix missing topic 0 in `.visualize_topics` ([#533](https://github.com/MaartenGr/BERTopic/issues/533))\n* Fix inconsistencies in `.get_topic_info` ([#572](https://github.com/MaartenGr/BERTopic/issues/572)) and ([#581](https://github.com/MaartenGr/BERTopic/issues/581))\n* Add `optimal_ordering` parameter to `.visualize_hierarchy` by [@rafaelvalero](https://github.com/rafaelvalero) in [#390](https://github.com/MaartenGr/BERTopic/pull/390)\n* Fix RuntimeError when used as sklearn estimator by [@simonfelding](https://github.com/simonfelding) in [#448](https://github.com/MaartenGr/BERTopic/pull/448)\n* Fix typo in visualization documentation by [@dwhdai](https://github.com/dwhdai) in [#475](https://github.com/MaartenGr/BERTopic/pull/475)\n* Fix typo in docstrings by [@xwwwwww](https://github.com/xwwwwww) in [#549](https://github.com/MaartenGr/BERTopic/pull/549)\n* Support higher Flair versions\n\n\n\n## **Version 0.10.0**\n*Release date: 30 April, 2022*\n\n\n**Highlights**:\n\n* Use any dimensionality reduction technique instead of UMAP:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.decomposition import PCA\n\ndim_model = PCA(n_components=5)\ntopic_model = BERTopic(umap_model=dim_model)\n```\n\n* Use any clustering technique instead of HDBSCAN:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.cluster import KMeans\n\ncluster_model = KMeans(n_clusters=50)\ntopic_model = BERTopic(hdbscan_model=cluster_model)\n```\n\n**Documentation**:\n\n* Add a CountVectorizer page with tips and tricks on how to create topic representations that fit your use case\n* Added pages on how to use other dimensionality reduction and clustering algorithms\n * Additional instructions on how to reduce outliers in the FAQ:\n\n```python\nimport numpy as np\nprobability_threshold = 0.01\nnew_topics = [np.argmax(prob) if max(prob) >= probability_threshold else -1 for prob in probs]\n```\n\n**Fixes**:\n\n* Fixed `None` being returned for probabilities when transforming unseen documents\n* Replaced all instances of `arg:` with `Arguments:` for consistency\n* Before saving a fitted BERTopic instance, we remove the stopwords in the fitted CountVectorizer model as it can get quite large due to the number of words that end in stopwords if `min_df` is set to a value larger than 1\n* Set `\"hdbscan>=0.8.28\"` to prevent numpy issues\n * Although this was already fixed by the new release of HDBSCAN, it is technically still possible to install 0.8.27 with BERTopic which leads to these numpy issues\n* Update gensim dependency to `>=4.0.0` ([#371](https://github.com/MaartenGr/BERTopic/issues/371))\n* Fix topic 0 not appearing in visualizations ([#472](https://github.com/MaartenGr/BERTopic/issues/472))\n* Fix ([#506](https://github.com/MaartenGr/BERTopic/issues/506))\n* Fix ([#429](https://github.com/MaartenGr/BERTopic/issues/429))\n* Fix typo in DTM documentation by [@hp0404](https://github.com/hp0404) in [#386](https://github.com/MaartenGr/BERTopic/pull/386)\n\n## **Version 0.9.4**\n*Release date: 14 December, 2021*\n\nA number of fixes, documentation updates, and small features:\n\n* Expose diversity parameter\n * Use `BERTopic(diversity=0.1)` to change how diverse the words in a topic representation are (ranges from 0 to 1)\n* Improve stability of topic reduction by only computing the cosine similarity within c-TF-IDF and not the topic embeddings\n* Added property to c-TF-IDF that all IDF values should be positive ([#351](https://github.com/MaartenGr/BERTopic/issues/351))\n* Improve stability of `.visualize_barchart()` and `.visualize_hierarchy()`\n* Major [documentation](https://maartengr.github.io/BERTopic/) overhaul (mkdocs, tutorials, FAQ, images, etc. ) ([#330](https://github.com/MaartenGr/BERTopic/issues/330))\n* Drop python 3.6 ([#333](https://github.com/MaartenGr/BERTopic/issues/333))\n* Relax plotly dependency ([#88](https://github.com/MaartenGr/BERTopic/issues/88))\n* Additional logging for `.transform` ([#356](https://github.com/MaartenGr/BERTopic/issues/356))\n\n\n## **Version 0.9.3**\n*Release date: 17 October, 2021*\n\n* Fix [#282](https://github.com/MaartenGr/BERTopic/issues/282)\n * As it turns out the old implementation of topic mapping was still found in the `transform` function\n* Fix [#285](https://github.com/MaartenGr/BERTopic/issues/285)\n * Fix getting all representative docs\n* Fix [#288](https://github.com/MaartenGr/BERTopic/issues/288)\n * A recent issue with the package `pyyaml` that can be found in Google Colab\n\n\n## **Version 0.9.2**\n*Release date: 12 October, 2021*\n\nA release focused on algorithmic optimization and fixing several issues:\n\n**Highlights**:\n\n* Update the non-multilingual paraphrase-* models to the all-* models due to improved [performance](https://www.sbert.net/docs/pretrained_models.html)\n* Reduce necessary RAM in c-TF-IDF top 30 word [extraction](https://stackoverflow.com/questions/49207275/finding-the-top-n-values-in-a-row-of-a-scipy-sparse-matrix)\n\n**Fixes**:\n\n* Fix topic mapping\n * When reducing the number of topics, these need to be mapped to the correct input/output which had some issues in the previous version\n * A new class was created as a way to track these mappings regardless of how many times they were executed\n * In other words, you can iteratively reduce the number of topics after training the model without the need to continuously train the model\n* Fix typo in embeddings page ([#200](https://github.com/MaartenGr/BERTopic/issues/200))\n* Fix link in README ([#233](https://github.com/MaartenGr/BERTopic/issues/233))\n* Fix documentation `.visualize_term_rank()` ([#253](https://github.com/MaartenGr/BERTopic/issues/253))\n* Fix getting correct representative docs ([#258](https://github.com/MaartenGr/BERTopic/issues/258))\n* Update [memory FAQ](https://maartengr.github.io/BERTopic/faq.html#i-am-facing-memory-issues-help) with [HDBSCAN pr](https://github.com/MaartenGr/BERTopic/issues/151)\n\n## **Version 0.9.1**\n*Release date: 1 September, 2021*\n\nA release focused on fixing several issues:\n\n**Fixes**:\n\n* Fix TypeError when auto-reducing topics ([#210](https://github.com/MaartenGr/BERTopic/issues/210))\n* Fix mapping representative docs when reducing topics ([#208](https://github.com/MaartenGr/BERTopic/issues/208))\n* Fix visualization issues with probabilities ([#205](https://github.com/MaartenGr/BERTopic/issues/205))\n* Fix missing `normalize_frequency` param in plots ([#213](https://github.com/MaartenGr/BERTopic/issues/208))\n\n\n## **Version 0.9.0**\n*Release date: 9 August, 2021*\n\n**Highlights**:\n\n* Implemented a [**Guided BERTopic**](https://maartengr.github.io/BERTopic/getting_started/guided/guided.html) -> Use seeds to steer the Topic Modeling\n* Get the most representative documents per topic: `topic_model.get_representative_docs(topic=1)`\n * This allows users to see which documents are good representations of a topic and better understand the topics that were created\n* Added `normalize_frequency` parameter to `visualize_topics_per_class` and `visualize_topics_over_time` in order to better compare the relative topic frequencies between topics\n* Return flat probabilities as default, only calculate the probabilities of all topics per document if `calculate_probabilities` is True\n* Added several FAQs\n\n**Fixes**:\n\n* Fix loading pre-trained BERTopic model\n* Fix mapping of probabilities\n* Fix [#190](https://github.com/MaartenGr/BERTopic/issues/190)\n\n\n**Guided BERTopic**:\n\nGuided BERTopic works in two ways:\n\nFirst, we create embeddings for each seeded topics by joining them and passing them through the document embedder.\nThese embeddings will be compared with the existing document embeddings through cosine similarity and assigned a label.\nIf the document is most similar to a seeded topic, then it will get that topic's label.\nIf it is most similar to the average document embedding, it will get the -1 label.\nThese labels are then passed through UMAP to create a semi-supervised approach that should nudge the topic creation to the seeded topics.\n\nSecond, we take all words in `seed_topic_list` and assign them a multiplier larger than 1.\nThose multipliers will be used to increase the IDF values of the words across all topics thereby increasing\nthe likelihood that a seeded topic word will appear in a topic. This does, however, also increase the chance of an\nirrelevant topic having unrelated words. In practice, this should not be an issue since the IDF value is likely to\nremain low regardless of the multiplier. The multiplier is now a fixed value but may change to something more elegant,\nlike taking the distribution of IDF values and its position into account when defining the multiplier.\n\n```python\nseed_topic_list = [[\"company\", \"billion\", \"quarter\", \"shrs\", \"earnings\"],\n [\"acquisition\", \"procurement\", \"merge\"],\n [\"exchange\", \"currency\", \"trading\", \"rate\", \"euro\"],\n [\"grain\", \"wheat\", \"corn\"],\n [\"coffee\", \"cocoa\"],\n [\"natural\", \"gas\", \"oil\", \"fuel\", \"products\", \"petrol\"]]\n\ntopic_model = BERTopic(seed_topic_list=seed_topic_list)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\n\n## **Version 0.8.1**\n*Release date: 8 June, 2021*\n\n**Highlights**:\n\n* Improved models:\n * For English documents the default is now: `\"paraphrase-MiniLM-L6-v2\"`\n * For Non-English or multi-lingual documents the default is now: `\"paraphrase-multilingual-MiniLM-L12-v2\"`\n * Both models show not only great performance but are much faster!\n* Add interactive visualizations to the `plotting` API documentation\n\nFor better performance, please use the following models:\n\n* English: `\"paraphrase-mpnet-base-v2\"`\n* Non-English or multi-lingual: `\"paraphrase-multilingual-mpnet-base-v2\"`\n\n**Fixes**:\n\n* Improved unit testing for more stability\n* Set transformers version for Flair\n\n## **Version 0.8.0**\n*Release date: 31 May, 2021*\n\n**Highlights**:\n\n* Additional visualizations:\n * Topic Hierarchy: `topic_model.visualize_hierarchy()`\n * Topic Similarity Heatmap: `topic_model.visualize_heatmap()`\n * Topic Representation Barchart: `topic_model.visualize_barchart()`\n * Term Score Decline: `topic_model.visualize_term_rank()`\n* Created `bertopic.plotting` library to easily extend visualizations\n* Improved automatic topic reduction by using HDBSCAN to detect similar topics\n* Sort topic ids by their frequency. -1 is the outlier class and contains typically the most documents. After that 0 is the largest topic, 1 the second largest, etc.\n\n**Fixes**:\n\n* Fix typo [#113](https://github.com/MaartenGr/BERTopic/pull/113), [#117](https://github.com/MaartenGr/BERTopic/pull/117)\n* Fix [#121](https://github.com/MaartenGr/BERTopic/issues/121) by removing [these](https://github.com/MaartenGr/BERTopic/blob/5c6cf22776fafaaff728370781a5d33727d3dc8f/bertopic/_bertopic.py#L359-L360) two lines\n* Fix mapping of topics after reduction (it now excludes 0) ([#103](https://github.com/MaartenGr/BERTopic/issues/103))\n\n## **Version 0.7.0**\n*Release date: 26 April, 2021*\n\nThe two main features are **(semi-)supervised topic modeling**\nand several **backends** to use instead of Flair and SentenceTransformers!\n\n**Highlights**:\n\n* (semi-)supervised topic modeling by leveraging supervised options in UMAP\n * `model.fit(docs, y=target_classes)`\n* Backends:\n * Added Spacy, Gensim, USE (TFHub)\n * Use a different backend for document embeddings and word embeddings\n * Create your own backends with `bertopic.backend.BaseEmbedder`\n * Click [here](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html) for an overview of all new backends\n* Calculate and visualize topics per class\n * Calculate: `topics_per_class = topic_model.topics_per_class(docs, topics, classes)`\n * Visualize: `topic_model.visualize_topics_per_class(topics_per_class)`\n* Several tutorials were updated and added:\n\n| Name | Link |\n|---|---|\n| Topic Modeling with BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1FieRA9fLdkQEGDIMYl0I3MCjSUKVF8C-?usp=sharing) |\n| (Custom) Embedding Models in BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/18arPPe50szvcCp_Y6xS56H2tY0m-RLqv?usp=sharing) |\n| Advanced Customization in BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1ClTYut039t-LDtlcd-oQAdXWgcsSGTw9?usp=sharing) |\n| (semi-)Supervised Topic Modeling with BERTopic | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bxizKzv5vfxJEB29sntU__ZC7PBSIPaQ?usp=sharing) |\n| Dynamic Topic Modeling with Trump's Tweets | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1un8ooI-7ZNlRoK0maVkYhmNRl0XGK88f?usp=sharing) |\n\n**Fixes**:\n\n* Fixed issues with Torch req\n* Prevent saving term frequency matrix in CTFIDF class\n* Fixed DTM not working when reducing topics ([#96](https://github.com/MaartenGr/BERTopic/issues/96))\n* Moved visualization dependencies to base BERTopic\n * `pip install bertopic[visualization]` becomes `pip install bertopic`\n* Allow precomputed embeddings in bertopic.find_topics() ([#79](https://github.com/MaartenGr/BERTopic/issues/79)):\n\n```python\nmodel = BERTopic(embedding_model=my_embedding_model)\nmodel.fit(docs, my_precomputed_embeddings)\nmodel.find_topics(search_term)\n```\n\n## **Version 0.6.0**\n*Release date: 1 March, 2021*\n\n**Highlights**:\n\n* DTM: Added a basic dynamic topic modeling technique based on the global c-TF-IDF representation\n * `model.topics_over_time(docs, timestamps, global_tuning=True)`\n* DTM: Option to evolve topics based on t-1 c-TF-IDF representation which results in evolving topics over time\n * Only uses topics at t-1 and skips evolution if there is a gap\n * `model.topics_over_time(docs, timestamps, evolution_tuning=True)`\n* DTM: Function to visualize topics over time\n * `model.visualize_topics_over_time(topics_over_time)`\n* DTM: Add binning of timestamps\n * `model.topics_over_time(docs, timestamps, nr_bins=10)`\n* Add function get general information about topics (id, frequency, name, etc.)\n * `get_topic_info()`\n* Improved stability of c-TF-IDF by taking the average number of words across all topics instead of the number of documents\n\n**Fixes**:\n\n* `_map_probabilities()` does not take into account that there is no probability of the outlier class and the probabilities are mutated instead of copied (#63, #64)\n\n## **Version 0.5.0**\n*Release date: 8 Februari, 2021*\n\n**Highlights**:\n\n* Add `Flair` to allow for more (custom) token/document embeddings, including πŸ€— transformers\n* Option to use custom UMAP, HDBSCAN, and CountVectorizer\n* Added `low_memory` parameter to reduce memory during computation\n* Improved verbosity (shows progress bar)\n* Return the figure of `visualize_topics()`\n* Expose all parameters with a single function: `get_params()`\n\n**Fixes**:\n\n* To simplify the API, the parameters stop_words and n_neighbors were removed. These can still be used when a custom UMAP or CountVectorizer is used.\n* Set `calculate_probabilities` to False as a default. Calculating probabilities with HDBSCAN significantly increases computation time and memory usage. Better to remove calculating probabilities or only allow it by manually turning this on.\n* Use the newest version of `sentence-transformers` as it speeds ups encoding significantly\n\n## **Version 0.4.2**\n*Release date: 10 Januari, 2021*\n\n**Fixes**:\n\n* Selecting `embedding_model` did not work when `language` was also used. This led to the user needing\nto set `language` to None before being able to use `embedding_model`. Fixed by using `embedding_model` when\n`language` is used (as a default parameter).\n\n## **Version 0.4.1**\n*Release date: 07 Januari, 2021*\n\n**Fixes**:\n\n* Simple fix by lowering the languages variable to match the lowered input language.\n\n## **Version 0.4.0**\n*Release date: 21 December, 2020*\n\n**Highlights**:\n\n* Visualize Topics similar to [LDAvis](https://github.com/cpsievert/LDAvis)\n* Added option to reduce topics after training\n* Added option to update topic representation after training\n* Added option to search topics using a search term\n* Significantly improved the stability of generating clusters\n* Finetune the topic words by selecting the most coherent words with the highest c-TF-IDF values\n* More extensive tutorials in the documentation\n\n**Notable Changes**:\n\n* Option to select language instead of sentence-transformers models to minimize the complexity of using BERTopic\n* Improved logging (remove duplicates)\n* Check if BERTopic is fitted\n* Added TF-IDF as an embedder instead of transformer models (see tutorial)\n* Numpy for Python 3.6 will be dropped and was therefore removed from the workflow.\n* Preprocess text before passing it through c-TF-IDF\n* Merged `get_topics_freq()` with `get_topic_freq()`\n\n**Fixes**:\n\n* Fix error handling topic probabilities\n\n## **Version 0.3.2**\n*Release date: 16 November, 2020*\n\n**Highlights**:\n\n* Fixed a bug with the topic reduction method that seems to reduce the number of topics but not to the nr_topics as defined in the class. Since this was, to a certain extend, breaking the topic reduction method a new release was necessary.\n\n## **Version 0.3.1**\n*Release date: 4 November, 2020*\n\n**Highlights**:\n\n* Adding the option to use custom embeddings or embeddings that you generated beforehand with whatever package you'd like to use. This allows users to further customize BERTopic to their liking.\n\n## **Version 0.3.0**\n*Release date: 29 October, 2020*\n\n**Highlights**:\n\n- transform() and fit_transform() now also return the topic probability distributions\n- Added visualize_distribution() which visualizes the topic probability distribution for a single document\n\n## **Version 0.2.2**\n*Release date: 17 October, 2020*\n\n**Highlights**:\n\n- Fixed n_gram_range not being used\n- Added option for using stopwords\n\n## **Version 0.2.1**\n*Release date: 11 October, 2020*\n\n**Highlights**:\n\n* Improved the calculation of the class-based TF-IDF procedure by limiting the calculation to sparse matrices. This prevents out-of-memory problems when faced with large datasets.\n\n## **Version 0.2.0**\n*Release date: 11 October, 2020*\n\n**Highlights**:\n\n- Changed c-TF-IDF procedure such that it implements a version of scikit-learns procedure. This should also speed up the calculation of the sparse matrix and prevent memory errors.\n- Added automated unit tests\n\n## **Version 0.1.2**\n*Release date: 1 October, 2020*\n\n**Highlights**:\n\n* When transforming new documents, self.mapped_topics seemed to be missing. Added to the init.\n\n## **Version 0.1.1**\n*Release date: 24 September, 2020*\n\n**Highlights**:\n\n* Fixed requirements --> Issue with pytorch\n* Update documentation\n\n## **Version 0.1.0**\n*Release date: 24 September, 2020*\n\n**Highlights**:\n\n- First release of `BERTopic`\n- Added parameters for UMAP and HDBSCAN\n- Option to choose sentence-transformer model\n- Method for transforming unseen documents\n- Save and load trained models (UMAP and HDBSCAN)\n- Extract topics and their sizes\n\n**Notable Changes**:\n\n- Optimized c-TF-IDF\n- Improved documentation\n- Improved topic reduction\n" }, { "path": "docs/faq.md", "content": "---\nhide:\n - navigation\n---\n\n# Frequently Asked Questions\n\n## **Why are the results not consistent between runs?**\nDue to the stochastic nature of UMAP, the results from BERTopic might differ even if you run the same code multiple times. Using custom embeddings allows you to try out BERTopic several times until you find the topics that suit you best. You only need to generate the embeddings themselves once and run BERTopic several times\nwith different parameters.\n\nIf you want to reproduce the results, at the expense of [performance](https://umap-learn.readthedocs.io/en/latest/reproducibility.html), you can set a `random_state` in UMAP to prevent\nany stochastic behavior:\n\n```python\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\numap_model = UMAP(n_neighbors=15, n_components=5,\n min_dist=0.0, metric='cosine', random_state=42)\ntopic_model = BERTopic(umap_model=umap_model)\n```\n\n## **Which embedding model should I choose?**\nUnfortunately, there is not a definitive list of the best models for each language, this highly depends on your data, the model, and your specific use case. However, the default model in BERTopic (`\"all-MiniLM-L6-v2\"`) works great for **English** documents. In contrast, for **multi-lingual** documents or any other language, `\"paraphrase-multilingual-MiniLM-L12-v2\"` has shown great performance.\n\nIf you want to use a model that provides a higher quality, but takes more computing time, then I would advise using `all-mpnet-base-v2` and `paraphrase-multilingual-mpnet-base-v2` instead.\n\n**MTEB Leaderboard**\nNew embedding models are released frequently and their performance keeps getting better. To keep track of the best embedding models out there, you can visit the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard). It is an excellent place for selecting the embedding that works best for you. For example, if you want the best of the best, then the top 5 models might the place to look.\n\nMany of these models can be used with `SentenceTransformers` in BERTopic, like so:\n\n```python\nfrom bertopic import BERTopic\nfrom sentence_transformers import SentenceTransformer\n\nembedding_model = SentenceTransformer(\"BAAI/bge-base-en-v1.5\")\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n**SentenceTransformers**\n[SentenceTransformers](https://www.sbert.net/docs/pretrained_models.html#sentence-embedding-models) work typically quite well\nand are the preferred models to use. They are great at generating document embeddings and have several\nmulti-lingual versions available.\n\n**πŸ€— transformers**\nBERTopic allows you to use any πŸ€— transformers model. These models are typically embeddings created on a word/sentence level but can easily be pooled using Flair (see Guides/Embeddings). If you have a specific language for which you want to generate embeddings, you can choose the model [here](https://huggingface.co/models).\n\n## **How do I reduce topic outliers?**\nThere are several ways we can reduce outliers.\n\nFirst, the amount of datapoint classified as outliers is handled by the `min_samples` parameters in HDBSCAN. This value is automatically set to the\nsame value of `min_cluster_size`. However, you can set it independently if you want to reduce the number of generated outliers. Lowering this value will\nresult in less noise being generated.\n\n```python\nfrom bertopic import BERTopic\nfrom hdbscan import HDBSCAN\n\nhdbscan_model = HDBSCAN(min_cluster_size=10, metric='euclidean',\n cluster_selection_method='eom', prediction_data=True, min_samples=5)\ntopic_model = BERTopic(hdbscan_model=hdbscan_model)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\n!!! note \"Note\"\n Although this will lower outliers found in the data, this might force outliers to be put into topics where they do not belong. So make\n sure to strike a balance between keeping noise and reducing outliers.\n\nSecond, after training our BERTopic model, we can assign outliers to topics by making use of the `.reduce_outliers` function in BERTopic. An advantage of using this approach is that there are four built in strategies one can choose for reducing outliers. Moreover, this technique allows the user to experiment with reducing outliers across a number of strategies and parameters without actually having to re-train the topic model each time. You can learn more about the `.reduce_outlier` function [here](https://maartengr.github.io/BERTopic/getting_started/outlier_reduction/outlier_reduction.html). The following is a minimal example of how to use this function:\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers\nnew_topics = topic_model.reduce_outliers(docs, topics)\n```\n\nThird, we can replace HDBSCAN with any other clustering algorithm that we want. So we can choose a clustering algorithm, like k-Means, that\ndoes not produce any outliers at all. Using k-Means instead of HDBSCAN is straightforward:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.cluster import KMeans\n\ncluster_model = KMeans(n_clusters=50)\ntopic_model = BERTopic(hdbscan_model=cluster_model)\n```\n\n\n## **How do I remove stop words?**\nAt times, stop words might end up in our topic representations. This is something we typically want to avoid as they contribute little to the interpretation of the topics. However, removing stop words as a preprocessing step is not advised as the transformer-based embedding models that we use need the full context to create accurate embeddings.\n\nInstead, we can use the `CountVectorizer` to preprocess our documents **after** having generated embeddings and clustered\nour documents. I have found almost no disadvantages to using the `CountVectorizer` to remove stop words and\nit is something I would strongly advise to try out:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer_model = CountVectorizer(stop_words=\"english\")\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\n```\n\nWe can also use the `ClassTfidfTransformer` to reduce the impact of frequent words. The result is very similar to explicitly removing stop words but this process does this automatically:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True)\ntopic_model = BERTopic(ctfidf_model=ctfidf_model)\n```\n\n## **How can I speed up BERTopic?**\nYou can speed up BERTopic by either generating your embeddings beforehand or by\nsetting `calculate_probabilities` to False. Calculating the probabilities is quite expensive and can significantly increase the computation time. Thus, only use it if you do not mind waiting a bit before the model is done running or if you have less than a couple of hundred thousand documents.\n\nAlso, make sure to use a GPU when extracting the sentence/document embeddings. Transformer models typically require a GPU and using only a CPU can slow down computation time quite a lot. However, if you do not have access to a GPU, looking into quantization might help.\n\nLastly, it is also possible to speed up BERTopic with [cuML's](https://rapids.ai/start.html#rapids-release-selector) GPU acceleration of UMAP and HDBSCAN:\n\n\n```python\nfrom bertopic import BERTopic\nfrom cuml.cluster import HDBSCAN\nfrom cuml.manifold import UMAP\n\n# Create instances of GPU-accelerated UMAP and HDBSCAN\numap_model = UMAP(n_components=5, n_neighbors=15, min_dist=0.0)\nhdbscan_model = HDBSCAN(min_samples=10, gen_min_span_tree=True, prediction_data=True)\n\n# Pass the above models to be used in BERTopic\ntopic_model = BERTopic(umap_model=umap_model, hdbscan_model=hdbscan_model)\n```\n\n\n## **I am facing memory issues. Help!**\nThere are several ways to perform computation with large datasets:\n\n* First, you can set `low_memory` to True when instantiating BERTopic.\nThis may prevent blowing up the memory in UMAP.\n\n* Second, setting `calculate_probabilities` to False when instantiating BERTopic prevents a huge document-topic\nprobability matrix from being created. Moreover, HDBSCAN is quite slow when it tries to calculate probabilities on large datasets.\n\n* Third, you can set the minimum frequency of words in the CountVectorizer class to reduce the size of the resulting\nsparse c-TF-IDF matrix. You can do this as follows:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer_model = CountVectorizer(ngram_range=(1, 2), stop_words=\"english\", min_df=10)\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\n```\n\nThe [min_df](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html)\nparameter is used to indicate the minimum frequency of words. Setting this value larger than 1 can significantly reduce memory.\n\n* Fourth, you can use online topic modeling instead to use BERTopic on big data by training the model in chunks\n\nIf the problem persists, then this could be an issue related to your available memory. The processing of millions of documents is quite computationally expensive and sufficient RAM is necessary.\n\n## **I have only a few topics, how do I increase them?**\nThere are several reasons why your topic model may result in only a few topics:\n\n* First, you might only have a few documents (~1000). This makes it very difficult to properly\nextract topics due to the little amount of data available. Increasing the number of documents\nmight solve your issues.\n\n* Second, `min_topic_size` might be simply too large for your number of documents. If you decrease\nthe minimum size of topics, then you are much more likely to increase the number of topics generated.\nYou could also decrease the `n_neighbors` parameter used in `UMAP` if this does not work.\n\n* Third, although this does not happen very often, there simply aren't that many topics to be found\nin your documents. You can often see this when you have many `-1` topics, which is not a topic\nbut a category of outliers.\n\n## **I have too many topics, how do I decrease them?**\nIf you have a large dataset, then it is possible to generate thousands of topics. Especially with large datasets, there is a good chance they contain many small topics. In practice, you might want a few hundred topics at most to interpret them nicely.\n\nThere are a few ways of decreasing the number of generated topics:\n\n* First, we can set the `min_topic_size` in the BERTopic initialization much higher (e.g., 300) to make sure that those small clusters will not be generated. This is an HDBSCAN parameter that specifies the minimum number of documents needed in a cluster. More documents in a cluster mean fewer topics will be generated.\n\n* Second, you can create a custom UMAP model and set `n_neighbors` much higher than the default 15 (e.g., 200). This also prevents those micro clusters to be generated as it will need many neighboring documents to create a cluster.\n\n* Third, we can set `nr_topics` to a value that seems logical to the user. Do note that topics are forced\nto merge which might result in a lower quality of topics. In practice, I would advise using\n`nr_topic=\"auto\"` as that will merge topics that are very similar. Dissimilar topics will\ntherefore remain separated.\n\n## **How do I calculate the probabilities of all topics in a document?**\nAlthough it is possible to calculate all the probabilities, the process of doing so is quite computationally\ninefficient and might significantly increase the computation time. To prevent this, the probabilities are\nnot calculated as a default. To calculate them, you will have to set `calculate_probabilities` to True:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(calculate_probabilities=True)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\n!!! note\n The `calculate_probabilities` parameter is only used when using HDBSCAN or cuML's HDBSCAN model. In other words, this will not work when using a model other than HDBSCAN. Instead, we can approximate the topic distributions across all documents with [`.approximate_distribution`](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html).\n\n## **Numpy gives me an error when running BERTopic**\nWith the release of Numpy 1.20.0, there have been significant issues with using that version (and previous ones) due to compilation issues and pypi.\n\nThis is a known issue with the order of installation using pypi. You can find more details about this issue\n[here](https://github.com/lmcinnes/umap/issues/567) and [here](https://github.com/scikit-learn-contrib/hdbscan/issues/457).\n\nI would suggest doing one of the following:\n\n* Install the newest version from BERTopic (>= v0.5).\n* You can install hdbscan with `pip install hdbscan --no-cache-dir --no-binary :all: --no-build-isolation` which might resolve the issue\n* Install BERTopic in a fresh environment using these steps.\n\n## **How can I run BERTopic without an internet connection?**\nThe great thing about using sentence-transformers is that it searches automatically for an embedding model locally.\nIf it cannot find one, it will download the pre-trained model from its servers.\nMake sure that you set the correct path for sentence-transformers to work. You can find a bit more about that\n[here](https://github.com/UKPLab/sentence-transformers/issues/888).\n\nYou can download the corresponding model [here](https://public.ukp.informatik.tu-darmstadt.de/reimers/sentence-transformers/v0.2/)\nand unzip it. Then, simply use the following to create your embedding model:\n\n```python\nfrom sentence_transformers import SentenceTransformer\nembedding_model = SentenceTransformer('path/to/unzipped/model')\n```\n\nThen, pass it to BERTopic:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n## **Can I use the GPU to speed up the model?**\nYes. The GPU is automatically used when you use a SentenceTransformer or Flair embedding model. Using\na CPU would then definitely slow things down. However, you can use other embeddings like TF-IDF or Doc2Vec\nembeddings in BERTopic which do not depend on GPU acceleration.\n\nYou can use [cuML](https://rapids.ai/start.html#rapids-release-selector) to speed up both\nUMAP and HDBSCAN through GPU acceleration:\n\n```python\nfrom bertopic import BERTopic\nfrom cuml.cluster import HDBSCAN\nfrom cuml.manifold import UMAP\n\n# Create instances of GPU-accelerated UMAP and HDBSCAN\numap_model = UMAP(n_components=5, n_neighbors=15, min_dist=0.0)\nhdbscan_model = HDBSCAN(min_samples=10, gen_min_span_tree=True, prediction_data=True)\n\n# Pass the above models to be used in BERTopic\ntopic_model = BERTopic(umap_model=umap_model, hdbscan_model=hdbscan_model)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nDepending on the embeddings you are using, you might want to normalize them first to force a cosine-related distance metric in UMAP:\n\n```python\nfrom cuml.preprocessing import normalize\nembeddings = normalize(embeddings)\n```\n\n## **How can I use BERTopic with Chinese documents?**\nCurrently, CountVectorizer tokenizes text by splitting whitespace which does not work for Chinese.\nTo get it to work, you will have to create a custom `CountVectorizer` with `jieba`:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport jieba\n\ndef tokenize_zh(text):\n words = jieba.lcut(text)\n return words\n\nvectorizer = CountVectorizer(tokenizer=tokenize_zh)\n```\n\nNext, we pass our custom vectorizer to BERTopic and create our topic model:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(embedding_model=model, verbose=True, vectorizer_model=vectorizer)\ntopics, _ = topic_model.fit_transform(docs, embeddings=embeddings)\n```\n\n## **Why does it take so long to import BERTopic?**\nThe main culprit here seems to be UMAP. After running tests with [Tuna](https://github.com/nschloe/tuna) we\ncan see that most of the resources when importing BERTopic can be dedicated to UMAP:\n\n\n\nUnfortunately, there currently is no fix for this issue. The most recent ticket regarding this\nissue can be found [here](https://github.com/lmcinnes/umap/issues/631).\n\n## **Should I preprocess the data?**\nNo. By using document embeddings there is typically no need to preprocess the data as all parts of a document\nare important in understanding the general topic of the document. Although this holds in 99% of cases, if you\nhave data that contains a lot of noise, for example, HTML-tags, then it would be best to remove them. HTML-tags\ntypically do not contribute to the meaning of a document and should therefore be removed. However, if you apply\ntopic modeling to HTML-code to extract topics of code, then it becomes important.\n\n## **I run into issues running on Apple Silicon. What should I do?**\nApple Silicon chips (M1 & M2) are based on `arm64` (aka [`AArch64`](https://apple.stackexchange.com/questions/451238/is-m1-chip-aarch64-or-amd64), not to be confused with `amd64`/`x86_64`). There are known issues with upstream dependencies for this architecture, for example [numba](https://github.com/numba/numba/issues/5520). You may not always run into this issue, depending on the extras that you need.\n\nOne possible solution is to use [VS Code Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers), which allow you to setup a Linux-based environment. To run BERTopic effectively you need to be aware of two things:\n\n1. Make sure to use a Docker image specifically built for arm64\n2. Make sure to use a *volume* instead of a *bind-mount*\n ℹ️ the latter significantly reduces disk I/O\n\nUsing the pre-configured [Data Science Dev Containers](https://github.com/b-data/data-science-devcontainers) makes sure these setting are optimized. To start using them, do the following:\n\n* Install and run Docker\n* Clone repository [data-science-devcontainers](https://github.com/b-data/data-science-devcontainers)\n* Open VS Code, build the `Python base` or `Python scipy` container and start working\n ℹ️ Change `PYTHON_VERSION` to `3.11` in the respective `devcontainer.json` to work with the latest patch release of Python 3.11\n* Note that data is persisted in the container\n * When using an unmodified `devcontainer.json`: Work in `/home/vscode`\n πŸ‘‰ This is the *home directory* of user `vscode`\n * Python packages are installed to the home directory by default\n πŸ‘‰ This is due to env variable `PIP_USER=1`\n * Note that the directory `/workspaces` is also persisted\n\n### **Do these Data Science Dev Containers support GPU acceleration?**\n\nYes, but only on Linux and Windows.\n\nThe CUDA-enabled variants require the following in addition to Docker:\n\n* NVIDIA GPU\n* NVIDIA driver\n* Linux: [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)\n* Windows: [GPU support in Docker Desktop](https://docs.docker.com/desktop/gpu/)\n\nℹ️ The host running the GPU accelerated Dev Containers only requires the NVIDIA driver, the CUDA toolkit does not have to be installed.\n\nSee the [CUDA Version Matrix](https://github.com/b-data/jupyterlab-python-docker-stack/blob/main/CUDA_VERSION_MATRIX.md) regarding Ubuntu/CUDA/Python versions and recommended NVIDIA drivers.\n" }, { "path": "docs/getting_started/best_practices/best_practices.md", "content": "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1BoQ_vakEVtojsd2x_U6-_x52OOuqruj2?usp=sharing) - Overview of Best Practices\n\nThrough the nature of BERTopic, its modularity, many variations of the topic modeling technique is possible. However, during the development and through the usage of the package, a set of best practices have been developed that generally lead to great results.\n\nThe following are a number of steps, parameters, and settings that you can use that will generally improve the quality of the resulting topics. In other words, after going through the quick start and getting a feeling for the API these steps should get you to the next level of performance.\n\n!!! Note\n Although these are called *best practices*, it does not necessarily mean that they work across all use cases perfectly. The underlying modular nature of BERTopic is meant to take different use cases into account. After going through these practices it is advised to fine-tune wherever necessary.\n\n\nTo showcase how these \"best practices\" work, we will go through an example dataset and apply all practices to it.\n\n## **Data**\n\nFor this example, we will use a dataset containing abstracts and metadata from [ArXiv articles](https://huggingface.co/datasets/arxiv_dataset).\n\n\n```python\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\n\n# Extract abstracts to train on and corresponding titles\nabstracts = dataset[\"abstract\"]\ntitles = dataset[\"title\"]\n```\n\n!!! Tip \"Sentence Splitter\"\n\n Whenever you have large documents, you typically want to split them up into either paragraphs or sentences. A nice way to do so is by using NLTK's sentence splitter which is nothing more than:\n\n ```python\n from nltk.tokenize import sent_tokenize, word_tokenize\n sentences = [sent_tokenize(abstract) for abstract in abstracts]\n sentences = [sentence for doc in sentences for sentence in doc]\n ```\n\n## **Pre-calculate Embeddings**\nAfter having created our data, namely `abstracts`, we can dive into the very first best practice, **pre-calculating embeddings**.\n\nBERTopic works by converting documents into numerical values, called embeddings. This process can be very costly, especially if we want to iterate over parameters. Instead, we can calculate those embeddings once and feed them to BERTopic to skip calculating embeddings each time.\n\n```python\nfrom sentence_transformers import SentenceTransformer\n\n# Pre-calculate embeddings\nembedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = embedding_model.encode(abstracts, show_progress_bar=True)\n```\n\n!!! Tip\n New embedding models are released frequently and their performance keeps getting better. To keep track of the best embedding models out there, you can visit the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard). It is an excellent place for selecting the embedding that works best for you. For example, if you want the best of the best, then the top 5 models might the place to look.\n\n\n## **Preventing Stochastic Behavior**\nIn BERTopic, we generally use a dimensionality reduction algorithm to reduce the size of the embeddings. This is done to prevent the [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) to a certain degree.\n\nAs a default, this is done with [UMAP](https://github.com/lmcinnes/umap) which is an incredible algorithm for reducing dimensional space. However, by default, it shows stochastic behavior which creates different results each time you run it. To prevent that, we will need to set a `random_state` of the model before passing it to BERTopic.\n\nAs a result, we can now fully reproduce the results each time we run the model.\n\n```python\nfrom umap import UMAP\n\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\n```\n\n## **Controlling Number of Topics**\nThere is a parameter to control the number of topics, namely `nr_topics`. This parameter, however, merges topics **after** they have been created. It is a parameter that supports creating a fixed number of topics.\n\nHowever, it is advised to control the number of topics through the cluster model which is by default HDBSCAN. HDBSCAN has a parameter, namely `min_cluster_size` that indirectly controls the number of topics that will be created.\n\nA higher `min_cluster_size` will generate fewer topics and a lower `min_cluster_size` will generate more topics.\n\nHere, we will go with `min_cluster_size=150` to prevent too many micro-clusters from being created:\n\n```python\nfrom hdbscan import HDBSCAN\n\nhdbscan_model = HDBSCAN(min_cluster_size=150, metric='euclidean', cluster_selection_method='eom', prediction_data=True)\n```\n\n## **Improving Default Representation**\nThe default representation of topics is calculated through [c-TF-IDF](https://maartengr.github.io/BERTopic/algorithm/algorithm.html#5-topic-representation). However, c-TF-IDF is powered by the [CountVectorizer](https://maartengr.github.io/BERTopic/getting_started/vectorizers/vectorizers.html) which converts text into tokens. Using the CountVectorizer, we can do a number of things:\n\n* Remove stopwords\n* Ignore infrequent words\n* Increase the n-gram range\n\nIn other words, we can preprocess the topic representations **after** documents are assigned to topics. This will not influence the clustering process in any way.\n\nHere, we will ignore English stopwords and infrequent words. Moreover, by increasing the n-gram range we will consider topic representations that are made up of one or two words.\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(stop_words=\"english\", min_df=2, ngram_range=(1, 2))\n```\n\n## **Additional Representations**\nPreviously, we have tuned the default representation but there are quite a number of [other topic representations](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) in BERTopic that we can choose from. From [KeyBERTInspired](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html#keybertinspired) and [PartOfSpeech](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html#partofspeech), to [OpenAI's ChatGPT](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#chatgpt) and [open-source](https://maartengr.github.io/BERTopic/getting_started/representation/llm.html#langchain) alternatives, many representations are possible.\n\nIn BERTopic, you can model many different topic representations simultaneously to test them out and get different perspectives of topic descriptions. This is called [multi-aspect](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) topic modeling.\n\nHere, we will demonstrate a number of interesting and useful representations in BERTopic:\n\n* KeyBERTInspired\n * A method that derives inspiration from how KeyBERT works\n* PartOfSpeech\n * Using SpaCy's POS tagging to extract words\n* MaximalMarginalRelevance\n * Diversify the topic words\n* OpenAI\n * Use ChatGPT to label our topics\n\n\n```python\nimport openai\nfrom bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, OpenAI, PartOfSpeech\n\n# KeyBERT\nkeybert_model = KeyBERTInspired()\n\n# Part-of-Speech\npos_model = PartOfSpeech(\"en_core_web_sm\")\n\n# MMR\nmmr_model = MaximalMarginalRelevance(diversity=0.3)\n\n# GPT-3.5\nclient = openai.OpenAI(api_key=\"sk-...\")\nprompt = \"\"\"\nI have a topic that contains the following documents:\n[DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS]\n\nBased on the information above, extract a short but highly descriptive topic label of at most 5 words. Make sure it is in the following format:\ntopic: \n\"\"\"\nopenai_model = OpenAI(client, model=\"gpt-4o-mini\", exponential_backoff=True, prompt=prompt)\n\n# All representation models\nrepresentation_model = {\n \"KeyBERT\": keybert_model,\n # \"OpenAI\": openai_model, # Uncomment if you will use OpenAI\n \"MMR\": mmr_model,\n \"POS\": pos_model\n}\n```\n\n## **Training**\nNow that we have a set of best practices, we can use them in our training loop. Here, several different representations, keywords and labels for our topics will be created. If you want to iterate over the topic model it is advised to use the pre-calculated embeddings as that significantly speeds up training.\n\n```python\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic(\n\n # Pipeline models\n embedding_model=embedding_model,\n umap_model=umap_model,\n hdbscan_model=hdbscan_model,\n vectorizer_model=vectorizer_model,\n representation_model=representation_model,\n\n # Hyperparameters\n top_n_words=10,\n verbose=True\n)\n\n# Train model\ntopics, probs = topic_model.fit_transform(abstracts, embeddings)\n\n# Show topics\ntopic_model.get_topic_info()\n```\n\nTo get all representations for a single topic, we simply run the following:\n\n```python\n>>> topic_model.get_topic(1, full=True)\n{'Main': [('adversarial', 0.028838938990764302),\n ('attacks', 0.021726302042463556),\n ('attack', 0.016803574415028524),\n ('robustness', 0.013046135743326167),\n ('adversarial examples', 0.01151254557995679),\n ('examples', 0.009920962487998853),\n ('perturbations', 0.009053305826870773),\n ('adversarial attacks', 0.008747627064844006),\n ('malware', 0.007675131707700338),\n ('defense', 0.007365955840313783)],\n 'KeyBERT': [('adversarial training', 0.76427937),\n ('adversarial attack', 0.74271905),\n ('vulnerable adversarial', 0.73302543),\n ('adversarial', 0.7311052),\n ('adversarial examples', 0.7179245),\n ('adversarial attacks', 0.7082),\n ('adversarially', 0.7005141),\n ('adversarial robustness', 0.69911957),\n ('adversarial perturbations', 0.6588783),\n ('adversary', 0.4467769)],\n 'OpenAI': [('Adversarial attacks and defense', 1)],\n 'MMR': [('adversarial', 0.028838938990764302),\n ('attacks', 0.021726302042463556),\n ('attack', 0.016803574415028524),\n ('robustness', 0.013046135743326167),\n ('adversarial examples', 0.01151254557995679),\n ('examples', 0.009920962487998853),\n ('perturbations', 0.009053305826870773),\n ('adversarial attacks', 0.008747627064844006),\n ('malware', 0.007675131707700338),\n ('defense', 0.007365955840313783)],\n 'POS': [('adversarial', 0.028838938990764302),\n ('attacks', 0.021726302042463556),\n ('attack', 0.016803574415028524),\n ('robustness', 0.013046135743326167),\n ('adversarial examples', 0.01151254557995679),\n ('examples', 0.009920962487998853),\n ('perturbations', 0.009053305826870773),\n ('adversarial attacks', 0.008747627064844006),\n ('malware', 0.007675131707700338),\n ('defense', 0.007365955840313783)]}\n```\n\n**NOTE**: The labels generated by OpenAI's **ChatGPT** are especially interesting to use throughout your model. Below, we will go into more detail how to set that as a custom label.\n\n!!! Tip \"Parameters\"\n If you would like to return the topic-document probability matrix, then it is advised to use `calculate_probabilities=True`. Do note that this can significantly slow down training. To speed it up, use [cuML's HDBSCAN](https://maartengr.github.io/BERTopic/getting_started/clustering/clustering.html#cuml-hdbscan) instead. You could also approximate the topic-document probability matrix with `.approximate_distribution` which will be discussed later.\n\n\n## **(Custom) Labels**\nThe default label of each topic are the top 3 words in each topic combined with an underscore between them.\n\nThis, of course, might not be the best label that you can think of for a certain topic. Instead, we can use `.set_topic_labels` to manually label all or certain topics.\n\nWe can also use `.set_topic_labels` to use one of the other topic representations that we had before, like `KeyBERTInspired` or even `OpenAI`.\n\n```python\n# Label the topics yourself\ntopic_model.set_topic_labels({1: \"Space Travel\", 7: \"Religion\"})\n\n# or use one of the other topic representations, like KeyBERTInspired\nkeybert_topic_labels = {topic: \" | \".join(list(zip(*values))[0][:3]) for topic, values in topic_model.topic_aspects_[\"KeyBERT\"].items()}\ntopic_model.set_topic_labels(keybert_topic_labels)\n\n# or ChatGPT's labels\nchatgpt_topic_labels = {topic: \" | \".join(list(zip(*values))[0]) for topic, values in topic_model.topic_aspects_[\"OpenAI\"].items()}\nchatgpt_topic_labels[-1] = \"Outlier Topic\"\ntopic_model.set_topic_labels(chatgpt_topic_labels)\n```\n\nNow that we have set the updated topic labels, we can access them with the many functions used throughout BERTopic. Most notably, you can show the updated labels in visualizations with the `custom_labels=True` parameters.\n\nIf we were to run `topic_model.get_topic_info()` it will now include the column `CustomName`. That is the custom label that we just created for each topic.\n\n## **Topic-Document Distribution**\nIf using `calculate_probabilities=True` is not possible, then you can [approximate the topic-document distributions](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html) using `.approximate_distribution`. It is a fast and flexible method for creating different topic-document distributions.\n\n```python\n# `topic_distr` contains the distribution of topics in each document\ntopic_distr, _ = topic_model.approximate_distribution(abstracts, window=8, stride=4)\n```\n\nNext, lets take a look at a specific abstract and see how the topic distribution was extracted:\n\n```python\n# Visualize the topic-document distribution for a single document\ntopic_model.visualize_distribution(topic_distr[abstract_id], custom_labels=True)\n```\n\nIt seems to have extracted a number of topics that are relevant and shows the distributions of these topics across the abstract. We can go one step further and visualize them on a token-level:\n\n```python\n# Calculate the topic distributions on a token-level\ntopic_distr, topic_token_distr = topic_model.approximate_distribution(abstracts[abstract_id], calculate_tokens=True)\n\n# Visualize the token-level distributions\ndf = topic_model.visualize_approximate_distribution(abstracts[abstract_id], topic_token_distr[0])\ndf\n```\n\n!!! Tip \"use_embedding_model\"\n As a default, we compare the c-TF-IDF calculations between the token sets and all topics. Due to its bag-of-word representation, this is quite fast. However, you might want to use the selected embedding_model instead to do this comparison. Do note that due to the many token sets, it is often computationally quite a bit slower:\n\n ```python\n topic_distr, _ = topic_model.approximate_distribution(docs, use_embedding_model=True)\n ```\n## **Outlier Reduction**\nBy default, HDBSCAN generates outliers which is a helpful mechanic in creating accurate topic representations. However, you might want to assign every single document to a topic. We can use `.reduce_outliers` to map some or all outliers to a topic:\n\n```python\n# Reduce outliers\nnew_topics = topic_model.reduce_outliers(abstracts, topics)\n\n# Reduce outliers with pre-calculate embeddings instead\nnew_topics = topic_model.reduce_outliers(abstracts, topics, strategy=\"embeddings\", embeddings=embeddings)\n```\n\n!!! Note \"Update Topics with Outlier Reduction\"\n After having generated updated topic assignments, we can pass them to BERTopic in order to update the topic representations:\n\n ```python\n topic_model.update_topics(docs, topics=new_topics)\n ```\n\n It is important to realize that updating the topics this way may lead to errors if topic reduction or topic merging techniques are used afterwards. The reason for this is that when you assign a -1 document to topic 1 and another -1 document to topic 2, it is unclear how you map the -1 documents. Is it matched to topic 1 or 2.\n\n\n## **Visualize Topics**\n\nWith visualizations, we are closing into the realm of subjective \"best practices\". These are things that I generally do because I like the representations but your experience might differ.\n\nHaving said that, there are two visualizations that are my go-to when visualizing the topics themselves:\n\n* `topic_model.visualize_topics()`\n* `topic_model.visualize_hierarchy()`\n\n```python\n# Visualize topics with custom labels\ntopic_model.visualize_topics(custom_labels=True)\n\n# Visualize hierarchy with custom labels\ntopic_model.visualize_hierarchy(custom_labels=True)\n```\n\n## **Visualize Documents**\n\nWhen visualizing documents, it helps to have embedded the documents beforehand to speed up computation. Fortunately, we have already done that as a \"best practice\".\n\nVisualizing documents in 2-dimensional space helps in understanding the underlying structure of the documents and topics.\n\n```python\n# Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\n```\n\nThe following plot is **interactive** which means that you can zoom in, double click on a label to only see that one and generally interact with the plot:\n\n```python\n# Visualize the documents in 2-dimensional space and show the titles on hover instead of the abstracts\n# NOTE: You can hide the hover with `hide_document_hover=True` which is especially helpful if you have a large dataset\n# NOTE: You can also hide the annotations with `hide_annotations=True` which is helpful to see the larger structure\ntopic_model.visualize_documents(titles, reduced_embeddings=reduced_embeddings, custom_labels=True)\n```\n\n!!! Note \"2-dimensional space\"\n Although visualizing the documents in 2-dimensional gives an idea of their underlying structure, there is a risk involved.\n\n Visualizing the documents in 2-dimensional space means that we have lost significant information since the original embeddings were more than 384 dimensions. Condensing all that information in 2 dimensions is simply not possible. In other words, it is merely an **approximation**, albeit quite an accurate one.\n\n## **Serialization**\n\nWhen saving a BERTopic model, there are several ways in doing so. You can either save the entire model with `pickle`, `pytorch`, or `safetensors`.\n\nPersonally, I would advise going with `safetensors` whenever possible. The reason for this is that the format allows for a very small topic model to be saved and shared.\n\nWhen saving a model with `safetensors`, it skips over saving the dimensionality reduction and clustering models. The `.transform` function will still work without these models but instead assign topics based on the similarity between document embeddings and the topic embeddings.\n\nAs a result, the `.transform` step might give different results but it is generally worth it considering the smaller and significantly faster model.\n\n```python\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"my_model_dir\", serialization=\"safetensors\", save_ctfidf=True, save_embedding_model=embedding_model)\n```\n\n!!! Note \"Embedding Model\"\n Using `safetensors`, we are not saving the underlying embedding model but merely a pointer to the model. For example, in the above example we are saving the string `\"sentence-transformers/all-MiniLM-L6-v2\"` so that we can load in the embedding model alongside the topic model.\n\n This currently only works if you are using a sentence transformer model. If you are using a different model, you can load it in when loading the topic model like this:\n\n ```python\n from sentence_transformers import SentenceTransformer\n\n # Define embedding model\n embedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\n # Load model and add embedding model\n loaded_model = BERTopic.load(\"my_model_dir\", embedding_model=embedding_model)\n ```\n\n## **Inference**\n\nTo speed up the inference, we can leverage a \"best practice\" that we used before, namely serialization. When you save a model as `safetensors` and then load it in, we are removing the dimensionality reduction and clustering steps from the pipeline.\n\nInstead, the assignment of topics is done through cosine similarity of document embeddings and topic embeddings. This speeds up inferences significantly.\n\nTo show its effect, let's start by disabling the logger:\n\n```python\nfrom bertopic._utils import MyLogger\nlogger = MyLogger()\nlogger.configure(\"ERROR\")\nloaded_model.verbose = False\ntopic_model.verbose = False\n```\n\nThen, we run inference on both the loaded model and the non-loaded model:\n\n```python\n>>> %timeit loaded_model.transform(abstracts[:100])\n343 ms Β± 31.1 ms per loop (mean Β± std. dev. of 7 runs, 1 loop each)\n```\n\n```python\n>>> %timeit topic_model.transform(abstracts[:100])\n1.37 s Β± 166 ms per loop (mean Β± std. dev. of 7 runs, 1 loop each)\n```\n\nBased on the above, the `loaded_model` seems to be quite a bit faster for inference than the original `topic_model`.\n" }, { "path": "docs/getting_started/clustering/clustering.md", "content": "After reducing the dimensionality of our input embeddings, we need to cluster them into groups of similar embeddings to extract our topics.\nThis process of clustering is quite important because the more performant our clustering technique the more accurate our topic representations are.\n\nIn BERTopic, we typically use HDBSCAN as it is quite capable of capturing structures with different densities. However, there is not one perfect\nclustering model and you might want to be using something entirely different for your use case. Moreover, what if a new state-of-the-art model\nis released tomorrow? We would like to be able to use that in BERTopic, right? Since BERTopic assumes some independence among steps, we can allow for this modularity:\n\n
\n ![Image title](clustering.svg)\n
\n
\n\nAs a result, the `hdbscan_model` parameter in BERTopic now allows for a variety of clustering models. To do so, the class should have\nthe following attributes:\n\n* `.fit(X)`\n * A function that can be used to fit the model\n* `.predict(X)`\n * A predict function that transforms the input to cluster labels\n* `.labels_`\n * The labels after fitting the model\n\n\nIn other words, it should have the following structure:\n\n```python\nclass ClusterModel:\n def fit(self, X):\n self.labels_ = None\n return self\n\n def predict(self, X):\n return X\n```\n\nIn this section, we will go through several examples of clustering algorithms and how they can be implemented.\n\n\n## **HDBSCAN**\nAs a default, BERTopic uses HDBSCAN to perform its clustering. To use a HDBSCAN model with custom parameters,\nwe simply define it and pass it to BERTopic:\n\n```python\nfrom bertopic import BERTopic\nfrom hdbscan import HDBSCAN\n\nhdbscan_model = HDBSCAN(min_cluster_size=15, metric='euclidean', cluster_selection_method='eom', prediction_data=True)\ntopic_model = BERTopic(hdbscan_model=hdbscan_model)\n```\n\nHere, we can define any parameters in HDBSCAN to optimize for the best performance based on whatever validation metrics you are using.\n\n## **k-Means**\nAlthough HDBSCAN works quite well in BERTopic and is typically advised, you might want to be using k-Means instead.\nIt allows you to select how many clusters you would like and forces every single point to be in a cluster. Therefore, no\noutliers will be created. This also has disadvantages. When you force every single point in a cluster, it will mean\nthat the cluster is highly likely to contain noise which can hurt the topic representations. As a small tip, using\nthe `vectorizer_model=CountVectorizer(stop_words=\"english\")` helps quite a bit to then improve the topic representation.\n\nHaving said that, using k-Means is quite straightforward:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.cluster import KMeans\n\ncluster_model = KMeans(n_clusters=50)\ntopic_model = BERTopic(hdbscan_model=cluster_model)\n```\n\n!!! note\n As you might have noticed, the `cluster_model` is passed to `hdbscan_model` which might be a bit confusing considering\n you are not passing an HDBSCAN model. For now, the name of the parameter is kept the same to adhere to the current\n state of the API. Changing the name could lead to deprecation issues, which I want to prevent as much as possible.\n\n## **Agglomerative Clustering**\nLike k-Means, there are a bunch more clustering algorithms in `sklearn` that you can be using. Some of these models do\nnot have a `.predict()` method but still can be used in BERTopic. However, using BERTopic's `.transform()` function\nwill then give errors.\n\nHere, we will demonstrate Agglomerative Clustering:\n\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.cluster import AgglomerativeClustering\n\ncluster_model = AgglomerativeClustering(n_clusters=50)\ntopic_model = BERTopic(hdbscan_model=cluster_model)\n```\n\n\n## **cuML HDBSCAN**\n\nAlthough the original HDBSCAN implementation is an amazing technique, it may have difficulty handling large amounts of data. Instead,\nwe can use [cuML](https://rapids.ai/start.html#rapids-release-selector) to speed up HDBSCAN through GPU acceleration:\n\n```python\nfrom bertopic import BERTopic\nfrom cuml.cluster import HDBSCAN\n\nhdbscan_model = HDBSCAN(min_samples=10, gen_min_span_tree=True, prediction_data=True)\ntopic_model = BERTopic(hdbscan_model=hdbscan_model)\n```\n\nThe great thing about using cuML's HDBSCAN implementation is that it supports many features of the original implementation. In other words,\n`calculate_probabilities=True` also works!\n\n!!! note\n As of the v0.13 release, it is not yet possible to calculate the topic-document probability matrix for unseen data (i.e., `.transform`) using cuML's HDBSCAN.\n However, it is still possible to calculate the topic-document probability matrix for the data on which the model was trained (i.e., `.fit` and `.fit_transform`).\n\n!!! note\n To install cuML with BERTopic, run these commands:\n\n **For CUDA 12:**\n ```bash\n !pip install cuml-cu12\n !pip install bertopic\n ```\n\n **For CUDA 13:**\n ```bash\n !pip install cuml-cu13\n !pip install bertopic\n ```\n\n !!! warning\n Install cuML first, then BERTopic. Installing both in a single command can fail due to pip resolver limitations with CUDA runtime dependencies.\n\n **Note:** cuML is already installed on Google Colab.\n\n For more detailed information on installing cuML, including additional dependencies and platform-specific instructions, see the [RAPIDS installation guide](https://docs.rapids.ai/install/).\n" }, { "path": "docs/getting_started/ctfidf/ctfidf.md", "content": "# c-TF-IDF\n\nIn BERTopic, in order to get an accurate representation of the topics from our bag-of-words matrix, TF-IDF was adjusted to work on a cluster/categorical/topic level instead of a document level. This adjusted TF-IDF representation is called **c-TF-IDF** and takes into account what makes the documents in one cluster different from documents in another cluster:\n\n\n\n
\nEach cluster is converted to a single document instead of a set of documents. Then, we extract the frequency of word `x` in class `c`, where `c` refers to the cluster we created before. This results in our class-based `tf` representation. This representation is L1-normalized to account for the differences in topic sizes.\n

\nThen, we take the logarithm of one plus the average number of words per class `A` divided by the frequency of word `x` across all classes. We add plus one within the logarithm to force values to be positive. This results in our class-based `idf` representation. Like with the classic TF-IDF, we then multiply `tf` with `idf` to get the importance score per word in each class. In other words, the classical TF-IDF procedure is **not** used here but a modified version of the algorithm that allows for a much better representation.\n\nSince the topic representation is somewhat independent of the clustering step, we can change how the c-TF-IDF representation will look like. This can be in the form of parameter tuning, different weighting schemes, or using a diversity metric on top of it. This allows for some modularity concerning the weighting scheme:\n\n
\n ![Image title](ctfidf.svg)\n
\n
\n\n\nThis class-based TF-IDF representation is enabled by default in BERTopic. However, we can explicitly pass it to BERTopic through the `ctfidf_model` allowing for parameter tuning and the customization of the topic extraction technique:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer()\ntopic_model = BERTopic(ctfidf_model=ctfidf_model )\n```\n\n## **Parameters**\nThere are two parameters worth exploring in the `ClassTfidfTransformer`, namely `bm25_weighting` and `reduce_frequent_words`.\n\n\n### bm25_weighting\n\nThe `bm25_weighting` is a boolean parameter that indicates whether a class-based BM-25 weighting measure is used instead of the default method as defined in the formula at the beginning of this page.\n\nInstead of using the following weighting scheme:\n\n\n\n\nthe class-based BM-25 weighting is used instead:\n\n\n\nAt smaller datasets, this variant can be more robust to stop words that appear in your data. It can be enabled as follows:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer(bm25_weighting=True)\ntopic_model = BERTopic(ctfidf_model=ctfidf_model )\n```\n\n\n### reduce_frequent_words\n\nSome words appear quite often in every topic but are generally not considered stop words as found in the `CountVectorizer(stop_words=\"english\")` list. To further reduce these frequent words, we can use `reduce_frequent_words` to take the square root of the term frequency after applying the weighting scheme.\n\nInstead of the default term frequency:\n\n\n\nwe take the square root of the term frequency after normalizing the frequency matrix:\n\n\n\nAlthough seemingly a small change, it can have quite a large effect on the number of stop words in the resulting topic representations. It can be enabled as follows:\n\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True)\ntopic_model = BERTopic(ctfidf_model=ctfidf_model )\n```\n\n!!! tip\n\tBoth parameters can be used simultaneously: `ClassTfidfTransformer(bm25_weighting=True, reduce_frequent_words=True)`\n" }, { "path": "docs/getting_started/dim_reduction/dim_reduction.md", "content": "An important aspect of BERTopic is the dimensionality reduction of the input embeddings. As embeddings are often high in dimensionality, clustering becomes difficult due to the curse of dimensionality.\n\nA solution is to reduce the dimensionality of the embeddings to a workable dimensional space (e.g., 5) for clustering algorithms to work with.\nUMAP is used as a default in BERTopic since it can capture both the local and global high-dimensional space in lower dimensions.\nHowever, there are other solutions out there, such as PCA that users might be interested in trying out. Since BERTopic allows assumes some independency between steps, we can\nuse any other dimensionality reduction algorithm. The image below illustrates this modularity:\n\n\n
\n ![Image title](dimensionality.svg)\n
\n
\n\n\n\nAs a result, the `umap_model` parameter in BERTopic now allows for a variety of dimensionality reduction models. To do so, the class should have\nthe following attributes:\n\n* `.fit(X)`\n * A function that can be used to fit the model\n* `.transform(X)`\n * A transform function that transforms the input to a lower dimensional size\n\nIn other words, it should have the following structure:\n\n```python\nclass DimensionalityReduction:\n def fit(self, X):\n return self\n\n def transform(self, X):\n return X\n```\n\nIn this section, we will go through several examples of dimensionality reduction techniques and how they can be implemented.\n\n\n## **UMAP**\nAs a default, BERTopic uses UMAP to perform its dimensionality reduction. To use a UMAP model with custom parameters,\nwe simply define it and pass it to BERTopic:\n\n```python\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine')\ntopic_model = BERTopic(umap_model=umap_model)\n```\n\nHere, we can define any parameters in UMAP to optimize for the best performance based on whatever validation metrics you are using.\n\n## **PCA**\nAlthough UMAP works quite well in BERTopic and is typically advised, you might want to be using PCA instead. It can be faster to train and perform\ninference. To use PCA, we can simply import it from `sklearn` and pass it to the `umap_model` parameter:\n\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.decomposition import PCA\n\ndim_model = PCA(n_components=5)\ntopic_model = BERTopic(umap_model=dim_model)\n```\n\nAs a small note, PCA and k-Means have worked quite well in my experiments and might be interesting to use instead of PCA and HDBSCAN.\n\n\n!!! note\n As you might have noticed, the `dim_model` is passed to `umap_model` which might be a bit confusing considering\n you are not passing a UMAP model. For now, the name of the parameter is kept the same to adhere to the current\n state of the API. Changing the name could lead to deprecation issues, which I want to prevent as much as possible.\n\n## **Truncated SVD**\nLike PCA, there are a bunch more dimensionality reduction techniques in `sklearn` that you can be using. Here, we will demonstrate Truncated SVD\nbut any model can be used as long as it has both a `.fit()` and `.transform()` method:\n\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.decomposition import TruncatedSVD\n\ndim_model = TruncatedSVD(n_components=5)\ntopic_model = BERTopic(umap_model=dim_model)\n```\n\n## **cuML UMAP**\n\nAlthough the original UMAP implementation is an amazing technique, it may have difficulty handling large amounts of data. Instead,\nwe can use [cuML](https://rapids.ai/start.html#rapids-release-selector) to speed up UMAP through GPU acceleration:\n\n```python\nfrom bertopic import BERTopic\nfrom cuml.manifold import UMAP\n\numap_model = UMAP(n_components=5, n_neighbors=15, min_dist=0.0)\ntopic_model = BERTopic(umap_model=umap_model)\n```\n\n!!! note\n To install cuML with BERTopic, run these commands:\n\n **For CUDA 12:**\n ```bash\n !pip install cuml-cu12\n !pip install bertopic\n ```\n\n **For CUDA 13:**\n ```bash\n !pip install cuml-cu13\n !pip install bertopic\n ```\n\n !!! warning\n Install cuML first, then BERTopic. Installing both in a single command can fail due to pip resolver limitations with CUDA runtime dependencies.\n\n **Note:** cuML is already installed on Google Colab.\n\n For more detailed information on installing cuML, including additional dependencies and platform-specific instructions, see the [RAPIDS installation guide](https://docs.rapids.ai/install/).\n\n\n## **Skip dimensionality reduction**\nAlthough BERTopic applies dimensionality reduction as a default in its pipeline, this is a step that you might want to skip. We generate an \"empty\" model that simply returns the data pass it to:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.dimensionality import BaseDimensionalityReduction\n\n# Fit BERTopic without actually performing any dimensionality reduction\nempty_dimensionality_model = BaseDimensionalityReduction()\ntopic_model = BERTopic(umap_model=empty_dimensionality_model)\n```\n\nIn other words, we go from this pipeline:\n\n
\n
\n--8<-- \"docs/getting_started/dim_reduction/default_pipeline.svg\"\n
\n
\n\nTo the following pipeline:\n\n
\n
\n--8<-- \"docs/getting_started/dim_reduction/no_dimensionality.svg\"\n
\n\n
\n" }, { "path": "docs/getting_started/distribution/distribution.md", "content": "BERTopic approaches topic modeling as a cluster task and attempts to cluster semantically similar documents to extract common topics. A disadvantage of using such a method is that each document is assigned to a single cluster and therefore also a single topic. In practice, documents may contain a mixture of topics. This can be accounted for by splitting up the documents into sentences and feeding those to BERTopic.\n\nAnother option is to use a cluster model that can perform soft clustering, like HDBSCAN. As BERTopic focuses on modularity, we may still want to model that mixture of topics even when we are using a hard-clustering model, like k-Means without the need to split up our documents. This is where `.approximate_distribution` comes in!\n\n
\n
\n--8<-- \"docs/getting_started/distribution/approximate_distribution.svg\"\n
\n
\n\nTo perform this approximation, each document is split into tokens according to the provided tokenizer in the `CountVectorizer`. Then, a **sliding window** is applied on each document creating subsets of the document. For example, with a window size of 3 and stride of 1, the document:\n\n> Solving the right problem is difficult.\n\ncan be split up into `solving the right`, `the right problem`, `right problem is`, and `problem is difficult`. These are called token sets.\nFor each of these token sets, we calculate their c-TF-IDF representation and find out how similar they are to the previously generated topics.\nThen, the similarities to the topics for each token set are summed to create a topic distribution for the entire document.\n\nAlthough it is often said that documents can contain a mixture of topics, these are often modeled by assigning each word to a single topic.\nWith this approach, we take into account that there may be multiple topics for a single word.\n\nWe can make this multiple-topic word assignment a bit more accurate by then splitting these token sets up into individual tokens and assigning\nthe topic distributions for each token set to each individual token. That way, we can visualize the extent to which a certain word contributes\nto a document's topic distribution.\n\n## **Example**\n\nTo calculate our topic distributions, we first need to fit a basic topic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic().fit(docs)\n```\n\nAfter doing so, we can approximate the topic distributions for your documents:\n\n```python\ntopic_distr, _ = topic_model.approximate_distribution(docs)\n```\n\nThe resulting `topic_distr` is a *n* x *m* matrix where *n* are the documents and *m* the topics. We can then visualize the distribution\nof topics in a document:\n\n```python\ntopic_model.visualize_distribution(topic_distr[1])\n```\n\n\n\nAlthough a topic distribution is nice, we may want to see how each token contributes to a specific topic. To do so, we need to first\ncalculate topic distributions on a token level and then visualize the results:\n\n```python\n# Calculate the topic distributions on a token-level\ntopic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)\n\n# Visualize the token-level distributions\ndf = topic_model.visualize_approximate_distribution(docs[1], topic_token_distr[1])\ndf\n```\n\n

\n\n

\n\n!!! tip\n You can also approximate the topic distributions for unseen documents. It will not be as accurate as `.transform` but it is quite fast and can serve you well in a production setting.\n\n!!! note\n To get the stylized dataframe for `.visualize_approximate_distribution` you will need to have Jinja installed. If you do not have this installed, an unstylized dataframe will be returned instead. You can install Jinja via `pip install jinja2`\n\n## **Parameters**\nThere are a few parameters that are of interest which will be discussed below.\n\n\n### **batch_size**\nCreating token sets for each document can result in quite a large list of token sets. The similarity of these token sets with the topics can result a large matrix that might not fit into memory anymore. To circumvent this, we can process batches of documents instead to minimize the memory overload. The value for `batch_size` indicates the number of documents that will be processed at once:\n\n```python\ntopic_distr, _ = topic_model.approximate_distribution(docs, batch_size=500)\n```\n\n### **window**\nThe number of tokens that are combined into token sets are defined by the `window` parameter. Seeing as we are performing a sliding window, we can change the size of the window. A larger window takes more tokens into account but setting it too large can result in considering too much information. Personally, I like to have this window between 4 and 8:\n\n```python\ntopic_distr, _ = topic_model.approximate_distribution(docs, window=4)\n```\n\n### **stride**\nThe sliding window that is performed on a document shifts, as a default, 1 token to the right each time to create its token sets. As a result, especially with large windows, a single token gets judged several times. We can use the `stride` parameter to increase the number of tokens the window shifts to the right. By increasing\nthis value, we are judging each token less frequently which often results in a much faster calculation. Combining this parameter with `window` is preferred. For example, if we have a very large dataset, we can set `stride=4` and `window=8` to judge token sets that contain 8 tokens but that are shifted with 4 steps\neach time. As a result, this increases the computational speed quite a bit:\n\n```python\ntopic_distr, _ = topic_model.approximate_distribution(docs, window=4)\n```\n\n### **use_embedding_model**\nAs a default, we compare the c-TF-IDF calculations between the token sets and all topics. Due to its bag-of-word representation, this is quite fast. However, you might want to use the selected `embedding_model` instead to do this comparison. Do note that due to the many token sets, it is often computationally quite a bit slower:\n\n```python\ntopic_distr, _ = topic_model.approximate_distribution(docs, use_embedding_model=True)\n```\n" }, { "path": "docs/getting_started/distribution/distribution_viz.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/embeddings/embeddings.md", "content": "# Embedding Models\nBERTopic starts with transforming our input documents into numerical representations. Although there are many ways this can be achieved, we typically use sentence-transformers (`\"all-MiniLM-L6-v2\"`) as it is quite capable of capturing the semantic similarity between documents.\n\nHowever, there is not one perfect\nembedding model and you might want to be using something entirely different for your use case. Since BERTopic assumes some independence among steps, we can allow for this modularity:\n\n
\n ![Image title](embeddings.svg)\n
\n
\n\n\nThis modularity allows us not only to choose any embedding model to convert our documents into numerical representations, we can use essentially any data to perform our clustering.\nWhen new state-of-the-art pre-trained embedding models are released, BERTopic will be able to use them. As a result, BERTopic grows with any new models being released.\nOut of the box, BERTopic supports several embedding techniques. In this section, we will go through several of them and how they can be implemented.\n\n## **Sentence Transformers**\nYou can select any model from sentence-transformers [here](https://www.sbert.net/docs/pretrained_models.html)\nand pass it through BERTopic with `embedding_model`:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(embedding_model=\"all-MiniLM-L6-v2\")\n```\n\nOr select a SentenceTransformer model with your parameters:\n\n```python\nfrom sentence_transformers import SentenceTransformer\n\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\ntopic_model = BERTopic(embedding_model=sentence_model)\n```\n\n!!! tip \"Tip 1!\"\n This embedding back-end was put here first for a reason, sentence-transformers works amazing out of the box! Playing around with different models can give you great results. Also, make sure to frequently visit [this](https://www.sbert.net/docs/pretrained_models.html) page as new models are often released.\n\n!!! tip \"Tip 2!\"\n New embedding models are released frequently and their performance keeps getting better. To keep track of the best embedding models out there, you can visit the [MTEB leaderboard](https://huggingface.co/spaces/mteb/leaderboard). It is an excellent place for selecting the embedding that works best for you. For example, if you want the best of the best, then the top 5 models might the place to look.\n\n Many of these models can be used with `SentenceTransformers` in BERTopic, like so:\n\n ```python\n from sentence_transformers import SentenceTransformer\n\n embedding_model = SentenceTransformer(\"BAAI/bge-base-en-v1.5\")\n topic_model = BERTopic(embedding_model=embedding_model)\n ```\n\n## **Model2Vec**\nTo use a blazingly fast [Model2Vec](https://github.com/MinishLab/model2vec) model, you first need to install model2vec:\n\n```\npip install model2vec\n```\n\nThen, you can load in any of their models and pass it to BERTopic like so:\n\n```python\nfrom model2vec import StaticModel\nembedding_model = StaticModel.from_pretrained(\"minishlab/potion-base-8M\")\n\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n### **Distillation**\n\nThese models are extremely versatile and can be distilled from existing embedding model (like those compatible with `sentence-transformers`).\nThis distillation process doesn't require a vocabulary (as it uses the tokenizer's vocabulary) but can benefit from having one. Fortunately, this allows you to\nuse the vocabulary from your input documents to distill a model yourself.\n\nDoing so requires you to install some additional dependencies of model2vec like so:\n\n```\npip install model2vec[distill]\n```\n\nTo then distill common embedding models, you need to import the `Model2VecBackend` from BERTopic:\n\n```python\nfrom bertopic.backend import Model2VecBackend\n\n# Choose a model to distill (a non-Model2Vec model)\nembedding_model = Model2VecBackend(\n \"sentence-transformers/all-MiniLM-L6-v2\",\n distill=True\n)\n\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\nYou can also choose a custom vectorizer for creating the vocabulary and define custom arguments for the distillatio process:\n\n```python\nfrom bertopic.backend import Model2VecBackend\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n# Choose a model to distill (a non-Model2Vec model)\nembedding_model = Model2VecBackend(\n \"sentence-transformers/all-MiniLM-L6-v2\",\n distill=True,\n distill_kwargs={\"pca_dims\": 256, \"apply_zipf\": True, \"use_subword\": True},\n distill_vectorizer=CountVectorizer(ngram_range=(1, 3))\n)\n\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n!!! tip \"Tip!\"\n You can save the resulting model with `topic_model.embedding_model.embedding_model.save_pretrained(\"m2v_model\")`.\n\n\n## **πŸ€— Hugging Face Transformers**\nTo use a Hugging Face transformers model, load in a pipeline and point\nto any model found on their model hub (https://huggingface.co/models):\n\n```python\nfrom transformers.pipelines import pipeline\n\nembedding_model = pipeline(\"feature-extraction\", model=\"distilbert-base-cased\")\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n!!! tip \"Tip!\"\n These transformers also work quite well using `sentence-transformers` which has great optimizations tricks that make using it a bit faster.\n\n**Langchain**\n[Langchain](https://python.langchain.com/docs/introduction) allows you to use different embedding models supported by various cloud providers. On top of that, it supports various integrations to open source models. To get started:\n\n```python\nfrom langchain_community.embeddings import HuggingFaceInstructEmbeddings\nfrom bertopic.backend import LangChainBackend\n\nhf_embedding = HuggingFaceInstructEmbeddings()\nlangchain_embedder = LangChainBackend(hf_embedding)\n```\n\nTo see what providers are being supported by Langchain, you can check the list [here](https://python.langchain.com/docs/integrations/providers/).\nFor more information, you can have a look on [Langchain's Embedding Models](https://python.langchain.com/docs/integrations/text_embedding/).\n\n## **Flair**\n[Flair](https://github.com/flairNLP/flair) allows you to choose almost any embedding model that\nis publicly available. Flair can be used as follows:\n\n```python\nfrom flair.embeddings import TransformerDocumentEmbeddings\n\nroberta = TransformerDocumentEmbeddings('roberta-base')\ntopic_model = BERTopic(embedding_model=roberta)\n```\n\nYou can select any πŸ€— transformers model [here](https://huggingface.co/models).\n\nMoreover, you can also use Flair to use word embeddings and pool them to create document embeddings.\nUnder the hood, Flair simply averages all word embeddings in a document. Then, we can easily\npass it to BERTopic to use those word embeddings as document embeddings:\n\n```python\nfrom flair.embeddings import WordEmbeddings, DocumentPoolEmbeddings\n\nglove_embedding = WordEmbeddings('crawl')\ndocument_glove_embeddings = DocumentPoolEmbeddings([glove_embedding])\n\ntopic_model = BERTopic(embedding_model=document_glove_embeddings)\n```\n\n## **Spacy**\n[Spacy](https://github.com/explosion/spaCy) is an amazing framework for processing text. There are\nmany models available across many languages for modeling text.\n\nTo use Spacy's non-transformer models in BERTopic:\n\n```python\nimport spacy\n\nnlp = spacy.load(\"en_core_web_md\", exclude=['tagger', 'parser', 'ner',\n 'attribute_ruler', 'lemmatizer'])\n\ntopic_model = BERTopic(embedding_model=nlp)\n```\n\nUsing spacy-transformer models:\n\n```python\nimport spacy\n\nspacy.prefer_gpu()\nnlp = spacy.load(\"en_core_web_trf\", exclude=['tagger', 'parser', 'ner',\n 'attribute_ruler', 'lemmatizer'])\n\ntopic_model = BERTopic(embedding_model=nlp)\n```\n\nIf you run into memory issues with spacy-transformer models, try:\n\n```python\nimport spacy\nfrom thinc.api import set_gpu_allocator, require_gpu\n\nnlp = spacy.load(\"en_core_web_trf\", exclude=['tagger', 'parser', 'ner',\n 'attribute_ruler', 'lemmatizer'])\nset_gpu_allocator(\"pytorch\")\nrequire_gpu(0)\n\ntopic_model = BERTopic(embedding_model=nlp)\n```\n\n## **Universal Sentence Encoder (USE)**\nThe Universal Sentence Encoder encodes text into high-dimensional vectors that are used here\nfor embedding the documents. The model is trained and optimized for greater-than-word length text,\nsuch as sentences, phrases, or short paragraphs.\n\nUsing USE in BERTopic is rather straightforward:\n\n```python\nimport tensorflow_hub\nembedding_model = tensorflow_hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n## **Gensim**\nBERTopic supports the `gensim.downloader` module, which allows it to download any word embedding model supported by Gensim.\nTypically, these are Glove, Word2Vec, or FastText embeddings:\n\n```python\nimport gensim.downloader as api\nft = api.load('fasttext-wiki-news-subwords-300')\ntopic_model = BERTopic(embedding_model=ft)\n```\n\n!!! tip \"Tip!\"\n Gensim is primarily used for Word Embedding models. This works typically best for short documents since the word embeddings are pooled.\n\n\n## **Scikit-Learn Embeddings**\nScikit-Learn is a framework for more than just machine learning.\nIt offers many preprocessing tools, some of which can be used to create representations\nfor text. Many of these tools are relatively lightweight and do not require a GPU.\nWhile the representations may be less expressive than many BERT models, the fact that\nit runs much faster can make it a relevant candidate to consider.\n\nIf you have a scikit-learn compatible pipeline that you'd like to use to embed\ntext then you can also pass this to BERTopic.\n\n```python\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\npipe = make_pipeline(\n TfidfVectorizer(),\n TruncatedSVD(100)\n)\n\ntopic_model = BERTopic(embedding_model=pipe)\n```\n\n!!! Warning\n One caveat to be aware of is that scikit-learns base `Pipeline` class does not\n support the `.partial_fit()`-API. If you have a pipeline that theoretically should\n be able to support online learning then you might want to explore\n the [scikit-partial](https://github.com/koaning/scikit-partial) project.\n Moreover, since this backend does not generate representations on a word level,\n it does not support the `bertopic.representation` models.\n\n\n## **OpenAI**\nTo use OpenAI's external API, we need to define our key and explicitly call `bertopic.backend.OpenAIBackend`\nto be used in our topic model:\n\n```python\nimport openai\nfrom bertopic.backend import OpenAIBackend\n\nclient = openai.OpenAI(api_key=\"sk-...\")\nembedding_model = OpenAIBackend(client, \"text-embedding-ada-002\")\n\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n\n## **Cohere**\nTo use Cohere's external API, we need to define our key and explicitly call `bertopic.backend.CohereBackend`\nto be used in our topic model:\n\n```python\nimport cohere\nfrom bertopic.backend import CohereBackend\n\nclient = cohere.Client(\"MY_API_KEY\")\nembedding_model = CohereBackend(client)\n\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n## **FastEmbed**\nFastEmbed[https://qdrant.tech/documentation/fastembed/] is a lightweight python library for embedding generation\nand it supports popular embedding models.\nYou can easily use it as in the example below:\n\n```python\nfrom bertopic.backend import FastEmbedBackend\n\nembedding_model = FastEmbedBackend(\"BAAI/bge-small-en-v1.5\")\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\n!!! tip \"Tip!\"\n Before to start check the supported FastEmbed text embedding models [here](https://qdrant.github.io/fastembed/examples/Supported_Models/).\n\n\n## **Multimodal**\nTo create embeddings for both text and images in the same vector space, we can use the `MultiModalBackend`.\nThis model uses a clip-vit based model that is capable of embedding text, images, or both:\n\n```python\nfrom bertopic.backend import MultiModalBackend\nmodel = MultiModalBackend('clip-ViT-B-32', batch_size=32)\n\n# Embed documents only\ndoc_embeddings = model.embed_documents(docs)\n\n# Embedding images only\nimage_embeddings = model.embed_images(images)\n\n# Embed both images and documents, then average them\ndoc_image_embeddings = model.embed(docs, images)\n```\n\n\n## **Custom Backend**\nIf your backend or model cannot be found in the ones currently available, you can use the `bertopic.backend.BaseEmbedder` class to\ncreate your backend. Below, you will find an example of creating a SentenceTransformer backend for BERTopic:\n\n```python\nfrom bertopic.backend import BaseEmbedder\nfrom sentence_transformers import SentenceTransformer\n\nclass CustomEmbedder(BaseEmbedder):\n def __init__(self, embedding_model):\n super().__init__()\n self.embedding_model = embedding_model\n\n def embed(self, documents, verbose=False):\n embeddings = self.embedding_model.encode(documents, show_progress_bar=verbose)\n return embeddings\n\n# Create custom backend\nembedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\ncustom_embedder = CustomEmbedder(embedding_model=embedding_model)\n\n# Pass custom backend to bertopic\ntopic_model = BERTopic(embedding_model=custom_embedder)\n```\n\n## **Custom Embeddings**\nThe base models in BERTopic are BERT-based models that work well with document similarity tasks. Your documents,\nhowever, might be too specific for a general pre-trained model to be used. Fortunately, you can use the embedding\nmodel in BERTopic to create document features.\n\nYou only need to prepare the document embeddings yourself and pass them through `fit_transform` of BERTopic:\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train our topic model using our pre-trained sentence-transformers embeddings\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs, embeddings)\n```\n\nAs you can see above, we used a SentenceTransformer model to create the embedding. You could also have used\n`πŸ€— transformers`, `Doc2Vec`, or any other embedding method.\n\n### **TF-IDF**\nAs mentioned above, any embedding technique can be used. However, when running UMAP, the typical distance metric is\n`cosine` which does not work quite well for a TF-IDF matrix. Instead, BERTopic will recognize that a sparse matrix\nis passed and use `hellinger` instead which works quite well for the similarity between probability distributions.\n\nWe simply create a TF-IDF matrix and use them as embeddings in our `fit_transform` method:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n# Create TF-IDF sparse matrix\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nvectorizer = TfidfVectorizer(min_df=5)\nembeddings = vectorizer.fit_transform(docs)\n\n# Train our topic model using TF-IDF vectors\ntopic_model = BERTopic(stop_words=\"english\")\ntopics, probs = topic_model.fit_transform(docs, embeddings)\n```\n\nHere, you will probably notice that creating the embeddings is quite fast whereas `fit_transform` is quite slow.\nThis is to be expected as reducing the dimensionality of a large sparse matrix takes some time. The inverse of using\ntransformer embeddings is true: creating the embeddings is slow whereas `fit_transform` is quite fast.\n" }, { "path": "docs/getting_started/guided/guided.md", "content": "Guided Topic Modeling or Seeded Topic Modeling is a collection of techniques that guides the topic modeling approach by setting several seed topics to which the model will converge to. These techniques allow the user to set a predefined number of topic representations that are sure to be in documents. For example, take an IT business that has a ticket system for the software their clients use. Those tickets may typically contain information about a specific bug regarding login issues that the IT business is aware of.\n\nTo model that bug, we can create a seed topic representation containing the words `bug`, `login`, `password`,\nand `username`. By defining those words, a Guided Topic Modeling approach will try to converge at least one topic to those words.\n\n
\n
\n--8<-- \"docs/getting_started/guided/guided.svg\"\n
\n
\n\nGuided BERTopic has two main steps:\n\nFirst, we create embeddings for each seeded topic by joining them and passing them through the document embedder. These embeddings will be compared with the existing document embeddings through cosine similarity and assigned a label. If the document is most similar to a seeded topic, then it will get that topic's label.\nIf it is most similar to the average document embedding, it will get the -1 label.\nThese labels are then passed through UMAP to create a semi-supervised approach that should nudge\nthe topic creation to the seeded topics.\n\nSecond, we take all words in seed_topic_list and assign them a multiplier larger than 1.\nThose multipliers will be used to increase the IDF values of the words across all topics thereby increasing\nthe likelihood that a seeded topic word will appear in a topic. This does, however, also increase the chance of an irrelevant topic having unrelated words. In practice, this should not be an issue since the IDF value is likely to remain low regardless of the multiplier. The multiplier is now a fixed value but may change to something more elegant, like taking the distribution of IDF values and its position into account when defining the multiplier.\n\n### **Example**\nTo demonstrate Guided BERTopic, we use the 20 Newsgroups dataset as our example. We have frequently used this\ndataset in BERTopic examples and we sometimes see a topic generated about health with words such as `drug` and `cancer`\nbeing important. However, due to the stochastic nature of UMAP, this topic is not always found.\n\nIn order to guide BERTopic to that topic, we create a seed topic list that we pass through our model. However,\nthere may be several other topics that we know should be in the documents. Let's also initialize those:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))[\"data\"]\n\nseed_topic_list = [[\"drug\", \"cancer\", \"drugs\", \"doctor\"],\n [\"windows\", \"drive\", \"dos\", \"file\"],\n [\"space\", \"launch\", \"orbit\", \"lunar\"]]\n\ntopic_model = BERTopic(seed_topic_list=seed_topic_list)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nAs you can see above, the `seed_topic_list` contains a list of topic representations. By defining the above topics\nBERTopic is more likely to model the defined seeded topics. However, BERTopic is merely nudged towards creating those\ntopics. In practice, if the seeded topics do not exist or might be divided into smaller topics, then they will\nnot be modeled. Thus, seed topics need to be accurate to accurately converge towards them.\n" }, { "path": "docs/getting_started/hierarchicaltopics/hierarchical_topics.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/hierarchicaltopics/hierarchicaltopics.md", "content": "When tweaking your topic model, the number of topics that are generated has a large effect on the quality of the topic representations. Some topics could be merged and having an understanding of the effect will help you understand which topics should and which should not be merged.\n\nThat is where hierarchical topic modeling comes in. It tries to model the possible hierarchical nature of the topics you have created to understand which topics are similar to each other. Moreover, you will have more insight into sub-topics that might exist in your data.\n\n
\n
\n--8<-- \"docs/getting_started/hierarchicaltopics/hierarchical.svg\"\n
\n
\n\nIn BERTopic, we can approximate this potential hierarchy by making use of our topic-term matrix (c-TF-IDF matrix). This matrix contains information about the importance of every word in every topic and makes for a nice numerical representation of our topics. The smaller the distance between two c-TF-IDF representations, the more similar we assume they are. In practice, this process of merging topics is done through the hierarchical clustering capabilities of `scipy` (see [here](https://docs.scipy.org/doc/scipy/reference/cluster.hierarchy.html)). It allows for several linkage methods through which we can approximate our topic hierarchy. As a default, we are using the [ward](https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.ward.html#scipy.cluster.hierarchy.ward) but many others are available.\n\nWhenever we merge two topics, we can calculate the c-TF-IDF representation of these two merged by summing their bag-of-words representation. We assume that two sets of topics are merged and that all others are kept the same, regardless of their location in the hierarchy. This helps us isolate the potential effect of merging sets of topics. As a result, we can see the topic representation at each level in the tree.\n\n## **Example**\nTo demonstrate hierarchical topic modeling with BERTopic, we use the 20 Newsgroups dataset to see how the topics that we uncover are represented in the 20 categories of documents.\n\nFirst, we train a basic BERTopic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))[\"data\"]\ntopic_model = BERTopic(verbose=True)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nNext, we can use our fitted BERTopic model to extract possible hierarchies from our c-TF-IDF matrix:\n\n```python\nhierarchical_topics = topic_model.hierarchical_topics(docs)\n```\n\nThe resulting `hierarchical_topics` is a dataframe in which merged topics are described. For example, if you would\nmerge two topics, what would the topic representation of the new topic be?\n\n## **Linkage functions**\n\nWhen creating the potential hierarchical nature of topics, we use Scipy's ward `linkage` function as a default\nto generate the hierarchy. However, you might want to use a [different linkage function](https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html)\nfor your use case, such as `single`, `complete`, `average`, `centroid`, or `median`. In BERTopic, you can define the\nlinkage function yourself, including the distance function that you would like to use:\n\n\n```python\nfrom scipy.cluster import hierarchy as sch\nfrom bertopic import BERTopic\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Hierarchical topics\nlinkage_function = lambda x: sch.linkage(x, 'single', optimal_ordering=True)\nhierarchical_topics = topic_model.hierarchical_topics(docs, linkage_function=linkage_function)\n```\n\n\n## **Visualizations**\nTo visualize these results, we can start by running a familiar function, namely `topic_model.visualize_hierarchy`:\n\n```python\ntopic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n```\n\n\nIf you **hover** over the black circles, you will see the topic representation at that level of the hierarchy. These representations\nhelp you understand the effect of merging certain topics. Some might be logical to merge whilst others might not. Moreover,\nwe can now see which sub-topics can be found within certain larger themes.\n\nAlthough this gives a nice overview of the potential hierarchy, hovering over all black circles can be tiresome. Instead, we can\nuse `topic_model.get_topic_tree` to create a text-based representation of this hierarchy. Although the general structure is more difficult\nto view, we can see better which topics could be logically merged:\n\n```python\n>>> tree = topic_model.get_topic_tree(hierarchical_topics)\n>>> print(tree)\n.\n└─atheists_atheism_god_moral_atheist\n β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n └─■──moral_morality_objective_immoral_morals ── Topic: 29\n```\n\n
\n Click here to view the full tree.\n\n ```bash\n .\n β”œβ”€people_armenian_said_god_armenians\n β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”‚ β”œβ”€jehovah_lord_mormon_mcconkie_god\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ra_satan_thou_god_lucifer ── Topic: 94\n β”‚ β”‚ β”‚ β”‚ └─■──jehovah_lord_mormon_mcconkie_unto ── Topic: 78\n β”‚ β”‚ β”‚ └─jesus_mary_god_hell_sin\n β”‚ β”‚ β”‚ β”œβ”€jesus_hell_god_eternal_heaven\n β”‚ β”‚ β”‚ β”‚ β”œβ”€hell_jesus_eternal_god_heaven\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jesus_tomb_disciples_resurrection_john ── Topic: 69\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──hell_eternal_god_jesus_heaven ── Topic: 53\n β”‚ β”‚ β”‚ β”‚ └─■──aaron_baptism_sin_law_god ── Topic: 89\n β”‚ β”‚ β”‚ └─■──mary_sin_maria_priest_conception ── Topic: 56\n β”‚ β”‚ └─■──marriage_married_marry_ceremony_marriages ── Topic: 110\n β”‚ └─people_armenian_armenians_said_mr\n β”‚ β”œβ”€people_armenian_armenians_said_israel\n β”‚ β”‚ β”œβ”€god_homosexual_homosexuality_atheists_sex\n β”‚ β”‚ β”‚ β”œβ”€homosexual_homosexuality_sex_gay_homosexuals\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€kinsey_sex_gay_men_sexual ── Topic: 44\n β”‚ β”‚ β”‚ β”‚ └─homosexuality_homosexual_sin_homosexuals_gay\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gay_homosexual_homosexuals_sexual_cramer ── Topic: 50\n β”‚ β”‚ β”‚ β”‚ └─■──homosexuality_homosexual_sin_paul_sex ── Topic: 27\n β”‚ β”‚ β”‚ └─god_atheists_atheism_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jim_context_challenges_articles_quote ── Topic: 36\n β”‚ β”‚ β”‚ β”‚ └─islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€islam_quran_islamic_rushdie_muslims ── Topic: 31\n β”‚ β”‚ β”‚ β”‚ └─■──judas_scripture_bible_books_greek ── Topic: 33\n β”‚ β”‚ β”‚ └─atheists_atheism_god_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ β”‚ β”‚ β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n β”‚ β”‚ β”‚ └─■──moral_morality_objective_immoral_morals ── Topic: 29\n β”‚ β”‚ └─armenian_armenians_people_israel_said\n β”‚ β”‚ β”œβ”€armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€tax_rights_government_income_taxes\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rights_right_slavery_slaves_residence ── Topic: 106\n β”‚ β”‚ β”‚ β”‚ └─tax_government_taxes_income_libertarians\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€government_libertarians_libertarian_regulation_party ── Topic: 58\n β”‚ β”‚ β”‚ β”‚ └─■──tax_taxes_income_billion_deficit ── Topic: 41\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€gun_guns_militia_firearms_amendment\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€blacks_penalty_death_cruel_punishment ── Topic: 55\n β”‚ β”‚ β”‚ β”‚ └─■──gun_guns_militia_firearms_amendment ── Topic: 7\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_jews_turkish\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€israel_israeli_jews_arab_jewish ── Topic: 4\n β”‚ β”‚ β”‚ └─■──armenian_armenians_turkish_armenia_azerbaijan ── Topic: 15\n β”‚ β”‚ └─stephanopoulos_president_mr_myers_ms\n β”‚ β”‚ β”œβ”€β– β”€β”€serbs_muslims_stephanopoulos_mr_bosnia ── Topic: 35\n β”‚ β”‚ └─■──myers_stephanopoulos_president_ms_mr ── Topic: 87\n β”‚ └─batf_fbi_koresh_compound_gas\n β”‚ β”œβ”€β– β”€β”€reno_workers_janet_clinton_waco ── Topic: 77\n β”‚ └─batf_fbi_koresh_gas_compound\n β”‚ β”œβ”€batf_koresh_fbi_warrant_compound\n β”‚ β”‚ β”œβ”€β– β”€β”€batf_warrant_raid_compound_fbi ── Topic: 42\n β”‚ β”‚ └─■──koresh_batf_fbi_children_compound ── Topic: 61\n β”‚ └─■──fbi_gas_tear_bds_building ── Topic: 23\n └─use_like_just_dont_new\n β”œβ”€game_team_year_games_like\n β”‚ β”œβ”€game_team_games_25_year\n β”‚ β”‚ β”œβ”€game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€window_printer_use_problem_mhz\n β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_wire_simms_wiring_battery\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_mhz_battery_cpu_heat\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_pds_simm_vram_lc\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€pds_nubus_lc_slot_card ── Topic: 119\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──simms_simm_vram_meg_dram ── Topic: 32\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─mhz_battery_cpu_heat_speed\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€fan_cpu_heat_sink_fans ── Topic: 92\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──mhz_speed_cpu_fpu_clock ── Topic: 22\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──monitor_turn_power_computer_electricity ── Topic: 91\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─battery_batteries_concrete_duo_discharge\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€duo_battery_apple_230_problem ── Topic: 121\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──battery_batteries_concrete_discharge_temperature ── Topic: 75\n β”‚ β”‚ β”‚ β”‚ β”‚ └─wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€leds_uv_blue_light_boards ── Topic: 66\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──wire_wiring_ground_neutral_outlets ── Topic: 120\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─scope_scopes_phone_dial_number\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€dial_number_phone_line_output ── Topic: 93\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──scope_scopes_motorola_generator_oscilloscope ── Topic: 113\n β”‚ β”‚ β”‚ β”‚ β”‚ └─celp_dsp_sampling_antenna_digital\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€antenna_antennas_receiver_cable_transmitter ── Topic: 70\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──celp_dsp_sampling_speech_voice ── Topic: 52\n β”‚ β”‚ β”‚ β”‚ └─window_printer_xv_mouse_windows\n β”‚ β”‚ β”‚ β”‚ β”œβ”€window_xv_error_widget_problem\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€error_symbol_undefined_xterm_rx\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€symbol_error_undefined_doug_parse ── Topic: 63\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──rx_remote_server_xdm_xterm ── Topic: 45\n β”‚ β”‚ β”‚ β”‚ β”‚ └─window_xv_widget_application_expose\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€window_widget_expose_application_event\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gc_mydisplay_draw_gxxor_drawing ── Topic: 103\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──window_widget_application_expose_event ── Topic: 25\n β”‚ β”‚ β”‚ β”‚ β”‚ └─xv_den_polygon_points_algorithm\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€den_polygon_points_algorithm_polygons ── Topic: 28\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──xv_24bit_image_bit_images ── Topic: 57\n β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_mouse_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€scanner_logitech_grayscale_ocr_scanman ── Topic: 108\n β”‚ β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€printer_print_deskjet_hp_ink ── Topic: 18\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──fonts_font_truetype_tt_atm ── Topic: 49\n β”‚ β”‚ β”‚ β”‚ └─mouse_ghostscript_midi_driver_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€ghostscript_midi_postscript_files_file\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ghostscript_postscript_pageview_ghostview_dsc ── Topic: 104\n β”‚ β”‚ β”‚ β”‚ β”‚ └─midi_sound_file_windows_driver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€location_mar_file_host_rwrr ── Topic: 83\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──midi_sound_driver_blaster_soundblaster ── Topic: 98\n β”‚ β”‚ β”‚ β”‚ └─■──mouse_driver_mice_ball_problem ── Topic: 68\n β”‚ β”‚ β”‚ └─game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€1st_sale_condition_comics_hulk\n β”‚ β”‚ β”‚ β”‚ β”œβ”€sale_condition_offer_asking_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€condition_stereo_amp_speakers_asking\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€miles_car_amfm_toyota_cassette ── Topic: 62\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──amp_speakers_condition_stereo_audio ── Topic: 24\n β”‚ β”‚ β”‚ β”‚ β”‚ └─games_sale_pom_cds_shipping\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€pom_cds_sale_shipping_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€size_shipping_sale_condition_mattress ── Topic: 100\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──pom_cds_cd_sale_picture ── Topic: 37\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──games_game_snes_sega_genesis ── Topic: 40\n β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”œβ”€1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€lens_tape_camera_backup_lenses\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€tape_backup_tapes_drive_4mm ── Topic: 107\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──lens_camera_lenses_zoom_pouch ── Topic: 114\n β”‚ β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€1st_hulk_comics_art_appears ── Topic: 105\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──books_book_cover_trek_chemistry ── Topic: 125\n β”‚ β”‚ β”‚ β”‚ └─tickets_hotel_ticket_voucher_package\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hotel_voucher_package_vacation_room ── Topic: 74\n β”‚ β”‚ β”‚ β”‚ └─■──tickets_ticket_june_airlines_july ── Topic: 84\n β”‚ β”‚ β”‚ └─game_team_games_season_hockey\n β”‚ β”‚ β”‚ β”œβ”€game_hockey_team_25_550\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€espn_pt_pts_game_la ── Topic: 17\n β”‚ β”‚ β”‚ β”‚ └─■──team_25_game_hockey_550 ── Topic: 2\n β”‚ β”‚ β”‚ └─■──year_game_hit_baseball_players ── Topic: 0\n β”‚ β”‚ └─bike_car_greek_insurance_msg\n β”‚ β”‚ β”œβ”€car_bike_insurance_cars_engine\n β”‚ β”‚ β”‚ β”œβ”€car_insurance_cars_radar_engine\n β”‚ β”‚ β”‚ β”‚ β”œβ”€insurance_health_private_care_canada\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€insurance_health_private_care_canada ── Topic: 99\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──insurance_car_accident_rates_sue ── Topic: 82\n β”‚ β”‚ β”‚ β”‚ └─car_cars_radar_engine_detector\n β”‚ β”‚ β”‚ β”‚ β”œβ”€car_radar_cars_detector_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€radar_detector_detectors_ka_alarm ── Topic: 39\n β”‚ β”‚ β”‚ β”‚ β”‚ └─car_cars_mustang_ford_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clutch_shift_shifting_transmission_gear ── Topic: 88\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──car_cars_mustang_ford_v8 ── Topic: 14\n β”‚ β”‚ β”‚ β”‚ └─oil_diesel_odometer_diesels_car\n β”‚ β”‚ β”‚ β”‚ β”œβ”€odometer_oil_sensor_car_drain\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€odometer_sensor_speedo_gauge_mileage ── Topic: 96\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──oil_drain_car_leaks_taillights ── Topic: 102\n β”‚ β”‚ β”‚ β”‚ └─■──diesel_diesels_emissions_fuel_oil ── Topic: 79\n β”‚ β”‚ β”‚ └─bike_riding_ride_bikes_motorcycle\n β”‚ β”‚ β”‚ β”œβ”€bike_ride_riding_bikes_lane\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€bike_ride_riding_lane_car ── Topic: 11\n β”‚ β”‚ β”‚ β”‚ └─■──bike_bikes_miles_honda_motorcycle ── Topic: 19\n β”‚ β”‚ β”‚ └─■──countersteering_bike_motorcycle_rear_shaft ── Topic: 46\n β”‚ β”‚ └─greek_msg_kuwait_greece_water\n β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_water\n β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_kuwait_greece_turkish_greeks\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€greek_greece_turkish_greeks_cyprus ── Topic: 71\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kuwait_iraq_iran_gulf_arabia ── Topic: 76\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_dog_drugs_drug_food\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clinton_bush_quayle_reagan_panicking ── Topic: 101\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€cooper_trial_weaver_spence_witnesses ── Topic: 90\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──dog_dogs_bike_trained_springer ── Topic: 67\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_drugs_drug_food_chinese\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€msg_food_chinese_foods_taste ── Topic: 30\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──drugs_drug_marijuana_cocaine_alcohol ── Topic: 72\n β”‚ β”‚ β”‚ β”‚ └─water_theory_universe_science_larsons\n β”‚ β”‚ β”‚ β”‚ β”œβ”€water_nuclear_cooling_steam_dept\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rocketry_rockets_engines_nuclear_plutonium ── Topic: 115\n β”‚ β”‚ β”‚ β”‚ β”‚ └─water_cooling_steam_dept_plants\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€water_dept_phd_environmental_atmospheric ── Topic: 97\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──cooling_water_steam_towers_plants ── Topic: 109\n β”‚ β”‚ β”‚ β”‚ └─theory_universe_larsons_larson_science\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€theory_universe_larsons_larson_science ── Topic: 54\n β”‚ β”‚ β”‚ β”‚ └─■──oort_cloud_grbs_gamma_burst ── Topic: 80\n β”‚ β”‚ β”‚ └─helmet_kirlian_photography_lock_wax\n β”‚ β”‚ β”‚ β”œβ”€helmet_kirlian_photography_leaf_mask\n β”‚ β”‚ β”‚ β”‚ β”œβ”€kirlian_photography_leaf_pictures_deleted\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€deleted_joke_stuff_maddi_nickname\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€joke_maddi_nickname_nicknames_frank ── Topic: 43\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──deleted_stuff_bookstore_joke_motto ── Topic: 81\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kirlian_photography_leaf_pictures_aura ── Topic: 85\n β”‚ β”‚ β”‚ β”‚ └─helmet_mask_liner_foam_cb\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€helmet_liner_foam_cb_helmets ── Topic: 112\n β”‚ β”‚ β”‚ β”‚ └─■──mask_goalies_77_santore_tl ── Topic: 123\n β”‚ β”‚ β”‚ └─lock_wax_paint_plastic_ear\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€lock_cable_locks_bike_600 ── Topic: 117\n β”‚ β”‚ β”‚ └─wax_paint_ear_plastic_skin\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€wax_paint_plastic_scratches_solvent ── Topic: 65\n β”‚ β”‚ β”‚ └─■──ear_wax_skin_greasy_acne ── Topic: 116\n β”‚ β”‚ └─m4_mp_14_mw_mo\n β”‚ β”‚ β”œβ”€m4_mp_14_mw_mo\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€m4_mp_14_mw_mo ── Topic: 111\n β”‚ β”‚ β”‚ └─■──test_ensign_nameless_deane_deanebinahccbrandeisedu ── Topic: 118\n β”‚ β”‚ └─■──ites_cheek_hello_hi_ken ── Topic: 3\n β”‚ └─space_medical_health_disease_cancer\n β”‚ β”œβ”€medical_health_disease_cancer_patients\n β”‚ β”‚ β”œβ”€β– β”€β”€cancer_centers_center_medical_research ── Topic: 122\n β”‚ β”‚ └─health_medical_disease_patients_hiv\n β”‚ β”‚ β”œβ”€patients_medical_disease_candida_health\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€candida_yeast_infection_gonorrhea_infections ── Topic: 48\n β”‚ β”‚ β”‚ └─patients_disease_cancer_medical_doctor\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hiv_medical_cancer_patients_doctor ── Topic: 34\n β”‚ β”‚ β”‚ └─■──pain_drug_patients_disease_diet ── Topic: 26\n β”‚ β”‚ └─■──health_newsgroup_tobacco_vote_votes ── Topic: 9\n β”‚ └─space_launch_nasa_shuttle_orbit\n β”‚ β”œβ”€space_moon_station_nasa_launch\n β”‚ β”‚ β”œβ”€β– β”€β”€sky_advertising_billboard_billboards_space ── Topic: 59\n β”‚ β”‚ └─■──space_station_moon_redesign_nasa ── Topic: 16\n β”‚ └─space_mission_hst_launch_orbit\n β”‚ β”œβ”€space_launch_nasa_orbit_propulsion\n β”‚ β”‚ β”œβ”€β– β”€β”€space_launch_nasa_propulsion_astronaut ── Topic: 47\n β”‚ β”‚ └─■──orbit_km_jupiter_probe_earth ── Topic: 86\n β”‚ └─■──hst_mission_shuttle_orbit_arrays ── Topic: 60\n └─drive_file_key_windows_use\n β”œβ”€key_file_jpeg_encryption_image\n β”‚ β”œβ”€key_encryption_clipper_chip_keys\n β”‚ β”‚ β”œβ”€β– β”€β”€key_clipper_encryption_chip_keys ── Topic: 1\n β”‚ β”‚ └─■──entry_file_ripem_entries_key ── Topic: 73\n β”‚ └─jpeg_image_file_gif_images\n β”‚ β”œβ”€motif_graphics_ftp_available_3d\n β”‚ β”‚ β”œβ”€motif_graphics_openwindows_ftp_available\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€openwindows_motif_xview_windows_mouse ── Topic: 20\n β”‚ β”‚ β”‚ └─■──graphics_widget_ray_3d_available ── Topic: 95\n β”‚ β”‚ └─■──3d_machines_version_comments_contact ── Topic: 38\n β”‚ └─jpeg_image_gif_images_format\n β”‚ β”œβ”€β– β”€β”€gopher_ftp_files_stuffit_images ── Topic: 51\n β”‚ └─■──jpeg_image_gif_format_images ── Topic: 13\n └─drive_db_card_scsi_windows\n β”œβ”€db_windows_dos_mov_os2\n β”‚ β”œβ”€β– β”€β”€copy_protection_program_software_disk ── Topic: 64\n β”‚ └─■──db_windows_dos_mov_os2 ── Topic: 8\n └─drive_card_scsi_drives_ide\n β”œβ”€drive_scsi_drives_ide_disk\n β”‚ β”œβ”€β– β”€β”€drive_scsi_drives_ide_disk ── Topic: 6\n β”‚ └─■──meg_sale_ram_drive_shipping ── Topic: 12\n └─card_modem_monitor_video_drivers\n β”œβ”€β– β”€β”€card_monitor_video_drivers_vga ── Topic: 5\n └─■──modem_port_serial_irq_com ── Topic: 10\n ```\n
\n\n\n## **Merge topics**\n\nAfter seeing the potential hierarchy of your topic, you might want to merge specific\ntopics. For example, if topic 1 is\n`1_space_launch_moon_nasa` and topic 2 is `2_spacecraft_solar_space_orbit` it might\nmake sense to merge those two topics as they are quite similar in meaning. In BERTopic,\nyou can use `.merge_topics` to manually select and merge those topics. Doing so will\nupdate their topic representation which in turn updates the entire model:\n\n```python\ntopics_to_merge = [1, 2]\ntopic_model.merge_topics(docs, topics_to_merge)\n```\n\nIf you have several groups of topics you want to merge, create a list of lists instead:\n\n```python\ntopics_to_merge = [[1, 2],\n [3, 4]]\ntopic_model.merge_topics(docs, topics_to_merge)\n```\n" }, { "path": "docs/getting_started/manual/manual.md", "content": "Although topic modeling is typically done by discovering topics in an unsupervised manner, there might be times when you already have a bunch of clusters or classes from which you want to model the topics. For example, the often used [20 NewsGroups dataset](https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html) is already split up into 20 classes. Here, we might want to see how we can transform those 20 classes into 20 topics. Instead of using BERTopic to discover previously unknown topics, we are now going to manually pass them to BERTopic without actually learning them.\n\nWe can view this as a manual topic modeling approach. There is no underlying algorithm for detecting these topics since you already have done that before. Whether that is simply because they are already available, like with the 20 NewsGroups dataset, or maybe because you have created clusters of documents before using packages like [human-learn](https://github.com/koaning/human-learn), [bulk](https://github.com/koaning/bulk), [thisnotthat](https://github.com/TutteInstitute/thisnotthat) or something entirely different.\n\nIn other words, we can pass our labels to BERTopic and it will try to transform those labels into topics by running the c-TF-IDF representations on the set of documents within each label. This process allows us to model the topics themselves and similarly gives us the option to use everything BERTopic has to offer.\n\n
\n
\n--8<-- \"docs/getting_started/manual/pipeline.svg\"\n
\n
\n\nTo do so, we need to skip over the dimensionality reduction and clustering steps since we already know the labels for our documents. We can use the documents and labels from the 20 NewsGroups dataset to create topics from those 20 labels:\n\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Get labeled data\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data['data']\ny = data['target']\n```\n\nThen, we make sure to create empty instances of the dimensionality reduction and clustering steps. We pass those to BERTopic to simply skip over them and go to the topic representation process:\n\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.backend import BaseEmbedder\nfrom bertopic.cluster import BaseCluster\nfrom bertopic.vectorizers import ClassTfidfTransformer\nfrom bertopic.dimensionality import BaseDimensionalityReduction\n\n# Prepare our empty sub-models and reduce frequent words while we are at it.\nempty_embedding_model = BaseEmbedder()\nempty_dimensionality_model = BaseDimensionalityReduction()\nempty_cluster_model = BaseCluster()\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True)\n\n# Fit BERTopic without actually performing any clustering\ntopic_model= BERTopic(\n embedding_model=empty_embedding_model,\n umap_model=empty_dimensionality_model,\n hdbscan_model=empty_cluster_model,\n ctfidf_model=ctfidf_model\n)\ntopics, probs = topic_model.fit_transform(docs, y=y)\n```\n\nLet's take a look at a few topics that we get out of training this way by running `topic_model.get_topic_info()`:\n\n
\n
\n--8<-- \"docs/getting_started/manual/table.svg\"\n
\n
\n\nWe can see several interesting topics appearing here. They seem to relate to the 20 classes we had as input. Now, let's map those topics to our original classes to view their relationship:\n\n```python\n# Map input `y` to topics\nmappings = topic_model.topic_mapper_.get_mappings()\nmappings = {value: data[\"target_names\"][key] for key, value in mappings.items()}\n\n# Assign original classes to our topics\ndf = topic_model.get_topic_info()\ndf[\"Class\"] = df.Topic.map(mappings)\ndf\n```\n\n
\n
\n--8<-- \"docs/getting_started/manual/table_classes.svg\"\n
\n
\n\nWe can see that the c-TF-IDF representations nicely extract the words that give a nice representation of our input classes. This is all done without actually embedding and clustering the data.\n\nAs a result, the entire \"training\" process only takes a couple of seconds. Moreover, we can still perform BERTopic-specific features like dynamic topic modeling, topics per class, hierarchical topic modeling, modeling topic distributions, etc.\n\n!!! note\n The resulting `topics` may be a different mapping from the `y` labels. To map `y` to `topics`, we can run the following:\n\n\n ```python\n mappings = topic_model.topic_mapper_.get_mappings()\n y_mapped = [mappings[val] for val in y]\n ```\n" }, { "path": "docs/getting_started/merge/merge.md", "content": "# Merge Multiple Fitted Models\n\nAfter you have trained a new BERTopic model on your data, new data might still be coming in. Although you can use [online BERTopic](https://maartengr.github.io/BERTopic/getting_started/online/online.html), you might prefer to use the default HDBSCAN and UMAP models since they do not support incremental learning out of the box.\n\nInstead, we you can train a new BERTopic on incoming data and merge it with your base model to detect whether new topics have appeared in the unseen documents. This is a great way of detecting whether your new model contains information that was not previously found in your base topic model.\n\nSimilarly, you might want to train multiple BERTopic models using different sets of settings, even though they might all be using the same underlying embedding model. Merging these models would also allow for a single model that you can use throughout your use cases.\n\nLastly, this methods also allows for a degree of `federated learning` where each node trains a topic model that are aggregated in a central server.\n\n## **Example**\nTo demonstrate merging different topic models with BERTopic, we use the ArXiv paper abstracts to see which topics they generally contain.\n\nFirst, we train three separate models on different parts of the data:\n\n```python\nfrom umap import UMAP\nfrom bertopic import BERTopic\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\n\n# Extract abstracts to train on and corresponding titles\nabstracts_1 = dataset[\"abstract\"][:5_000]\nabstracts_2 = dataset[\"abstract\"][5_000:10_000]\nabstracts_3 = dataset[\"abstract\"][10_000:15_000]\n\n# Create topic models\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\ntopic_model_1 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_1)\ntopic_model_2 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_2)\ntopic_model_3 = BERTopic(umap_model=umap_model, min_topic_size=20).fit(abstracts_3)\n```\n\nThen, we can combine all three models into one with `.merge_models`:\n\n```python\n# Combine all models into one\nmerged_model = BERTopic.merge_models([topic_model_1, topic_model_2, topic_model_3])\n```\n\nWhen we inspect the first model, we can see it has 52 topics:\n\n```python\n>>> len(topic_model_1.get_topic_info())\n52\n```\n\nNow, we inspect the merged model, we can see it has 57 topics:\n\n```python\n>>> len(merged_model.get_topic_info())\n57\n```\n\nIt seems that by merging these three models, there were 6 undiscovered topics that we could add to the very first model.\n\n!!! Note\n Note that the models are merged sequentially. This means that the comparison starts with `topic_model_1` and that\n each new topic from `topic_model_2` and `topic_model_3` will be added to `topic_model_1`.\n\nWe can check the newly added topics in the `merged_model` by simply looking at the 6 latest topics that were added. The order of topics from `topic_model_1`\nremains the same. All new topics are simply added on top of them.\n\nLet's inspect them:\n\n```python\n>>> merged_model.get_topic_info().tail(5)\n```\n\n| | Topic | Count | Name | Representation | Representative_Docs |\n|---:|--------:|--------:|:---------------------------------------|:-----------------------------------------------------------------------------------------------------------------------|----------------------:|\n| 52 | 51 | 47 | 50_activity_mobile_wearable_sensors | ['activity', 'mobile', 'wearable', 'sensors', 'falls', 'human', 'phone', 'recognition', 'activities', 'accelerometer'] | nan |\n| 53 | 52 | 48 | 25_music_musical_audio_chord | ['music', 'musical', 'audio', 'chord', 'and', 'we', 'to', 'that', 'of', 'for'] | nan |\n| 54 | 53 | 32 | 36_fairness_discrimination_fair_groups | ['fairness', 'discrimination', 'fair', 'groups', 'protected', 'decision', 'we', 'of', 'classifier', 'to'] | nan |\n| 55 | 54 | 30 | 38_traffic_driver_prediction_flow | ['traffic', 'driver', 'prediction', 'flow', 'trajectory', 'the', 'and', 'congestion', 'of', 'transportation'] | nan |\n| 56 | 55 | 22 | 50_spiking_neurons_networks_learning | ['spiking', 'neurons', 'networks', 'learning', 'neural', 'snn', 'dynamics', 'plasticity', 'snns', 'of'] | nan |\n\n\nIt seems that topics about activity, music, fairness, traffic, and spiking networks were added to the base topic model! Two things that you might have noticed. First,\nthe representative documents were not added to the model. This is because of privacy reasons, you might want to combine models that were trained on different stations which\nwould allow for a degree of `federated learning`. Second, the names of the new topics contain topic ids that refer to one of the old models. They were purposefully left this way\nso that the user can identify which topics were newly added which you could inspect in the original models.\n\n\n## **min_similarity**\n\nThe way the models are merged is through comparison of their topic embeddings. If topics between models are similar enough, then they will be regarded as the same topics\nand the topic of the first model in the list will be chosen. However, if topics between models are dissimilar enough, then the topic of the latter model will be added to the former.\n\nThis (dis)similarity is can be tweaked using the `min_similarity` parameter. Increasing this value will increase the chance of adding new topics. In contrast, decreasing this value\nwill make it more strict and threfore decrease the chance of adding new topics. The value is set to `0.7` by default, so let's see what happens if we were to increase this value to\n`0.9``:\n\n```python\n# Combine all models into one\nmerged_model = BERTopic.merge_models([topic_model_1, topic_model_2, topic_model_3], min_similarity=0.9)\n```\n\nWhen we inspect the number of topics in our new model, we can see that they have increased quite a bit:\n\n```python\n>>> len(merged_model.get_topic_info())\n102\n```\n\nThis demonstrates the influence of `min_similarity` on the number of new topics that are added to the base model.\n" }, { "path": "docs/getting_started/multiaspect/multiaspect.md", "content": "During the development of BERTopic, many different types of representations can be created, from keywords and phrases to summaries and custom labels. There is a variety of techniques that one can choose from to represent a topic. As such, there are a number of interesting and creative ways one can summarize topics. A topic is more than just a single representation.\n\nTherefore, `multi-aspect topic modeling` is introduced! During the `.fit` or `.fit_transform` stages, you can now get multiple representations of a single topic. In practice, it works by generating and storing all kinds of different topic representations (see image below).\n\n
\n ![Image title](multiaspect.svg)\n
\n
\n\nThe approach is rather straightforward. We might want to represent our topics using a `PartOfSpeech` representation model but we might also want to try out `KeyBERTInspired` and compare those representation models. We can do this as follows:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic.representation import PartOfSpeech\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Documents to train on\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\n# The main representation of a topic\nmain_representation = KeyBERTInspired()\n\n# Additional ways of representing a topic\naspect_model1 = PartOfSpeech(\"en_core_web_sm\")\naspect_model2 = [KeyBERTInspired(top_n_words=30), MaximalMarginalRelevance(diversity=.5)]\n\n# Add all models together to be run in a single `fit`\nrepresentation_model = {\n \"Main\": main_representation,\n \"Aspect1\": aspect_model1,\n \"Aspect2\": aspect_model2\n}\ntopic_model = BERTopic(representation_model=representation_model).fit(docs)\n```\n\nAs show above, to perform multi-aspect topic modeling, we make sure that `representation_model` is a dictionary where each representation model pipeline is defined.\nThe main pipeline, that is used in most visualization options, is defined with the `\"Main\"` key. All other aspects can be defined however you want. In the example above, the two additional aspects that we are interested in are defined as `\"Aspect1\"` and `\"Aspect2\"`.\n\nAfter we have fitted our model, we can access all representations with `topic_model.get_topic_info()`:\n\n

\n\n

\n\nAs you can see, there are a number of different representations for our topics that we can inspect. All aspects are found in `topic_model.topic_aspects_`.\n" }, { "path": "docs/getting_started/multimodal/multimodal.md", "content": "Documents or text are often accompanied by imagery or the other way around. For example, social media images with captions and products with descriptions. Topic modeling has traditionally focused on creating topics from textual representations. However, as more multimodal representations are created, the need for multimodal topics increases.\n\nBERTopic can perform **multimodal topic modeling** in a number of ways during `.fit` and `.fit_transform` stages.\n\n## **Text + Images**\n\nThe most basic example of multimodal topic modeling in BERTopic is when you have images that accompany your documents. This means that it is expected that each document has an image and vice versa. Instagram pictures, for example, almost always have some descriptions to them.\n\n
\n ![Image title](images_and_text.svg)\n
\n
\n\nIn this example, we are going to use images from `flickr` that each have a caption associated to it:\n\n```python\n# NOTE: This requires the `datasets` package which you can\n# install with `pip install datasets`\nfrom datasets import load_dataset\n\nds = load_dataset(\"maderix/flickr_bw_rgb\")\nimages = ds[\"train\"][\"image\"]\ndocs = ds[\"train\"][\"caption\"]\n```\n\nThe `docs` variable contains the captions for each image in `images`. We can now use these variables to run our multimodal example:\n\n!!! Tip\n Do note that it is better to pass the paths of the images instead of the images themselves as there is no need to keep all images in memory. When passing the paths of the images, they are only opened temporarily when they are needed.\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import VisualRepresentation\n\n# Additional ways of representing a topic\nvisual_model = VisualRepresentation()\n\n# Make sure to add the `visual_model` to a dictionary\nrepresentation_model = {\n \"Visual_Aspect\": visual_model,\n}\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\nIn this example, we are clustering the documents and are then looking for the best matching images to the resulting clusters.\n\nWe can now access our image representations for each topic with `topic_model.topic_aspects_[\"Visual_Aspect\"]`.\nIf you want an overview of the topic images together with their textual representations in jupyter, you can run the following:\n\n```python\nimport base64\nfrom io import BytesIO\nfrom IPython.display import HTML\n\ndef image_base64(im):\n if isinstance(im, str):\n im = get_thumbnail(im)\n with BytesIO() as buffer:\n im.save(buffer, 'jpeg')\n return base64.b64encode(buffer.getvalue()).decode()\n\n\ndef image_formatter(im):\n return f''\n\n# Extract dataframe\ndf = topic_model.get_topic_info().drop(\"Representative_Docs\", 1).drop(\"Name\", 1)\n\n# Visualize the images\nHTML(df.to_html(formatters={'Visual_Aspect': image_formatter}, escape=False))\n```\n\n

\n\n

\n\n!!! Tip\n In the example above, we are clustering the documents but since you have\n images, you might want to cluster those or cluster an aggregation of both\n images and documents. For that, you can use the new `MultiModalBackend`\n to generate embeddings:\n\n ```python\n from bertopic.backend import MultiModalBackend\n model = MultiModalBackend('clip-ViT-B-32', batch_size=32)\n\n # Embed documents only\n doc_embeddings = model.embed_documents(docs)\n\n # Embedding images only\n image_embeddings = model.embed_images(images)\n\n # Embed both images and documents, then average them\n doc_image_embeddings = model.embed(docs, images)\n ```\n\n## **Images Only**\n\nTraditional topic modeling techniques can only be run on textual data, as is shown in the example above. However, there are plenty of cases where textual data is not available but images are. BERTopic allows topic modeling to be performed using only images as your input data.\n\n
\n ![Image title](images_only.svg)\n
\n
\n\nTo run BERTopic on images only, we first need to embed our images and then define a model that convert images to text. To do so, we are going to need some images. We will take the same images as the above but instead save them locally and pass the paths to the images instead. As mentioned before, this will make sure that we do not hold too many images in memory whilst only a small subset is needed:\n\n\n```python\nimport os\nimport glob\nimport zipfile\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom sentence_transformers import util\n\n# Flickr 8k images\nimg_folder = 'photos/'\ncaps_folder = 'captions/'\nif not os.path.exists(img_folder) or len(os.listdir(img_folder)) == 0:\n os.makedirs(img_folder, exist_ok=True)\n\n if not os.path.exists('Flickr8k_Dataset.zip'): #Download dataset if does not exist\n util.http_get('https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_Dataset.zip', 'Flickr8k_Dataset.zip')\n util.http_get('https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_text.zip', 'Flickr8k_text.zip')\n\n for folder, file in [(img_folder, 'Flickr8k_Dataset.zip'), (caps_folder, 'Flickr8k_text.zip')]:\n with zipfile.ZipFile(file, 'r') as zf:\n for member in tqdm(zf.infolist(), desc='Extracting'):\n zf.extract(member, folder)\nimages = list(glob.glob('photos/Flicker8k_Dataset/*.jpg'))\n```\n\nNext, we can run our pipeline:\n\n\n```python\nfrom bertopic.representation import KeyBERTInspired, VisualRepresentation\nfrom bertopic.backend import MultiModalBackend\n\n# Image embedding model\nembedding_model = MultiModalBackend('clip-ViT-B-32', batch_size=32)\n\n# Image to text representation model\nrepresentation_model = {\n \"Visual_Aspect\": VisualRepresentation(image_to_text_model=\"nlpconnect/vit-gpt2-image-captioning\")\n}\n\n```\n\nUsing these models, we can run our pipeline:\n\n```python\nfrom bertopic import BERTopic\n\n# Train our model with images only\ntopic_model = BERTopic(embedding_model=embedding_model, representation_model=representation_model, min_topic_size=30)\ntopics, probs = topic_model.fit_transform(documents=None, images=images)\n```\n\nWe can now access our image representations for each topic with `topic_model.topic_aspects_[\"Visual_Aspect\"]`.\nIf you want an overview of the topic images together with their textual representations in jupyter, you can run the following:\n\n```python\nimport base64\nfrom io import BytesIO\nfrom IPython.display import HTML\n\ndef image_base64(im):\n if isinstance(im, str):\n im = get_thumbnail(im)\n with BytesIO() as buffer:\n im.save(buffer, 'jpeg')\n return base64.b64encode(buffer.getvalue()).decode()\n\n\ndef image_formatter(im):\n return f''\n\n# Extract dataframe\ndf = topic_model.get_topic_info().drop(\"Representative_Docs\", 1).drop(\"Name\", 1)\n\n# Visualize the images\nHTML(df.to_html(formatters={'Visual_Aspect': image_formatter}, escape=False))\n```\n\n

\n\n

\n" }, { "path": "docs/getting_started/online/online.md", "content": "Online topic modeling (sometimes called \"incremental topic modeling\") is the ability to learn incrementally from a mini-batch of instances. Essentially, it is a way to update your topic model with data on which it was not trained before. In Scikit-Learn, this technique is often modeled through a `.partial_fit` function, which is also used in BERTopic.\n\n!!! Tip\n Another method for online topic modeling can be found with the [**.merge_models**](https://maartengr.github.io/BERTopic/getting_started/merge/merge.html) functionality of BERTopic. It allows for merging multiple BERTopic models to create a single new one. This method can be used to discover new topics by training a new model and exploring whether that new model added new topics to the original model when merging. A major benefit, compared to `.partial_fit` is that you can keep using the original UMAP and HDBSCAN models which tends result in improved performance and gives you significant more flexibility.\n\nIn BERTopic, there are three main goals for using this technique.\n\n* To reduce the memory necessary for training a topic model.\n* To continuously update the topic model as new data comes in.\n* To continuously find new topics as new data comes in.\n\nIn BERTopic, online topic modeling can be a bit tricky as there are several steps involved in which online learning needs to be made available. To recap, BERTopic consists of the following 6 steps:\n\n1. Extract embeddings\n2. Reduce dimensionality\n3. Cluster reduced embeddings\n4. Tokenize topics\n5. Extract topic words\n6. (Optional) Fine-tune topic words\n\nFor some steps, an online variant is more important than others. Typically, in step 1 we use pre-trained language models that are in less need of continuous updates. This means that we can use an embedding model like Sentence-Transformers for extracting the embeddings and still use it in an online setting. Similarly, steps 5 and 6 do not necessarily need online variants since they are built upon step 4, tokenization. If that tokenization is by itself incremental, then so will steps 5 and 6.\n\n
\n
\n--8<-- \"docs/getting_started/online/online.svg\"\n
\n
\n\nThis means that we will need online variants for steps 2 through 4. Steps 2 and 3, dimensionality reduction and clustering, can be modeled through the use of Scikit-Learn's `.partial_fit` function. In other words, it supports any algorithm that can be trained using `.partial_fit` since these algorithms can be trained incrementally. For example, incremental dimensionality reduction can be achieved using Scikit-Learn's `IncrementalPCA` and incremental clustering with `MiniBatchKMeans`.\n\nLastly, we need to develop an online variant for step 5, tokenization. In this step, a Bag-of-words representation is created through the `CountVectorizer`. However, as new data comes in, its vocabulary will need to be updated. For that purpose, `bertopic.vectorizers.OnlineCountVectorizer` was created that not only updates out-of-vocabulary words but also implements decay and cleaning functions to prevent the sparse bag-of-words matrix to become too large. Most notably, the `decay` parameter is a value between 0 and 1 to weigh the percentage of frequencies that the previous bag-of-words matrix should be reduced to. For example, a value of `.1` will decrease the frequencies in the bag-of-words matrix by 10% at each iteration. This will make sure that recent data has more weight than previous iterations. Similarly, `delete_min_df` will remove certain words from its vocabulary if their frequency is lower than a set value. This ties together with the `decay` parameter as some words will decay over time if not used. For more information regarding the `OnlineCountVectorizer`, please see the [vectorizers documentation](https://maartengr.github.io/BERTopic/getting_started/vectorizers/vectorizers.html#onlinecountvectorizer).\n\n\n\n## **Example**\n\nOnline topic modeling in BERTopic is rather straightforward. We first need to have our documents split into chunks such that we can train and update our topic model incrementally.\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Prepare documents\nall_docs = fetch_20newsgroups(subset=subset, remove=('headers', 'footers', 'quotes'))[\"data\"]\ndoc_chunks = [all_docs[i:i+1000] for i in range(0, len(all_docs), 1000)]\n```\n\nHere, we created chunks of 1000 documents to be fed in BERTopic. Then, we will need to define several sub-models that support online learning. Specifically, we are going to be using `IncrementalPCA`, `MiniBatchKMeans`, and the `OnlineCountVectorizer`:\n\n```python\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.decomposition import IncrementalPCA\nfrom bertopic.vectorizers import OnlineCountVectorizer\n\n# Prepare sub-models that support online learning\numap_model = IncrementalPCA(n_components=5)\ncluster_model = MiniBatchKMeans(n_clusters=50, random_state=0)\nvectorizer_model = OnlineCountVectorizer(stop_words=\"english\", decay=.01)\n```\n\nAfter having defined our sub-models, we can start training our topic model incrementally by looping over our document chunks:\n\n```python\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic(umap_model=umap_model,\n hdbscan_model=cluster_model,\n vectorizer_model=vectorizer_model)\n\n# Incrementally fit the topic model by training on 1000 documents at a time\nfor docs in doc_chunks:\n topic_model.partial_fit(docs)\n```\n\nAnd that is it! During each iteration, you can access the predicted topics through the `.topics_` attribute.\n\n!!! note\n Do note that in BERTopic it is not possible to use `.partial_fit` after the `.fit` as they work quite differently concerning internally updating topics, frequencies, representations, etc.\n\n!!! tip Tip\n You can use any other dimensionality reduction and clustering algorithm as long as they have a `.partial_fit` function. Moreover, you can use dimensionality reduction algorithms that do not support `.partial_fit` functions but do have a `.fit` function to first train it on a large amount of data and then continuously add documents. The dimensionality reduction will not be updated but may be trained sufficiently to properly reduce the embeddings without the need to continuously add documents.\n\n!!! warning\n Only the most recent batch of documents is tracked. If you want to be using online topic modeling for low-memory use cases, then it is advised to also update the `.topics_` attribute. Otherwise, variations such as **hierarchical topic modeling** will not work.\n\n ```python\n # Incrementally fit the topic model by training on 1000 documents at a time and track the topics in each iteration\n topics = []\n for docs in doc_chunks:\n topic_model.partial_fit(docs)\n topics.extend(topic_model.topics_)\n\n topic_model.topics_ = topics\n ```\n\n\n## **River**\n\nTo continuously find new topics as they come in, we can use the package [river](https://github.com/online-ml/river). It contains several clustering models that can create new clusters as new data comes in. To make sure we can use their models, we first need to create a class that has a `.partial_fit` function and the option to extract labels through `.labels_`:\n\n```python\nfrom river import stream\nfrom river import cluster\n\nclass River:\n def __init__(self, model):\n self.model = model\n\n def partial_fit(self, umap_embeddings):\n for umap_embedding, _ in stream.iter_array(umap_embeddings):\n self.model.learn_one(umap_embedding)\n\n labels = []\n for umap_embedding, _ in stream.iter_array(umap_embeddings):\n label = self.model.predict_one(umap_embedding)\n labels.append(label)\n\n self.labels_ = labels\n return self\n```\n\nThen, we can choose any `river.cluster` model that we are interested in and pass it to the `River` class before using it in BERTopic:\n\n```python\n# Using DBSTREAM to detect new topics as they come in\ncluster_model = River(cluster.DBSTREAM())\nvectorizer_model = OnlineCountVectorizer(stop_words=\"english\")\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True, bm25_weighting=True)\n\n# Prepare model\ntopic_model = BERTopic(\n hdbscan_model=cluster_model,\n vectorizer_model=vectorizer_model,\n ctfidf_model=ctfidf_model,\n)\n\n\n# Incrementally fit the topic model by training on 1000 documents at a time\nfor docs in doc_chunks:\n topic_model.partial_fit(docs)\n```\n" }, { "path": "docs/getting_started/outlier_reduction/fig_base.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/outlier_reduction/fig_reduced.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/outlier_reduction/outlier_reduction.md", "content": "When using HDBSCAN, DBSCAN, or OPTICS, a number of outlier documents might be created\nthat do not fall within any of the created topics. These are labeled as -1. Depending on your use case, you might want\nto decrease the number of documents that are labeled as outliers. Fortunately, there are a number of strategies one might\nuse to reduce the number of outliers after you have trained your BERTopic model.\n\nThe main way to reduce your outliers in BERTopic is by using the `.reduce_outliers` function. To make it work without too much tweaking, you will only need to pass the `docs` and their corresponding `topics`. You can pass outlier and non-outlier documents together since it will only try to reduce outlier documents and label them to a non-outlier topic.\n\nThe following is a minimal example:\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers\nnew_topics = topic_model.reduce_outliers(docs, topics)\n```\n\n!!! note\n You can use the `threshold` parameter to select the minimum distance or similarity when matching outlier documents with non-outlier topics. This allows the user to change the amount of outlier documents are assigned to non-outlier topics.\n\n\n## **Strategies**\n\nThe default method for reducing outliers is by calculating the c-TF-IDF representations of outlier documents and assigning them\nto the best matching c-TF-IDF representations of non-outlier topics.\n\nHowever, there are a number of other strategies one can use, either separately or in conjunction that are worthwhile to explore:\n\n* Using the topic-document probabilities to assign topics\n* Using the topic-document distributions to assign topics\n* Using c-TF-IDF representations to assign topics\n* Using document and topic embeddings to assign topics\n\n### **Probabilities**\nThis strategy uses the soft-clustering as performed by HDBSCAN to find the\nbest matching topic for each outlier document. To use this, make\nsure to calculate the `probabilities` beforehand by instantiating\nBERTopic with `calculate_probabilities=True`.\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model and calculate the document-topic probabilities\ntopic_model = BERTopic(calculate_probabilities=True)\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers using the `probabilities` strategy\nnew_topics = topic_model.reduce_outliers(docs, topics, probabilities=probs, strategy=\"probabilities\")\n```\n\n### **Topic Distributions**\nUse the topic distributions, as calculated with `.approximate_distribution`\nto find the most frequent topic in each outlier document. You can use the\n`distributions_params` variable to tweak the parameters of\n`.approximate_distribution`.\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers using the `distributions` strategy\nnew_topics = topic_model.reduce_outliers(docs, topics, strategy=\"distributions\")\n```\n\n### **c-TF-IDF**\nCalculate the c-TF-IDF representation for each outlier document and\nfind the best matching c-TF-IDF topic representation using\ncosine similarity.\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers using the `c-tf-idf` strategy\nnew_topics = topic_model.reduce_outliers(docs, topics, strategy=\"c-tf-idf\")\n```\n\n### **Embeddings**\nUsing the embeddings of each outlier documents, find the best\nmatching topic embedding using cosine similarity.\n\n```python\nfrom bertopic import BERTopic\n\n# Train your BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Reduce outliers using the `embeddings` strategy\nnew_topics = topic_model.reduce_outliers(docs, topics, strategy=\"embeddings\")\n```\n\n!!! note\n If you have pre-calculated the documents embeddings you can speed up the outlier\n reduction process for the `\"embeddings\"` strategy as it will prevent re-calculating\n the document embeddings.\n\n### **Chain Strategies**\n\nSince the `.reduce_outliers` function does not internally update the topics, we can easily try out different strategies but also chain them together.\nYou might want to do a first pass with the `\"c-tf-idf\"` strategy as it is quite fast. Then, we can perform the `\"distributions\"` strategy on the\noutliers that are left since this method is typically much slower:\n\n```python\n# Use the \"c-TF-IDF\" strategy with a threshold\nnew_topics = topic_model.reduce_outliers(docs, topics , strategy=\"c-tf-idf\", threshold=0.1)\n\n# Reduce all outliers that are left with the \"distributions\" strategy\nnew_topics = topic_model.reduce_outliers(docs, new_topics, strategy=\"distributions\")\n```\n\n\n## **Update Topics**\n\nAfter generating our updated topics, we can feed them back into BERTopic in one of two ways. We can either update the topic representations themselves based on the documents that now belong to new topics or we can only update the topic frequency without updating the topic representations themselves.\n\n!!! warning\n In both cases, it is important to realize that\n updating the topics this way may lead to errors if topic reduction or topic merging techniques are used afterwards. The reason for this is that when you assign a -1 document to topic 1 and another -1 document to topic 2, it is unclear how you map the -1 documents. Is it matched to topic 1 or 2.\n\n\n### **Update Topic Representation**\n\nWhen outlier documents are generated, they are not used when modeling the topic representations. These documents are completely ignored when finding good descriptions of topics. Thus, after having reduced the number of outliers in your topic model, you might want to update the topic representations with the documents that now belong to actual topics. To do so, we can make use of the `.update_topics` function:\n\n```python\ntopic_model.update_topics(docs, topics=new_topics)\n```\n\nAs seen above, you will only need to pass the documents on which the model was trained including the new topics that were generated using one of the above four strategies.\n\n### **Exploration**\n\nWhen you are reducing the number of topics, it might be worthwhile to iteratively visualize the results in order to get an intuitive understanding of the effect of the above four strategies. Making use of `.visualize_documents`, we can quickly iterate over the different strategies and view their effects. Here, an example will be shown on how to approach such a pipeline.\n\nFirst, we train our model:\n\n```python\nfrom umap import UMAP\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n# Prepare data, extract embeddings, and prepare sub-models\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\nvectorizer_model = CountVectorizer(stop_words=\"english\")\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=True)\n\n# We reduce our embeddings to 2D as it will allows us to quickly iterate later on\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2,\n min_dist=0.0, metric='cosine').fit_transform(embeddings)\n\n# Train our topic model\ntopic_model = BERTopic(embedding_model=sentence_model, umap_model=umap_model,\n vectorizer_model=vectorizer_model, calculate_probabilities=True, nr_topics=40)\ntopics, probs = topic_model.fit_transform(docs, embeddings)\n```\n\nAfter having trained our model, let us take a look at the 2D representation of the generated topics:\n\n```python\ntopic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings,\n hide_document_hover=True, hide_annotations=True)\n```\n\n\n\n\nNext, we reduce the number of outliers using the `probabilities` strategy:\n\n```python\nnew_topics = reduce_outliers(topic_model, docs, topics, probabilities=probs,\n threshold=0.05, strategy=\"probabilities\")\ntopic_model.update_topics(docs, topics=new_topics)\n```\n\nAnd finally, we visualize the results:\n\n```python\ntopic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings,\n hide_document_hover=True, hide_annotations=True)\n```\n\n\n" }, { "path": "docs/getting_started/parameter tuning/parametertuning.md", "content": "# Hyperparameter Tuning\n\nAlthough BERTopic works quite well out of the box, there are a number of hyperparameters to tune according to your use case.\nThis section will focus on important parameters directly accessible in BERTopic but also hyperparameter optimization in sub-models\nsuch as HDBSCAN and UMAP.\n\n## **BERTopic**\n\nWhen instantiating BERTopic, there are several hyperparameters that you can directly adjust that could significantly improve the performance of your topic model. In this section, we will go through the most impactful parameters in BERTopic and directions on how to optimize them.\n\n### **language**\nThe `language` parameter is used to simplify the selection of models for those who are not familiar with sentence-transformers models.\n\nIn essence, there are two options to choose from:\n\n* `language = \"english\"` or\n* `language = \"multilingual\"`\n\nThe English model is \"all-MiniLM-L6-v2\" and can be found [here](https://www.sbert.net/docs/pretrained_models.html). It is the default model that is used in BERTopic and works great for English documents.\n\nThe multilingual model is \"paraphrase-multilingual-MiniLM-L12-v2\" and supports over 50+ languages which can be found [here](https://www.sbert.net/docs/pretrained_models.html). The model is very similar to the base model but is trained on many languages and has a slightly different architecture.\n\n### **top_n_words**\n\n`top_n_words` refers to the number of words per topic that you want to be extracted. In practice, I would advise you to keep this value below 30 and preferably between 10 and 20. The reasoning for this is that the more words you put in a topic the less coherent it can become. The top words are the most representative of the topic and should be focused on.\n\n### **n_gram_range**\nThe `n_gram_range` parameter refers to the CountVectorizer used when creating the topic representation. It relates to the number of words you want in your topic representation. For example, \"New\" and \"York\" are two separate words but are often used as \"New York\" which represents an n-gram of 2. Thus, the `n_gram_range` should be set to (1, 2) if you want \"New York\" in your topic representation.\n\n### **min_topic_size**\n`min_topic_size` is an important parameter! It is used to specify what the minimum size of a topic can be. The lower this value the more topics are created. If you set this value too high, then it is possible that simply no topics will be created! Set this value too low and you will get many microclusters.\n\nIt is advised to play around with this value depending on the size of your dataset. If it nears a million documents, then it is advised to set it much higher than the default of 10, for example, 100 or even 500.\n\n### **nr_topics**\n`nr_topics` can be a tricky parameter. It specifies, after training the topic model, the number of topics that will be reduced. For example, if your topic model results in 100 topics but you have set `nr_topics` to 20 then the topic model will try to reduce the number of topics from 100 to 20.\n\nThis reduction can take a while as each reduction in topics activates a c-TF-IDF calculation. If this is set to None, no reduction is applied. Use \"auto\" to automatically reduce topics using HDBSCAN.\n\n### **low_memory**\n`low_memory` sets UMAP's `low_memory` to True to make sure that less memory is used in the computation. This slows down computation but allows UMAP to be run on low-memory machines.\n\n### **calculate_probabilities**\n`calculate_probabilities` lets you calculate the probabilities of each topic in each document. This is computationally quite expensive and is turned off by default.\n\n## **UMAP**\n\nUMAP is an amazing technique for dimensionality reduction. In BERTopic, it is used to reduce the dimensionality of document embedding into something easier to use with HDBSCAN to create good clusters.\n\nHowever, it does has a significant number of parameters you could take into account. As exposing all parameters in BERTopic would be difficult to manage, we can instantiate our UMAP model and pass it to BERTopic:\n\n```python\nfrom umap import UMAP\n\numap_model = UMAP(n_neighbors=15, n_components=10, metric='cosine', low_memory=False)\ntopic_model = BERTopic(umap_model=umap_model).fit(docs)\n```\n\n### **n_neighbors**\n`n_neighbors` is the number of neighboring sample points used when making the manifold approximation. Increasing this value typically results in a\nmore global view of the embedding structure whilst smaller values result in a more local view. Increasing this value often results in larger clusters\nbeing created.\n\n### **n_components**\n`n_components` refers to the dimensionality of the embeddings after reducing them. This is set as a default to `5` to reduce dimensionality\nas much as possible whilst trying to maximize the information kept in the resulting embeddings. Although lowering or increasing this value influences the quality of embeddings, its effect is largest on the performance of HDBSCAN. Increasing this value too much and HDBSCAN will have a\nhard time clustering the high-dimensional embeddings. Lower this value too much and too little information in the resulting embeddings are available\nto create proper clusters. If you want to increase this value, I would advise setting using a metric for HDBSCAN that works well in high dimensional data.\n\n### **metric**\n`metric` refers to the method used to compute the distances in high dimensional space. The default is `cosine` as we are dealing with high dimensional data. However, BERTopic is also able to use any input, even regular tabular data, to cluster the documents. Thus, you might want to change the metric\nto something that fits your use case.\n\n### **low_memory**\n`low_memory` is used when datasets may consume a lot of memory. Using millions of documents can lead to memory issues and setting this value to `True`\nmight alleviate some of the issues.\n\n## **HDBSCAN**\nAfter reducing the embeddings with UMAP, we use HDBSCAN to cluster our documents into clusters of similar documents. Similar to UMAP, HDBSCAN has many parameters that could be tweaked to improve the cluster's quality.\n\n```python\nfrom hdbscan import HDBSCAN\n\nhdbscan_model = HDBSCAN(min_cluster_size=10, metric='euclidean', prediction_data=True)\ntopic_model = BERTopic(hdbscan_model=hdbscan_model).fit(docs)\n```\n\n### **min_cluster_size**\n`min_cluster_size` is arguably the most important parameter in HDBSCAN. It controls the minimum size of a cluster and thereby the number of clusters\nthat will be generated. It is set to `10` as a default. Increasing this value results in fewer clusters but of larger size whereas decreasing this value\nresults in more micro clusters being generated. Typically, I would advise increasing this value rather than decreasing it.\n\n### **min_samples**\n`min_samples` is automatically set to `min_cluster_size` and controls the number of outliers generated. Setting this value significantly lower than\n`min_cluster_size` might help you reduce the amount of noise you will get. Do note that outliers are to be expected and forcing the output\nto have no outliers may not properly represent the data.\n\n### **metric**\n`metric`, like with HDBSCAN is used to calculate the distances. Here, we went with `euclidean` as, after reducing the dimensionality, we have\nlow dimensional data and not much optimization is necessary. However, if you increase `n_components` in UMAP, then it would be advised to look into\nmetrics that work with high dimensional data.\n\n### **prediction_data**\nMake sure you always set this value to `True` as it is needed to predict new points later on. You can set this to False if you do not wish to predict\nany unseen data points.\n" }, { "path": "docs/getting_started/quickstart/quickstart.md", "content": "## **Installation**\n\nInstallation, with sentence-transformers, can be done using [pypi](https://pypi.org/project/bertopic/):\n\n```bash\npip install bertopic\n```\n\nYou may want to install more depending on the transformers and language backends that you will be using.\nThe possible installations are:\n\n```bash\n# Choose an embedding backend\npip install bertopic[flair, gensim, spacy, use]\n\n# Topic modeling with images\npip install bertopic[vision]\n```\n\n## **Quick Start**\nWe start by extracting topics from the well-known 20 newsgroups dataset which is comprised of English documents:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nAfter generating topics, we can access the frequent topics that were generated:\n\n```python\n>>> topic_model.get_topic_info()\n\nTopic\tCount\tName\n-1\t 4630\t-1_can_your_will_any\n0\t 693\t 49_windows_drive_dos_file\n1\t 466\t 32_jesus_bible_christian_faith\n2\t 441\t 2_space_launch_orbit_lunar\n3\t 381\t 22_key_encryption_keys_encrypted\n```\n\n-1 refers to all outliers and should typically be ignored. Next, let's take a look at the most\nfrequent topic that was generated, topic 0:\n\n```python\n>>> topic_model.get_topic(0)\n\n[('windows', 0.006152228076250982),\n ('drive', 0.004982897610645755),\n ('dos', 0.004845038866360651),\n ('file', 0.004140142872194834),\n ('disk', 0.004131678774810884),\n ('mac', 0.003624848635985097),\n ('memory', 0.0034840976976789903),\n ('software', 0.0034415334250699077),\n ('email', 0.0034239554442333257),\n ('pc', 0.003047105930670237)]\n```\n\nUsing `.get_document_info`, we can also extract information on a document level, such as their corresponding topics, probabilities, whether they are representative documents for a topic, etc.:\n\n```python\n>>> topic_model.get_document_info(docs)\n\nDocument Topic\tName\t Top_n_words Probability ...\nI am sure some bashers of Pens...\t 0\t 0_game_team_games_season\tgame - team - games...\t 0.200010 ...\nMy brother is in the market for... -1 -1_can_your_will_any\t can - your - will...\t 0.420668 ...\nFinally you said what you dream...\t -1 -1_can_your_will_any\t can - your - will... 0.807259 ...\nThink! It is the SCSI card doing...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.071746 ...\n1) I have an old Jasmine drive...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.038983 ...\n```\n\n!!! Tip \"Multilingual\"\n Use `BERTopic(language=\"multilingual\")` to select a model that supports 50+ languages.\n\n\n## **Fine-tune Topic Representations**\n\nIn BERTopic, there are a number of different [topic representations](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) that we can choose from. They are all quite different from one another and give interesting perspectives and variations of topic representations. A great start is `KeyBERTInspired`, which for many users increases the coherence and reduces stopwords from the resulting topic representations:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\n\n# Fine-tune your topic representations\nrepresentation_model = KeyBERTInspired()\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nHowever, you might want to use something more powerful to describe your clusters. You can even use ChatGPT or other models from OpenAI to generate labels, summaries, phrases, keywords, and more:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\n\n# Fine-tune topic representations with GPT\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(client, model=\"gpt-4o-mini\", chat=True)\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n!!! tip \"Multi-aspect Topic Modeling\"\n Instead of iterating over all of these different topic representations, you can model them simultaneously with [multi-aspect topic representations](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) in BERTopic.\n\n\n## **Visualizations**\nAfter having trained our BERTopic model, we can iteratively go through hundreds of topics to get a good\nunderstanding of the topics that were extracted. However, that takes quite some time and lacks a global representation. Instead, we can use one of the [many visualization options](https://maartengr.github.io/BERTopic/getting_started/visualization/visualization.html) in BERTopic. For example, we can visualize the topics that were generated in a way very similar to\n[LDAvis](https://github.com/cpsievert/LDAvis):\n\n```python\ntopic_model.visualize_topics()\n```\n\n\n\n## **Save/Load BERTopic model**\n\nThere are three methods for saving BERTopic:\n\n1. A light model with `.safetensors` and config files\n2. A light model with pytorch `.bin` and config files\n3. A full model with `.pickle`\n\nMethod 3 allows for saving the entire topic model but has several drawbacks:\n\n* Arbitrary code can be run from `.pickle` files\n* The resulting model is rather large (often > 500MB) since all sub-models need to be saved\n* Explicit and specific version control is needed as they typically only run if the environment is exactly the same\n\n> **It is advised to use methods 1 or 2 for saving.**\n\nThese methods have a number of advantages:\n\n* `.safetensors` is a relatively **safe format**\n* The resulting model can be **very small** (often < 20MB) since no sub-models need to be saved\n* Although version control is important, there is a bit more **flexibility** with respect to specific versions of packages\n* More easily used in **production**\n* **Share** models with the HuggingFace Hub\n\n\n!!! Tip \"Tip\"\n For more detail about how to load in a custom vectorizer, representation model, and more, it is highly advised to checkout the [serialization](https://maartengr.github.io/BERTopic/getting_started/serialization/serialization.html) page. It contains more examples, details, and some tips and tricks for loading and saving your environment.\n\n\nThe methods are as used as follows:\n\n```python\ntopic_model = BERTopic().fit(my_docs)\n\n# Method 1 - safetensors\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"safetensors\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 2 - pytorch\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"pytorch\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 3 - pickle\ntopic_model.save(\"my_model\", serialization=\"pickle\")\n```\n\nTo load a model:\n\n```python\n# Load from directory\nloaded_model = BERTopic.load(\"path/to/my/model_dir\")\n\n# Load from file\nloaded_model = BERTopic.load(\"my_model\")\n\n# Load from HuggingFace\nloaded_model = BERTopic.load(\"MaartenGr/BERTopic_Wikipedia\")\n```\n\n!!! Warning \"Warning\"\n When saving the model, make sure to also keep track of the versions of dependencies and Python used.\n Loading and saving the model should be done using the same dependencies and Python. Moreover, models\n saved in one version of BERTopic should not be loaded in other versions.\n" }, { "path": "docs/getting_started/quickstart/viz.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/representation/llm.md", "content": "As we have seen in the [previous section](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html), the topics that you get from BERTopic can be fine-tuned using a number of approaches. Here, we are going to focus on text generation Large Language Models such as ChatGPT, GPT-4, and open-source solutions.\n\nUsing these techniques, we can further fine-tune topics to generate labels, summaries, poems of topics, and more. To do so, we first generate a set of keywords and documents that describe a topic best using BERTopic's c-TF-IDF calculate. Then, these candidate keywords and documents are passed to the text generation model and asked to generate output that fits the topic best.\n\nA huge benefit of this is that we can describe a topic with only a few documents and we therefore do not need to pass all documents to the text generation model. Not only speeds this the generation of topic labels up significantly, you also do not need a massive amount of credits when using an external API, such as Cohere or OpenAI.\n\n\n## **Prompt Engineering**\n\nIn most of the examples below, we use certain tags to customize our prompts. There are currently two tags, namely `\"[KEYWORDS]\"` and `\"[DOCUMENTS]\"`.\nThese tags indicate where in the prompt they are to be replaced with a topics keywords and top 4 most representative documents respectively.\nFor example, if we have the following prompt:\n\n```python\nprompt = \"\"\"\nI have topic that contains the following documents: \\n[DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS]\n\nBased on the above information, can you give a short label of the topic?\n\"\"\"\n```\n\nthen that will be rendered as follows:\n\n```python\n\"\"\"\nI have a topic that contains the following documents:\n- Our videos are also made possible by your support on patreon.co.\n- If you want to help us make more videos, you can do so on patreon.com or get one of our posters from our shop.\n- If you want to help us make more videos, you can do so there.\n- And if you want to support us in our endeavor to survive in the world of online video, and make more videos, you can do so on patreon.com.\n\nThe topic is described by the following keywords: videos video you our support want this us channel patreon make on we if facebook to patreoncom can for and more watch\n\nBased on the above information, can you give a short label of the topic?\n\"\"\"\n```\n\n!!! tip \"Tip 1\"\n You can access the default prompts of these models with `representation_model.default_prompt_`. The prompts that were generated after training can be accessed with `topic_model.representation_model.prompts_`.\n\n### **Selecting Documents**\n\nBy default, four of the most representative documents will be passed to `[DOCUMENTS]`. These documents are selected by calculating their similarity (through c-TF-IDF representations) with the main c-TF-IDF representation of the topics. The four best matching documents per topic are selected.\n\nTo increase the number of documents passed to `[DOCUMENTS]`, we can use the `nr_docs` parameter which is accessible in all LLMs on this page. Using this value allows you to select the top *n* most representative documents instead. If you have a long enough context length, then you could even give the LLM dozens of documents.\n\nHowever, some of these documents might be very similar to one another and might be near duplicates. They will not provide much additional information about the content of the topic. Instead, we can use the `diversity` parameter in each LLM to only select documents that are sufficiently diverse. It takes values between 0 and 1 but a value of 0.1 already does wonders!\n\n### **Truncating Documents**\n\nWe can truncate the input documents in `[DOCUMENTS]` in order to reduce the number of tokens that we have in our input prompt. To do so, all text generation modules have two parameters that we can tweak:\n\n* `doc_length`\n * The maximum length of each document. If a document is longer, it will be truncated. If None, the entire document is passed.\n* `tokenizer`\n * The tokenizer used to calculate to split the document into segments used to count the length of a document.\n * If tokenizer is `'char'`, then the document is split up into characters which are counted to adhere to `doc_length`\n * If tokenizer is `'whitespace'`, the document is split up into words separated by whitespaces. These words are counted and truncated depending on `doc_length`\n * If tokenizer is `'vectorizer'`, then the internal CountVectorizer is used to tokenize the document. These tokens are counted and truncated depending on `doc_length`\n * If tokenizer is a callable, then that callable is used to tokenized the document. These tokens are counted and truncated depending on `doc_length`\n\nThis means that the definition of `doc_length` changes depending on what constitutes a token in the `tokenizer` parameter. If a token is a character, then `doc_length` refers to max length in characters. If a token is a word, then `doc_length` refers to the max length in words.\n\nLet's illustrate this with an example. In the code below, we will use [`tiktoken`](https://github.com/openai/tiktoken) to count the number of tokens in each document and limit them to 100 tokens. All documents that have more than 100 tokens will be truncated.\n\nWe start by installing the relevant packages:\n\n```bash\npip install tiktoken openai\n```\n\nThen, we use `bertopic.representation.OpenAI` to represent our topics with nicely written labels. We specify that documents that we put in the prompt cannot exceed 100 tokens each. Since we will put 4 documents in the prompt, they will total roughly 400 tokens:\n\n```python\nimport openai\nimport tiktoken\nfrom bertopic.representation import OpenAI\nfrom bertopic import BERTopic\n\n# Tokenizer\ntokenizer= tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n\n# Create your representation model\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(\n client,\n model=\"gpt-3.5-turbo\",\n delay_in_seconds=2,\n chat=True,\n nr_docs=4,\n doc_length=100,\n tokenizer=tokenizer\n)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n## **πŸ€— Transformers**\n\nNearly every week, there are new and improved models released on the πŸ€— [Model Hub](https://huggingface.co/models) that, with some creativity, allow for\nfurther fine-tuning of our c-TF-IDF based topics. These models range from text generation to zero-classification. In BERTopic, wrappers around these\nmethods are created as a way to support whatever might be released in the future.\n\nUsing a GPT-like model from the huggingface hub is rather straightforward:\n\n```python\nfrom bertopic.representation import TextGeneration\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = TextGeneration('gpt2')\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nGPT2, however, is not the most accurate model out there on HuggingFace models. You can get\nmuch better results with a `flan-T5` like model:\n\n```python\nfrom transformers import pipeline\nfrom bertopic.representation import TextGeneration\n\nprompt = \"I have a topic described by the following keywords: [KEYWORDS]. Based on the previous keywords, what is this topic about?\"\n\n# Create your representation model\ngenerator = pipeline('text2text-generation', model='google/flan-t5-base')\nrepresentation_model = TextGeneration(generator)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/hf.svg\"\n
\n
\n\nAs can be seen from the example above, if you would like to use a `text2text-generation` model, you will to\npass a `transformers.pipeline` with the `\"text2text-generation\"` parameter. Moreover, you can use a custom prompt and decide where the keywords should\nbe inserted by using the `[KEYWORDS]` or documents with the `[DOCUMENTS]` tag.\n\n### **Mistral (GGUF)**\n\nWe can go a step further with open-source Large Language Models (LLMs) that have shown to match the performance of closed-source LLMs like ChatGPT.\n\nIn this example, we will show you how to use Zephyr, a fine-tuning version of Mistral 7B. Mistral 7B outperforms other open-source LLMs at a much smaller scale and is a worthwhile solution for use cases such as topic modeling. We want to keep inference as fast as possible and a relatively small model helps with that. Zephyr is a fine-tuned version of Mistral 7B that was trained on a mix of publicly available and synthetic datasets using Direct Preference Optimization (DPO).\n\nTo use Zephyr in BERTopic, we will first need to install and update a couple of packages that can handle quantized versions of Zephyr:\n\n```python\npip install ctransformers[cuda]\npip install --upgrade git+https://github.com/huggingface/transformers\n```\n\nInstead of loading in the full model, we can instead load a quantized model which is a compressed version of the original model:\n\n```python\nfrom ctransformers import AutoModelForCausalLM\nfrom transformers import AutoTokenizer, pipeline\n\n# Set gpu_layers to the number of layers to offload to GPU. Set to 0 if no GPU acceleration is available on your system.\nmodel = AutoModelForCausalLM.from_pretrained(\n \"TheBloke/zephyr-7B-alpha-GGUF\",\n model_file=\"zephyr-7b-alpha.Q4_K_M.gguf\",\n model_type=\"mistral\",\n gpu_layers=50,\n hf=True\n)\ntokenizer = AutoTokenizer.from_pretrained(\"HuggingFaceH4/zephyr-7b-alpha\")\n\n# Pipeline\ngenerator = pipeline(\n model=model, tokenizer=tokenizer,\n task='text-generation',\n max_new_tokens=50,\n repetition_penalty=1.1\n)\n```\n\nThis Zephyr model requires a specific prompt template in order to work:\n\n```python\nprompt = \"\"\"<|system|>You are a helpful, respectful and honest assistant for labeling topics..\n<|user|>\nI have a topic that contains the following documents:\n[DOCUMENTS]\n\nThe topic is described by the following keywords: '[KEYWORDS]'.\n\nBased on the information about the topic above, please create a short label of this topic. Make sure you to only return the label and nothing more.\n<|assistant|>\"\"\"\n```\n\nAfter creating this prompt template, we can create our representation model to be used in BERTopic:\n\n\n```python\nfrom bertopic.representation import TextGeneration\n\n# Text generation with Zephyr\nzephyr = TextGeneration(generator, prompt=prompt)\nrepresentation_model = {\"Zephyr\": zephyr}\n\n# Topic Modeling\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\n### **Llama (Manual Quantization)**\n\nFull Llama Tutorial: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1QCERSMUjqGetGGujdrvv_6_EeoIcd_9M?usp=sharing)\n\nOpen-source LLMs are starting to become more and more popular. Here, we will go through a minimal example of using [Llama 2](https://huggingface.co/meta-llama/Llama-2-13b-chat-hf) together with BERTopic.\n\n!!! Note\n Although this is an example of the older Llama 2 model, you can use the code below for any Llama variant.\n\n\nFirst, we need to load in our Llama model:\n\n```python\nfrom torch import bfloat16\nimport transformers\n\n# set quantization configuration to load large model with less GPU memory\n# this requires the `bitsandbytes` library\nbnb_config = transformers.BitsAndBytesConfig(\n load_in_4bit=True, # 4-bit quantization\n bnb_4bit_quant_type='nf4', # Normalized float 4\n bnb_4bit_use_double_quant=True, # Second quantization after the first\n bnb_4bit_compute_dtype=bfloat16 # Computation type\n)\n\n# Llama Tokenizer\ntokenizer = transformers.AutoTokenizer.from_pretrained(model_id)\n\n# Llama Model\nmodel = transformers.AutoModelForCausalLM.from_pretrained(\n model_id,\n trust_remote_code=True,\n quantization_config=bnb_config,\n device_map='auto',\n)\nmodel.eval()\n\n# Our text generator\ngenerator = transformers.pipeline(\n model=model, tokenizer=tokenizer,\n task='text-generation',\n temperature=0.1,\n max_new_tokens=500,\n repetition_penalty=1.1\n)\n```\n\nAfter doing so, we will need to define a prompt that works with both Llama as well as BERTopic:\n\n\n```python\n# System prompt describes information given to all conversations\nsystem_prompt = \"\"\"\n[INST] <>\nYou are a helpful, respectful and honest assistant for labeling topics.\n<>\n\"\"\"\n\n# Example prompt demonstrating the output we are looking for\nexample_prompt = \"\"\"\nI have a topic that contains the following documents:\n- Traditional diets in most cultures were primarily plant-based with a little meat on top, but with the rise of industrial style meat production and factory farming, meat has become a staple food.\n- Meat, but especially beef, is the word food in terms of emissions.\n- Eating meat doesn't make you a bad person, not eating meat doesn't make you a good one.\n\nThe topic is described by the following keywords: 'meat, beef, eat, eating, emissions, steak, food, health, processed, chicken'.\n\nBased on the information about the topic above, please create a short label of this topic. Make sure you to only return the label and nothing more.\n\n[/INST] Environmental impacts of eating meat\n\"\"\"\n\n# Our main prompt with documents ([DOCUMENTS]) and keywords ([KEYWORDS]) tags\nmain_prompt = \"\"\"\n[INST]\nI have a topic that contains the following documents:\n[DOCUMENTS]\n\nThe topic is described by the following keywords: '[KEYWORDS]'.\n\nBased on the information about the topic above, please create a short label of this topic. Make sure you to only return the label and nothing more.\n[/INST]\n\"\"\"\n\nprompt = system_prompt + example_prompt + main_prompt\n```\n\nThree pieces of the prompt were created:\n\n* `system_prompt` helps us guide the model during a conversation. For example, we can say that it is a helpful assistant that is specialized in labeling topics.\n* `example_prompt` gives an example of a correctly labeled topic to guide Llama\n* `main_prompt` contains the main question we are going to ask it, namely to label a topic. Note that it uses the `[DOCUMENTS]` and `[KEYWORDS]` to provide the most relevant documents and keywords as additional context\n\nAfter having generated our prompt template, we can start running our topic model:\n\n```python\nfrom bertopic.representation import TextGeneration\nfrom bertopic import BERTopic\n\n# Text generation with Llama\nllama2 = TextGeneration(generator, prompt=prompt)\nrepresentation_model = {\n \"Llama2\": llama2,\n}\n\n# Create our BERTopic model\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\n## **llama.cpp**\n\nAn amazing framework for using LLMs for inference is [`llama.cpp`](https://github.com/ggerganov/llama.cpp) which has [python bindings](https://github.com/abetlen/llama-cpp-python) that we can use in BERTopic. To start with, we first need to install `llama-cpp-python`:\n\n```bash\npip install llama-cpp-python\n```\n\nor using the following for hardware acceleration:\n\n```bash\nCMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python\n```\n\n!!! Note\n There are a number of [installation options](https://github.com/abetlen/llama-cpp-python#installation-with-hardware-acceleration) depending on your hardware and OS. Make sure that you select the correct one to optimize your performance.\n\nAfter installation, you need to download your LLM locally before we use it in BERTopic, like so:\n\n```bash\nwget https://huggingface.co/TheBloke/zephyr-7B-alpha-GGUF/resolve/main/zephyr-7b-alpha.Q4_K_M.gguf\n```\n\nFinally, we can now use the model with BERTopic in just a couple of lines:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import LlamaCPP\n\n# Use llama.cpp to load in a 4-bit quantized version of Zephyr 7B Alpha\nrepresentation_model = LlamaCPP(\"zephyr-7b-alpha.Q4_K_M.gguf\")\n\n# Create our BERTopic model\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\nIf you want to have more control over the LLMs parameters, you can run it like so:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import LlamaCPP\nfrom llama_cpp import Llama\n\n# Use llama.cpp to load in a 4-bit quantized version of Zephyr 7B Alpha\nllm = Llama(model_path=\"zephyr-7b-alpha.Q4_K_M.gguf\", n_gpu_layers=-1, n_ctx=4096, stop=\"Q:\")\nrepresentation_model = LlamaCPP(llm)\n\n# Create our BERTopic model\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\n!!! Note\n The default template that is being used uses a \"Q: ... A: ... \" type of structure which is why the `stop` is set at `\"Q:\"`.\n The default template is:\n ```python\n \"\"\"\n Q: I have a topic that contains the following documents:\n [DOCUMENTS]\n\n The topic is described by the following keywords: '[KEYWORDS]'.\n\n Based on the above information, can you give a short label of the topic?\n A:\n \"\"\"\n ```\n\n\n## **OpenAI**\n\nInstead of using a language model from πŸ€— transformers, we can use external APIs instead that\ndo the work for you. Here, we can use [OpenAI](https://openai.com/api/) to extract our topic labels from the candidate documents and keywords.\nTo use this, you will need to install openai first:\n\n```bash\npip install openai\n```\n\nThen, get yourself an API key and use OpenAI's API as follows:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\nfrom bertopic import BERTopic\n\n# Create your representation model\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(client)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/openai.svg\"\n
\n
\n\nYou can also use a custom prompt:\n\n```python\nprompt = \"I have the following documents: [DOCUMENTS] \\nThese documents are about the following topic: '\"\nrepresentation_model = OpenAI(client, prompt=prompt)\n```\n\n### **GPT-4o**\n\nTo choose a specific model from OpenAI's offering:\n\n```python\nrepresentation_model = OpenAI(client, model=\"gpt-4o-mini\", delay_in_seconds=10)\n```\n\nPrompting with their models is very satisfying and is customizable as follows:\n\n```python\nprompt = \"\"\"\nI have a topic that contains the following documents:\n[DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS]\n\nBased on the information above, extract a short topic label in the following format:\ntopic: \n\"\"\"\n```\n\n!!! note\n Whenever you create a custom prompt, it is important to add\n ```\n Based on the information above, extract a short topic label in the following format:\n topic: \n ```\n at the end of your prompt as BERTopic extracts everything that comes after `topic: `. Having\n said that, if `topic: ` is not in the output, then it will simply extract the entire response, so\n feel free to experiment with the prompts.\n\n### **Summarization**\n\nDue to the structure of the prompts in OpenAI's chat models, we can extract different types of topic representations from their GPT models.\nInstead of extracting a topic label, we can instead ask it to extract a short description of the topic instead:\n\n```python\nsummarization_prompt = \"\"\"\nI have a topic that is described by the following keywords: [KEYWORDS]\nIn this topic, the following documents are a small but representative subset of all documents in the topic:\n[DOCUMENTS]\n\nBased on the information above, please give a description of this topic in the following format:\ntopic: \n\"\"\"\n\nrepresentation_model = OpenAI(client, model=\"gpt-4o-mini\", prompt=summarization_prompt, nr_docs=5, delay_in_seconds=3)\n```\n\nThe above is not constrained to just creating a short description or summary of the topic, we can extract labels, keywords, poems, example documents, extensitive descriptions, and more using this method!\nIf you want to have multiple representations of a single topic, it might be worthwhile to also check out [**multi-aspect**](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) topic modeling with BERTopic.\n\n## **Ollama**\n\nTo use [Ollama](https://github.com/ollama/ollama) within BERTopic, it is advised to use the `openai` package as it allows to pass through a model using the url on which the model is running.\n\nYou will first need to install `openai`:\n\n```bash\npip install openai\n```\n\nAfter installation, usage is straightforward and you can select any model that you have prepared in your `ollama` model list. You can see all models by running `ollama list`.\n\nSelect one from the list and you can use it in BERTopic as follows:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\nfrom bertopic import BERTopic\n\nclient = openai.OpenAI(\n base_url = 'http://localhost:11434/v1', #wherever ollama is running\n api_key='ollama', # required, but unused\n)\n\n\n# Create your representation model\nrepresentation_model = OpenAI(client, model='phi3:14b-medium-128k-instruct-q4_K_M')\n\n# Create your BERTopic model\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\n## **LiteLLM**\n\nAn amazing framework to simplify connecting to external LLMs, is [LiteLLM](https://docs.litellm.ai). This package allows you to connect to OpenAI, Cohere, Anthropic, etc. all within one package. This makes iteration and testing out different models a breeze!\n\no start with, we first need to install `litellm`:\n\n```bash\npip install litellm\n```\n\nAfter installation, usage is straightforward and you can select any model found in their [docs](https://docs.litellm.ai/docs/providers).\nLet's show an example with OpenAI:\n\n```python\nimport os\nfrom bertopic import BERTopic\nfrom bertopic.representation import LiteLLM\n\n# set ENV variables\nos.environ[\"OPENAI_API_KEY\"] = \"MY_KEY\"\n\n# Create your representation model\nrepresentation_model = LiteLLM(model=\"gpt-4o-mini\")\n\n# Create our BERTopic model\ntopic_model = BERTopic(representation_model=representation_model, verbose=True)\n```\n\n## **LangChain**\n\n[Langchain](https://github.com/hwchase17/langchain) is a package that helps users with chaining large language models.\nIn BERTopic, we can leverage this package in order to more efficiently combine external knowledge. Here, this\nexternal knowledge are the most representative documents in each topic.\n\nTo use langchain, you will need to install the langchain package first. Additionally, you will need an underlying LLM to support langchain,\nlike openai:\n\n```bash\npip install langchain, openai\n```\n\nThen, you can create your chain as follows:\n\n```python\nfrom langchain.chains.question_answering import load_qa_chain\nfrom langchain.llms import OpenAI\nchain = load_qa_chain(OpenAI(temperature=0, openai_api_key=my_openai_api_key), chain_type=\"stuff\")\n```\n\nFinally, you can pass the chain to BERTopic as follows:\n\n```python\nfrom bertopic.representation import LangChain\n\n# Create your representation model\nrepresentation_model = LangChain(chain)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nYou can also use a custom prompt:\n\n```python\nprompt = \"What are these documents about? Please give a single label.\"\nrepresentation_model = LangChain(chain, prompt=prompt)\n```\n\n!!! note Note\n The prompt does not make use of `[KEYWORDS]` and `[DOCUMENTS]` tags as\n the documents are already used within langchain's `load_qa_chain`.\n\n## **Cohere**\n\nInstead of using a language model from πŸ€— transformers, we can use external APIs instead that\ndo the work for you. Here, we can use [Cohere](https://docs.cohere.ai/) to extract our topic labels from the candidate documents and keywords.\nTo use this, you will need to install cohere first:\n\n```bash\npip install cohere\n```\n\nThen, get yourself an API key and use Cohere's API as follows:\n\n```python\nimport cohere\nfrom bertopic.representation import Cohere\nfrom bertopic import BERTopic\n\n# Create your representation model\nco = cohere.Client(my_api_key)\nrepresentation_model = Cohere(co)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/cohere.svg\"\n
\n
\n\nYou can also use a custom prompt:\n\n```python\nprompt = \"\"\"\nI have topic that contains the following documents: [DOCUMENTS]\nThe topic is described by the following keywords: [KEYWORDS].\nBased on the above information, can you give a short label of the topic?\n\"\"\"\nrepresentation_model = Cohere(co, prompt=prompt)\n```\n" }, { "path": "docs/getting_started/representation/representation.md", "content": "One of the core components of BERTopic is its Bag-of-Words representation and weighting with c-TF-IDF. This method is fast and can quickly generate a number of keywords for a topic without depending on the clustering task. As a result, topics can easily and quickly be updated after training the model without the need to re-train it.\nAlthough these give good topic representations, we may want to further fine-tune the topic representations.\n\nAs such, there are a number of representation models implemented in BERTopic that allows for further fine-tuning of the topic representations. These are optional\nand are **not used by default**. You are not restrained by the how the representation can be fine-tuned, from GPT-like models to fast keyword extraction\nwith KeyBERT-like models:\n\n\n\nFor each model below, an example will be shown on how it may change or improve upon the default topic keywords that are generated. The dataset used in these examples can be found [here](https://www.kaggle.com/datasets/maartengr/kurzgesagt-transcriptions).\n\nIf you want to have multiple representations of a single topic, it might be worthwhile to also check out [**multi-aspect**](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) topic modeling with BERTopic.\n\n\n## **KeyBERTInspired**\n\nAfter having generated our topics with c-TF-IDF, we might want to do some fine-tuning based on the semantic\nrelationship between keywords/keyphrases and the set of documents in each topic. Although we can use a centroid-based\ntechnique for this, it can be costly and does not take the structure of a cluster into account. Instead, we leverage\nc-TF-IDF to create a set of representative documents per topic and use those as our updated topic embedding. Then, we calculate\nthe similarity between candidate keywords and the topic embedding using the same embedding model that embedded the documents.\n\n
\n
\n--8<-- \"docs/getting_started/representation/keybertinspired.svg\"\n
\n
\n\nThus, the algorithm follows some principles of [KeyBERT](https://github.com/MaartenGr/KeyBERT) but does some optimization in\norder to speed up inference. Usage is straightforward:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = KeyBERTInspired()\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/keybert.svg\"\n
\n
\n\n## **PartOfSpeech**\nOur candidate topics, as extracted with c-TF-IDF, do not take into account a keyword's part of speech as extracting noun-phrases from\nall documents can be computationally quite expensive. Instead, we can leverage c-TF-IDF to perform part of speech on a subset of\nkeywords and documents that best represent a topic.\n\n
\n
\n--8<-- \"docs/getting_started/representation/partofspeech.svg\"\n
\n
\n\nMore specifically, we find documents that contain the keywords from our candidate topics as calculated with c-TF-IDF. These documents serve\nas the representative set of documents from which the Spacy model can extract a set of candidate keywords for each topic.\nThese candidate keywords are first put through Spacy's POS module to see whether they match with the `DEFAULT_PATTERNS`:\n\n```python\nDEFAULT_PATTERNS = [\n [{'POS': 'ADJ'}, {'POS': 'NOUN'}],\n [{'POS': 'NOUN'}],\n [{'POS': 'ADJ'}]\n]\n```\n\nThese patterns follow Spacy's [Rule-Based Matching](https://spacy.io/usage/rule-based-matching). Then, the resulting keywords are sorted by\ntheir respective c-TF-IDF values.\n\n```python\nfrom bertopic.representation import PartOfSpeech\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = PartOfSpeech(\"en_core_web_sm\")\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/pos.svg\"\n
\n
\n\nYou can define custom POS patterns to be extracted:\n\n```python\npos_patterns = [\n [{'POS': 'ADJ'}, {'POS': 'NOUN'}],\n [{'POS': 'NOUN'}], [{'POS': 'ADJ'}]\n]\nrepresentation_model = PartOfSpeech(\"en_core_web_sm\", pos_patterns=pos_patterns)\n```\n\n\n## **MaximalMarginalRelevance**\nWhen we calculate the weights of keywords, we typically do not consider whether we already have similar keywords in our topic. Words like \"car\" and \"cars\"\nessentially represent the same information and often redundant.\n\n
\n
\n--8<-- \"docs/getting_started/representation/mmr.svg\"\n
\n
\n\n\n\nTo decrease this redundancy and improve the diversity of keywords, we can use an algorithm called Maximal Marginal Relevance (MMR). MMR considers the similarity of keywords/keyphrases with the document, along with the similarity of already selected keywords and keyphrases. This results in a selection of keywords\nthat maximize their within diversity with respect to the document.\n\n\n```python\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = MaximalMarginalRelevance(diversity=0.3)\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/mmr_output.svg\"\n
\n
\n\n## **Zero-Shot Classification**\n\nFor some use cases, you might already have a set of candidate labels that you would like to automatically assign to some of the topics.\nAlthough we can use guided or supervised BERTopic for that, we can also use zero-shot classification to assign labels to our topics.\nFor that, we can make use of πŸ€— transformers on their models on the [model hub](https://huggingface.co/models?pipeline_tag=zero-shot-classification&sort=downloads).\n\nTo perform this classification, we feed the model with the keywords as generated through c-TF-IDF and a set of candidate labels.\nIf, for a certain topic, we find a similar enough label, then it is assigned. If not, then we keep the original c-TF-IDF keywords.\n\nWe use it in BERTopic as follows:\n\n```python\nfrom bertopic.representation import ZeroShotClassification\nfrom bertopic import BERTopic\n\n# Create your representation model\ncandidate_topics = [\"space and nasa\", \"bicycles\", \"sports\"]\nrepresentation_model = ZeroShotClassification(candidate_topics, model=\"facebook/bart-large-mnli\")\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n
\n
\n--8<-- \"docs/getting_started/representation/zero.svg\"\n
\n
\n\n## **Chain Models**\n\nAll of the above models can make use of the candidate topics, as generated by c-TF-IDF, to further fine-tune the topic representations. For example, `MaximalMarginalRelevance` takes the keywords in the candidate topics and re-ranks them. Similarly, the keywords in the candidate topic can be used as the input for GPT-prompts in `OpenAI`.\n\nAlthough the default candidate topics are generated by c-TF-IDF, what if we were to chain these models? For example, we can use `MaximalMarginalRelevance` to improve upon the keywords in each topic before passing them to `OpenAI`.\n\nThis is supported in BERTopic by simply passing a list of representation models when instantiation the topic model:\n\n```python\nfrom bertopic.representation import MaximalMarginalRelevance, OpenAI\nfrom bertopic import BERTopic\nimport openai\n\n# Create your representation models\nclient = openai.OpenAI(api_key=\"sk-...\")\nopenai_generator = OpenAI(client)\nmmr = MaximalMarginalRelevance(diversity=0.3)\nrepresentation_models = [mmr, openai_generator]\n\n# Use the chained models\ntopic_model = BERTopic(representation_model=representation_models)\n```\n\n## **Custom Model**\n\nAlthough several representation models have been implemented in BERTopic, new technologies get released often and we should not have to wait until they get implemented in BERTopic. Therefore, you can create your own representation model and use that to fine-tune the topics.\n\nThe following is the basic structure for creating your custom model. Note that it returns the same topics as the those\ncalculated with c-TF-IDF:\n\n```python\nfrom bertopic.representation._base import BaseRepresentation\n\n\nclass CustomRepresentationModel(BaseRepresentation):\n def extract_topics(self, topic_model, documents, c_tf_idf, topics\n ) -> Mapping[str, List[Tuple[str, float]]]:\n \"\"\" Extract topics\n\n Arguments:\n topic_model: The BERTopic model\n documents: A dataframe of documents with their related topics\n c_tf_idf: The c-TF-IDF matrix\n topics: The candidate topics as calculated with c-TF-IDF\n\n Returns:\n updated_topics: Updated topic representations\n \"\"\"\n updated_topics = topics.copy()\n return updated_topics\n```\n\nThen, we can use that model as follows:\n\n```python\nfrom bertopic import BERTopic\n\n# Create our custom representation model\nrepresentation_model = CustomRepresentationModel()\n\n# Pass our custom representation model to BERTopic\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nThere are a few things to take into account when creating your custom model:\n\n* It needs to have the exact same parameter input: `topic_model`, `documents`, `c_tf_idf`, `topics`.\n* Make sure that `updated_topics` has the exact same structure as `topics`:\n\n```python\nupdated_topics = {\n \"1\", [(\"space\", 0.9), (\"nasa\", 0.7)],\n \"2\": [(\"science\", 0.66), (\"article\", 0.6)]\n}\n```\n\n!!! Tip\n You can change the `__init__` however you want, it does not influence the underlying structure. This\n also means that you can save data/embeddings/representations/sentiment in your custom representation\n model.\n" }, { "path": "docs/getting_started/search/search.md", "content": "After having created a BERTopic model, you might end up with over a hundred topics. Searching through those\ncan be quite cumbersome especially if you are searching for a specific topic. Fortunately, BERTopic allows you\nto search for topics using search terms. First, let's create and train a BERTopic model:\n\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Create topics\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nAfter having trained our model, we can use `find_topics` to search for topics that are similar\nto an input search_term. Here, we are going to be searching for topics that closely relate the\nsearch term \"motor\". Then, we extract the most similar topic and check the results:\n\n```python\n>>> similar_topics, similarity = topic_model.find_topics(\"motor\", top_n=5)\n>>> topic_model.get_topic(similar_topics[0])\n[('bike', 0.02275997701645559),\n ('motorcycle', 0.011391202866080292),\n ('bikes', 0.00981187573649205),\n ('dod', 0.009614623748226669),\n ('honda', 0.008247663662558535),\n ('ride', 0.0064683227888861945),\n ('harley', 0.006355502638631013),\n ('riding', 0.005766601561614182),\n ('motorcycles', 0.005596372493714447),\n ('advice', 0.005534544418830091)]\n```\n\nIt definitely seems that a topic was found that closely matches \"motor\". The topic seems to be motorcycle\nrelated and therefore matches our \"motor\" input. You can use the `similarity` variable to see how similar\nthe extracted topics are to the search term.\n\n!!! note\n You can only use this method if an embedding model was supplied to BERTopic using `embedding_model`.\n" }, { "path": "docs/getting_started/seed_words/seed_words.md", "content": "When performing Topic Modeling, you are often faced with data that you are familiar with to a certain extend or that speaks a very specific language. In those cases, topic modeling techniques might have difficulties capturing and representing the semantic nature of domain specific abbreviations, slang, short form, acronyms, etc. For example, the *\"TNM\"* classification is a method for identifying the stage of most cancers. The word *\"TNM\"* is an abbreviation and might not be correctly captured in generic embedding models.\n\nTo make sure that certain domain specific words are weighted higher and are more often used in topic representations, you can set any number of `seed_words` in the `bertopic.vectorizer.ClassTfidfTransformer`. The `ClassTfidfTransformer` is the base representation of BERTopic and essentially represents each topic as a bag of words. As such, we can choose to increase the importance of certain words, such as *\"TNM\"*.\n\nTo do so, let's take a look at an example. We have a dataset of article abstracts and want to perform some topic modeling. Since we might be familiar with the data, there are certain words that we know should be generally important. Let's assume that we have in-depth knowledge about reinforcement learning and know that words like \"agent\" and \"robot\" should be important in such a topic were it to be found. Using the `ClassTfidfTransformer`, we can define those `seed_words` and also choose by how much their values are multiplied.\n\nThe full example is then as follows:\n\n```python\nfrom umap import UMAP\nfrom datasets import load_dataset\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\n# Let's take a subset of ArXiv abstracts as the training data\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\nabstracts = dataset[\"abstract\"][:5_000]\n\n# For illustration purposes, we make sure the output is fixed when running this code multiple times\numap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)\n\n# We can choose any number of seed words for which we want their representation\n# to be strengthen. We increase the importance of these words as we want them to be more\n# likely to end up in the topic representations.\nctfidf_model = ClassTfidfTransformer(\n seed_words=[\"agent\", \"robot\", \"behavior\", \"policies\", \"environment\"],\n seed_multiplier=2\n)\n\n# We run the topic model with the seeded words\ntopic_model = BERTopic(\n umap_model=umap_model,\n min_topic_size=15,\n ctfidf_model=ctfidf_model,\n).fit(abstracts)\n```\n\nThen, when we run `topic_model.get_topic(0)`, we get the following output:\n\n```python\n[('policy', 0.023413102511982354),\n ('reinforcement', 0.021796126795834238),\n ('agent', 0.021131601305431902),\n ('policies', 0.01888385271486409),\n ('environment', 0.017819874593917057),\n ('learning', 0.015321710504308708),\n ('robot', 0.013881115279230468),\n ('control', 0.013297705894983875),\n ('the', 0.013247933839985382),\n ('to', 0.013058208312484141)]\n```\n\nAs we can see, the output includes some of the seed words that we assigned. However, if a word is not found to be important in a topic than we can still multiply its importance but it will remain relatively low. This is a great feature as it allows you to improve their importance with less risk of making words important in topics that really should not be.\n\nA benefit of this method is that this often influences all other representation methods, like KeyBERTInspired and OpenAI. The reason for this is that each representation model uses the words generated by the `ClassTfidfTransformer` as candidate words to be further optimized. In many cases, words like *\"TNM\"* might not end up in the candidate words. By increasing their importance, they are more likely to end up as candidate words in representation models.\n\nAnother benefit of using this method is that it artificially increases the interpretability of topics. Sure, some words might be more important than others but there might not mean something to a domain expert. For them, certain words, like *\"TNM\"* are highly descriptive and that is something difficult to capture using any method (embedding model, large language model, etc.).\n\nMoreover, these `seed_words` can be defined together with the domain expert as they can decide what type of words are generally important and might need a nudge from you the algorithmic developer.\n" }, { "path": "docs/getting_started/semisupervised/semisupervised.md", "content": "In BERTopic, you have several options to nudge the creation of topics toward certain pre-specified topics. Here, we will be looking at semi-supervised topic modeling with BERTopic.\n\nSemi-supervised modeling allows us to steer the dimensionality reduction of the embeddings into a space that closely follows any labels you might already have.\n\n
\n
\n--8<-- \"docs/getting_started/semisupervised/semisupervised.svg\"\n
\n
\n\nIn other words, we use a semi-supervised UMAP instance to reduce the dimensionality of embeddings before clustering the documents\nwith HDBSCAN.\n\nFirst, let us prepare the data needed for our topic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data[\"data\"]\ncategories = data[\"target\"]\ncategory_names = data[\"target_names\"]\n```\n\nWe are using the popular 20 Newsgroups dataset which contains roughly 18000 newsgroups posts that each is\nassigned to one of 20 categories. Using this dataset we can try to extract its corresponding topic model whilst\ntaking its underlying categories into account. These categories are here the variable `targets`.\n\nEach document can be put into one of the following categories:\n\n```python\n>>> category_names\n\n['alt.atheism',\n 'comp.graphics',\n 'comp.os.ms-windows.misc',\n 'comp.sys.ibm.pc.hardware',\n 'comp.sys.mac.hardware',\n 'comp.windows.x',\n 'misc.forsale',\n 'rec.autos',\n 'rec.motorcycles',\n 'rec.sport.baseball',\n 'rec.sport.hockey',\n 'sci.crypt',\n 'sci.electronics',\n 'sci.med',\n 'sci.space',\n 'soc.religion.christian',\n 'talk.politics.guns',\n 'talk.politics.mideast',\n 'talk.politics.misc',\n 'talk.religion.misc']\n```\n\nTo perform this semi-supervised approach, we can take in some pre-defined topics and simply pass those to the `y` parameter when fitting BERTopic. These labels can be pre-defined topics or simply documents that you feel belong together regardless of their content. BERTopic will nudge the creation of topics toward these categories\nusing the pre-defined labels.\n\nTo perform supervised topic modeling, we simply use all categories:\n\n```python\ntopic_model = BERTopic(verbose=True).fit(docs, y=categories)\n```\n\nThe topic model will be much more attuned to the categories that were defined previously. However, this does not mean that only topics for these categories will be found. BERTopic is likely to find more specific topics in those you have already defined. This allows you to discover previously unknown topics!\n\n## **Partial labels**\n\nAt times, you might only have labels for a subset of documents. Fortunately, we can still use those labels to at least nudge the documents for which those labels exist. The documents for which we do not have labels are assigned a -1. For this example, imagine we only have the labels of categories that are related to computers and we want to create a topic model using semi-supervised modeling:\n\n```python\nlabels_to_add = ['comp.graphics', 'comp.os.ms-windows.misc',\n 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware',\n 'comp.windows.x',]\nindices = [category_names.index(label) for label in labels_to_add]\ny = [label if label in indices else -1 for label in categories]\n```\n\nThe `y` variable contains many -1 values since we do not know all the categories.\n\nNext, we use those newly constructed labels to again BERTopic semi-supervised:\n\n```python\ntopic_model = BERTopic(verbose=True).fit(docs, y=y)\n```\n\nAnd that is it! By defining certain classes for our documents, we can steer the topic modeling towards modeling the pre-defined categories.\n" }, { "path": "docs/getting_started/serialization/serialization.md", "content": "Saving, loading, and sharing a BERTopic model can be done in several ways. It is generally advised to go with `.safetensors` as that allows for a small, safe, and fast method for saving your BERTopic model. However, other formats, such as `.pickle` and pytorch `.bin` are also possible.\n\n## **Saving**\n\nThere are three methods for saving BERTopic:\n\n1. A light model with `.safetensors` and config files\n2. A light model with pytorch `.bin` and config files\n3. A full model with `.pickle`\n\n\n!!! Tip \"Tip\"\n It is advised to use methods 1 or 2 for saving as they generated very small models. Especially method 1 (`safetensors`)\n allows for a relatively safe format compared to the other methods.\n\nThe methods are used as follows:\n\n```python\ntopic_model = BERTopic().fit(my_docs)\n\n# Method 1 - safetensors\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"safetensors\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 2 - pytorch\nembedding_model = \"sentence-transformers/all-MiniLM-L6-v2\"\ntopic_model.save(\"path/to/my/model_dir\", serialization=\"pytorch\", save_ctfidf=True, save_embedding_model=embedding_model)\n\n# Method 3 - pickle\ntopic_model.save(\"my_model\", serialization=\"pickle\")\n```\n\n!!! Warning \"Warning\"\n When saving the model, make sure to also keep track of the versions of dependencies and Python used.\n Loading and saving the model should be done using the same dependencies and Python. Moreover, models\n saved in one version of BERTopic are not guaranteed to load in other versions.\n\n\n### **Pickle Drawbacks**\nSaving the model with `pickle` allows for saving the entire topic model, including dimensionality reduction and clustering algorithms, but has several drawbacks:\n\n* Arbitrary code can be run from `.pickle` files\n* The resulting model is rather large (often > 500MB) since all sub-models need to be saved\n* Explicit and specific version control is needed as they typically only run if the environment is exactly the same\n\n\n### **Safetensors and Pytorch Advantages**\nSaving the topic modeling with `.safetensors` or `pytorch` has a number of advantages:\n\n* `.safetensors` is a relatively **safe format**\n* The resulting model can be **very small** (often < 20MB>) since no sub-models need to be saved\n* Although version control is important, there is a bit more **flexibility** with respect to specific versions of packages\n* More easily used in **production**\n* **Share** models with the HuggingFace Hub\n\n

\n\n

\n\nThe above image, a model trained on 100,000 documents, demonstrates the differences in sizes comparing `safetensors`, `pytorch`, and `pickle`. The difference in sizes can mostly be explained due to the efficient saving procedure and that the clustering and dimensionality reductions are not saved in safetensors/pytorch since inference can be done based on the topic embeddings.\n\n\n## **HuggingFace Hub**\n\nWhen you have created a BERTopic model, you can easily share it with other through the HuggingFace Hub. First, you need to log in to your HuggingFace account which you can do in a number of ways:\n\n* Log in to your Hugging Face account with the command below\n\n```bash\nhuggingface-cli login\n\n# or using an environment variable\nhuggingface-cli login --token $HUGGINGFACE_TOKEN\n```\n\n* Alternatively, you can programmatically login using login() in a notebook or a script\n\n```python\nfrom huggingface_hub import login\nlogin()\n```\n\n* Or you can give a token with the `token` variable\n\nWhen you have logged in to your HuggingFace account, you can save and upload the model as follows:\n\n```python\nfrom bertopic import BERTopic\n\n# Train model\ntopic_model = BERTopic().fit(my_docs)\n\n# Push to HuggingFace Hub\ntopic_model.push_to_hf_hub(\n repo_id=\"MaartenGr/BERTopic_ArXiv\",\n save_ctfidf=True\n)\n\n# Load from HuggingFace\nloaded_model = BERTopic.load(\"MaartenGr/BERTopic_ArXiv\")\n```\n\n### **Parameters**\nThere are number of parameters that may be worthwhile to know:\n\n* `private`\n * Whether to create a private repository\n* `serialization`\n * The type of serialization. Either `safetensors` or `pytorch`. Make sure to run `pip install safetensors` for safetensors.\n* `save_embedding_model`\n * A pointer towards a HuggingFace model to be loaded in with SentenceTransformers. E.g., `sentence-transformers/all-MiniLM-L6-v2`\n* `save_ctfidf`\n * Whether to save c-TF-IDF information\n\n\n## **Loading**\n\nTo load a model:\n\n```python\n# Load from directory\nloaded_model = BERTopic.load(\"path/to/my/model_dir\")\n\n# Load from file\nloaded_model = BERTopic.load(\"my_model\")\n\n# Load from HuggingFace\nloaded_model = BERTopic.load(\"MaartenGr/BERTopic_Wikipedia\")\n```\n\nThe embedding model cannot always be saved using a non-pickle method if, for example, you are using OpenAI embeddings. Instead, you can load them in as follows:\n\n\n```python\n# Define embedding model\nimport openai\nfrom bertopic.backend import OpenAIBackend\n\nclient = openai.OpenAI(api_key=\"sk-...\")\nembedding_model = OpenAIBackend(client, \"text-embedding-ada-002\")\n\n# Load model and add embedding model\nloaded_model = BERTopic.load(\"path/to/my/model_dir\", embedding_model=embedding_model)\n```\n" }, { "path": "docs/getting_started/supervised/supervised.md", "content": "Although topic modeling is typically done by discovering topics in an unsupervised manner, there might be times when you already have a bunch of clusters or classes from which you want to model the topics. For example, the often used [20 NewsGroups dataset](https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html) is already split up into 20 classes. Similarly, you might already have created some labels yourself through packages like [human-learn](https://github.com/koaning/human-learn), [bulk](https://github.com/koaning/bulk), [thisnotthat](https://github.com/TutteInstitute/thisnotthat) or something entirely different.\n\nInstead of using BERTopic to discover previously unknown topics, we are now going to manually pass them to BERTopic and try to learn the relationship between those topics and the input documents.\n\n> In other words, we are going to be performing classification instead!\n\nWe can view this as a supervised topic modeling approach. Instead of using a clustering algorithm, we are going to be using a classification algorithm instead.\n\nGenerally, we have the following pipeline:\n\n
\n
\n--8<-- \"docs/getting_started/supervised/default_pipeline.svg\"\n
\n
\n\nInstead, we are now going to skip over the dimensionality reduction step and replace the clustering step with a classification model:\n\n
\n
\n--8<-- \"docs/getting_started/supervised/classification_pipeline.svg\"\n
\n
\n\nIn other words, we can pass our labels to BERTopic and it will not only learn how to predict labels for new instances, but it also transforms those labels into topics by running the c-TF-IDF representations on the set of documents within each label. This process allows us to model the topics themselves and similarly gives us the option to use everything BERTopic has to offer.\n\nTo do so, we need to skip over the dimensionality reduction step and replace the clustering step with a classification algorithm. We can use the documents and labels from the 20 NewsGroups dataset to create topics from those 20 labels:\n\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Get labeled data\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data['data']\ny = data['target']\n```\n\nThen, we make sure to create empty instances of the dimensionality reduction and clustering steps. We pass those to BERTopic to simply skip over them and go to the topic representation process:\n\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\nfrom bertopic.dimensionality import BaseDimensionalityReduction\nfrom sklearn.linear_model import LogisticRegression\n\n# Get labeled data\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data['data']\ny = data['target']\n\n# Skip over dimensionality reduction, replace cluster model with classifier,\n# and reduce frequent words while we are at it.\nempty_dimensionality_model = BaseDimensionalityReduction()\nclf = LogisticRegression()\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True)\n\n# Create a fully supervised BERTopic instance\ntopic_model= BERTopic(\n umap_model=empty_dimensionality_model,\n hdbscan_model=clf,\n ctfidf_model=ctfidf_model\n)\ntopics, probs = topic_model.fit_transform(docs, y=y)\n```\n\nLet's take a look at a few topics that we get out of training this way by running `topic_model.get_topic_info()`:\n\n
\n
\n--8<-- \"docs/getting_started/supervised/table.svg\"\n
\n
\n\nWe can see several interesting topics appearing here. They seem to relate to the 20 classes we had as input. Now, let's map those topics to our original classes to view their relationship:\n\n```python\n# Map input `y` to topics\nmappings = topic_model.topic_mapper_.get_mappings()\nmappings = {value: data[\"target_names\"][key] for key, value in mappings.items()}\n\n# Assign original classes to our topics\ndf = topic_model.get_topic_info()\ndf[\"Class\"] = df.Topic.map(mappings)\ndf\n```\n
\n--8<-- \"docs/getting_started/supervised/table_classes.svg\"\n
\n\n
\n\nWe can see that the c-TF-IDF representations extract the words that give a good representation of our input classes. This is all done directly from the labeling. A welcome side-effect is that we now have a classification algorithm that allows us to predict the topics of unseen data:\n\n```python\n>>> topic, _ = topic_model.transform(\"this is a document about cars\")\n>>> topic_model.get_topic(topic)\n[('car', 0.4407600315538472),\n ('cars', 0.32348015696446325),\n ('engine', 0.28032518444946686),\n ('ford', 0.2500224508115155),\n ('oil', 0.2325984913598611),\n ('dealer', 0.2310723968585826),\n ('my', 0.22045777551991935),\n ('it', 0.21327993649430219),\n ('tires', 0.20420842634292657),\n ('brake', 0.20246902481367085)]\n```\n\n Moreover, we can still perform BERTopic-specific features like dynamic topic modeling, topics per class, hierarchical topic modeling, modeling topic distributions, etc.\n\n!!! note\n The resulting `topics` may be a different mapping from the `y` labels. To map `y` to `topics`, we can run the following:\n\n\n ```python\n mappings = topic_model.topic_mapper_.get_mappings()\n y_mapped = [mappings[val] for val in y]\n ```\n" }, { "path": "docs/getting_started/tips_and_tricks/tips_and_tricks.md", "content": "# Tips & Tricks\n\n\n## **Document length**\nAs a default, we are using sentence-transformers to embed our documents. However, as the name implies, the embedding model works best for either sentences or paragraphs. This means that whenever you have a set of documents, where each documents contains several paragraphs, the document is truncated and the topic model is only trained on a small part of the data.\n\nOne way to solve this issue is by splitting up longer documents into either sentences or paragraphs before embedding them. Another solution is to approximate the [topic distributions](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html) of topics after having trained your topic model.\n\n\n## **Removing stop words**\nAt times, stop words might end up in our topic representations. This is something we typically want to avoid as they contribute little to the interpretation of the topics. However, removing stop words as a preprocessing step is not advised as the transformer-based embedding models that we use need the full context in order to create accurate embeddings.\n\nInstead, we can use the `CountVectorizer` to preprocess our documents **after** having generated embeddings and clustered\nour documents. Personally, I have found almost no disadvantages to using the `CountVectorizer` to remove stopwords and\nit is something I would strongly advise to try out:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer_model = CountVectorizer(stop_words=\"english\")\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\n```\n\nWe can also use the `ClassTfidfTransformer` to reduce the impact of frequent words. The end result is very similar to explicitly removing stopwords but this process does this automatically:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\nctfidf_model = ClassTfidfTransformer(reduce_frequent_words=True)\ntopic_model = BERTopic(ctfidf_model=ctfidf_model)\n```\n\nLastly, we can use a KeyBERT-Inspired model to reduce the appearance of stop words. This also often improves the topic representation:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\nfrom bertopic import BERTopic\n\n# Create your representation model\nrepresentation_model = KeyBERTInspired()\n\n# Use the representation model in BERTopic on top of the default pipeline\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n## **Diversify topic representation**\nAfter having calculated our top *n* words per topic there might be many words that essentially\nmean the same thing. As a little bonus, we can use `bertopic.representation.MaximalMarginalRelevance` in BERTopic to\ndiversify words in each topic such that we limit the number of duplicate words we find in each topic.\nThis is done using an algorithm called Maximal Marginal Relevance which compares word embeddings\nwith the topic embedding.\n\nWe do this by specifying a value between 0 and 1, with 0 being not at all diverse and 1 being completely diverse:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import MaximalMarginalRelevance\n\nrepresentation_model = MaximalMarginalRelevance(diversity=0.2)\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nSince MMR is using word embeddings to diversify the topic representations, it is necessary to pass the embedding model to BERTopic if you are using pre-computed embeddings:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.representation import MaximalMarginalRelevance\nfrom sentence_transformers import SentenceTransformer\n\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\nrepresentation_model = MaximalMarginalRelevance(diversity=0.2)\ntopic_model = BERTopic(embedding_model=sentence_model, representation_model=representation_model)\n```\n\n\n## **Topic-term matrix**\nAlthough BERTopic focuses on clustering our documents, the end result does contain a topic-term matrix.\nThis topic-term matrix is calculated using c-TF-IDF, a TF-IDF procedure optimized for class-based analyses.\n\nTo extract the topic-term matrix (or c-TF-IDF matrix) with the corresponding words, we can simply do the following:\n\n```python\ntopic_term_matrix = topic_model.c_tf_idf_\nwords = topic_model.vectorizer_model.get_feature_names()\n```\n\n\n## **Pre-compute embeddings**\nTypically, we want to iterate fast over different versions of our BERTopic model whilst we are trying to optimize it to a specific use case. To speed up this process, we can pre-compute the embeddings, save them,\nand pass them to BERTopic so it does not need to calculate the embeddings each time:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train our topic model using our pre-trained sentence-transformers embeddings\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs, embeddings)\n```\n\n## **Speed up UMAP**\nAt times, UMAP may take a while to fit on the embeddings that you have. This often happens when you have\nthe embeddings millions of documents that you want to reduce in dimensionality. There is a trick that\ncan speed up this process somewhat: Initializing UMAP with rescaled PCA embeddings.\n\nWithout going in too much detail (look [here](https://github.com/lmcinnes/umap/issues/771#issuecomment-931886015) for more information), you can reduce the embeddings using PCA\nand use that as a starting point. This can speed up the dimensionality reduction a bit:\n\n\n```python\nimport numpy as np\nfrom umap import UMAP\nfrom bertopic import BERTopic\nfrom sklearn.decomposition import PCA\n\n\ndef rescale(x, inplace=False):\n \"\"\" Rescale an embedding so optimization will not have convergence issues.\n \"\"\"\n if not inplace:\n x = np.array(x, copy=True)\n\n x /= np.std(x[:, 0]) * 10000\n\n return x\n\n\n# Initialize and rescale PCA embeddings\npca_embeddings = rescale(PCA(n_components=5).fit_transform(embeddings))\n\n# Start UMAP from PCA embeddings\numap_model = UMAP(\n n_neighbors=15,\n n_components=5,\n min_dist=0.0,\n metric=\"cosine\",\n init=pca_embeddings,\n)\n\n# Pass the model to BERTopic:\ntopic_model = BERTopic(umap_model=umap_model)\n```\n\n## **GPU acceleration**\n\nYou can use [cuML](https://rapids.ai/start.html#rapids-release-selector) to speed up both\nUMAP and HDBSCAN through GPU acceleration:\n\n```python\nfrom bertopic import BERTopic\nfrom cuml.cluster import HDBSCAN\nfrom cuml.manifold import UMAP\n\n# Create instances of GPU-accelerated UMAP and HDBSCAN\numap_model = UMAP(n_components=5, n_neighbors=15, min_dist=0.0)\nhdbscan_model = HDBSCAN(min_samples=10, gen_min_span_tree=True, prediction_data=True)\n\n# Pass the above models to be used in BERTopic\ntopic_model = BERTopic(umap_model=umap_model, hdbscan_model=hdbscan_model)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nDepending on the embeddings you are using, you might want to normalize them first in order to\nforce a cosine-related distance metric in UMAP:\n\n```python\nfrom cuml.preprocessing import normalize\nembeddings = normalize(embeddings)\n```\n\n!!! note\n As of the v0.13 release, it is not yet possible to calculate the topic-document probability matrix for unseen data (i.e., `.transform`) using cuML's HDBSCAN.\n However, it is still possible to calculate the topic-document probability matrix for the data on which the model was trained (i.e., `.fit` and `.fit_transform`).\n\n!!! note\n To install cuML with BERTopic, run these commands:\n\n **For CUDA 12:**\n ```bash\n !pip install cuml-cu12\n !pip install bertopic\n ```\n\n **For CUDA 13:**\n ```bash\n !pip install cuml-cu13\n !pip install bertopic\n ```\n\n !!! warning\n Install cuML first, then BERTopic. Installing both in a single command can fail due to pip resolver limitations with CUDA runtime dependencies.\n\n **Note:** cuML is already installed on Google Colab.\n\n For more detailed information on installing cuML, including additional dependencies and platform-specific instructions, see the [RAPIDS installation guide](https://docs.rapids.ai/install/).\n\n\n## **Lightweight installation**\n\nThe default embedding model in BERTopic is one of the amazing sentence-transformers models, namely `\"all-MiniLM-L6-v2\"`. Although this model performs well out of the box, it typically needs a GPU to transform the documents into embeddings in a reasonable time. Moreover, the installation requires `pytorch` which often results in a rather large environment, memory-wise.\n\nFortunately, it is possible to install BERTopic without `sentence-transformers`, `UMAP`, `HDBSCAN` and/or `plotly`. This can be to reduce your docker images for inference or when you do not use `pytorch` but for instance [Model2Vec](https://github.com/MinishLab/model2vec) instead. The installation can be done as follows:\n\n```bash\npip install --no-deps bertopic\npip install --upgrade numpy pandas scikit-learn tqdm pyyaml\n```\n\nThis installs a bare-bones version of BERTopic. If you want to use UMAP and Model2Vec for instance, you'll need to first install them:\n\n`pip install model2vec umap-learn`\n\nThen, you can BERTopic without needing to have a GPU:\n\n```python\nfrom bertopic import BERTopic\nfrom model2vec import StaticModel\n\n# Model2Vec\nembedding_model = StaticModel.from_pretrained(\"minishlab/potion-base-8M\")\n\n# BERTopic\ntopic_model = BERTopic(embedding_model=embedding_model)\n```\n\nAs a result, the entire package and resulting model can be run quickly on the CPU and no GPU is necessary!\n\n!!! Note\n If you have an alternative embedding model, you can use that instead of Model2Vec. Likewise, if you have a different method for dimensionality reduction that you want to use, you can use that instead of UMAP.\n\n\n## **WordCloud**\nTo minimize the number of dependencies in BERTopic, it is not possible to generate wordclouds out-of-the-box. However,\nthere is a minimal script that you can use to generate wordclouds in BERTopic. First, you will need to install\nthe [wordcloud](https://github.com/amueller/word_cloud) package with `pip install wordcloud`. Then, run the following code\nto generate the wordcloud for a specific topic:\n\n```python\nfrom wordcloud import WordCloud\nimport matplotlib.pyplot as plt\n\ndef create_wordcloud(model, topic):\n text = {word: value for word, value in model.get_topic(topic)}\n wc = WordCloud(background_color=\"white\", max_words=1000)\n wc.generate_from_frequencies(text)\n plt.imshow(wc, interpolation=\"bilinear\")\n plt.axis(\"off\")\n plt.show()\n\n# Show wordcloud\ncreate_wordcloud(topic_model, topic=1)\n```\n\n![](wordcloud.jpg)\n\n\n!!! tip Tip\n To increase the number of words shown in the wordcloud, you can increase the `top_n_words`\n parameter when instantiating BERTopic. You can also increase the number of words in a topic\n after training the model using `.update_topics()`.\n\n\n## **Finding similar topics between models**\n\nWhenever you have trained separate BERTopic models on different datasets, it might\nbe worthful to find the similarities among these models. Is there overlap between\ntopics in model A and topic in model B? In other words, can we find topics in model A that are similar to those in model B?\n\nWe can compare the topic representations of several models in two ways. First, by comparing the topic embeddings that are created when using the same embedding model across both fitted BERTopic instances. Second, we can compare the c-TF-IDF representations instead assuming we have fixed the vocabulary in both instances.\n\nThis example will go into the former, using the same embedding model across two BERTopic instances. To do this comparison, let's first create an example where I trained two models, one on an English dataset and one on a Dutch dataset:\n\n```python\nfrom datasets import load_dataset\nfrom bertopic import BERTopic\nfrom sentence_transformers import SentenceTransformer\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\n# The same embedding model needs to be used for both topic models\n# and since we are dealing with multiple languages, the model needs to be multi-lingual\nsentence_model = SentenceTransformer(\"paraphrase-multilingual-MiniLM-L12-v2\")\n\n# To make this example reproducible\numap_model = UMAP(n_neighbors=15, n_components=5,\n min_dist=0.0, metric='cosine', random_state=42)\n\n# English\nen_dataset = load_dataset(\"stsb_multi_mt\", name=\"en\", split=\"train\").to_pandas().sentence1.tolist()\nen_model = BERTopic(embedding_model=sentence_model, umap_model=umap_model)\nen_model.fit(en_dataset)\n\n# Dutch\nnl_dataset = load_dataset(\"stsb_multi_mt\", name=\"nl\", split=\"train\").to_pandas().sentence1.tolist()\nnl_model = BERTopic(embedding_model=sentence_model, umap_model=umap_model)\nnl_model.fit(nl_dataset)\n```\n\nIn the code above, there is one important thing to note and that is the `sentence_model`. This model needs to be exactly the same in all BERTopic models, otherwise, it is not possible to compare topic models.\n\nNext, we can calculate the similarity between topics in the English topic model `en_model` and the Dutch model `nl_model`. To do so, we can simply calculate the cosine similarity between the `topic_embedding` of both models:\n\n```python\nfrom sklearn.metrics.pairwise import cosine_similarity\nsim_matrix = cosine_similarity(en_model.topic_embeddings_, nl_model.topic_embeddings_)\n```\n\nNow that we know which topics are similar to each other, we can extract the most similar topics. Let's say that we have topic 10 in the `en_model` which represents a topic related to trains:\n\n```python\n>>> topic = 10\n>>> en_model.get_topic(topic)\n[('train', 0.2588080580844999),\n ('tracks', 0.1392140438801078),\n ('station', 0.12126454635946024),\n ('passenger', 0.058057876475695866),\n ('engine', 0.05123717127783682),\n ('railroad', 0.048142847325312044),\n ('waiting', 0.04098973702226946),\n ('track', 0.03978248702913929),\n ('subway', 0.03834661195748458),\n ('steam', 0.03834661195748458)]\n```\n\nTo find the matching topic, we extract the most similar topic in the `sim_matrix`:\n\n```python\n>>> most_similar_topic = np.argmax(sim_matrix[topic + 1])-1\n>>> nl_model.get_topic(most_similar_topic)\n[('trein', 0.24186603209316418),\n ('spoor', 0.1338118418551581),\n ('sporen', 0.07683661859111401),\n ('station', 0.056990389779394225),\n ('stoommachine', 0.04905829711711234),\n ('zilveren', 0.04083879598477808),\n ('treinen', 0.03534099197032758),\n ('treinsporen', 0.03534099197032758),\n ('staat', 0.03481332997324445),\n ('zwarte', 0.03179591746822408)]\n```\n\nIt seems to be working as, for example, `trein` is a translation of `train` and `sporen` a translation of `tracks`! You can do this for every single topic to find out which topic in the `en_model` might belong to a model in the `nl_model`.\n\n## **Multimodal data**\n[Concept](https://github.com/MaartenGr/Concept) is a variation\nof BERTopic for multimodal data, such as images with captions. Although we can use that\npackage for multimodal data, we can perform a small trick with BERTopic to have a similar feature.\n\nBERTopic is a relatively modular approach that attempts to isolate steps from one another. This means,\nfor example, that you can use k-Means instead of HDBSCAN or PCA instead of UMAP as it does not make\nany assumptions with respect to the nature of the clustering.\n\nSimilarly, you can pass pre-calculated embeddings to BERTopic that represent the documents that you have.\nHowever, it does not make any assumption with respect to the relationship between those embeddings and\nthe documents. This means that we could pass any metadata to BERTopic to cluster on instead of document\nembeddings. In this example, we can separate our embeddings from our documents so that the embeddings\nare generated from images instead of their corresponding images. Thus, we will cluster image embeddings but\ncreate the topic representation from the related captions.\n\nIn this example, we first need to fetch our data, namely the Flickr 8k dataset that contains images\nwith captions:\n\n```python\nimport os\nimport glob\nimport zipfile\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sentence_transformers import SentenceTransformer, util\n\n# Flickr 8k images\nimg_folder = 'photos/'\ncaps_folder = 'captions/'\nif not os.path.exists(img_folder) or len(os.listdir(img_folder)) == 0:\n os.makedirs(img_folder, exist_ok=True)\n\n if not os.path.exists('Flickr8k_Dataset.zip'): #Download dataset if does not exist\n util.http_get('https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_Dataset.zip', 'Flickr8k_Dataset.zip')\n util.http_get('https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_text.zip', 'Flickr8k_text.zip')\n\n for folder, file in [(img_folder, 'Flickr8k_Dataset.zip'), (caps_folder, 'Flickr8k_text.zip')]:\n with zipfile.ZipFile(file, 'r') as zf:\n for member in tqdm(zf.infolist(), desc='Extracting'):\n zf.extract(member, folder)\nimages = list(glob.glob('photos/Flicker8k_Dataset/*.jpg'))\n\n# Prepare dataframe\ncaptions = pd.read_csv(\"captions/Flickr8k.lemma.token.txt\",sep='\\t',names=[\"img_id\",\"img_caption\"])\ncaptions.img_id = captions.apply(lambda row: \"photos/Flicker8k_Dataset/\" + row.img_id.split(\".jpg\")[0] + \".jpg\", 1)\ncaptions = captions.groupby([\"img_id\"])[\"img_caption\"].apply(','.join).reset_index()\ncaptions = pd.merge(captions, pd.Series(images, name=\"img_id\"), on=\"img_id\")\n\n# Extract images together with their documents/captions\nimages = captions.img_id.to_list()\ndocs = captions.img_caption.to_list()\n```\n\nNow that we have our images and captions, we need to generate our image embeddings:\n\n```python\nmodel = SentenceTransformer('clip-ViT-B-32')\n\n# Prepare images\nbatch_size = 32\nnr_iterations = int(np.ceil(len(images) / batch_size))\n\n# Embed images per batch\nembeddings = []\nfor i in tqdm(range(nr_iterations)):\n start_index = i * batch_size\n end_index = (i * batch_size) + batch_size\n\n images_to_embed = [Image.open(filepath) for filepath in images[start_index:end_index]]\n img_emb = model.encode(images_to_embed, show_progress_bar=False)\n embeddings.extend(img_emb.tolist())\n\n # Close images\n for image in images_to_embed:\n image.close()\nembeddings = np.array(embeddings)\n```\n\nFinally, we can fit BERTopic the way we are used to, with documents and embeddings:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.cluster import KMeans\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer_model = CountVectorizer(stop_words=\"english\")\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\ntopics, probs = topic_model.fit_transform(docs, embeddings)\ncaptions[\"Topic\"] = topics\n```\n\nAfter fitting our model, let's inspect a topic about skateboarders:\n\n```python\n>>> topic_model.get_topic(2)\n[('skateboard', 0.09592033177340711),\n ('skateboarder', 0.07792520092546491),\n ('trick', 0.07481578896400298),\n ('ramp', 0.056952605147927216),\n ('skate', 0.03745127816149923),\n ('perform', 0.036546213623432654),\n ('bicycle', 0.03453483070441857),\n ('bike', 0.033233021253898994),\n ('jump', 0.026709362981948037),\n ('air', 0.025422798170830936)]\n```\n\nBased on the above output, we can take an image to see if the representation makes sense:\n\n```python\nimage = captions.loc[captions.Topic == 2, \"img_id\"].values.tolist()[0]\nImage.open(image)\n```\n\n![](skateboarders.jpg)\n\n## **KeyBERT** & **BERTopic**\n\nAlthough BERTopic focuses on topic extraction methods that does not assume specific structures for the generated clusters, it is possible to do this on a more local level. More specifically, we can use KeyBERT to generate a number of keywords for each document and then build a vocabulary on top of that as the input for BERTopic. This way, we can select words that we know have meaning to a topic, without focusing on the centroid of that cluster. This also allows more frequent words to pop-up regardless of the structure and density of a cluster.\n\nTo do this, we first need to run [KeyBERT](https://github.com/MaartenGr/KeyBERT) on our data and create our vocabulary:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom keybert import KeyBERT\n\n# Prepare documents\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\n# Extract keywords\nkw_model = KeyBERT()\nkeywords = kw_model.extract_keywords(docs)\n\n# Create our vocabulary\nvocabulary = [k[0] for keyword in keywords for k in keyword]\nvocabulary = list(set(vocabulary))\n```\n\nThen, we pass our `vocabulary` to BERTopic and train the model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer_model= CountVectorizer(vocabulary=vocabulary)\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\ntopics, probs = topic_model.fit_transform(docs)\n```\n" }, { "path": "docs/getting_started/topicreduction/topicreduction.md", "content": "BERTopic uses HDBSCAN for clustering the data and it cannot specify the number of clusters you would want. To a certain extent,\nthis is an advantage, as we can trust HDBSCAN to be better in finding the number of clusters than we are.\nInstead, we can try to reduce the number of topics that have been created. Below, you will find three methods of doing\nso.\n\n!!! Warning\n For all cases of topic reduction it is generally advised to create the number of topics you would first through the clustering algorithm. That tends to be the most stable technique and often gives you the best results. This also applies with algorithms that do not allow you to select the number of topics beforehands, like HDBSCAN where you can make sure of the `min_cluster_size` parameter to control the number of topics.\n Therefore, it is **highly** advised to not use `nr_topics` before you have attempted to control the number of topics through the clustering algorithm!\n\n### **Manual Topic Reduction**\nEach resulting topic has its feature vector constructed from c-TF-IDF. Using those feature vectors, we can find the most similar\ntopics and merge them. Using `sklearn.cluster.AgglomerativeClustering`, the resulting feature vectors are clustered to get to the set value of `nr_topics` by finding out which topics are most similar to one another through cosine similarity.\n\nTo do so, you can make sure of the `nr_topics` parameter:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(nr_topics=20)\n```\n\nIt is also possible to manually select certain topics that you believe should be merged.\nFor example, if topic 1 is `1_space_launch_moon_nasa` and topic 2 is `2_spacecraft_solar_space_orbit`\nit might make sense to merge those two topics:\n\n```python\ntopics_to_merge = [1, 2]\ntopic_model.merge_topics(docs, topics_to_merge)\n```\n\nIf you have several groups of topics you want to merge, create a list of lists instead:\n\n```python\ntopics_to_merge = [[1, 2]\n [3, 4]]\ntopic_model.merge_topics(docs, topics_to_merge)\n```\n\n### **Automatic Topic Reduction**\nOne issue with the approach above is that it will merge topics regardless of whether they are very similar. They\nare simply the most similar out of all options. This can be resolved by reducing the number of topics automatically.\nTo do this, we can use HDBSCAN to cluster our topics using each c-TF-IDF representation. Then, we merge topics that are clustered together.\nAnother benefit of HDBSCAN is that it generates outliers. These outliers prevent topics from being merged if no other topics are similar.\n\nTo use this option, we simply set `nr_topics` to `\"auto\"`:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(nr_topics=\"auto\")\n```\n\n### **Topic Reduction after Training**\nFinally, we can also reduce the number of topics after having trained a BERTopic model. The advantage of doing so is that you can decide the number of topics after knowing how many are created. It is difficult to predict before training your model how many topics that are in your documents and how many will be extracted.\nInstead, we can decide afterward how many topics seem realistic:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Create topics -> Typically over 50 topics\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Further reduce topics\ntopic_model.reduce_topics(docs, nr_topics=30)\n\n# Access updated topics\ntopics = topic_model.topics_\n```\n\nThe reasoning for putting `docs` as a parameter is that the documents are not saved within\nBERTopic on purpose. If you were to have a million documents, it is very inefficient to save those in BERTopic instead of a dedicated database.\n" }, { "path": "docs/getting_started/topicrepresentation/topicrepresentation.md", "content": "The topics that are extracted from BERTopic are represented by words. These words are extracted from the documents\noccupying their topics using a class-based TF-IDF. This allows us to extract words that are interesting to a topic but\nless so to another.\n\n### **Update Topic Representation after Training**\nWhen you have trained a model and viewed the topics and the words that represent them,\nyou might not be satisfied with the representation. Perhaps you forgot to remove\nstop_words or you want to try out a different n_gram_range. We can use the function `update_topics` to update\nthe topic representation with new parameters for `c-TF-IDF`:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Create topics\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic(n_gram_range=(2, 3))\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nFrom the model created above, one of the most frequent topics is the following:\n\n```python\n>>> topic_model.get_topic(31)[:10]\n[('clipper chip', 0.007240771542316232),\n ('key escrow', 0.004601603973377443),\n ('law enforcement', 0.004277247929596332),\n ('intercon com', 0.0035961920238955824),\n ('amanda walker', 0.003474856425297157),\n ('serial number', 0.0029876119137150358),\n ('com amanda', 0.002789303096817983),\n ('intercon com amanda', 0.0027386688593327084),\n ('amanda intercon', 0.002585262048515583),\n ('amanda intercon com', 0.002585262048515583)]\n```\n\nAlthough there does seems to be some relation between words, it is difficult, at least for me, to intuitively understand\nwhat the topic is about. Instead, let's simplify the topic representation by setting `n_gram_range` to (1, 3) to\nalso allow for single words.\n\n```python\n>>> topic_model.update_topics(docs, n_gram_range=(1, 3))\n>>> topic_model.get_topic(31)[:10]\n[('encryption', 0.008021846079148017),\n ('clipper', 0.00789642647602742),\n ('chip', 0.00637127942464045),\n ('key', 0.006363124787175884),\n ('escrow', 0.005030980365244285),\n ('clipper chip', 0.0048271268437973395),\n ('keys', 0.0043245812747907545),\n ('crypto', 0.004311198708675516),\n ('intercon', 0.0038772934659295076),\n ('amanda', 0.003516026493904586)]\n```\n\nTo me, the combination of the words above seem a bit more intuitive than the words we previously had! You can play\naround with `n_gram_range` or use your own custom `sklearn.feature_extraction.text.CountVectorizer` and pass that\ninstead:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(stop_words=\"english\", ngram_range=(1, 5))\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\n!!! Tip \"Tip!\"\n If you want to change the topics to something else, whether that is merging them or removing outliers, you can pass\n a custom list of topics to update them: `topic_model.update_topics(docs, topics=my_updated_topics)`\n\n### **Custom labels**\n\nThe topic labels are currently automatically generated by taking the top 3 words and combining them\nusing the `_` separator. Although this is an informative label, in practice, this is definitely not the prettiest nor necessarily the most accurate label. For example, although the topic label\n`1_space_nasa_orbit` is informative, but we would prefer to have a bit more intuitive label, such as\n`space travel`. The difficulty with creating such topic labels is that much of the interpretation is left to the user. Would `space travel` be more accurate or perhaps `space explorations`? To truly understand which labels are most suited, going into some of the documents in topics is especially helpful.\n\nAlthough we can go through every single topic ourselves and try to label them, we can start by creating an overview of labels that have the length and number of words that we are looking for. To do so, we can generate our list of topic labels with `.generate_topic_labels` and define the number of words, the separator, word length, etc:\n\n```python\ntopic_labels = topic_model.generate_topic_labels(nr_words=3,\n topic_prefix=False,\n word_length=10,\n separator=\", \")\n```\n\n!!! Tip\n If you created [**multiple topic representations**](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) or aspects, you can choose one of these aspects with `aspect=\"Aspect1\"` or whatever you named the aspect.\n\nIn the above example, `1_space_nasa_orbit` would turn into `space, nasa, orbit` since we selected 3 words, no topic prefix, and the `, ` separator. We can then either change our `topic_labels` to whatever we want or directly pass them to `.set_topic_labels` so that they can be used across most visualization functions:\n\n```python\ntopic_model.set_topic_labels(topic_labels)\n```\n\nIt is also possible to only change a few topic labels at a time by passing a dictionary\nwhere the key represents the *topic ID* and the value is the *topic label*:\n\n```python\ntopic_model.set_topic_labels({1: \"Space Travel\", 7: \"Religion\"})\n```\n\nThen, to make use of those custom topic labels across visualizations, such as `.visualize_hierarchy()`,\nwe can use the `custom_labels=True` parameter that is found in most visualizations.\n\n```python\nfig = topic_model.visualize_barchart(custom_labels=True)\n```\n\n#### Optimize labels\nThe great advantage of passing custom labels to BERTopic is that when more accurate zero-shot are released,\nwe can simply use those on top of BERTopic to further fine-tune the labeling. For example, let's say you\nhave a set of potential topic labels that you want to use instead of the ones generated by BERTopic. You could\nuse the [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) model to find which user-defined\nlabels best represent the BERTopic-generated labels:\n\n\n```python\nfrom transformers import pipeline\nclassifier = pipeline(\"zero-shot-classification\", model=\"facebook/bart-large-mnli\")\n\n# A selected topic representation\n# 'god jesus atheists atheism belief atheist believe exist beliefs existence'\nsequence_to_classify = \" \".join([word for word, _ in topic_model.get_topic(1)])\n\n# Our set of potential topic labels\ncandidate_labels = ['cooking', 'dancing', 'religion']\nclassifier(sequence_to_classify, candidate_labels)\n\n#{'labels': ['cooking', 'dancing', 'religion'],\n# 'scores': [0.086, 0.063, 0.850],\n# 'sequence': 'god jesus atheists atheism belief atheist believe exist beliefs existence'}\n```\n" }, { "path": "docs/getting_started/topicsovertime/topicsovertime.md", "content": "Dynamic topic modeling (DTM) is a collection of techniques aimed at analyzing the evolution of topics\nover time. These methods allow you to understand how a topic is represented across different times.\nFor example, in 1995 people may talk differently about environmental awareness than those in 2015. Although the\ntopic itself remains the same, environmental awareness, the exact representation of that topic might differ.\n\nBERTopic allows for DTM by calculating the topic representation at each timestep without the need to\nrun the entire model several times. To do this, we first need to fit BERTopic as if there were no temporal\naspect in the data. Thus, a general topic model will be created. We use the global representation as to the main topics that can be found at, most likely, different timesteps. For each topic and timestep, we calculate the c-TF-IDF representation. This will result in a specific topic representation at each timestep without the need to create clusters from embeddings as they were already created.\n\n
\n
\n--8<-- \"docs/getting_started/topicsovertime/topicsovertime.svg\"\n
\n
\n\nNext, there are two main ways to further fine-tune these specific topic representations,\nnamely **globally** and **evolutionary**.\n\nA topic representation at timestep *t* can be fine-tuned **globally** by averaging its c-TF-IDF representation with that of the global representation. This allows each topic representation to move slightly towards the global representation whilst still keeping some of its specific words.\n\nA topic representation at timestep *t* can be fine-tuned **evolutionary** by averaging its c-TF-IDF representation with that of the c-TF-IDF representation at timestep *t-1*. This is done for each topic representation allowing for the representations to evolve over time.\n\nBoth fine-tuning methods are set to `True` as a default and allow for interesting representations to be created.\n\n## **Example**\nTo demonstrate DTM in BERTopic, we first need to prepare our data. A good example of where DTM is useful is topic\nmodeling on Twitter data. We can analyze how certain people have talked about certain topics in the years\nthey have been on Twitter. Due to the controversial nature of his tweets, we are going to be using all\ntweets by Donald Trump.\n\nFirst, we need to load the data and do some very basic cleaning. For example, I am not interested in his\nre-tweets for this use-case:\n\n```python\nimport re\nimport pandas as pd\n\n# Prepare data\ntrump = pd.read_csv('https://drive.google.com/uc?export=download&id=1xRKHaP-QwACMydlDnyFPEaFdtskJuBa6')\ntrump.text = trump.apply(lambda row: re.sub(r\"http\\S+\", \"\", row.text).lower(), 1)\ntrump.text = trump.apply(lambda row: \" \".join(filter(lambda x:x[0]!=\"@\", row.text.split())), 1)\ntrump.text = trump.apply(lambda row: \" \".join(re.sub(\"[^a-zA-Z]+\", \" \", row.text).split()), 1)\ntrump = trump.loc[(trump.isRetweet == \"f\") & (trump.text != \"\"), :]\ntimestamps = trump.date.to_list()\ntweets = trump.text.to_list()\n```\n\nThen, we need to extract the global topic representations by simply creating and training a BERTopic model:\n\n```python\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic(verbose=True)\ntopics, probs = topic_model.fit_transform(tweets)\n```\n\nFrom these topics, we are going to generate the topic representations at each timestamp for each topic. We do this\nby simply calling `topics_over_time` and passing the tweets, the corresponding timestamps, and the related topics:\n\n```python\ntopics_over_time = topic_model.topics_over_time(tweets, timestamps, nr_bins=20)\n```\n\nAnd that is it! Aside from what you always need for BERTopic, you now only need to add `timestamps`\nto quickly calculate the topics over time.\n\n## **Parameters**\nThere are a few parameters that are of interest which will be discussed below.\n\n### **Tuning**\nBoth `global_tuning` and `evolutionary_tuning` are set to True as a default, but can easily be changed. Perhaps\nyou do not want the representations to be influenced by the global representation and merely see how they\nevolved over time:\n\n```python\ntopics_over_time = topic_model.topics_over_time(tweets, timestamps,\n global_tuning=True, evolution_tuning=True, nr_bins=20)\n```\n\n### **Bins**\nIf you have more than 100 unique timestamps, then there will be topic representations created for each of those\ntimestamps which can negatively affect the topic representations. It is advised to keep the number of unique\ntimestamps below 50. To do this, you can simply set the number of bins that are created when calculating the\ntopic representations. The timestamps will be taken and put into equal-sized bins:\n\n```python\ntopics_over_time = topic_model.topics_over_time(tweets, timestamps, nr_bins=20)\n```\n\n### **Datetime format**\nIf you are passing strings (dates) instead of integers, then BERTopic will try to automatically detect\nwhich datetime format your strings have. Unfortunately, this will not always work if they are in an unexpected format.\nWe can use `datetime_format` to pass the format the timestamps have:\n\n```python\ntopics_over_time = topic_model.topics_over_time(tweets, timestamps, datetime_format=\"%b%M\", nr_bins=20)\n```\n\n## **Visualization**\nTo me, DTM becomes truly interesting when you have a good way of visualizing how topics have changed over time.\nA nice way of doing so is by leveraging the interactive abilities of Plotly. Plotly allows us to show the frequency\nof topics over time whilst giving the option of hovering over the points to show the time-specific topic representations.\nSimply call `visualize_topics_over_time` with the newly created topics over time:\n\n```python\ntopic_model.visualize_topics_over_time(topics_over_time, top_n_topics=20)\n```\n\nI used `top_n_topics` to only show the top 20 most frequent topics. If I were to visualize all topics, which is possible by\nleaving `top_n_topics` empty, there is a chance that hundreds of lines will fill the plot.\n\nYou can also use `topics` to show specific topics:\n\n```python\ntopic_model.visualize_topics_over_time(topics_over_time, topics=[9, 10, 72, 83, 87, 91])\n```\n\n\n" }, { "path": "docs/getting_started/topicsovertime/trump.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/topicsperclass/topics_per_class.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/topicsperclass/topicsperclass.md", "content": "In some cases, you might be interested in how certain topics are represented over certain categories. Perhaps\nthere are specific groups of users for which you want to see how they talk about certain topics.\n\nInstead of running the topic model per class, we can simply create a topic model and then extract, for each topic, its representation per class. This allows you to see how certain topics, calculated over all documents, are represented for certain subgroups.\n\n
\n
\n--8<-- \"docs/getting_started/topicsperclass/class_modeling.svg\"\n
\n
\n\n\nTo do so, we use the 20 Newsgroups dataset to see how the topics that we uncover are represented in the 20 categories of documents.\n\nFirst, let's prepare the data:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data[\"data\"]\ntargets = data[\"target\"]\ntarget_names = data[\"target_names\"]\nclasses = [data[\"target_names\"][i] for i in data[\"target\"]]\n```\n\nNext, we want to extract the topics across all documents without taking the categories into account:\n\n```python\ntopic_model = BERTopic(verbose=True)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nNow that we have created our global topic model, let us calculate the topic representations across each category:\n\n```python\ntopics_per_class = topic_model.topics_per_class(docs, classes=classes)\n```\n\nThe `classes` variable contains the class for each document. Then, we simply visualize these topics per class:\n\n```python\ntopic_model.visualize_topics_per_class(topics_per_class, top_n_topics=10)\n```\n\n\nYou can hover over the bars to see the topic representation per class.\n\nAs you can see in the visualization above, the topics `93_homosexual_homosexuality_sex` and `58_bike_bikes_motorcycle`\nare somewhat distributed over all classes.\n\nYou can see that the topic representation between rec.motorcycles and rec.autos in `58_bike_bikes_motorcycle` clearly\ndiffers from one another. It seems that BERTopic has tried to combine those two categories into a single topic. However,\nsince they do contain two separate topics, the topic representation in those two categories differs.\n\nWe see something similar for `93_homosexual_homosexuality_sex`, where the topic is distributed among several categories\nand is represented slightly differently.\n\nThus, you can see that although in certain categories the topic is similar, the way the topic is represented can differ.\n" }, { "path": "docs/getting_started/vectorizers/vectorizers.md", "content": "In topic modeling, the quality of the topic representations is key for interpreting the topics, communicating results, and understanding patterns. It is of utmost\nimportance to make sure that the topic representations fit with your use case.\n\nIn practice, there is not one correct way of creating topic representations. Some use cases might opt for higher n-grams, whereas others might focus more on single\nwords without any stop words. The diversity in use cases also means that we need to have some flexibility in BERTopic to make sure it can be used across most use cases.\nThe image below illustrates this modularity:\n\n
\n ![Image title](vectorizers.svg)\n
\n
\n\nIn this section, we will go through several examples of vectorization algorithms and how they can be implemented.\n\n## **CountVectorizer**\n\nOne often underestimated component of BERTopic is the `CountVectorizer` and `c-TF-IDF` calculation. Together, they are responsible for creating the topic representations and luckily\ncan be quite flexible in parameter tuning. Here, we will go through tips and tricks for tuning your `CountVectorizer` and see how they might affect the topic representations.\n\nBefore starting, it should be noted that you can pass the `CountVectorizer` before and after training your topic model. Passing it before training allows you to\nminimize the size of the resulting `c-TF-IDF` matrix:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n# Train BERTopic with a custom CountVectorizer\nvectorizer_model = CountVectorizer(min_df=10)\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nPassing it after training allows you to fine-tune the topic representations by using `.update_topics()`:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n# Train a BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n\n# Fine-tune topic representations after training BERTopic\nvectorizer_model = CountVectorizer(stop_words=\"english\", ngram_range=(1, 3), min_df=10)\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\nThe great thing about using `.update_topics()` is that it allows you to tweak the topic representations without re-training your model! Thus, here we will be focusing\non fine-tuning our topic representations after training our model.\n\n\n!!! note\n The great thing about processing our topic representations with the `CountVectorizer` is that it does **not** influence the quality of clusters as that is\n\tbeing performed before generating the topic representations.\n\n### **Basic Usage**\n\nFirst, let's start with defining our documents and training our topic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Prepare documents\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\n# Train a BERTopic model\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nNow, let's see the top 10 most frequent topics that have been generated:\n\n```python\n>>> topic_model.get_topic_info()[1:11]\nTopic\tCount\tName\n1\t0\t1822\t0_game_team_games_he\n2\t1\t580\t1_key_clipper_chip_encryption\n3\t2\t532\t2_ites_hello_cheek_hi\n4\t3\t493\t3_israel_israeli_jews_arab\n5\t4\t453\t4_card_monitor_video_drivers\n6\t5\t438\t5_you_your_post_jim\n7\t6\t314\t6_car_cars_engine_ford\n8\t7\t279\t7_health_newsgroup_cancer_1993\n9\t8\t218\t8_fbi_koresh_fire_gas\n10\t9\t174\t9_amp_audio_condition_asking\n```\n\nThe topic representations generated already seem quite interpretable! However, I am quite sure we do much better without having\nto re-train our model. Next, we will go through common parameters in `CountVectorizer` and focus on the effects that they might have. As a baseline, we will be comparing\nthem to the topic representation above.\n\n### **Parameters**\n\nThere are several basic parameters in the CountVectorizer that we can use to improve upon the quality of the resulting topic representations.\n\n#### ngram_range\n\nThe `ngram_range` parameter allows us to decide how many tokens each entity is in a topic representation. For example, we have words like\n`game` and `team` with a length of 1 in a topic but it would also make sense to have words like `hockey league` with a length of 2. To allow for these words to be generated,\nwe can set the `ngram_range` parameter:\n\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(ngram_range=(1, 3), stop_words=\"english\")\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\nAs you might have noticed, I also added `stop_words=\"english\"`. This is necessary as longer words tend to have many stop words and removing them allows\nfor nicer topic representations:\n\n```python\n>>> topic_model.get_topic_info()[1:11]\n\tTopic\tCount\tName\n1\t0\t1822\t0_game_team_games_players\n2\t1\t580\t1_key_clipper_chip_encryption\n3\t2\t532\t2_hello ites_forget hello_ites 15_huh hi\n4\t3\t493\t3_israel_israeli_jews_arab\n5\t4\t453\t4_card_monitor_video_drivers\n6\t5\t438\t5_post_jim_context_forged\n7\t6\t314\t6_car_cars_engine_ford\n8\t7\t279\t7_health_newsgroup_cancer_1993\n9\t8\t218\t8_fbi_koresh_gas_compound\n10\t9\t174\t9_amp_audio_condition_asking\n```\n\nAlthough they look very similar, if we zoom in on topic 8, we can see longer words in our representation:\n\n```python\n>>> topic_model.get_topic(8)\n[('fbi', 0.019637149205975653),\n ('koresh', 0.019054514637064403),\n ('gas', 0.014156057632897179),\n ('compound', 0.012381224868591681),\n ('batf', 0.010349992314076047),\n ('children', 0.009336408916322387),\n ('tear gas', 0.008941747802855279),\n ('tear', 0.008446786597564537),\n ('davidians', 0.007911119583253022),\n ('started', 0.007398687505638955)]\n```\n\n`tear` and `gas` have now been combined into a single representation. This helps us understand what those individual words might have been representing.\n\n#### stop_words\n\nIn some of the topics, we can see stop words appearing like `he` or `the`.\nStop words are something we typically want to prevent in our topic representations as they do not give additional information to the topic.\nTo prevent those stop words, we can use the `stop_words` parameter in the `CountVectorizer` to remove them from the representations:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(stop_words=\"english\")\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\nAfter running the above, we get the following output:\n\n```python\n>>> topic_model.get_topic_info()[1:11]\n\tTopic\tCount\tName\n1\t0\t1822\t0_game_team_games_players\n2\t1\t580\t1_key_clipper_chip_encryption\n3\t2\t532\t2_ites_cheek_hello_hi\n4\t3\t493\t3_israel_israeli_jews_arab\n5\t4\t453\t4_monitor_card_video_vga\n6\t5\t438\t5_post_jim_context_forged\n7\t6\t314\t6_car_cars_engine_ford\n8\t7\t279\t7_health_newsgroup_cancer_tobacco\n9\t8\t218\t8_fbi_koresh_gas_compound\n10\t9\t174\t9_amp_audio_condition_stereo\n```\n\nAs you can see, the topic representations already look much better! Stop words are removed and the representations are more interpretable.\nWe can also pass in a list of stop words if you have multiple languages to take into account.\n\n\n#### min_df\n\nOne important parameter to keep in mind is the `min_df`. This is typically an integer representing how frequent a word must be before\nbeing added to our representation. You can imagine that if we have a million documents and a certain word only appears a single time across all of them, then\nit would be highly unlikely to be representative of a topic. Typically, the `c-TF-IDF` calculation removes that word from the topic representation but when\nyou have millions of documents, that will also lead to a very large topic-term matrix. To prevent a huge vocabulary, we can set the `min_df` to only accept\nwords that have a minimum frequency.\n\nWhen you have millions of documents or error issues, I would advise increasing the value of `min_df` as long as the topic representations might sense:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(min_df=10)\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\nWith the following topic representation:\n\n```python\n>>> topic_model.get_topic_info()[1:11]\n\tTopic\tCount\tName\n1\t0\t1822\t0_game_team_games_he\n2\t1\t580\t1_key_clipper_chip_encryption\n3\t2\t532\t2_hello_hi_yep_huh\n4\t3\t493\t3_israel_jews_jewish_peace\n5\t4\t453\t4_card_monitor_video_drivers\n6\t5\t438\t5_you_your_post_jim\n7\t6\t314\t6_car_cars_engine_ford\n8\t7\t279\t7_health_newsgroup_cancer_1993\n9\t8\t218\t8_fbi_koresh_fire_gas\n10\t9\t174\t9_audio_condition_stereo_asking\n```\n\nAs you can see, the output is nearly the same which is what we would like to achieve. All words that appear less than 10 times are now removed\nfrom our topic-term matrix (i.e., `c-TF-IDF` matrix) which drastically lowers the matrix in size.\n\n\n#### max_features\n\nA parameter similar to `min_df` is `max_features` which allows you to select the top n most frequent words to be used in the topic representation.\nSetting this, for example, to `10_000` creates a topic-term matrix with `10_000` terms. This helps you control the size of the topic-term matrix\ndirectly without having to fiddle around with the `min_df` parameter:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nvectorizer_model = CountVectorizer(max_features=10_000)\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\nWith the following representation:\n\n```python\n>>> topic_model.get_topic_info()[1:11]\nTopic\tCount\tName\n1\t0\t1822\t0_game_team_games_he\n2\t1\t580\t1_key_clipper_chip_encryption\n3\t2\t532\t2_hello_hi_yep_huh\n4\t3\t493\t3_israel_israeli_jews_arab\n5\t4\t453\t4_card_monitor_video_drivers\n6\t5\t438\t5_you_your_post_jim\n7\t6\t314\t6_car_cars_engine_ford\n8\t7\t279\t7_health_newsgroup_cancer_1993\n9\t8\t218\t8_fbi_koresh_fire_gas\n10\t9\t174\t9_amp_audio_condition_asking\n```\n\nAs with `min_df`, we would like the topic representations to be very similar.\n\n#### tokenizer\n\nThe default tokenizer in the CountVectorizer works well for western languages but fails to tokenize some non-western languages, like Chinese.\nFortunately, we can use the `tokenizer` variable in the CountVectorizer to use [`jieba`](https://github.com/fxsjy/jieba), which is a package\nfor Chinese text segmentation. Using it is straightforward:\n\n```python\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport jieba\n\ndef tokenize_zh(text):\n words = jieba.lcut(text)\n return words\n\nvectorizer = CountVectorizer(tokenizer=tokenize_zh)\n```\n\nThen, we can simply pass the vectorizer to update our topic representations:\n\n```python\ntopic_model.update_topics(docs, vectorizer_model=vectorizer_model)\n```\n\n\n## **OnlineCountVectorizer**\n\nWhen using the online/incremental variant of BERTopic, we need a `CountVectorizer` than can incrementally update its representation. For that purpose, `OnlineCountVectorizer` was created that not only updates out-of-vocabulary words but also implements decay and cleaning functions to prevent the sparse bag-of-words matrix to become too large. It is a class that can be found in `bertopic.vectorizers` which extends `sklearn.feature_extraction.text.CountVectorizer`. In other words, you can use the exact same parameter in `OnlineCountVectorizer` as found in Scikit-Learn's `CountVectorizer`. We can use it as follows:\n\n```python\nfrom bertopic import BERTopic\nfrom bertopic.vectorizers import OnlineCountVectorizer\n\n# Train BERTopic with a custom OnlineCountVectorizer\nvectorizer_model = OnlineCountVectorizer()\ntopic_model = BERTopic(vectorizer_model=vectorizer_model)\n```\n\n### **Parameters**\n\nOther than parameters found in `CountVectorizer`, such as `stop_words` and `ngram_range`, we can two parameters in `OnlineCountVectorizer` to adjust the way old data is processed and kept.\n\n#### decay\n\nAt each iteration, we sum the bag-of-words representation of the new documents with the bag-of-words representation of all documents processed thus far. In other words, the bag-of-words matrix keeps increasing with each iteration. However, especially in a streaming setting, older documents might become less and less relevant as time goes on. Therefore, a `decay` parameter was implemented that decays the bag-of-words' frequencies at each iteration before adding the document frequencies of new documents. The `decay` parameter is a value between 0 and 1 and indicates the percentage of frequencies the previous bag-of-words matrix should be reduced to. For example, a value of `.1` will decrease the frequencies in the bag-of-words matrix by 10% at each iteration before adding the new bag-of-words matrix. This will make sure that recent data has more weight than previous iterations.\n\n#### delete_min_df\n\nIn BERTopic, we might want to remove words from the topic representation that appear infrequently. The `min_df` in the `CountVectorizer` works quite well for that. However, when we have a streaming setting, the `min_df` does not work as well since a word's frequency might start below `min_df` but will end up higher than that over time. Setting that value high might not always be advised.\n\nAs a result, the vocabulary of the resulting bag-of-words matrix can become quite large. Similarly, if we implement the `decay` parameter, then some values will decrease over time until they are below `min_df`. For these reasons, the `delete_min_df` parameter was implemented. The parameter takes positive integers and indicates, at each iteration, which words will be removed. If the value is set to 5, it will check after each iteration if the total frequency of a word is exceeded by that value. If so, the word will be removed in its entirety from the bag-of-words matrix. This helps to keep the bag-of-words matrix of a manageable size.\n\n!!! note\n\tAlthough the `delete_min_df` parameter removes words from the bag-of-words matrix, it is not permanent. If new documents come in where those previously deleted words are used frequently, they get added back to the matrix.\n" }, { "path": "docs/getting_started/visualization/bar_chart.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/datamapplot.html", "content": "\n\n \n \n Interactive Data Map\n \n \n \n \n\n\n\n\n \n \n\n \n \n \n \n \n \n
\n \n
\n
\n
\n
\n
\n\n\n\n
\n \n
\n\n
\n
\n\n
\n
\n\n
\n
\n
\n
\n
\n \n Point Data: 0%\n \n
\n
\n \n Label Data: 0%\n \n
\n
\n \n Meta Data: 0%\n \n
\n
\n
\n \n \n \n \n" }, { "path": "docs/getting_started/visualization/documents.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/heatmap.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/hierarchical_documents.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/hierarchical_topics.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/hierarchy.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/probabilities.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/term_rank.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/term_rank_log.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/topics_per_class.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/trump.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/visualization/visualization.md", "content": "Visualizing BERTopic and its derivatives is important in understanding the model, how it works, and more importantly, where it works.\nSince topic modeling can be quite a subjective field it is difficult for users to validate their models. Looking at the topics and seeing\nif they make sense is an important factor in alleviating this issue.\n\n## **Visualize Topics**\nAfter having trained our `BERTopic` model, we can iteratively go through hundreds of topics to get a good\nunderstanding of the topics that were extracted. However, that takes quite some time and lacks a global representation.\nInstead, we can visualize the topics that were generated in a way very similar to\n[LDAvis](https://github.com/cpsievert/LDAvis).\n\nWe embed our c-TF-IDF representation of the topics in 2D using Umap and then visualize the two dimensions using\nplotly such that we can create an interactive view.\n\nFirst, we need to train our model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nThen, we can call `.visualize_topics` to create a 2D representation of your topics. The resulting graph is a\nplotly interactive graph which can be converted to HTML:\n\n```python\ntopic_model.visualize_topics()\n```\n\n\n\nYou can use the slider to select the topic which then lights up red. If you hover over a topic, then general\ninformation is given about the topic, including the size of the topic and its corresponding words.\n\n## **Visualize Documents**\nUsing the previous method, we can visualize the topics and get insight into their relationships. However,\nyou might want a more fine-grained approach where we can visualize the documents inside the topics to see\nif they were assigned correctly or whether they make sense. To do so, we can use the `topic_model.visualize_documents()`\nfunction. This function recalculates the document embeddings and reduces them to 2-dimensional space for easier visualization\npurposes. This process can be quite expensive, so it is advised to adhere to the following pipeline:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train BERTopic\ntopic_model = BERTopic().fit(docs, embeddings)\n\n# Run the visualization with the original embeddings\ntopic_model.visualize_documents(docs, embeddings=embeddings)\n\n# Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n```\n\n\n\n\n!!! note\n The visualization above was generated with the additional parameter `hide_document_hover=True` which disables the\n option to hover over the individual points and see the content of the documents. This was done for demonstration purposes\n as saving all those documents in the visualization can be quite expensive and result in large files. However,\n it might be interesting to set `hide_document_hover=False` in order to hover over the points and see the content of the documents.\n\n### **Custom Hover**\n\nWhen you visualize the documents, you might not always want to see the complete document over hover. Many documents have shorter information that might be more interesting to visualize, such as its title. To create the hover based on a documents' title instead of its content, you can simply pass a variable (`titles`) containing the title for each document:\n\n```python\ntopic_model.visualize_documents(titles, reduced_embeddings=reduced_embeddings)\n```\n\n## **Visualize Topic Hierarchy**\nThe topics that were created can be hierarchically reduced. In order to understand the potential hierarchical\nstructure of the topics, we can use `scipy.cluster.hierarchy` to create clusters and visualize how\nthey relate to one another. This might help to select an appropriate `nr_topics` when reducing the number\nof topics that you have created. To visualize this hierarchy, run the following:\n\n```python\ntopic_model.visualize_hierarchy()\n```\n\n\n\n!!! note\n Do note that this is not the actual procedure of `.reduce_topics()` when `nr_topics` is set to\n auto since HDBSCAN is used to automatically extract topics. The visualization above closely resembles\n the actual procedure of `.reduce_topics()` when any number of `nr_topics` is selected.\n\n### **Hierarchical labels**\n\nAlthough visualizing this hierarchy gives us information about the structure, it would be helpful to see what happens\nto the topic representations when merging topics. To do so, we first need to calculate the representations of the\nhierarchical topics:\n\n\nFirst, we train a basic BERTopic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))[\"data\"]\ntopic_model = BERTopic(verbose=True)\ntopics, probs = topic_model.fit_transform(docs)\nhierarchical_topics = topic_model.hierarchical_topics(docs)\n```\n\nTo visualize these results, we simply need to pass the resulting `hierarchical_topics` to our `.visualize_hierarchy` function:\n\n```python\ntopic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n```\n\n\n\nIf you **hover** over the black circles, you will see the topic representation at that level of the hierarchy. These representations\nhelp you understand the effect of merging certain topics. Some might be logical to merge whilst others might not. Moreover,\nwe can now see which sub-topics can be found within certain larger themes.\n\n### **Text-based topic tree**\n\nAlthough this gives a nice overview of the potential hierarchy, hovering over all black circles can be tiresome. Instead, we can\nuse `topic_model.get_topic_tree` to create a text-based representation of this hierarchy. Although the general structure is more difficult\nto view, we can see better which topics could be logically merged:\n\n```python\n>>> tree = topic_model.get_topic_tree(hierarchical_topics)\n>>> print(tree)\n.\n└─atheists_atheism_god_moral_atheist\n β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n └─■──moral_morality_objective_immoral_morals ── Topic: 29\n```\n\n
\n Click here to view the full tree.\n\n ```bash\n .\n β”œβ”€people_armenian_said_god_armenians\n β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”‚ β”œβ”€jehovah_lord_mormon_mcconkie_god\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ra_satan_thou_god_lucifer ── Topic: 94\n β”‚ β”‚ β”‚ β”‚ └─■──jehovah_lord_mormon_mcconkie_unto ── Topic: 78\n β”‚ β”‚ β”‚ └─jesus_mary_god_hell_sin\n β”‚ β”‚ β”‚ β”œβ”€jesus_hell_god_eternal_heaven\n β”‚ β”‚ β”‚ β”‚ β”œβ”€hell_jesus_eternal_god_heaven\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jesus_tomb_disciples_resurrection_john ── Topic: 69\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──hell_eternal_god_jesus_heaven ── Topic: 53\n β”‚ β”‚ β”‚ β”‚ └─■──aaron_baptism_sin_law_god ── Topic: 89\n β”‚ β”‚ β”‚ └─■──mary_sin_maria_priest_conception ── Topic: 56\n β”‚ β”‚ └─■──marriage_married_marry_ceremony_marriages ── Topic: 110\n β”‚ └─people_armenian_armenians_said_mr\n β”‚ β”œβ”€people_armenian_armenians_said_israel\n β”‚ β”‚ β”œβ”€god_homosexual_homosexuality_atheists_sex\n β”‚ β”‚ β”‚ β”œβ”€homosexual_homosexuality_sex_gay_homosexuals\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€kinsey_sex_gay_men_sexual ── Topic: 44\n β”‚ β”‚ β”‚ β”‚ └─homosexuality_homosexual_sin_homosexuals_gay\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gay_homosexual_homosexuals_sexual_cramer ── Topic: 50\n β”‚ β”‚ β”‚ β”‚ └─■──homosexuality_homosexual_sin_paul_sex ── Topic: 27\n β”‚ β”‚ β”‚ └─god_atheists_atheism_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jim_context_challenges_articles_quote ── Topic: 36\n β”‚ β”‚ β”‚ β”‚ └─islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€islam_quran_islamic_rushdie_muslims ── Topic: 31\n β”‚ β”‚ β”‚ β”‚ └─■──judas_scripture_bible_books_greek ── Topic: 33\n β”‚ β”‚ β”‚ └─atheists_atheism_god_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ β”‚ β”‚ β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n β”‚ β”‚ β”‚ └─■──moral_morality_objective_immoral_morals ── Topic: 29\n β”‚ β”‚ └─armenian_armenians_people_israel_said\n β”‚ β”‚ β”œβ”€armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€tax_rights_government_income_taxes\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rights_right_slavery_slaves_residence ── Topic: 106\n β”‚ β”‚ β”‚ β”‚ └─tax_government_taxes_income_libertarians\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€government_libertarians_libertarian_regulation_party ── Topic: 58\n β”‚ β”‚ β”‚ β”‚ └─■──tax_taxes_income_billion_deficit ── Topic: 41\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€gun_guns_militia_firearms_amendment\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€blacks_penalty_death_cruel_punishment ── Topic: 55\n β”‚ β”‚ β”‚ β”‚ └─■──gun_guns_militia_firearms_amendment ── Topic: 7\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_jews_turkish\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€israel_israeli_jews_arab_jewish ── Topic: 4\n β”‚ β”‚ β”‚ └─■──armenian_armenians_turkish_armenia_azerbaijan ── Topic: 15\n β”‚ β”‚ └─stephanopoulos_president_mr_myers_ms\n β”‚ β”‚ β”œβ”€β– β”€β”€serbs_muslims_stephanopoulos_mr_bosnia ── Topic: 35\n β”‚ β”‚ └─■──myers_stephanopoulos_president_ms_mr ── Topic: 87\n β”‚ └─batf_fbi_koresh_compound_gas\n β”‚ β”œβ”€β– β”€β”€reno_workers_janet_clinton_waco ── Topic: 77\n β”‚ └─batf_fbi_koresh_gas_compound\n β”‚ β”œβ”€batf_koresh_fbi_warrant_compound\n β”‚ β”‚ β”œβ”€β– β”€β”€batf_warrant_raid_compound_fbi ── Topic: 42\n β”‚ β”‚ └─■──koresh_batf_fbi_children_compound ── Topic: 61\n β”‚ └─■──fbi_gas_tear_bds_building ── Topic: 23\n └─use_like_just_dont_new\n β”œβ”€game_team_year_games_like\n β”‚ β”œβ”€game_team_games_25_year\n β”‚ β”‚ β”œβ”€game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€window_printer_use_problem_mhz\n β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_wire_simms_wiring_battery\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_mhz_battery_cpu_heat\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_pds_simm_vram_lc\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€pds_nubus_lc_slot_card ── Topic: 119\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──simms_simm_vram_meg_dram ── Topic: 32\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─mhz_battery_cpu_heat_speed\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€fan_cpu_heat_sink_fans ── Topic: 92\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──mhz_speed_cpu_fpu_clock ── Topic: 22\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──monitor_turn_power_computer_electricity ── Topic: 91\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─battery_batteries_concrete_duo_discharge\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€duo_battery_apple_230_problem ── Topic: 121\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──battery_batteries_concrete_discharge_temperature ── Topic: 75\n β”‚ β”‚ β”‚ β”‚ β”‚ └─wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€leds_uv_blue_light_boards ── Topic: 66\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──wire_wiring_ground_neutral_outlets ── Topic: 120\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─scope_scopes_phone_dial_number\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€dial_number_phone_line_output ── Topic: 93\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──scope_scopes_motorola_generator_oscilloscope ── Topic: 113\n β”‚ β”‚ β”‚ β”‚ β”‚ └─celp_dsp_sampling_antenna_digital\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€antenna_antennas_receiver_cable_transmitter ── Topic: 70\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──celp_dsp_sampling_speech_voice ── Topic: 52\n β”‚ β”‚ β”‚ β”‚ └─window_printer_xv_mouse_windows\n β”‚ β”‚ β”‚ β”‚ β”œβ”€window_xv_error_widget_problem\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€error_symbol_undefined_xterm_rx\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€symbol_error_undefined_doug_parse ── Topic: 63\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──rx_remote_server_xdm_xterm ── Topic: 45\n β”‚ β”‚ β”‚ β”‚ β”‚ └─window_xv_widget_application_expose\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€window_widget_expose_application_event\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gc_mydisplay_draw_gxxor_drawing ── Topic: 103\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──window_widget_application_expose_event ── Topic: 25\n β”‚ β”‚ β”‚ β”‚ β”‚ └─xv_den_polygon_points_algorithm\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€den_polygon_points_algorithm_polygons ── Topic: 28\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──xv_24bit_image_bit_images ── Topic: 57\n β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_mouse_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€scanner_logitech_grayscale_ocr_scanman ── Topic: 108\n β”‚ β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€printer_print_deskjet_hp_ink ── Topic: 18\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──fonts_font_truetype_tt_atm ── Topic: 49\n β”‚ β”‚ β”‚ β”‚ └─mouse_ghostscript_midi_driver_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€ghostscript_midi_postscript_files_file\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ghostscript_postscript_pageview_ghostview_dsc ── Topic: 104\n β”‚ β”‚ β”‚ β”‚ β”‚ └─midi_sound_file_windows_driver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€location_mar_file_host_rwrr ── Topic: 83\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──midi_sound_driver_blaster_soundblaster ── Topic: 98\n β”‚ β”‚ β”‚ β”‚ └─■──mouse_driver_mice_ball_problem ── Topic: 68\n β”‚ β”‚ β”‚ └─game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€1st_sale_condition_comics_hulk\n β”‚ β”‚ β”‚ β”‚ β”œβ”€sale_condition_offer_asking_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€condition_stereo_amp_speakers_asking\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€miles_car_amfm_toyota_cassette ── Topic: 62\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──amp_speakers_condition_stereo_audio ── Topic: 24\n β”‚ β”‚ β”‚ β”‚ β”‚ └─games_sale_pom_cds_shipping\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€pom_cds_sale_shipping_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€size_shipping_sale_condition_mattress ── Topic: 100\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──pom_cds_cd_sale_picture ── Topic: 37\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──games_game_snes_sega_genesis ── Topic: 40\n β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”œβ”€1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€lens_tape_camera_backup_lenses\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€tape_backup_tapes_drive_4mm ── Topic: 107\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──lens_camera_lenses_zoom_pouch ── Topic: 114\n β”‚ β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€1st_hulk_comics_art_appears ── Topic: 105\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──books_book_cover_trek_chemistry ── Topic: 125\n β”‚ β”‚ β”‚ β”‚ └─tickets_hotel_ticket_voucher_package\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hotel_voucher_package_vacation_room ── Topic: 74\n β”‚ β”‚ β”‚ β”‚ └─■──tickets_ticket_june_airlines_july ── Topic: 84\n β”‚ β”‚ β”‚ └─game_team_games_season_hockey\n β”‚ β”‚ β”‚ β”œβ”€game_hockey_team_25_550\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€espn_pt_pts_game_la ── Topic: 17\n β”‚ β”‚ β”‚ β”‚ └─■──team_25_game_hockey_550 ── Topic: 2\n β”‚ β”‚ β”‚ └─■──year_game_hit_baseball_players ── Topic: 0\n β”‚ β”‚ └─bike_car_greek_insurance_msg\n β”‚ β”‚ β”œβ”€car_bike_insurance_cars_engine\n β”‚ β”‚ β”‚ β”œβ”€car_insurance_cars_radar_engine\n β”‚ β”‚ β”‚ β”‚ β”œβ”€insurance_health_private_care_canada\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€insurance_health_private_care_canada ── Topic: 99\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──insurance_car_accident_rates_sue ── Topic: 82\n β”‚ β”‚ β”‚ β”‚ └─car_cars_radar_engine_detector\n β”‚ β”‚ β”‚ β”‚ β”œβ”€car_radar_cars_detector_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€radar_detector_detectors_ka_alarm ── Topic: 39\n β”‚ β”‚ β”‚ β”‚ β”‚ └─car_cars_mustang_ford_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clutch_shift_shifting_transmission_gear ── Topic: 88\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──car_cars_mustang_ford_v8 ── Topic: 14\n β”‚ β”‚ β”‚ β”‚ └─oil_diesel_odometer_diesels_car\n β”‚ β”‚ β”‚ β”‚ β”œβ”€odometer_oil_sensor_car_drain\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€odometer_sensor_speedo_gauge_mileage ── Topic: 96\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──oil_drain_car_leaks_taillights ── Topic: 102\n β”‚ β”‚ β”‚ β”‚ └─■──diesel_diesels_emissions_fuel_oil ── Topic: 79\n β”‚ β”‚ β”‚ └─bike_riding_ride_bikes_motorcycle\n β”‚ β”‚ β”‚ β”œβ”€bike_ride_riding_bikes_lane\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€bike_ride_riding_lane_car ── Topic: 11\n β”‚ β”‚ β”‚ β”‚ └─■──bike_bikes_miles_honda_motorcycle ── Topic: 19\n β”‚ β”‚ β”‚ └─■──countersteering_bike_motorcycle_rear_shaft ── Topic: 46\n β”‚ β”‚ └─greek_msg_kuwait_greece_water\n β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_water\n β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_kuwait_greece_turkish_greeks\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€greek_greece_turkish_greeks_cyprus ── Topic: 71\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kuwait_iraq_iran_gulf_arabia ── Topic: 76\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_dog_drugs_drug_food\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clinton_bush_quayle_reagan_panicking ── Topic: 101\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€cooper_trial_weaver_spence_witnesses ── Topic: 90\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──dog_dogs_bike_trained_springer ── Topic: 67\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_drugs_drug_food_chinese\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€msg_food_chinese_foods_taste ── Topic: 30\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──drugs_drug_marijuana_cocaine_alcohol ── Topic: 72\n β”‚ β”‚ β”‚ β”‚ └─water_theory_universe_science_larsons\n β”‚ β”‚ β”‚ β”‚ β”œβ”€water_nuclear_cooling_steam_dept\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rocketry_rockets_engines_nuclear_plutonium ── Topic: 115\n β”‚ β”‚ β”‚ β”‚ β”‚ └─water_cooling_steam_dept_plants\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€water_dept_phd_environmental_atmospheric ── Topic: 97\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──cooling_water_steam_towers_plants ── Topic: 109\n β”‚ β”‚ β”‚ β”‚ └─theory_universe_larsons_larson_science\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€theory_universe_larsons_larson_science ── Topic: 54\n β”‚ β”‚ β”‚ β”‚ └─■──oort_cloud_grbs_gamma_burst ── Topic: 80\n β”‚ β”‚ β”‚ └─helmet_kirlian_photography_lock_wax\n β”‚ β”‚ β”‚ β”œβ”€helmet_kirlian_photography_leaf_mask\n β”‚ β”‚ β”‚ β”‚ β”œβ”€kirlian_photography_leaf_pictures_deleted\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€deleted_joke_stuff_maddi_nickname\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€joke_maddi_nickname_nicknames_frank ── Topic: 43\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──deleted_stuff_bookstore_joke_motto ── Topic: 81\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kirlian_photography_leaf_pictures_aura ── Topic: 85\n β”‚ β”‚ β”‚ β”‚ └─helmet_mask_liner_foam_cb\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€helmet_liner_foam_cb_helmets ── Topic: 112\n β”‚ β”‚ β”‚ β”‚ └─■──mask_goalies_77_santore_tl ── Topic: 123\n β”‚ β”‚ β”‚ └─lock_wax_paint_plastic_ear\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€lock_cable_locks_bike_600 ── Topic: 117\n β”‚ β”‚ β”‚ └─wax_paint_ear_plastic_skin\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€wax_paint_plastic_scratches_solvent ── Topic: 65\n β”‚ β”‚ β”‚ └─■──ear_wax_skin_greasy_acne ── Topic: 116\n β”‚ β”‚ └─m4_mp_14_mw_mo\n β”‚ β”‚ β”œβ”€m4_mp_14_mw_mo\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€m4_mp_14_mw_mo ── Topic: 111\n β”‚ β”‚ β”‚ └─■──test_ensign_nameless_deane_deanebinahccbrandeisedu ── Topic: 118\n β”‚ β”‚ └─■──ites_cheek_hello_hi_ken ── Topic: 3\n β”‚ └─space_medical_health_disease_cancer\n β”‚ β”œβ”€medical_health_disease_cancer_patients\n β”‚ β”‚ β”œβ”€β– β”€β”€cancer_centers_center_medical_research ── Topic: 122\n β”‚ β”‚ └─health_medical_disease_patients_hiv\n β”‚ β”‚ β”œβ”€patients_medical_disease_candida_health\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€candida_yeast_infection_gonorrhea_infections ── Topic: 48\n β”‚ β”‚ β”‚ └─patients_disease_cancer_medical_doctor\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hiv_medical_cancer_patients_doctor ── Topic: 34\n β”‚ β”‚ β”‚ └─■──pain_drug_patients_disease_diet ── Topic: 26\n β”‚ β”‚ └─■──health_newsgroup_tobacco_vote_votes ── Topic: 9\n β”‚ └─space_launch_nasa_shuttle_orbit\n β”‚ β”œβ”€space_moon_station_nasa_launch\n β”‚ β”‚ β”œβ”€β– β”€β”€sky_advertising_billboard_billboards_space ── Topic: 59\n β”‚ β”‚ └─■──space_station_moon_redesign_nasa ── Topic: 16\n β”‚ └─space_mission_hst_launch_orbit\n β”‚ β”œβ”€space_launch_nasa_orbit_propulsion\n β”‚ β”‚ β”œβ”€β– β”€β”€space_launch_nasa_propulsion_astronaut ── Topic: 47\n β”‚ β”‚ └─■──orbit_km_jupiter_probe_earth ── Topic: 86\n β”‚ └─■──hst_mission_shuttle_orbit_arrays ── Topic: 60\n └─drive_file_key_windows_use\n β”œβ”€key_file_jpeg_encryption_image\n β”‚ β”œβ”€key_encryption_clipper_chip_keys\n β”‚ β”‚ β”œβ”€β– β”€β”€key_clipper_encryption_chip_keys ── Topic: 1\n β”‚ β”‚ └─■──entry_file_ripem_entries_key ── Topic: 73\n β”‚ └─jpeg_image_file_gif_images\n β”‚ β”œβ”€motif_graphics_ftp_available_3d\n β”‚ β”‚ β”œβ”€motif_graphics_openwindows_ftp_available\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€openwindows_motif_xview_windows_mouse ── Topic: 20\n β”‚ β”‚ β”‚ └─■──graphics_widget_ray_3d_available ── Topic: 95\n β”‚ β”‚ └─■──3d_machines_version_comments_contact ── Topic: 38\n β”‚ └─jpeg_image_gif_images_format\n β”‚ β”œβ”€β– β”€β”€gopher_ftp_files_stuffit_images ── Topic: 51\n β”‚ └─■──jpeg_image_gif_format_images ── Topic: 13\n └─drive_db_card_scsi_windows\n β”œβ”€db_windows_dos_mov_os2\n β”‚ β”œβ”€β– β”€β”€copy_protection_program_software_disk ── Topic: 64\n β”‚ └─■──db_windows_dos_mov_os2 ── Topic: 8\n └─drive_card_scsi_drives_ide\n β”œβ”€drive_scsi_drives_ide_disk\n β”‚ β”œβ”€β– β”€β”€drive_scsi_drives_ide_disk ── Topic: 6\n β”‚ └─■──meg_sale_ram_drive_shipping ── Topic: 12\n └─card_modem_monitor_video_drivers\n β”œβ”€β– β”€β”€card_monitor_video_drivers_vga ── Topic: 5\n └─■──modem_port_serial_irq_com ── Topic: 10\n ```\n
\n\n## **Visualize Hierarchical Documents**\nWe can extend the previous method by calculating the topic representation at different levels of the hierarchy and\nplotting them on a 2D plane. To do so, we first need to calculate the hierarchical topics:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train BERTopic and extract hierarchical topics\ntopic_model = BERTopic().fit(docs, embeddings)\nhierarchical_topics = topic_model.hierarchical_topics(docs)\n```\nThen, we can visualize the hierarchical documents by either supplying it with our embeddings or by\nreducing their dimensionality ourselves:\n\n```python\n# Run the visualization with the original embeddings\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)\n\n# Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n```\n\n\n\n!!! note\n The visualization above was generated with the additional parameter `hide_document_hover=True` which disables the\n option to hover over the individual points and see the content of the documents. This makes the resulting visualization\n smaller and fit into your RAM. However, it might be interesting to set `hide_document_hover=False` to hover\n over the points and see the content of the documents.\n\n## **Visualize Terms**\nWe can visualize the selected terms for a few topics by creating bar charts out of the c-TF-IDF scores\nfor each topic representation. Insights can be gained from the relative c-TF-IDF scores between and within\ntopics. Moreover, you can easily compare topic representations to each other.\nTo visualize this hierarchy, run the following:\n\n```python\ntopic_model.visualize_barchart()\n```\n\n\n\n\n## **Visualize Topic Similarity**\nHaving generated topic embeddings, through both c-TF-IDF and embeddings, we can create a similarity\nmatrix by simply applying cosine similarities through those topic embeddings. The result will be a\nmatrix indicating how similar certain topics are to each other.\nTo visualize the heatmap, run the following:\n\n```python\ntopic_model.visualize_heatmap()\n```\n\n\n\n\n!!! note\n You can set `n_clusters` in `visualize_heatmap` to order the topics by their similarity.\n This will result in blocks being formed in the heatmap indicating which clusters of topics are\n similar to each other. This step is very much recommended as it will make reading the heatmap easier.\n\n\n## **Visualize Term Score Decline**\nTopics are represented by a number of words starting with the best representative word.\nEach word is represented by a c-TF-IDF score. The higher the score, the more representative a word\nto the topic is. Since the topic words are sorted by their c-TF-IDF score, the scores slowly decline\nwith each word that is added. At some point adding words to the topic representation only marginally\nincreases the total c-TF-IDF score and would not be beneficial for its representation.\n\nTo visualize this effect, we can plot the c-TF-IDF scores for each topic by the term rank of each word.\nIn other words, the position of the words (term rank), where the words with\nthe highest c-TF-IDF score will have a rank of 1, will be put on the x-axis. Whereas the y-axis\nwill be populated by the c-TF-IDF scores. The result is a visualization that shows you the decline\nof c-TF-IDF score when adding words to the topic representation. It allows you, using the elbow method,\nthe select the best number of words in a topic.\n\nTo visualize the c-TF-IDF score decline, run the following:\n\n```python\ntopic_model.visualize_term_rank()\n```\n\n\n\nTo enable the log scale on the y-axis for a better view of individual topics, run the following:\n\n```python\ntopic_model.visualize_term_rank(log_scale=True)\n```\n\n\n\nThis visualization was heavily inspired by the \"Term Probability Decline\" visualization found in an\nanalysis by the amazing [tmtoolkit](https://tmtoolkit.readthedocs.io/).\nReference to that specific analysis can be found\n[here](https://wzbsocialsciencecenter.github.io/tm_corona/tm_analysis.html).\n\n## **Visualize Topics over Time**\nAfter creating topics over time with Dynamic Topic Modeling, we can visualize these topics by\nleveraging the interactive abilities of Plotly. Plotly allows us to show the frequency\nof topics over time whilst giving the option of hovering over the points to show the time-specific topic representations.\nSimply call `.visualize_topics_over_time` with the newly created topics over time:\n\n\n```python\nimport re\nimport pandas as pd\nfrom bertopic import BERTopic\n\n# Prepare data\ntrump = pd.read_csv('https://drive.google.com/uc?export=download&id=1xRKHaP-QwACMydlDnyFPEaFdtskJuBa6')\ntrump.text = trump.apply(lambda row: re.sub(r\"http\\S+\", \"\", row.text).lower(), 1)\ntrump.text = trump.apply(lambda row: \" \".join(filter(lambda x:x[0]!=\"@\", row.text.split())), 1)\ntrump.text = trump.apply(lambda row: \" \".join(re.sub(\"[^a-zA-Z]+\", \" \", row.text).split()), 1)\ntrump = trump.loc[(trump.isRetweet == \"f\") & (trump.text != \"\"), :]\ntimestamps = trump.date.to_list()\ntweets = trump.text.to_list()\n\n# Create topics over time\nmodel = BERTopic(verbose=True)\ntopics, probs = model.fit_transform(tweets)\ntopics_over_time = model.topics_over_time(tweets, timestamps)\n```\n\nThen, we visualize some interesting topics:\n\n```python\nmodel.visualize_topics_over_time(topics_over_time, topics=[9, 10, 72, 83, 87, 91])\n```\n\n\n## **Visualize Topics per Class**\nYou might want to extract and visualize the topic representation per class. For example, if you have\nspecific groups of users that might approach topics differently, then extracting them would help understanding\nhow these users talk about certain topics. In other words, this is simply creating a topic representation for\ncertain classes that you might have in your data.\n\nFirst, we need to train our model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Prepare data and classes\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data[\"data\"]\nclasses = [data[\"target_names\"][i] for i in data[\"target\"]]\n\n# Create topic model and calculate topics per class\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\ntopics_per_class = topic_model.topics_per_class(docs, classes=classes)\n```\n\nThen, we visualize the topic representation of major topics per class:\n\n```python\ntopic_model.visualize_topics_per_class(topics_per_class)\n```\n\n\n\n\n## **Visualize Probabilities or Distribution**\n\nWe can generate the topic-document probability matrix by simply setting `calculate_probabilities=True` if a HDBSCAN model is used:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(calculate_probabilities=True)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nThe resulting `probs` variable contains the soft-clustering as done through HDBSCAN.\n\nIf a non-HDBSCAN model is used, we can estimate the topic distributions after training our model:\n\n```python\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic()\ntopics, _ = topic_model.fit_transform(docs)\ntopic_distr, _ = topic_model.approximate_distribution(docs, min_similarity=0)\n```\n\nThen, we either pass the `probs` or `topic_distr` variable to `.visualize_distribution` to visualize either the probability distributions or the topic distributions:\n\n```python\n# To visualize the probabilities of topic assignment\ntopic_model.visualize_distribution(probs[0])\n\n# To visualize the topic distributions in a document\ntopic_model.visualize_distribution(topic_distr[0])\n```\n\n\n\nAlthough a topic distribution is nice, we may want to see how each token contributes to a specific topic. To do so, we need to first calculate topic distributions on a token level and then visualize the results:\n\n```python\n# Calculate the topic distributions on a token-level\ntopic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)\n\n# Visualize the token-level distributions\ndf = topic_model.visualize_approximate_distribution(docs[1], topic_token_distr[1])\ndf\n```\n\n

\n\n

\n\n!!! note\n To get the stylized dataframe for `.visualize_approximate_distribution` you will need to have Jinja installed. If you do not have this installed, an unstylized dataframe will be returned instead. You can install Jinja via `pip install jinja2`\n\n!!! note\n The distribution of the probabilities does not give an indication to\n the distribution of the frequencies of topics across a document. It merely shows\n how confident BERTopic is that certain topics can be found in a document.\n" }, { "path": "docs/getting_started/visualization/visualize_documents.md", "content": "## **Visualize documents with Plotly**\n\nUsing the `.visualize_topics`, we can visualize the topics and get insight into their relationships. However,\nyou might want a more fine-grained approach where we can visualize the documents inside the topics to see\nif they were assigned correctly or whether they make sense. To do so, we can use the `topic_model.visualize_documents()`\nfunction. This function recalculates the document embeddings and reduces them to 2-dimensional space for easier visualization\npurposes. This process can be quite expensive, so it is advised to adhere to the following pipeline:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train BERTopic\ntopic_model = BERTopic().fit(docs, embeddings)\n\n# Run the visualization with the original embeddings\ntopic_model.visualize_documents(docs, embeddings=embeddings)\n\n# Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_documents(docs, reduced_embeddings=reduced_embeddings)\n```\n\n\n\n\n!!! note\n The visualization above was generated with the additional parameter `hide_document_hover=True` which disables the\n option to hover over the individual points and see the content of the documents. This was done for demonstration purposes\n as saving all those documents in the visualization can be quite expensive and result in large files. However,\n it might be interesting to set `hide_document_hover=False` in order to hover over the points and see the content of the documents.\n\n### **Custom Hover**\n\nWhen you visualize the documents, you might not always want to see the complete document over hover. Many documents have shorter information that might be more interesting to visualize, such as its title. To create the hover based on a documents' title instead of its content, you can simply pass a variable (`titles`) containing the title for each document:\n\n```python\ntopic_model.visualize_documents(titles, reduced_embeddings=reduced_embeddings)\n```\n\n## **Visualize documents with DataMapPlot**\n\n`.visualize_document_datamap` provides an alternative way to visualize the documents inside the topics as a static [DataMapPlot](https://datamapplot.readthedocs.io/en/latest/intro_splash.html). Using the same pipeline as above, you can generate a DataMapPlot by running:\n\n```python\n\n# with the original embeddings\ntopic_model.visualize_document_datamap(docs, embeddings=embeddings)\n\n# with the reduced embeddings\ntopic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\n```\n\n

\n\n

\n\nOr if you want to save the resulting figure:\n\n```python\nfig = topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings)\nfig.savefig(\"path/to/file.png\", bbox_inches=\"tight\")\n```\n\n### Interactive DataMapPlot\n\nDataMapPlot has the amazing ability to also generate interactive plots. These plots generate HTML files that allow you zoom in on the generated topics and explore the data.\n\nUsage is straightforward, simply set `interactive=True`:\n\n```python\nfig = topic_model.visualize_document_datamap(docs, reduced_embeddings=reduced_embeddings, interactive=True)\n```\n\n\n\n## **Visualize Probabilities or Distribution**\n\nWe can generate the topic-document probability matrix by simply setting `calculate_probabilities=True` if a HDBSCAN model is used:\n\n```python\nfrom bertopic import BERTopic\ntopic_model = BERTopic(calculate_probabilities=True)\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nThe resulting `probs` variable contains the soft-clustering as done through HDBSCAN.\n\nIf a non-HDBSCAN model is used, we can estimate the topic distributions after training our model:\n\n```python\nfrom bertopic import BERTopic\n\ntopic_model = BERTopic()\ntopics, _ = topic_model.fit_transform(docs)\ntopic_distr, _ = topic_model.approximate_distribution(docs, min_similarity=0)\n```\n\nThen, we either pass the `probs` or `topic_distr` variable to `.visualize_distribution` to visualize either the probability distributions or the topic distributions:\n\n```python\n# To visualize the probabilities of topic assignment\ntopic_model.visualize_distribution(probs[0])\n\n# To visualize the topic distributions in a document\ntopic_model.visualize_distribution(topic_distr[0])\n```\n\n\n\nAlthough a topic distribution is nice, we may want to see how each token contributes to a specific topic. To do so, we need to first calculate topic distributions on a token level and then visualize the results:\n\n```python\n# Calculate the topic distributions on a token-level\ntopic_distr, topic_token_distr = topic_model.approximate_distribution(docs, calculate_tokens=True)\n\n# Visualize the token-level distributions\ndf = topic_model.visualize_approximate_distribution(docs[1], topic_token_distr[1])\ndf\n```\n\n

\n\n

\n\n!!! note\n To get the stylized dataframe for `.visualize_approximate_distribution` you will need to have Jinja installed. If you do not have this installed, an unstylized dataframe will be returned instead. You can install Jinja via `pip install jinja2`\n\n!!! note\n The distribution of the probabilities does not give an indication to\n the distribution of the frequencies of topics across a document. It merely shows\n how confident BERTopic is that certain topics can be found in a document.\n" }, { "path": "docs/getting_started/visualization/visualize_hierarchy.md", "content": "The topics that you create can be hierarchically reduced. In order to understand the potential hierarchical\nstructure of the topics, we can use `scipy.cluster.hierarchy` to create clusters and visualize how\nthey relate to one another. This might help to select an appropriate `nr_topics` when reducing the number\nof topics that you have created. To visualize this hierarchy, run the following:\n\n```python\ntopic_model.visualize_hierarchy()\n```\n\n\n\n!!! note\n Do note that this is not the actual procedure of `.reduce_topics()` when `nr_topics` is set to\n auto since HDBSCAN is used to automatically extract topics. The visualization above closely resembles\n the actual procedure of `.reduce_topics()` when any number of `nr_topics` is selected.\n\n### **Hierarchical labels**\n\nAlthough visualizing this hierarchy gives us information about the structure, it would be helpful to see what happens\nto the topic representations when merging topics. To do so, we first need to calculate the representations of the\nhierarchical topics:\n\n\nFirst, we train a basic BERTopic model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))[\"data\"]\ntopic_model = BERTopic(verbose=True)\ntopics, probs = topic_model.fit_transform(docs)\nhierarchical_topics = topic_model.hierarchical_topics(docs)\n```\n\nTo visualize these results, we simply need to pass the resulting `hierarchical_topics` to our `.visualize_hierarchy` function:\n\n```python\ntopic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)\n```\n\n\n\nIf you **hover** over the black circles, you will see the topic representation at that level of the hierarchy. These representations\nhelp you understand the effect of merging certain topics. Some might be logical to merge whilst others might not. Moreover,\nwe can now see which sub-topics can be found within certain larger themes.\n\n### **Text-based topic tree**\n\nAlthough this gives a nice overview of the potential hierarchy, hovering over all black circles can be tiresome. Instead, we can\nuse `topic_model.get_topic_tree` to create a text-based representation of this hierarchy. Although the general structure is more difficult\nto view, we can see better which topics could be logically merged:\n\n```python\n>>> tree = topic_model.get_topic_tree(hierarchical_topics)\n>>> print(tree)\n.\n└─atheists_atheism_god_moral_atheist\n β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n └─■──moral_morality_objective_immoral_morals ── Topic: 29\n```\n\n
\n Click here to view the full tree.\n\n ```bash\n .\n β”œβ”€people_armenian_said_god_armenians\n β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”œβ”€god_jesus_jehovah_lord_christ\n β”‚ β”‚ β”‚ β”œβ”€jehovah_lord_mormon_mcconkie_god\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ra_satan_thou_god_lucifer ── Topic: 94\n β”‚ β”‚ β”‚ β”‚ └─■──jehovah_lord_mormon_mcconkie_unto ── Topic: 78\n β”‚ β”‚ β”‚ └─jesus_mary_god_hell_sin\n β”‚ β”‚ β”‚ β”œβ”€jesus_hell_god_eternal_heaven\n β”‚ β”‚ β”‚ β”‚ β”œβ”€hell_jesus_eternal_god_heaven\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jesus_tomb_disciples_resurrection_john ── Topic: 69\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──hell_eternal_god_jesus_heaven ── Topic: 53\n β”‚ β”‚ β”‚ β”‚ └─■──aaron_baptism_sin_law_god ── Topic: 89\n β”‚ β”‚ β”‚ └─■──mary_sin_maria_priest_conception ── Topic: 56\n β”‚ β”‚ └─■──marriage_married_marry_ceremony_marriages ── Topic: 110\n β”‚ └─people_armenian_armenians_said_mr\n β”‚ β”œβ”€people_armenian_armenians_said_israel\n β”‚ β”‚ β”œβ”€god_homosexual_homosexuality_atheists_sex\n β”‚ β”‚ β”‚ β”œβ”€homosexual_homosexuality_sex_gay_homosexuals\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€kinsey_sex_gay_men_sexual ── Topic: 44\n β”‚ β”‚ β”‚ β”‚ └─homosexuality_homosexual_sin_homosexuals_gay\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gay_homosexual_homosexuals_sexual_cramer ── Topic: 50\n β”‚ β”‚ β”‚ β”‚ └─■──homosexuality_homosexual_sin_paul_sex ── Topic: 27\n β”‚ β”‚ β”‚ └─god_atheists_atheism_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€jim_context_challenges_articles_quote ── Topic: 36\n β”‚ β”‚ β”‚ β”‚ └─islam_quran_judas_islamic_book\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€islam_quran_islamic_rushdie_muslims ── Topic: 31\n β”‚ β”‚ β”‚ β”‚ └─■──judas_scripture_bible_books_greek ── Topic: 33\n β”‚ β”‚ β”‚ └─atheists_atheism_god_moral_atheist\n β”‚ β”‚ β”‚ β”œβ”€atheists_atheism_god_atheist_argument\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€atheists_atheism_god_atheist_argument ── Topic: 21\n β”‚ β”‚ β”‚ β”‚ └─■──br_god_exist_genetic_existence ── Topic: 124\n β”‚ β”‚ β”‚ └─■──moral_morality_objective_immoral_morals ── Topic: 29\n β”‚ β”‚ └─armenian_armenians_people_israel_said\n β”‚ β”‚ β”œβ”€armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€tax_rights_government_income_taxes\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rights_right_slavery_slaves_residence ── Topic: 106\n β”‚ β”‚ β”‚ β”‚ └─tax_government_taxes_income_libertarians\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€government_libertarians_libertarian_regulation_party ── Topic: 58\n β”‚ β”‚ β”‚ β”‚ └─■──tax_taxes_income_billion_deficit ── Topic: 41\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_people_jews\n β”‚ β”‚ β”‚ β”œβ”€gun_guns_militia_firearms_amendment\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€blacks_penalty_death_cruel_punishment ── Topic: 55\n β”‚ β”‚ β”‚ β”‚ └─■──gun_guns_militia_firearms_amendment ── Topic: 7\n β”‚ β”‚ β”‚ └─armenian_armenians_israel_jews_turkish\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€israel_israeli_jews_arab_jewish ── Topic: 4\n β”‚ β”‚ β”‚ └─■──armenian_armenians_turkish_armenia_azerbaijan ── Topic: 15\n β”‚ β”‚ └─stephanopoulos_president_mr_myers_ms\n β”‚ β”‚ β”œβ”€β– β”€β”€serbs_muslims_stephanopoulos_mr_bosnia ── Topic: 35\n β”‚ β”‚ └─■──myers_stephanopoulos_president_ms_mr ── Topic: 87\n β”‚ └─batf_fbi_koresh_compound_gas\n β”‚ β”œβ”€β– β”€β”€reno_workers_janet_clinton_waco ── Topic: 77\n β”‚ └─batf_fbi_koresh_gas_compound\n β”‚ β”œβ”€batf_koresh_fbi_warrant_compound\n β”‚ β”‚ β”œβ”€β– β”€β”€batf_warrant_raid_compound_fbi ── Topic: 42\n β”‚ β”‚ └─■──koresh_batf_fbi_children_compound ── Topic: 61\n β”‚ └─■──fbi_gas_tear_bds_building ── Topic: 23\n └─use_like_just_dont_new\n β”œβ”€game_team_year_games_like\n β”‚ β”œβ”€game_team_games_25_year\n β”‚ β”‚ β”œβ”€game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€window_printer_use_problem_mhz\n β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_wire_simms_wiring_battery\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_mhz_battery_cpu_heat\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€simms_pds_simm_vram_lc\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€pds_nubus_lc_slot_card ── Topic: 119\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──simms_simm_vram_meg_dram ── Topic: 32\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─mhz_battery_cpu_heat_speed\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€mhz_cpu_speed_heat_fan\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€fan_cpu_heat_sink_fans ── Topic: 92\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──mhz_speed_cpu_fpu_clock ── Topic: 22\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──monitor_turn_power_computer_electricity ── Topic: 91\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─battery_batteries_concrete_duo_discharge\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€duo_battery_apple_230_problem ── Topic: 121\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──battery_batteries_concrete_discharge_temperature ── Topic: 75\n β”‚ β”‚ β”‚ β”‚ β”‚ └─wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€wire_wiring_ground_neutral_outlets\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€leds_uv_blue_light_boards ── Topic: 66\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──wire_wiring_ground_neutral_outlets ── Topic: 120\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─scope_scopes_phone_dial_number\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€dial_number_phone_line_output ── Topic: 93\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──scope_scopes_motorola_generator_oscilloscope ── Topic: 113\n β”‚ β”‚ β”‚ β”‚ β”‚ └─celp_dsp_sampling_antenna_digital\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€antenna_antennas_receiver_cable_transmitter ── Topic: 70\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──celp_dsp_sampling_speech_voice ── Topic: 52\n β”‚ β”‚ β”‚ β”‚ └─window_printer_xv_mouse_windows\n β”‚ β”‚ β”‚ β”‚ β”œβ”€window_xv_error_widget_problem\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€error_symbol_undefined_xterm_rx\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€symbol_error_undefined_doug_parse ── Topic: 63\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──rx_remote_server_xdm_xterm ── Topic: 45\n β”‚ β”‚ β”‚ β”‚ β”‚ └─window_xv_widget_application_expose\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€window_widget_expose_application_event\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€gc_mydisplay_draw_gxxor_drawing ── Topic: 103\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──window_widget_application_expose_event ── Topic: 25\n β”‚ β”‚ β”‚ β”‚ β”‚ └─xv_den_polygon_points_algorithm\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€den_polygon_points_algorithm_polygons ── Topic: 28\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──xv_24bit_image_bit_images ── Topic: 57\n β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_mouse_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€scanner_logitech_grayscale_ocr_scanman ── Topic: 108\n β”‚ β”‚ β”‚ β”‚ β”‚ └─printer_fonts_print_font_deskjet\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€printer_print_deskjet_hp_ink ── Topic: 18\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──fonts_font_truetype_tt_atm ── Topic: 49\n β”‚ β”‚ β”‚ β”‚ └─mouse_ghostscript_midi_driver_postscript\n β”‚ β”‚ β”‚ β”‚ β”œβ”€ghostscript_midi_postscript_files_file\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€ghostscript_postscript_pageview_ghostview_dsc ── Topic: 104\n β”‚ β”‚ β”‚ β”‚ β”‚ └─midi_sound_file_windows_driver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€location_mar_file_host_rwrr ── Topic: 83\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──midi_sound_driver_blaster_soundblaster ── Topic: 98\n β”‚ β”‚ β”‚ β”‚ └─■──mouse_driver_mice_ball_problem ── Topic: 68\n β”‚ β”‚ β”‚ └─game_team_games_25_season\n β”‚ β”‚ β”‚ β”œβ”€1st_sale_condition_comics_hulk\n β”‚ β”‚ β”‚ β”‚ β”œβ”€sale_condition_offer_asking_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€condition_stereo_amp_speakers_asking\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€miles_car_amfm_toyota_cassette ── Topic: 62\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──amp_speakers_condition_stereo_audio ── Topic: 24\n β”‚ β”‚ β”‚ β”‚ β”‚ └─games_sale_pom_cds_shipping\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€pom_cds_sale_shipping_cd\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€size_shipping_sale_condition_mattress ── Topic: 100\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──pom_cds_cd_sale_picture ── Topic: 37\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──games_game_snes_sega_genesis ── Topic: 40\n β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”œβ”€1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€lens_tape_camera_backup_lenses\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€tape_backup_tapes_drive_4mm ── Topic: 107\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──lens_camera_lenses_zoom_pouch ── Topic: 114\n β”‚ β”‚ β”‚ β”‚ β”‚ └─1st_hulk_comics_art_appears\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€1st_hulk_comics_art_appears ── Topic: 105\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──books_book_cover_trek_chemistry ── Topic: 125\n β”‚ β”‚ β”‚ β”‚ └─tickets_hotel_ticket_voucher_package\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hotel_voucher_package_vacation_room ── Topic: 74\n β”‚ β”‚ β”‚ β”‚ └─■──tickets_ticket_june_airlines_july ── Topic: 84\n β”‚ β”‚ β”‚ └─game_team_games_season_hockey\n β”‚ β”‚ β”‚ β”œβ”€game_hockey_team_25_550\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€espn_pt_pts_game_la ── Topic: 17\n β”‚ β”‚ β”‚ β”‚ └─■──team_25_game_hockey_550 ── Topic: 2\n β”‚ β”‚ β”‚ └─■──year_game_hit_baseball_players ── Topic: 0\n β”‚ β”‚ └─bike_car_greek_insurance_msg\n β”‚ β”‚ β”œβ”€car_bike_insurance_cars_engine\n β”‚ β”‚ β”‚ β”œβ”€car_insurance_cars_radar_engine\n β”‚ β”‚ β”‚ β”‚ β”œβ”€insurance_health_private_care_canada\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€insurance_health_private_care_canada ── Topic: 99\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──insurance_car_accident_rates_sue ── Topic: 82\n β”‚ β”‚ β”‚ β”‚ └─car_cars_radar_engine_detector\n β”‚ β”‚ β”‚ β”‚ β”œβ”€car_radar_cars_detector_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€radar_detector_detectors_ka_alarm ── Topic: 39\n β”‚ β”‚ β”‚ β”‚ β”‚ └─car_cars_mustang_ford_engine\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clutch_shift_shifting_transmission_gear ── Topic: 88\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──car_cars_mustang_ford_v8 ── Topic: 14\n β”‚ β”‚ β”‚ β”‚ └─oil_diesel_odometer_diesels_car\n β”‚ β”‚ β”‚ β”‚ β”œβ”€odometer_oil_sensor_car_drain\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€odometer_sensor_speedo_gauge_mileage ── Topic: 96\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──oil_drain_car_leaks_taillights ── Topic: 102\n β”‚ β”‚ β”‚ β”‚ └─■──diesel_diesels_emissions_fuel_oil ── Topic: 79\n β”‚ β”‚ β”‚ └─bike_riding_ride_bikes_motorcycle\n β”‚ β”‚ β”‚ β”œβ”€bike_ride_riding_bikes_lane\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€bike_ride_riding_lane_car ── Topic: 11\n β”‚ β”‚ β”‚ β”‚ └─■──bike_bikes_miles_honda_motorcycle ── Topic: 19\n β”‚ β”‚ β”‚ └─■──countersteering_bike_motorcycle_rear_shaft ── Topic: 46\n β”‚ β”‚ └─greek_msg_kuwait_greece_water\n β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_water\n β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_msg_kuwait_greece_dog\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€greek_kuwait_greece_turkish_greeks\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€greek_greece_turkish_greeks_cyprus ── Topic: 71\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kuwait_iraq_iran_gulf_arabia ── Topic: 76\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_dog_drugs_drug_food\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€clinton_bush_quayle_reagan_panicking ── Topic: 101\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─dog_dogs_cooper_trial_weaver\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€cooper_trial_weaver_spence_witnesses ── Topic: 90\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──dog_dogs_bike_trained_springer ── Topic: 67\n β”‚ β”‚ β”‚ β”‚ β”‚ └─msg_drugs_drug_food_chinese\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€msg_food_chinese_foods_taste ── Topic: 30\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──drugs_drug_marijuana_cocaine_alcohol ── Topic: 72\n β”‚ β”‚ β”‚ β”‚ └─water_theory_universe_science_larsons\n β”‚ β”‚ β”‚ β”‚ β”œβ”€water_nuclear_cooling_steam_dept\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€rocketry_rockets_engines_nuclear_plutonium ── Topic: 115\n β”‚ β”‚ β”‚ β”‚ β”‚ └─water_cooling_steam_dept_plants\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€water_dept_phd_environmental_atmospheric ── Topic: 97\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──cooling_water_steam_towers_plants ── Topic: 109\n β”‚ β”‚ β”‚ β”‚ └─theory_universe_larsons_larson_science\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€theory_universe_larsons_larson_science ── Topic: 54\n β”‚ β”‚ β”‚ β”‚ └─■──oort_cloud_grbs_gamma_burst ── Topic: 80\n β”‚ β”‚ β”‚ └─helmet_kirlian_photography_lock_wax\n β”‚ β”‚ β”‚ β”œβ”€helmet_kirlian_photography_leaf_mask\n β”‚ β”‚ β”‚ β”‚ β”œβ”€kirlian_photography_leaf_pictures_deleted\n β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€deleted_joke_stuff_maddi_nickname\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€joke_maddi_nickname_nicknames_frank ── Topic: 43\n β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ └─■──deleted_stuff_bookstore_joke_motto ── Topic: 81\n β”‚ β”‚ β”‚ β”‚ β”‚ └─■──kirlian_photography_leaf_pictures_aura ── Topic: 85\n β”‚ β”‚ β”‚ β”‚ └─helmet_mask_liner_foam_cb\n β”‚ β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€helmet_liner_foam_cb_helmets ── Topic: 112\n β”‚ β”‚ β”‚ β”‚ └─■──mask_goalies_77_santore_tl ── Topic: 123\n β”‚ β”‚ β”‚ └─lock_wax_paint_plastic_ear\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€lock_cable_locks_bike_600 ── Topic: 117\n β”‚ β”‚ β”‚ └─wax_paint_ear_plastic_skin\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€wax_paint_plastic_scratches_solvent ── Topic: 65\n β”‚ β”‚ β”‚ └─■──ear_wax_skin_greasy_acne ── Topic: 116\n β”‚ β”‚ └─m4_mp_14_mw_mo\n β”‚ β”‚ β”œβ”€m4_mp_14_mw_mo\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€m4_mp_14_mw_mo ── Topic: 111\n β”‚ β”‚ β”‚ └─■──test_ensign_nameless_deane_deanebinahccbrandeisedu ── Topic: 118\n β”‚ β”‚ └─■──ites_cheek_hello_hi_ken ── Topic: 3\n β”‚ └─space_medical_health_disease_cancer\n β”‚ β”œβ”€medical_health_disease_cancer_patients\n β”‚ β”‚ β”œβ”€β– β”€β”€cancer_centers_center_medical_research ── Topic: 122\n β”‚ β”‚ └─health_medical_disease_patients_hiv\n β”‚ β”‚ β”œβ”€patients_medical_disease_candida_health\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€candida_yeast_infection_gonorrhea_infections ── Topic: 48\n β”‚ β”‚ β”‚ └─patients_disease_cancer_medical_doctor\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€hiv_medical_cancer_patients_doctor ── Topic: 34\n β”‚ β”‚ β”‚ └─■──pain_drug_patients_disease_diet ── Topic: 26\n β”‚ β”‚ └─■──health_newsgroup_tobacco_vote_votes ── Topic: 9\n β”‚ └─space_launch_nasa_shuttle_orbit\n β”‚ β”œβ”€space_moon_station_nasa_launch\n β”‚ β”‚ β”œβ”€β– β”€β”€sky_advertising_billboard_billboards_space ── Topic: 59\n β”‚ β”‚ └─■──space_station_moon_redesign_nasa ── Topic: 16\n β”‚ └─space_mission_hst_launch_orbit\n β”‚ β”œβ”€space_launch_nasa_orbit_propulsion\n β”‚ β”‚ β”œβ”€β– β”€β”€space_launch_nasa_propulsion_astronaut ── Topic: 47\n β”‚ β”‚ └─■──orbit_km_jupiter_probe_earth ── Topic: 86\n β”‚ └─■──hst_mission_shuttle_orbit_arrays ── Topic: 60\n └─drive_file_key_windows_use\n β”œβ”€key_file_jpeg_encryption_image\n β”‚ β”œβ”€key_encryption_clipper_chip_keys\n β”‚ β”‚ β”œβ”€β– β”€β”€key_clipper_encryption_chip_keys ── Topic: 1\n β”‚ β”‚ └─■──entry_file_ripem_entries_key ── Topic: 73\n β”‚ └─jpeg_image_file_gif_images\n β”‚ β”œβ”€motif_graphics_ftp_available_3d\n β”‚ β”‚ β”œβ”€motif_graphics_openwindows_ftp_available\n β”‚ β”‚ β”‚ β”œβ”€β– β”€β”€openwindows_motif_xview_windows_mouse ── Topic: 20\n β”‚ β”‚ β”‚ └─■──graphics_widget_ray_3d_available ── Topic: 95\n β”‚ β”‚ └─■──3d_machines_version_comments_contact ── Topic: 38\n β”‚ └─jpeg_image_gif_images_format\n β”‚ β”œβ”€β– β”€β”€gopher_ftp_files_stuffit_images ── Topic: 51\n β”‚ └─■──jpeg_image_gif_format_images ── Topic: 13\n └─drive_db_card_scsi_windows\n β”œβ”€db_windows_dos_mov_os2\n β”‚ β”œβ”€β– β”€β”€copy_protection_program_software_disk ── Topic: 64\n β”‚ └─■──db_windows_dos_mov_os2 ── Topic: 8\n └─drive_card_scsi_drives_ide\n β”œβ”€drive_scsi_drives_ide_disk\n β”‚ β”œβ”€β– β”€β”€drive_scsi_drives_ide_disk ── Topic: 6\n β”‚ └─■──meg_sale_ram_drive_shipping ── Topic: 12\n └─card_modem_monitor_video_drivers\n β”œβ”€β– β”€β”€card_monitor_video_drivers_vga ── Topic: 5\n └─■──modem_port_serial_irq_com ── Topic: 10\n ```\n
\n\n## **Visualize Hierarchical Documents**\nWe can extend the previous method by calculating the topic representation at different levels of the hierarchy and\nplotting them on a 2D plane. To do so, we first need to calculate the hierarchical topics:\n\n```python\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom bertopic import BERTopic\nfrom umap import UMAP\n\n# Prepare embeddings\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\nsentence_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\nembeddings = sentence_model.encode(docs, show_progress_bar=False)\n\n# Train BERTopic and extract hierarchical topics\ntopic_model = BERTopic().fit(docs, embeddings)\nhierarchical_topics = topic_model.hierarchical_topics(docs)\n```\nThen, we can visualize the hierarchical documents by either supplying it with our embeddings or by\nreducing their dimensionality ourselves:\n\n```python\n# Run the visualization with the original embeddings\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, embeddings=embeddings)\n\n# Reduce dimensionality of embeddings, this step is optional but much faster to perform iteratively:\nreduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric='cosine').fit_transform(embeddings)\ntopic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings)\n```\n\n\n\n!!! note\n The visualization above was generated with the additional parameter `hide_document_hover=True` which disables the\n option to hover over the individual points and see the content of the documents. This makes the resulting visualization\n smaller and fit into your RAM. However, it might be interesting to set `hide_document_hover=False` to hover\n over the points and see the content of the documents.\n" }, { "path": "docs/getting_started/visualization/visualize_terms.md", "content": "We can visualize the selected terms for a few topics by creating bar charts out of the c-TF-IDF scores\nfor each topic representation. Insights can be gained from the relative c-TF-IDF scores between and within\ntopics. Moreover, you can easily compare topic representations to each other.\nTo visualize this hierarchy, run the following:\n\n```python\ntopic_model.visualize_barchart()\n```\n\n\n\n\n## **Visualize Term Score Decline**\nTopics are represented by a number of words starting with the best representative word.\nEach word is represented by a c-TF-IDF score. The higher the score, the more representative a word\nto the topic is. Since the topic words are sorted by their c-TF-IDF score, the scores slowly decline\nwith each word that is added. At some point adding words to the topic representation only marginally\nincreases the total c-TF-IDF score and would not be beneficial for its representation.\n\nTo visualize this effect, we can plot the c-TF-IDF scores for each topic by the term rank of each word.\nIn other words, the position of the words (term rank), where the words with\nthe highest c-TF-IDF score will have a rank of 1, will be put on the x-axis. Whereas the y-axis\nwill be populated by the c-TF-IDF scores. The result is a visualization that shows you the decline\nof c-TF-IDF score when adding words to the topic representation. It allows you, using the elbow method,\nthe select the best number of words in a topic.\n\nTo visualize the c-TF-IDF score decline, run the following:\n\n```python\ntopic_model.visualize_term_rank()\n```\n\n\n\nTo enable the log scale on the y-axis for a better view of individual topics, run the following:\n\n```python\ntopic_model.visualize_term_rank(log_scale=True)\n```\n\n\n\nThis visualization was heavily inspired by the \"Term Probability Decline\" visualization found in an\nanalysis by the amazing [tmtoolkit](https://tmtoolkit.readthedocs.io/).\nReference to that specific analysis can be found\n[here](https://wzbsocialsciencecenter.github.io/tm_corona/tm_analysis.html).\n" }, { "path": "docs/getting_started/visualization/visualize_topics.md", "content": "Visualizing BERTopic and its derivatives is important in understanding the model, how it works, and more importantly, where it works.\nSince topic modeling can be quite a subjective field it is difficult for users to validate their models. Looking at the topics and seeing\nif they make sense is an important factor in alleviating this issue.\n\n## **Visualize Topics**\nAfter having trained our `BERTopic` model, we can iteratively go through hundreds of topics to get a good\nunderstanding of the topics that were extracted. However, that takes quite some time and lacks a global representation.\nInstead, we can visualize the topics that were generated in a way very similar to\n[LDAvis](https://github.com/cpsievert/LDAvis).\n\nWe embed our c-TF-IDF representation of the topics in 2D using Umap and then visualize the two dimensions using\nplotly such that we can create an interactive view.\n\nFirst, we need to train our model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nThen, we can call `.visualize_topics` to create a 2D representation of your topics. The resulting graph is a\nplotly interactive graph which can be converted to HTML:\n\n```python\ntopic_model.visualize_topics()\n```\n\n\n\nYou can use the slider to select the topic which then lights up red. If you hover over a topic, then general\ninformation is given about the topic, including the size of the topic and its corresponding words.\n\n\n## **Visualize Topic Similarity**\nHaving generated topic embeddings, through both c-TF-IDF and embeddings, we can create a similarity\nmatrix by simply applying cosine similarities through those topic embeddings. The result will be a\nmatrix indicating how similar certain topics are to each other.\nTo visualize the heatmap, run the following:\n\n```python\ntopic_model.visualize_heatmap()\n```\n\n\n\n\n!!! note\n You can set `n_clusters` in `visualize_heatmap` to order the topics by their similarity.\n This will result in blocks being formed in the heatmap indicating which clusters of topics are\n similar to each other. This step is very much recommended as it will make reading the heatmap easier.\n\n## **Visualize Topics over Time**\nAfter creating topics over time with Dynamic Topic Modeling, we can visualize these topics by\nleveraging the interactive abilities of Plotly. Plotly allows us to show the frequency\nof topics over time whilst giving the option of hovering over the points to show the time-specific topic representations.\nSimply call `.visualize_topics_over_time` with the newly created topics over time:\n\n\n```python\nimport re\nimport pandas as pd\nfrom bertopic import BERTopic\n\n# Prepare data\ntrump = pd.read_csv('https://drive.google.com/uc?export=download&id=1xRKHaP-QwACMydlDnyFPEaFdtskJuBa6')\ntrump.text = trump.apply(lambda row: re.sub(r\"http\\S+\", \"\", row.text).lower(), 1)\ntrump.text = trump.apply(lambda row: \" \".join(filter(lambda x:x[0]!=\"@\", row.text.split())), 1)\ntrump.text = trump.apply(lambda row: \" \".join(re.sub(\"[^a-zA-Z]+\", \" \", row.text).split()), 1)\ntrump = trump.loc[(trump.isRetweet == \"f\") & (trump.text != \"\"), :]\ntimestamps = trump.date.to_list()\ntweets = trump.text.to_list()\n\n# Create topics over time\nmodel = BERTopic(verbose=True)\ntopics, probs = model.fit_transform(tweets)\ntopics_over_time = model.topics_over_time(tweets, timestamps)\n```\n\nThen, we visualize some interesting topics:\n\n```python\nmodel.visualize_topics_over_time(topics_over_time, topics=[9, 10, 72, 83, 87, 91])\n```\n\n\n## **Visualize Topics per Class**\nYou might want to extract and visualize the topic representation per class. For example, if you have\nspecific groups of users that might approach topics differently, then extracting them would help understanding\nhow these users talk about certain topics. In other words, this is simply creating a topic representation for\ncertain classes that you might have in your data.\n\nFirst, we need to train our model:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\n# Prepare data and classes\ndata = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))\ndocs = data[\"data\"]\nclasses = [data[\"target_names\"][i] for i in data[\"target\"]]\n\n# Create topic model and calculate topics per class\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\ntopics_per_class = topic_model.topics_per_class(docs, classes=classes)\n```\n\nThen, we visualize the topic representation of major topics per class:\n\n```python\ntopic_model.visualize_topics_per_class(topics_per_class)\n```\n\n\n" }, { "path": "docs/getting_started/visualization/viz.html", "content": "\n\n\n
\n
\n\n" }, { "path": "docs/getting_started/zeroshot/zeroshot.md", "content": "Zero-shot Topic Modeling is a technique that allows you to find topics in large amounts of documents that were predefined. When faced with many documents, you often have an idea of which topics will definitely be in there. Whether that is a result of simply knowing your data or if a domain expert is involved in defining those topics.\n\nThis method allows you to not only find those specific topics but also create new topics for documents that would not fit with your predefined topics.\nThis allows for extensive flexibility as there are three scenario's to explore:\n\n* First, both zero-shot topics and clustered topics were detected. This means that some documents would fit with the predefined topics where others would not. For the latter, new topics were found.\n* Second, only zero-shot topics were detected. Here, we would not need to find additional topics since all original documents were assigned to one of the predefined topics.\n* Third, no zero-shot topics were detected. This means that none of the documents would fit with the predefined topics and a regular BERTopic would be run.\n\n
\n--8<-- \"docs/getting_started/zeroshot/zeroshot.svg\"\n
\n\nThis method works as follows. First, we create a number of labels for our predefined topics and embed them using any embedding model. Then, we compare the embeddings of the documents with the predefined labels using cosine similarity. If they pass a user-defined threshold, the zero-shot topic is assigned to a document. If it does not, then that document, along with others, will follow the regular BERTopic pipeline and attempt to find clusters that do not fit with the zero-shot topics.\n\n### **Example**\nIn order to use zero-shot BERTopic, we create a list of topics that we want to assign to our documents. However,\nthere may be several other topics that we know should be in the documents. The dataset that we use is small subset of ArXiv papers.\nWe know the data and believe there to be at least the following topics: *clustering*, *topic modeling*, and *large language models*.\nHowever, we are not sure whether other topics exist and want to explore those.\n\nZero-shot BERTopic needs two parameters:\n* `zeroshot_topic_list` - The names of the topics to assign documents to. Making sure this is as descriptive as possible helps improve the assignment since they are based on cosine similarities between embeddings.\n* `zeroshot_min_similarity` - The minimum cosine similarity needed to match a document to a document. It is a value between 0 and 1.\n\n\nUsing this feature is straightforward:\n\n```python\nfrom datasets import load_dataset\n\nfrom bertopic import BERTopic\nfrom bertopic.representation import KeyBERTInspired\n\n# We select a subsample of 5000 abstracts from ArXiv\ndataset = load_dataset(\"CShorten/ML-ArXiv-Papers\")[\"train\"]\ndocs = dataset[\"abstract\"][:5_000]\n\n# We define a number of topics that we know are in the documents\nzeroshot_topic_list = [\"Clustering\", \"Topic Modeling\", \"Large Language Models\"]\n\n# We fit our model using the zero-shot topics\n# and we define a minimum similarity. For each document,\n# if the similarity does not exceed that value, it will be used\n# for clustering instead.\ntopic_model = BERTopic(\n embedding_model=\"thenlper/gte-small\",\n min_topic_size=15,\n zeroshot_topic_list=zeroshot_topic_list,\n zeroshot_min_similarity=.85,\n representation_model=KeyBERTInspired()\n)\ntopics, _ = topic_model.fit_transform(docs)\n```\n\nWhen we run `topic_model.get_topic_info()` you will see something like this:\n\n\n
\n\nThe `zeroshot_min_similarity` parameter controls how many of the documents are assigned to the predefined zero-shot topics. Lower this value and you will have more documents assigned to zero-shot topics and fewer documents will be clustered. Increase this value you will have fewer documents assigned to zero-shot topics and more documents will be clustered.\n\n!!! Note\n Setting the `zeroshot_min_similarity` parameter requires a bit of experimentation. Some embedding\n models have different similarity distributions, so trying out the values manually and exploring the results\n is highly advised.\n\n\n!!! Tip\n Because zero-shot topic modeling is essentially merging two different topic models, the\n `probs` will be empty initially. If you want to have the probabilities of topics across documents,\n you can run `topic_model.transform` on your documents to extract the updated `probs`.\n" }, { "path": "docs/img/probabilities.html", "content": "\n\n\n
\n\n" }, { "path": "docs/index.md", "content": "---\nhide:\n - navigation\n---\n\n# BERTopic\n\n\n\nBERTopic is a topic modeling technique that leverages πŸ€— transformers and c-TF-IDF to create dense clusters\nallowing for easily interpretable topics whilst keeping important words in the topic descriptions.\n\nBERTopic supports all kinds of topic modeling techniques:\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
GuidedSupervisedSemi-supervised
ManualMulti-topic distributionsHierarchical
Class-basedDynamicOnline/Incremental
MultimodalMulti-aspectText Generation/LLM
Zero-shot (new!)Merge Models (new!)Seed Words (new!)
\n\nCorresponding medium posts can be found [here](https://medium.com/data-science/topic-modeling-with-bert-779f7db187e6?sk=0b5a470c006d1842ad4c8a3057063a99\n), [here](https://medium.com/data-science/using-whisper-and-bertopic-to-model-kurzgesagts-videos-7d8a63139bdf?sk=b1e0fd46f70cb15e8422b4794a81161d\n) and [here](https://medium.com/data-science/interactive-topic-modeling-with-bertopic-1ea55e7d73d8?sk=03c2168e9e74b6bda2a1f3ed953427e4\n). For a more detailed overview, you can read the [paper](https://arxiv.org/abs/2203.05794) or see a [brief overview](https://maartengr.github.io/BERTopic/algorithm/algorithm.html).\n\n## **Installation**\n\nInstallation, with sentence-transformers, can be done using [pypi](https://pypi.org/project/bertopic/):\n\n```bash\npip install bertopic\n```\n\nYou may want to install more depending on the transformers and language backends that you will be using.\nThe possible installations are:\n\n```bash\n# Choose an embedding backend\npip install bertopic[flair, gensim, spacy, use]\n\n# Topic modeling with images\npip install bertopic[vision]\n```\n\n## **Quick Start**\nWe start by extracting topics from the well-known 20 newsgroups dataset containing English documents:\n\n```python\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\n\ndocs = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))['data']\n\ntopic_model = BERTopic()\ntopics, probs = topic_model.fit_transform(docs)\n```\n\nAfter generating topics and their probabilities, we can access the frequent topics that were generated:\n\n```python\n>>> topic_model.get_topic_info()\n\nTopic\tCount\tName\n-1\t 4630\t-1_can_your_will_any\n0\t 693\t 49_windows_drive_dos_file\n1\t 466\t 32_jesus_bible_christian_faith\n2\t 441\t 2_space_launch_orbit_lunar\n3\t 381\t 22_key_encryption_keys_encrypted\n```\n\n-1 refers to all outliers and should typically be ignored. Next, let's take a look at the most\nfrequent topic that was generated, topic 0:\n\n```python\n>>> topic_model.get_topic(0)\n\n[('windows', 0.006152228076250982),\n ('drive', 0.004982897610645755),\n ('dos', 0.004845038866360651),\n ('file', 0.004140142872194834),\n ('disk', 0.004131678774810884),\n ('mac', 0.003624848635985097),\n ('memory', 0.0034840976976789903),\n ('software', 0.0034415334250699077),\n ('email', 0.0034239554442333257),\n ('pc', 0.003047105930670237)]\n```\n\nUsing `.get_document_info`, we can also extract information on a document level, such as their corresponding topics, probabilities, whether they are representative documents for a topic, etc.:\n\n```python\n>>> topic_model.get_document_info(docs)\n\nDocument Topic\tName\t Top_n_words Probability ...\nI am sure some bashers of Pens...\t 0\t 0_game_team_games_season\tgame - team - games...\t 0.200010 ...\nMy brother is in the market for... -1 -1_can_your_will_any\t can - your - will...\t 0.420668 ...\nFinally you said what you dream...\t -1 -1_can_your_will_any\t can - your - will... 0.807259 ...\nThink! It is the SCSI card doing...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.071746 ...\n1) I have an old Jasmine drive...\t 49 49_windows_drive_dos_file\twindows - drive - docs...\t 0.038983 ...\n```\n\n!!! tip \"Multilingual\"\n\n Use `BERTopic(language=\"multilingual\")` to select a model that supports 50+ languages.\n\n\n## **Fine-tune Topic Representations**\n\nIn BERTopic, there are a number of different [topic representations](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) that we can choose from. They are all quite different from one another and give interesting perspectives and variations of topic representations. A great start is `KeyBERTInspired`, which for many users increases the coherence and reduces stopwords from the resulting topic representations:\n\n```python\nfrom bertopic.representation import KeyBERTInspired\n\n# Fine-tune your topic representations\nrepresentation_model = KeyBERTInspired()\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\nHowever, you might want to use something more powerful to describe your clusters. You can even use ChatGPT or other models from OpenAI to generate labels, summaries, phrases, keywords, and more:\n\n```python\nimport openai\nfrom bertopic.representation import OpenAI\n\n# Fine-tune topic representations with GPT\nclient = openai.OpenAI(api_key=\"sk-...\")\nrepresentation_model = OpenAI(client, model=\"gpt-4o-mini\", chat=True)\ntopic_model = BERTopic(representation_model=representation_model)\n```\n\n!!! tip \"Multi-aspect Topic Modeling\"\n Instead of iterating over all of these different topic representations, you can model them simultaneously with [multi-aspect topic representations](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) in BERTopic.\n\n\n## **Modularity**\n\nBy default, the main steps for topic modeling with BERTopic are sentence-transformers, UMAP, HDBSCAN, and c-TF-IDF run in sequence. However, it assumes some independence between these steps which makes BERTopic quite modular. In other words, BERTopic not only allows you to build your own topic model but to explore several topic modeling techniques on top of your customized topic model:\n\n\n\nYou can swap out any of these models or even remove them entirely. The following steps are completely modular:\n\n1. [Embedding](https://maartengr.github.io/BERTopic/getting_started/embeddings/embeddings.html) documents\n2. [Reducing dimensionality](https://maartengr.github.io/BERTopic/getting_started/dim_reduction/dim_reduction.html) of embeddings\n3. [Clustering](https://maartengr.github.io/BERTopic/getting_started/clustering/clustering.html) reduced embeddings into topics\n4. [Tokenization](https://maartengr.github.io/BERTopic/getting_started/vectorizers/vectorizers.html) of topics\n5. [Weight](https://maartengr.github.io/BERTopic/getting_started/ctfidf/ctfidf.html) tokens\n6. [Represent topics](https://maartengr.github.io/BERTopic/getting_started/representation/representation.html) with one or [multiple](https://maartengr.github.io/BERTopic/getting_started/multiaspect/multiaspect.html) representations\n\nTo find more about the underlying algorithm and assumptions [here](https://maartengr.github.io/BERTopic/algorithm/algorithm.html).\n\n\n## **Overview**\nBERTopic has many functions that quickly can become overwhelming. To alleviate this issue, you will find an overview\nof all methods and a short description of its purpose.\n\n### Common\nBelow, you will find an overview of common functions in BERTopic.\n\n| Method | Code |\n|-----------------------|---|\n| Fit the model | `.fit(docs)` |\n| Fit the model and predict documents | `.fit_transform(docs)` |\n| Predict new documents | `.transform([new_doc])` |\n| Access single topic | `.get_topic(topic=12)` |\n| Access all topics | `.get_topics()` |\n| Get topic freq | `.get_topic_freq()` |\n| Get all topic information| `.get_topic_info()` |\n| Get all document information| `.get_document_info(docs)` |\n| Get representative docs per topic | `.get_representative_docs()` |\n| Update topic representation | `.update_topics(docs, n_gram_range=(1, 3))` |\n| Generate topic labels | `.generate_topic_labels()` |\n| Set topic labels | `.set_topic_labels(my_custom_labels)` |\n| Merge topics | `.merge_topics(docs, topics_to_merge)` |\n| Reduce nr of topics | `.reduce_topics(docs, nr_topics=30)` |\n| Reduce outliers | `.reduce_outliers(docs, topics)` |\n| Find topics | `.find_topics(\"vehicle\")` |\n| Save model | `.save(\"my_model\", serialization=\"safetensors\")` |\n| Load model | `BERTopic.load(\"my_model\")` |\n| Get parameters | `.get_params()` |\n\n\n### Attributes\nAfter having trained your BERTopic model, several are saved within your model. These attributes, in part,\nrefer to how model information is stored on an estimator during fitting. The attributes that you see below all end in `_` and are\npublic attributes that can be used to access model information.\n\n| Attribute | Description |\n|------------------------|---------------------------------------------------------------------------------------------|\n| `.topics_` | The topics that are generated for each document after training or updating the topic model. |\n| `.probabilities_` | The probabilities that are generated for each document if HDBSCAN is used. |\n| `.topic_sizes_` | The size of each topic |\n| `.topic_mapper_` | A class for tracking topics and their mappings anytime they are merged/reduced. |\n| `.topic_representations_` | The top *n* terms per topic and their respective c-TF-IDF values. |\n| `.c_tf_idf_` | The topic-term matrix as calculated through c-TF-IDF. |\n| `.topic_aspects_` | The different aspects, or representations, of each topic. |\n| `.topic_labels_` | The default labels for each topic. |\n| `.custom_labels_` | Custom labels for each topic as generated through `.set_topic_labels`. |\n| `.topic_embeddings_` | The embeddings for each topic if `embedding_model` was used. |\n| `.representative_docs_` | The representative documents for each topic if HDBSCAN is used. |\n\n### Variations\nThere are many different use cases in which topic modeling can be used. As such, several variations of BERTopic have been developed such that one package can be used across many use cases.\n\n| Method | Code |\n|-----------------------|---|\n| [Topic Distribution Approximation](https://maartengr.github.io/BERTopic/getting_started/distribution/distribution.html) | `.approximate_distribution(docs)` |\n| [Online Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/online/online.html) | `.partial_fit(doc)` |\n| [Semi-supervised Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/semisupervised/semisupervised.html) | `.fit(docs, y=y)` |\n| [Supervised Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/supervised/supervised.html) | `.fit(docs, y=y)` |\n| [Manual Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/manual/manual.html) | `.fit(docs, y=y)` |\n| [Multimodal Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/multimodal/multimodal.html) | ``.fit(docs, images=images)`` |\n| [Topic Modeling per Class](https://maartengr.github.io/BERTopic/getting_started/topicsperclass/topicsperclass.html) | `.topics_per_class(docs, classes)` |\n| [Dynamic Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/topicsovertime/topicsovertime.html) | `.topics_over_time(docs, timestamps)` |\n| [Hierarchical Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/hierarchicaltopics/hierarchicaltopics.html) | `.hierarchical_topics(docs)` |\n| [Guided Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/guided/guided.html) | `BERTopic(seed_topic_list=seed_topic_list)` |\n| [Zero-shot Topic Modeling](https://maartengr.github.io/BERTopic/getting_started/zeroshot/zeroshot.html) | `BERTopic(zeroshot_topic_list=zeroshot_topic_list)` |\n| [Merge Multiple Models](https://maartengr.github.io/BERTopic/getting_started/merge/merge.html) | `BERTopic.merge_models([topic_model_1, topic_model_2])` |\n\n\n### Visualizations\nEvaluating topic models can be rather difficult due to the somewhat subjective nature of evaluation.\nVisualizing different aspects of the topic model helps in understanding the model and makes it easier\nto tweak the model to your liking.\n\n| Method | Code |\n|-----------------------|---|\n| Visualize Topics | `.visualize_topics()` |\n| Visualize Documents | `.visualize_documents()` |\n| Visualize Document with DataMapPlot | `.visualize_document_datamap()` |\n| Visualize Document Hierarchy | `.visualize_hierarchical_documents()` |\n| Visualize Topic Hierarchy | `.visualize_hierarchy()` |\n| Visualize Topic Tree | `.get_topic_tree(hierarchical_topics)` |\n| Visualize Topic Terms | `.visualize_barchart()` |\n| Visualize Topic Similarity | `.visualize_heatmap()` |\n| Visualize Term Score Decline | `.visualize_term_rank()` |\n| Visualize Topic Probability Distribution | `.visualize_distribution(probs[0])` |\n| Visualize Topics over Time | `.visualize_topics_over_time(topics_over_time)` |\n| Visualize Topics per Class | `.visualize_topics_per_class(topics_per_class)` |\n\n\n\n## **Citation**\nTo cite the [BERTopic paper](https://arxiv.org/abs/2203.05794), please use the following bibtex reference:\n\n```bibtext\n@article{grootendorst2022bertopic,\n title={BERTopic: Neural topic modeling with a class-based TF-IDF procedure},\n author={Grootendorst, Maarten},\n journal={arXiv preprint arXiv:2203.05794},\n year={2022}\n}\n```\n" }, { "path": "docs/stylesheets/extra.css", "content": ":root {\n --md-primary-fg-color: #234E70;\n\n }\n\n:root>* {\n--md-typeset-a-color: #016198;\n--md-text-link-color: #000000;\n}\n\nbody[data-md-color-primary=\"black\"] .svg_image svg {\n filter: invert(100%) hue-rotate(180deg);\n}\n\nbody[data-md-color-primary=\"black\"] .svg_image svg rect {\n fill: transparent;\n}\n\n.svg_image {\n text-align: center;\n width: 100%;\n}\n\n.center {\n display: block;\n margin: 0 auto;\n}\n" }, { "path": "docs/usecases.md", "content": "---\nhide:\n - navigation\n---\n\nOver the last few years, BERTopic has been used on a wide variety of use cases and domains, from cancer research and voice perception, to employee surveys and social media. This diversity allows for interesting use cases but it might quickly become overwhelming. This page is meant to demonstrate how, when, and why BERTopic is used in practice.\n\n
\n\n
\n\n## **Examples**\n\nBelow are a number of use cases that have been applied in practice. These use cases are collected from and written by data-professionals.\n\n!!! note\n If you would like to add your use case, feel free open up a PR! You only need to update [this file](https://github.com/MaartenGr/BERTopic/blob/master/docs/usecases.md) and add your example.\n You can just copy-paste one of the existing examples and adjust it contain a description of your use case.\n\n\n\n\n\n\n#### App User Feedback\n\n!!! quote \"\"\n

\n
\n \"Analyzing user reviews from the App Store and Play Store helps us reveal valuable customer information, fix technical or usability issues, and help constantly improve customer experience. We utilize BERTopic for topic modeling and supervised classification of predefined categories.\"\n
\n β€’β€’β€’
Tibor Fabian, Ph.D.
Lead/Master Data Scientist
TelefΓ³nica Germany
\n

\n\n#### Employee Surveys\n\n!!! quote \"\"\n

\n
\n \"We are using BERTopic to support analysis of employee surveys. Here, we use BERTopic to compute the topics of discussion found in employee responses to open-ended survey questions. To further understand how employees feel about certain topics, we combined BERTopic with sentiment analysis to identify the sentiments associated with different topics and vice versa.\"\n
\n β€’β€’β€’
Steve Quirolgico, Ph.D.
Principal Engineer
U.S. Department of Homeland Security
\n

\n\n#### Voice Perception\n\n!!! quote \"\"\n

\n
\n \"A research project on voice perception to categorize what people describe when they make first impressions based on hearing people say, \"Hi\".\" [preprint](https://psyarxiv.com/9ysqx/) | [code](https://osf.io/5t4b7/)
\n β€’β€’β€’
David Feinberg
Associate Professor
McMaster University
\n

\n\n#### Social Media\n\n!!! quote \"\"\n

\n
\n \"We use BERTopic to detect trending topics in social media, Our product (AIM Insights) is a social media monitoring tool so detecting trending topics in social media helps our clients to capitalize on them for their campaigns.

\n We use BERTopic to group social media posts into clusters, sort them by engagement to detect the ones that are trending, and then use OpenAI's GPT-3 to generate a label for each of the top clusters based on the most relevant documents in it. This is all done on Arabic posts using an in-house sentence embeddings model.\"
\n β€’β€’β€’
Ahmed Rashwan
AI lead
AIM Technologies
\n

\n\n#### IT Service Management\n\n!!! quote \"\"\n

\n


\n \"In IT Service Management systems (e.g., Service Now) we receive Incidents, Problems, Change requests etc. We use BERTopic to categorize them into a group of topics/clusters to understand the distribution of the work requests over the period of time to plan and act accordingly for the future.\"
\n β€’β€’β€’
Rajesh Thanaseelan
Data Science Consultant
DXC Technology
\n

\n\n#### Colon Cancer\n\n!!! quote \"\"\n

\n
\n

\n \"We use BERTopic to evaluate P53 in Ovarian cancer for Computational backgrounds researchers, who find it easier to relate Artificial Intelligence with advancing the transformer model and unstructured medical data. The paper explores the heterogeneity of keyBERT, BERTopic, PyCaret, and LDAs as key phrase generators and topic model extractors, with P53 in ovarian cancer as a use case.\"
\n β€’β€’β€’
\n Mary Adewunmi
PhD Student in Colon Cancer and AI
UTAS
\n

\n\n#### Telephone Help Line\n\n!!! quote \"\"\n

\n
\n \"We analyzed 100K+ phone call memos from a telephone help line. The Help Line is open to all people, regardless of religion, culture, and origin. It follows the principles of IFOTES (International Federation Of Telephone Emergency Services). The regional offices each offer independent counseling services via telephone or online.

\n The phone call memos are written by hundreds of independent volunteers and come in various shapes, lengths, forms, and wordings - additionally to have them in multiple languages. While using BERTopic we ran a few tests to figure out if the topic modeling works. Selecting only one language with ~60K data points and a mixed language model we achieved good results. It helped identify topics within the calls and therefore show the organization what reasons there are for people calling them. We identified in a workshop a few interesting topics, which they were not aware of, for example, religious topics.

\n The identification of existing and new, arising topics is crucial for the service quality of the organization. It furthermore helps detect trends over time, which can then be reported directly to Public Health institutions, which can then come up with campaigns to inform the public and help reduce certain psychological concerns. It acts as a representative psychological health barometer of the population.\"
\n β€’β€’β€’
Kevin Kuhn
Chief Executive Officer
gopf
\n

\n\n#### Regional Newspaper\n\n!!! quote \"\"\n

\n

\n \"Recently, we wanted to evaluate our overall section structure, especially our local news section. As you can imagine, local news is quite a big part of what we do in a regional newspaper. We used BERTopic on a year's worth of local news data to explore the topics in local news and define a new section structure. The results from this analysis helped to define the new section structure, which was implemented this month. \"
\n β€’β€’β€’
Thomas Husken
Data Scientist
Bergens Tidende
\n

\n\n#### Intelligent Virtual Assistants\n\n!!! quote \"\"\n

\n \"We have been using BERTopic as an early step in our exploratory analysis for intelligent virtual assistants. It helps us get a quick read on what some of the intents may be. The results help in the design discussions with customers.\"
\n β€’β€’β€’
Stephen Drew
VP, AI and Automation Solutions
Five9
\n

\n\n#### Electronic Health Records\n\n!!! quote \"\"\n

\n


\n \"Given physician-created documents from hospitals, find themes in the text as well as differentiate between \"relevant\" and \"irrelevant\" text, and disambiguate homonyms. \"
\n β€’β€’β€’
\n Alexis Raykhel
Senior NLP Engineer
Iodine Software
\n

\n\n#### Teaching\n\n!!! quote \"\"\n

\n \"BERTopic was used to determine a taxonomy of climate change risks discussed in financial news, and to compute firms' related exposure. It was used in a context a course offering on Climate Risks modelling with NLP.\"
\n β€’β€’β€’
\n Thomas Lorans
Senior Associate, Quantitative Analyst
\n

\n\n#### Zero Hunger Lab\n\n!!! quote \"\"\n

\n

\n \"I am a PhD student at Tilburg University, at a lab called Zero Hunger Lab, where we try to use data science methods to improve food insecurity. One key issue is classifying and predicting food insecurity in food-insecure nations. The Integrated Food Security Phase Classification (IPC) system serves this purpose. The IPC categorizes food insecurity into five phases, ranging from minimal food insecurity to famine, and serves as a guide for directing humanitarian resources to the most affected regions.

\n The IPC system strives to be based on evidence, however, obtaining accurate information about food insecurity in remote regions can prove challenging. Despite the availability of weather data, data in the socio-economic domain, such as food prices and conflict, can be scarce or unreliable due to limited infrastructure and bureaucratic obstacles. These complications often result in infrequent releases of IPC classifications and projections, making it difficult to effectively respond to food insecurity in these areas.

\n One large source of daily-updated information is local news. Thus, one can build a model that classifies/predicts IPC by relying on news features obtained by NLP methods in addition to stuff like weather data. Previous research shows this is possible (see https://arxiv.org/pdf/2111.15602.pdf). The authors find words related to food insecurity using semantic frame parsing. After which, they count the occurrence of these words to create features. The features are put into a linear classifier. We wanted to apply more advanced methods and use local news sources (which we suppose contain more localized information). We used BERTopic on over a million articles scraped from Somali news websites. Because articles are both in English and Somali, we use a multilingual sentence encoder (LaBSE, which outperforms newer models in Somali). The results are quite nice. For example, topics most strongly correlated with known conflict casualty data are topics about terrorist attacks, car bombings, etc. And topics most negatively correlated with known conflict casualty data are about peace talks. We can also get an indication of food price development and forced migration. Most importantly, we can track the development of topics relating to food insecurity over time. While topic modelling cannot replace evidence-based food insecurity assessment, it can give a quick insight into a local situation when 'hard data' is lacking.

\n I applaud you on your success with BERTopic. The package is incredibly clean and easy to use, and the method works well with little parameter tuning. To me, the fact that you were able to deliver such a useful tool on your own is incredible, especially in the field of NLP, which is dominated by large organizations such as Google and Meta. \"
\n β€’β€’β€’
Cascha van Wanrooij
PhD Student
Tilburg University
\n

\n\n\n## **Papers**\n\nBERTopic has also been adopted more and more in the academic field. Here are a few from all different kinds of research domains with interesting applications:\n\n\n* *Adewunmi, M., Sharma, S. K., Sharma, N., Sushma, N. S., & Mounmo, B.* (2022). [Cancer Health Disparities drivers with BERTopic modelling and PyCaret Evaluation](https://companyofscientists.com/index.php/chd/article/download/217/232). Cancer Health Disparities, 6.\n* *Ebeling, R., SΓ‘enz, C. A. C., Nobre, J. C., & Becker, K.* (2022, May). [Analysis of the influence of political polarization in the vaccination stance: the Brazilian COVID-19 scenario](https://ojs.aaai.org/index.php/ICWSM/article/download/19281/19053). In Proceedings of the International AAAI Conference on Web and Social Media (Vol. 16, pp. 159-170).\n* *Hoseini, M., Melo, P., Benevenuto, F., Feldmann, A., & Zannettou, S.* (2021). [On the globalization of the QAnon conspiracy theory through Telegram.](https://arxiv.org/pdf/2105.13020) arXiv preprint arXiv:2105.13020.\n* *Falkenberg, M., Galeazzi, A., Torricelli, M., Di Marco, N., Larosa, F., Sas, M., ... & Baronchelli, A.* (2022). [Growing polarization around climate change on social media.](https://www.nature.com/articles/s41558-022-01527-x) Nature Climate Change, 1-8.\n* *SΓ‘nchez‐Franco, M. J., & Rey‐Moreno, M.* (2022). [Do travelers' reviews depend on the destination? An analysis in coastal and urban peer‐to‐peer lodgings.](https://onlinelibrary.wiley.com/doi/pdf/10.1002/mar.21608) Psychology & Marketing, 39(2), 441-459.\n* Zhunis, A., Lima, G., Song, H., Han, J., & Cha, M. (2022, April). [Emotion bubbles: Emotional composition of online discourse before and after the COVID-19 outbreak.](https://dl.acm.org/doi/pdf/10.1145/3485447.3512132) In Proceedings of the ACM Web Conference 2022 (pp. 2603-2613).\n* *Alhaj, F., Al-Haj, A., Sharieh, A., & Jabri, R.* (2022). [Improving Arabic cognitive distortion classification in Twitter using BERTopic.](https://oars.uos.ac.uk/2327/1/Paper_99-Improving_Arabic_Cognitive_Distortion_Classification_in_Twitter.pdf) International Journal of Advanced Computer Science and Applications, 13(1), 854-860.\n\n\nClick [here](https://scholar.google.nl/scholar?oi=bibs&hl=nl&cites=17397195143750028634,8180869421575866889,8997387995186833349,18341994629392979746,1103958012065447153,735156116265912963,513933600215578884) for a full overview of papers citing BERTopic.\n" }, { "path": "mkdocs.yml", "content": "site_name: BERTopic\nrepo_url: https://github.com/MaartenGr/BERTopic\nsite_url: https://maartengr.github.io/BERTopic/\nsite_description: Leveraging BERT and a class-based TF-IDF to create easily interpretable topics.\nsite_author: Maarten P. Grootendorst\nuse_directory_urls: false\nextra_css:\n - stylesheets/extra.css\n\nnav:\n - Home: index.md\n - The Algorithm: algorithm/algorithm.md\n - Getting Started:\n - Quick Start: getting_started/quickstart/quickstart.md\n - Serialization: getting_started/serialization/serialization.md\n - Search Topics: getting_started/search/search.md\n - Best Practices: getting_started/best_practices/best_practices.md\n - In-depth:\n - Visualizations:\n - Topics: getting_started/visualization/visualize_topics.md\n - Documents: getting_started/visualization/visualize_documents.md\n - Terms: getting_started/visualization/visualize_terms.md\n - Hierarchy: getting_started/visualization/visualize_hierarchy.md\n - Update Topics:\n - Topic Reduction: getting_started/topicreduction/topicreduction.md\n - Update Topic Representations: getting_started/topicrepresentation/topicrepresentation.md\n - Outlier reduction: getting_started/outlier_reduction/outlier_reduction.md\n - Parameter tuning: getting_started/parameter tuning/parametertuning.md\n - Tips & Tricks: getting_started/tips_and_tricks/tips_and_tricks.md\n - Sub-models:\n - 1. Embeddings: getting_started/embeddings/embeddings.md\n - 2. Dimensionality Reduction: getting_started/dim_reduction/dim_reduction.md\n - 3. Clustering: getting_started/clustering/clustering.md\n - 4. Vectorizers: getting_started/vectorizers/vectorizers.md\n - 5. c-TF-IDF: getting_started/ctfidf/ctfidf.md\n - 6. Fine-tune Topics:\n - 6A. Representation Models: getting_started/representation/representation.md\n - 6B. LLM & Generative AI: getting_started/representation/llm.md\n - 6C. Multiple Representations: getting_started/multiaspect/multiaspect.md\n - Variations:\n - Dynamic Topic Modeling: getting_started/topicsovertime/topicsovertime.md\n - Hierarchical Topic Modeling: getting_started/hierarchicaltopics/hierarchicaltopics.md\n - Multimodal Topic Modeling: getting_started/multimodal/multimodal.md\n - Online Topic Modeling: getting_started/online/online.md\n - Merge Multiple Models: getting_started/merge/merge.md\n - (semi)-supervised:\n - Semi-supervised Topic Modeling: getting_started/semisupervised/semisupervised.md\n - Supervised Topic Modeling: getting_started/supervised/supervised.md\n - Manual Topic Modeling: getting_started/manual/manual.md\n - Guided Topic Modeling: getting_started/guided/guided.md\n - Zero-shot Topic Modeling: getting_started/zeroshot/zeroshot.md\n - Topic Distributions: getting_started/distribution/distribution.md\n - Topics per Class: getting_started/topicsperclass/topicsperclass.md\n - Seed Words: getting_started/seed_words/seed_words.md\n - FAQ: faq.md\n - Use Cases: usecases.md\n - API:\n - BERTopic: api/bertopic.md\n - Sub-models:\n - 1. Backends: api/backends.md\n - 2. Dimensionality Reduction: api/dimensionality.md\n - 3. Clustering: api/cluster.md\n - 4. Vectorizers: api/vectorizers.md\n - 5. c-TF-IDF: api/ctfidf.md\n - 6. Fine-Tune Topic Representation: api/representations.md\n - Plotting:\n - Barchart: api/plotting/barchart.md\n - Documents: api/plotting/documents.md\n - Documents with DataMapPlot: api/plotting/document_datamap.md\n - DTM: api/plotting/dtm.md\n - Hierarchical documents: api/plotting/hierarchical_documents.md\n - Hierarchical topics: api/plotting/hierarchy.md\n - Distribution: api/plotting/distribution.md\n - Heatmap: api/plotting/heatmap.md\n - Term Scores: api/plotting/term.md\n - Topics: api/plotting/topics.md\n - Topics per Class: api/plotting/topics_per_class.md\n - Changelog: changelog.md\n\nplugins:\n - mkdocstrings:\n watch:\n - bertopic\n - search\ncopyright: Copyright © 2024 Maintained by Maarten Grootendorst.\n\ntheme:\n custom_dir: images/\n name: material\n icon:\n logo: material/library\n font:\n text: Ubuntu\n code: Ubuntu Mono\n favicon: icon.png\n logo: img/icon.png\n feature:\n tabs: true\n features:\n - navigation.tabs\n - navigation.sections\n - navigation.instant\n - navigation.top\n - navigation.tracking\n - toc.follow\n - content.code.copy\n palette:\n - media: \"(prefers-color-scheme: light)\"\n primary: custom\n scheme: black\n toggle:\n icon: material/weather-sunny\n name: Switch to dark mode\n - media: \"(prefers-color-scheme: dark)\"\n scheme: slate\n primary: black\n toggle:\n icon: material/weather-night\n name: Switch to light mode\n\nmarkdown_extensions:\n - admonition\n - md_in_html\n - pymdownx.details\n - pymdownx.highlight\n - pymdownx.superfences\n - pymdownx.snippets\n - toc:\n permalink: true\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"bertopic\"\nversion = \"0.17.4\"\ndescription = \"BERTopic performs topic Modeling with state-of-the-art transformer models.\"\nreadme = \"README.md\"\nlicense = {file = \"LICENSE\"}\nrequires-python = \">=3.10\"\nauthors = [\n { name = \"Maarten P. Grootendorst\", email = \"maartengrootendorst@gmail.com\" },\n]\nkeywords = [\n \"nlp\",\n \"bert\",\n \"topic\",\n \"modeling\",\n \"embeddings\",\n]\nclassifiers = [\n \"Intended Audience :: Developers\",\n \"Intended Audience :: Science/Research\",\n \"License :: OSI Approved :: MIT License\",\n \"Operating System :: MacOS\",\n \"Operating System :: Microsoft :: Windows\",\n \"Operating System :: POSIX\",\n \"Operating System :: Unix\",\n \"Programming Language :: Python\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n \"Topic :: Scientific/Engineering\",\n \"Topic :: Scientific/Engineering :: Artificial Intelligence\",\n]\ndependencies = [\n \"hdbscan>=0.8.29\",\n \"umap-learn>=0.5.0\",\n \"numpy>=1.20.0\",\n \"pandas>=1.1.5\",\n \"plotly>=4.7.0\",\n \"scikit-learn>=1.0\",\n \"sentence-transformers>=0.4.1\",\n \"tqdm>=4.41.1\",\n \"llvmlite>0.36.0\" # To prevent conflicts with `uv` dependency manager\n]\n\n[project.optional-dependencies]\ndatamap = [\n \"datamapplot>=0.1\",\n \"matplotlib>=3.8\",\n]\ndev = [\n \"bertopic[docs,test]\",\n]\ndocs = [\n \"mkdocs==1.5.3\",\n \"mkdocs-material==9.5.18\",\n \"mkdocstrings-python==1.10.0\",\n \"mkdocstrings==0.24.3\",\n]\nfastembed = [\n \"fastembed>=0.6.0\",\n]\nflair = [\n \"flair>=0.7\",\n \"torch>=1.4.0\",\n \"transformers>=3.5.1\",\n]\ngensim = [\n \"gensim>=4.0.0\",\n]\nspacy = [\n \"spacy>=3.0.1\",\n]\ntest = [\n \"pytest>=5.4.3\",\n \"pytest-cov>=2.6.1\",\n \"ruff~=0.14.0\",\n]\nuse = [\n \"tensorflow\",\n \"tensorflow_hub\",\n \"tensorflow_text\",\n]\nvision = [\n \"accelerate>=0.19.0\", # To prevent \"cannot import name 'PartialState' from 'accelerate'\"\n \"Pillow>=9.2.0\",\n]\n\n[project.urls]\nDocumentation = \"https://maartengr.github.io/BERTopic/\"\nHomepage = \"https://github.com/MaartenGr/BERTopic\"\nIssues = \"https://github.com/MaartenGr/BERTopic/issues\"\nRepository = \"https://github.com/MaartenGr/BERTopic.git\"\n\n[tool.setuptools.packages.find]\ninclude = [\"bertopic*\"]\nexclude = [\"tests\"]\n\n[tool.ruff]\nline-length = 120\n\n[tool.ruff.lint]\nselect = [\n \"E4\",\n \"E7\",\n \"E9\",\n \"F\", # pyflakes\n \"D\", # pydocstyle\n \"PD\", # pandas-vet\n \"RUF\", # ruff\n]\n\nignore = [\n \"D100\", # Missing docstring in public module\n \"D104\", # Missing docstring in public package\n \"D205\", # 1 blank line required between summary line and description\n \"E731\", # Do not assign a lambda expression, use a def\n]\n\n[tool.ruff.lint.per-file-ignores]\n\"**/tests/*\" = [\"D\"] # Ignore all docstring errors in tests\n\n[tool.ruff.lint.pydocstyle]\nconvention = \"google\"\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/conftest.py", "content": "import copy\nimport pytest\nfrom umap import UMAP\nfrom hdbscan import HDBSCAN\nfrom bertopic import BERTopic\nfrom sklearn.datasets import fetch_20newsgroups\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.cluster import KMeans, MiniBatchKMeans\nfrom sklearn.decomposition import PCA\nfrom bertopic.vectorizers import OnlineCountVectorizer\nfrom bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance\nfrom bertopic.dimensionality import BaseDimensionalityReduction\nfrom sklearn.linear_model import LogisticRegression\n\n\n@pytest.fixture(scope=\"session\")\ndef embedding_model():\n model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef document_embeddings(documents, embedding_model):\n embeddings = embedding_model.encode(documents)\n return embeddings\n\n\n@pytest.fixture(scope=\"session\")\ndef reduced_embeddings(document_embeddings):\n reduced_embeddings = UMAP(n_neighbors=10, n_components=2, min_dist=0.0, metric=\"cosine\").fit_transform(\n document_embeddings\n )\n return reduced_embeddings\n\n\n@pytest.fixture(scope=\"session\")\ndef documents():\n newsgroup_docs = fetch_20newsgroups(subset=\"all\", remove=(\"headers\", \"footers\", \"quotes\"))[\"data\"][:1000]\n return newsgroup_docs\n\n\n@pytest.fixture(scope=\"session\")\ndef targets():\n data = fetch_20newsgroups(subset=\"all\", remove=(\"headers\", \"footers\", \"quotes\"))\n y = data[\"target\"][:1000]\n return y\n\n\n@pytest.fixture(scope=\"session\")\ndef base_topic_model(documents, document_embeddings, embedding_model):\n model = BERTopic(embedding_model=embedding_model, calculate_probabilities=True)\n model.umap_model.random_state = 42\n model.hdbscan_model.min_cluster_size = 3\n model.fit(documents, document_embeddings)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef zeroshot_topic_model(documents, document_embeddings, embedding_model):\n zeroshot_topic_list = [\"religion\", \"cars\", \"electronics\"]\n model = BERTopic(\n embedding_model=embedding_model,\n calculate_probabilities=True,\n zeroshot_topic_list=zeroshot_topic_list,\n zeroshot_min_similarity=0.3,\n )\n model.umap_model.random_state = 42\n model.hdbscan_model.min_cluster_size = 2\n model.fit(documents, document_embeddings)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef custom_topic_model(documents, document_embeddings, embedding_model):\n umap_model = UMAP(n_neighbors=15, n_components=6, min_dist=0.0, metric=\"cosine\", random_state=42)\n hdbscan_model = HDBSCAN(\n min_cluster_size=3,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n )\n model = BERTopic(\n umap_model=umap_model,\n hdbscan_model=hdbscan_model,\n embedding_model=embedding_model,\n calculate_probabilities=True,\n ).fit(documents, document_embeddings)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef representation_topic_model(documents, document_embeddings, embedding_model):\n umap_model = UMAP(n_neighbors=15, n_components=6, min_dist=0.0, metric=\"cosine\", random_state=42)\n hdbscan_model = HDBSCAN(\n min_cluster_size=3,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n )\n representation_model = {\n \"Main\": KeyBERTInspired(),\n \"MMR\": [KeyBERTInspired(top_n_words=30), MaximalMarginalRelevance()],\n }\n model = BERTopic(\n umap_model=umap_model,\n hdbscan_model=hdbscan_model,\n embedding_model=embedding_model,\n representation_model=representation_model,\n calculate_probabilities=True,\n ).fit(documents, document_embeddings)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef reduced_topic_model(custom_topic_model, documents):\n model = copy.deepcopy(custom_topic_model)\n model.reduce_topics(documents, nr_topics=\"auto\")\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef merged_topic_model(custom_topic_model, documents):\n model = copy.deepcopy(custom_topic_model)\n\n # Merge once\n topics_to_merge = [[1, 2], [3, 4]]\n model.merge_topics(documents, topics_to_merge)\n\n # Merge second time\n topics_to_merge = [[5, 6, 7]]\n model.merge_topics(documents, topics_to_merge)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef kmeans_pca_topic_model(documents, document_embeddings):\n hdbscan_model = KMeans(n_clusters=15, random_state=42)\n dim_model = PCA(n_components=5)\n model = BERTopic(\n hdbscan_model=hdbscan_model,\n umap_model=dim_model,\n embedding_model=embedding_model,\n ).fit(documents, document_embeddings)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef supervised_topic_model(documents, document_embeddings, embedding_model, targets):\n empty_dimensionality_model = BaseDimensionalityReduction()\n clf = LogisticRegression()\n\n model = BERTopic(\n embedding_model=embedding_model,\n umap_model=empty_dimensionality_model,\n hdbscan_model=clf,\n ).fit(documents, embeddings=document_embeddings, y=targets)\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef online_topic_model(documents, document_embeddings, embedding_model):\n umap_model = PCA(n_components=5)\n cluster_model = MiniBatchKMeans(n_clusters=50, random_state=0)\n vectorizer_model = OnlineCountVectorizer(stop_words=\"english\", decay=0.01)\n model = BERTopic(\n umap_model=umap_model,\n hdbscan_model=cluster_model,\n vectorizer_model=vectorizer_model,\n embedding_model=embedding_model,\n )\n\n topics = []\n for index in range(0, len(documents), 50):\n model.partial_fit(documents[index : index + 50], document_embeddings[index : index + 50])\n topics.extend(model.topics_)\n model.topics_ = topics\n return model\n\n\n@pytest.fixture(scope=\"session\")\ndef cuml_base_topic_model(documents, document_embeddings, embedding_model):\n from cuml.cluster import HDBSCAN as cuml_hdbscan\n from cuml.manifold import UMAP as cuml_umap\n\n model = BERTopic(\n embedding_model=embedding_model,\n calculate_probabilities=True,\n umap_model=cuml_umap(n_components=5, n_neighbors=5, random_state=42),\n hdbscan_model=cuml_hdbscan(min_cluster_size=3, prediction_data=True),\n )\n model.fit(documents, document_embeddings)\n return model\n" }, { "path": "tests/test_bertopic.py", "content": "import copy\nimport pytest\nfrom bertopic import BERTopic\nimport importlib.util\n\n\ndef cuml_available():\n try:\n return importlib.util.find_spec(\"cuml\") is not None\n except ImportError:\n return False\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"base_topic_model\"),\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n (\"supervised_topic_model\"),\n (\"representation_topic_model\"),\n (\"zeroshot_topic_model\"),\n pytest.param(\n \"cuml_base_topic_model\",\n marks=pytest.mark.skipif(not cuml_available(), reason=\"cuML not available\"),\n ),\n ],\n)\ndef test_full_model(model, documents, request):\n \"\"\"Tests the entire pipeline in one go. This serves as a sanity check to see if the default\n settings result in a good separation of topics.\n\n NOTE: This does not cover all cases but merely combines it all together\n \"\"\"\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n if model == \"base_topic_model\":\n topic_model.save(\n \"model_dir\",\n serialization=\"pytorch\",\n save_ctfidf=True,\n save_embedding_model=\"sentence-transformers/all-MiniLM-L6-v2\",\n )\n topic_model = BERTopic.load(\"model_dir\")\n\n if model == \"cuml_base_topic_model\":\n assert \"cuml\" in str(type(topic_model.umap_model)).lower()\n assert \"cuml\" in str(type(topic_model.hdbscan_model)).lower()\n\n topics = topic_model.topics_\n\n for topic in set(topics):\n words = topic_model.get_topic(topic)[:10]\n assert len(words) == 10\n\n for topic in topic_model.get_topic_freq().Topic:\n words = topic_model.get_topic(topic)[:10]\n assert len(words) == 10\n\n assert len(topic_model.get_topic_freq()) > 2\n assert len(topic_model.get_topics()) == len(topic_model.get_topic_freq())\n\n # Test extraction of document info\n document_info = topic_model.get_document_info(documents)\n assert len(document_info) == len(documents)\n\n # Test transform\n doc = \"This is a new document to predict.\"\n topics_test, _probs_test = topic_model.transform([doc, doc])\n\n assert len(topics_test) == 2\n\n # Test zero-shot topic modeling\n if topic_model._is_zeroshot():\n if topic_model._outliers:\n assert set(topic_model.topic_labels_.keys()) == set(range(-1, len(topic_model.topic_labels_) - 1))\n else:\n assert set(topic_model.topic_labels_.keys()) == set(range(len(topic_model.topic_labels_)))\n\n # Test topics over time\n timestamps = [i % 10 for i in range(len(documents))]\n topics_over_time = topic_model.topics_over_time(documents, timestamps)\n\n assert topics_over_time.Frequency.sum() == len(documents)\n assert len(topics_over_time.Topic.unique()) == len(set(topics))\n\n # Test hierarchical topics\n hier_topics = topic_model.hierarchical_topics(documents)\n\n assert len(hier_topics) > 0\n assert hier_topics.Parent_ID.astype(int).min() > max(topics)\n\n # Test creation of topic tree\n tree = topic_model.get_topic_tree(hier_topics, tight_layout=False)\n assert isinstance(tree, str)\n assert len(tree) > 10\n\n # Test find topic\n similar_topics, similarity = topic_model.find_topics(\"query\", top_n=2)\n assert len(similar_topics) == 2\n assert len(similarity) == 2\n assert max(similarity) <= 1\n\n # Test topic reduction\n nr_topics = len(set(topics))\n nr_topics = 2 if nr_topics < 2 else nr_topics - 1\n topic_model.reduce_topics(documents, nr_topics=nr_topics)\n\n assert len(topic_model.get_topic_freq()) == nr_topics\n assert len(topic_model.topics_) == len(topics)\n\n # Test update topics\n topic = topic_model.get_topic(1)[:10]\n vectorizer_model = topic_model.vectorizer_model\n topic_model.update_topics(documents, n_gram_range=(2, 2))\n\n updated_topic = topic_model.get_topic(1)[:10]\n\n topic_model.update_topics(documents, vectorizer_model=vectorizer_model)\n original_topic = topic_model.get_topic(1)[:10]\n\n assert topic != updated_topic\n if topic_model.representation_model is not None:\n assert topic != original_topic\n\n # Test updating topic labels\n topic_labels = topic_model.generate_topic_labels(nr_words=3, topic_prefix=False, word_length=10, separator=\", \")\n assert len(topic_labels) == len(set(topic_model.topics_))\n\n # Test setting topic labels\n topic_model.set_topic_labels(topic_labels)\n assert topic_model.custom_labels_ == topic_labels\n\n # Test merging topics\n freq = topic_model.get_topic_freq(0)\n topics_to_merge = [0, 1]\n topic_model.merge_topics(documents, topics_to_merge)\n assert freq < topic_model.get_topic_freq(0)\n\n # Test reduction of outliers\n if -1 in topics:\n new_topics = topic_model.reduce_outliers(documents, topics, threshold=0.0)\n nr_outliers_topic_model = sum([1 for topic in topic_model.topics_ if topic == -1])\n nr_outliers_new_topics = sum([1 for topic in new_topics if topic == -1])\n\n if topic_model._outliers == 1:\n assert nr_outliers_topic_model > nr_outliers_new_topics\n\n # Combine models\n topic_model1 = BERTopic.load(\"model_dir\")\n merged_model = BERTopic.merge_models([topic_model, topic_model1])\n\n assert len(merged_model.get_topic_info()) > len(topic_model.get_topic_info())\n" }, { "path": "tests/test_other.py", "content": "from bertopic import BERTopic\nfrom bertopic.dimensionality import BaseDimensionalityReduction\n\ntry:\n from plotly.graph_objects import Figure\nexcept ImportError:\n Figure = None\n\n\ndef test_load_save_model():\n model = BERTopic(language=\"Dutch\", embedding_model=None)\n model.save(\"test\", serialization=\"pickle\")\n loaded_model = BERTopic.load(\"test\")\n assert type(model) is type(loaded_model)\n assert model.language == loaded_model.language\n assert model.embedding_model == loaded_model.embedding_model\n assert model.top_n_words == loaded_model.top_n_words\n\n\ndef test_get_params():\n model = BERTopic()\n params = model.get_params()\n assert not params[\"embedding_model\"]\n assert not params[\"low_memory\"]\n assert not params[\"nr_topics\"]\n assert params[\"n_gram_range\"] == (1, 1)\n assert params[\"min_topic_size\"] == 10\n assert params[\"language\"] == \"english\"\n\n\ndef test_no_plotly():\n model = BERTopic(\n language=\"Dutch\",\n embedding_model=None,\n min_topic_size=2,\n top_n_words=1,\n umap_model=BaseDimensionalityReduction(),\n )\n model.fit([\"hello\", \"hi\", \"goodbye\", \"goodbye\", \"whats up\"] * 10)\n\n try:\n out = model.visualize_topics()\n assert isinstance(out, Figure) if Figure else False\n except ImportError as e:\n assert \"Plotly is required to use\" in str(e)\n" }, { "path": "tests/test_plotting/__init__.py", "content": "" }, { "path": "tests/test_plotting/test_approximate.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\"batch_size\", [50, None])\n@pytest.mark.parametrize(\"padding\", [True, False])\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n ],\n)\ndef test_approximate_distribution(batch_size, padding, model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n\n # Calculate only on a document-level based on tokensets\n topic_distr, _ = topic_model.approximate_distribution(documents, padding=padding, batch_size=batch_size)\n assert topic_distr.shape[1] == len(topic_model.topic_labels_) - topic_model._outliers\n\n # Use the distribution visualization\n for i in range(3):\n topic_model.visualize_distribution(topic_distr[i])\n\n # Calculate distribution on a token-level\n topic_distr, topic_token_distr = topic_model.approximate_distribution(documents[:100], calculate_tokens=True)\n assert topic_distr.shape[1] == len(topic_model.topic_labels_) - topic_model._outliers\n assert len(topic_token_distr) == len(documents[:100])\n\n for token_distr in topic_token_distr:\n assert token_distr.shape[1] == len(topic_model.topic_labels_) - topic_model._outliers\n" }, { "path": "tests/test_plotting/test_bar.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_barchart(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n fig = topic_model.visualize_barchart()\n\n assert len(fig.to_dict()[\"layout\"][\"annotations\"]) == 8\n for annotation in fig.to_dict()[\"layout\"][\"annotations\"]:\n assert int(annotation[\"text\"].split(\" \")[-1]) != -1\n\n fig = topic_model.visualize_barchart(top_n_topics=5)\n\n assert len(fig.to_dict()[\"layout\"][\"annotations\"]) == 5\n for annotation in fig.to_dict()[\"layout\"][\"annotations\"]:\n assert int(annotation[\"text\"].split(\" \")[-1]) != -1\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_barchart_outlier(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topic_model.topic_sizes_[-1] = 4\n fig = topic_model.visualize_barchart()\n\n assert len(fig.to_dict()[\"layout\"][\"annotations\"]) == 8\n for annotation in fig.to_dict()[\"layout\"][\"annotations\"]:\n assert int(annotation[\"text\"].split(\" \")[-1]) != -1\n\n fig = topic_model.visualize_barchart(top_n_topics=5)\n\n assert len(fig.to_dict()[\"layout\"][\"annotations\"]) == 5\n for annotation in fig.to_dict()[\"layout\"][\"annotations\"]:\n assert int(annotation[\"text\"].split(\" \")[-1]) != -1\n" }, { "path": "tests/test_plotting/test_documents.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n ],\n)\ndef test_documents(model, reduced_embeddings, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics = set(topic_model.topics_)\n if -1 in topics:\n topics.remove(-1)\n fig = topic_model.visualize_documents(documents, embeddings=reduced_embeddings, hide_document_hover=True)\n fig_topics = [int(data[\"name\"].split(\"_\")[0]) for data in fig.to_dict()[\"data\"][1:]]\n assert set(fig_topics) == topics\n" }, { "path": "tests/test_plotting/test_dynamic.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_dynamic(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n timestamps = [i % 10 for i in range(len(documents))]\n topics_over_time = topic_model.topics_over_time(documents, timestamps)\n fig = topic_model.visualize_topics_over_time(topics_over_time)\n\n assert len(fig.to_dict()[\"data\"]) == len(set(topic_model.topics_)) - topic_model._outliers\n" }, { "path": "tests/test_plotting/test_heatmap.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n ],\n)\ndef test_heatmap(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics = set(topic_model.topics_)\n if -1 in topics:\n topics.remove(-1)\n fig = topic_model.visualize_heatmap()\n fig_topics = [int(topic.split(\"_\")[0]) for topic in fig.to_dict()[\"data\"][0][\"x\"]]\n\n assert set(fig_topics) == topics\n" }, { "path": "tests/test_plotting/test_term_rank.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\"model\", [(\"kmeans_pca_topic_model\"), (\"base_topic_model\"), (\"custom_topic_model\")])\ndef test_term_rank(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topic_model.visualize_term_rank()\n" }, { "path": "tests/test_plotting/test_topics.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_topics(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n fig = topic_model.visualize_topics()\n for slider in fig.to_dict()[\"layout\"][\"sliders\"]:\n for step in slider[\"steps\"]:\n assert int(step[\"label\"].split(\" \")[-1]) != -1\n\n fig = topic_model.visualize_topics(top_n_topics=5)\n for slider in fig.to_dict()[\"layout\"][\"sliders\"]:\n for step in slider[\"steps\"]:\n assert int(step[\"label\"].split(\" \")[-1]) != -1\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_topics_outlier(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topic_model.topic_sizes_[-1] = 4\n fig = topic_model.visualize_topics()\n\n for slider in fig.to_dict()[\"layout\"][\"sliders\"]:\n for step in slider[\"steps\"]:\n assert int(step[\"label\"].split(\" \")[-1]) != -1\n\n fig = topic_model.visualize_topics(top_n_topics=5)\n for slider in fig.to_dict()[\"layout\"][\"sliders\"]:\n for step in slider[\"steps\"]:\n assert int(step[\"label\"].split(\" \")[-1]) != -1\n" }, { "path": "tests/test_reduction/__init__.py", "content": "" }, { "path": "tests/test_reduction/test_delete.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_delete(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n nr_topics = len(set(topic_model.topics_))\n length_documents = len(topic_model.topics_)\n\n # First deletion\n topics_to_delete = [1, 2]\n topic_model.delete_topics(topics_to_delete)\n mappings = topic_model.topic_mapper_.get_mappings(list(topic_model.hdbscan_model.labels_))\n mapped_labels = [mappings[label] for label in topic_model.hdbscan_model.labels_]\n\n if model == \"online_topic_model\" or model == \"kmeans_pca_topic_model\":\n assert nr_topics == len(set(topic_model.topics_)) + 1\n assert topic_model.get_topic_info().Count.sum() == length_documents\n else:\n assert nr_topics == len(set(topic_model.topics_)) + 2\n assert topic_model.get_topic_info().Count.sum() == length_documents\n\n if model == \"online_topic_model\":\n assert mapped_labels == topic_model.topics_[950:]\n else:\n assert mapped_labels == topic_model.topics_\n\n # Find two existing topics for second deletion\n remaining_topics = sorted(list(set(topic_model.topics_)))\n remaining_topics = [t for t in remaining_topics if t != -1] # Exclude outlier topic\n topics_to_delete = remaining_topics[:2] # Take first two remaining topics\n\n # Second deletion\n topic_model.delete_topics(topics_to_delete)\n mappings = topic_model.topic_mapper_.get_mappings(list(topic_model.hdbscan_model.labels_))\n mapped_labels = [mappings[label] for label in topic_model.hdbscan_model.labels_]\n\n if model == \"online_topic_model\" or model == \"kmeans_pca_topic_model\":\n assert nr_topics == len(set(topic_model.topics_)) + 3\n assert topic_model.get_topic_info().Count.sum() == length_documents\n else:\n assert nr_topics == len(set(topic_model.topics_)) + 4\n assert topic_model.get_topic_info().Count.sum() == length_documents\n\n if model == \"online_topic_model\":\n assert mapped_labels == topic_model.topics_[950:]\n else:\n assert mapped_labels == topic_model.topics_\n" }, { "path": "tests/test_reduction/test_merge.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_merge(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n nr_topics = len(set(topic_model.topics_))\n\n topics_to_merge = [1, 2]\n topic_model.merge_topics(documents, topics_to_merge)\n mappings = topic_model.topic_mapper_.get_mappings(list(topic_model.hdbscan_model.labels_))\n mapped_labels = [mappings[label] for label in topic_model.hdbscan_model.labels_]\n\n assert nr_topics == len(set(topic_model.topics_)) + 1\n assert topic_model.get_topic_info().Count.sum() == len(documents)\n if model == \"online_topic_model\":\n assert mapped_labels == topic_model.topics_[950:]\n else:\n assert mapped_labels == topic_model.topics_\n\n topics_to_merge = [1, 2]\n topic_model.merge_topics(documents, topics_to_merge)\n mappings = topic_model.topic_mapper_.get_mappings(list(topic_model.hdbscan_model.labels_))\n mapped_labels = [mappings[label] for label in topic_model.hdbscan_model.labels_]\n\n assert nr_topics == len(set(topic_model.topics_)) + 2\n assert topic_model.get_topic_info().Count.sum() == len(documents)\n if model == \"online_topic_model\":\n assert mapped_labels == topic_model.topics_[950:]\n else:\n assert mapped_labels == topic_model.topics_\n" }, { "path": "tests/test_representation/__init__.py", "content": "" }, { "path": "tests/test_representation/test_get.py", "content": "import copy\nimport pytest\nimport numpy as np\nimport pandas as pd\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_get_topic(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics = [topic_model.get_topic(topic) for topic in set(topic_model.topics_)]\n unknown_topic = topic_model.get_topic(500)\n\n for topic in topics:\n assert topic is not False\n\n assert len(topics) == len(topic_model.get_topic_info())\n assert not unknown_topic\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_get_topics(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics = topic_model.get_topics()\n\n assert topics == topic_model.topic_representations_\n assert len(topics.keys()) == len(set(topic_model.topics_))\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_get_topic_freq(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n for topic in set(topic_model.topics_):\n assert not isinstance(topic_model.get_topic_freq(topic), pd.DataFrame)\n\n topic_freq = topic_model.get_topic_freq()\n unique_topics = set(topic_model.topics_)\n topics_in_mapper = set(np.array(topic_model.topic_mapper_.mappings_)[:, -1])\n\n assert isinstance(topic_freq, pd.DataFrame)\n\n assert len(topic_freq) == len(set(topic_model.topics_))\n assert len(topics_in_mapper.difference(unique_topics)) == 0\n assert len(unique_topics.difference(topics_in_mapper)) == 0\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n ],\n)\ndef test_get_representative_docs(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n all_docs = topic_model.get_representative_docs()\n unique_topics = set(topic_model.topics_)\n topics_in_mapper = set(np.array(topic_model.topic_mapper_.mappings_)[:, -1])\n\n assert len(all_docs) == len(topic_model.topic_sizes_.keys())\n assert len(all_docs) == len(topics_in_mapper)\n assert len(all_docs) == topic_model.c_tf_idf_.shape[0]\n assert len(all_docs) == len(topic_model.topic_labels_)\n assert all([True if len(docs) == 3 else False for docs in all_docs.values()])\n\n topics = set(list(all_docs.keys()))\n\n assert len(topics.difference(unique_topics)) == 0\n assert len(topics.difference(topics_in_mapper)) == 0\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_get_topic_info(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n info = topic_model.get_topic_info()\n\n if topic_model._outliers:\n assert info.iloc[0].Topic == -1\n else:\n assert info.iloc[0].Topic == 0\n\n for topic in set(topic_model.topics_):\n assert len(topic_model.get_topic_info(topic)) == 1\n\n assert len(topic_model.get_topic_info(200)) == 0\n" }, { "path": "tests/test_representation/test_labels.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_generate_topic_labels(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n labels = topic_model.generate_topic_labels(topic_prefix=False)\n\n assert sum([label[0].isdigit() for label in labels[1:]]) / len(labels) < 0.2\n\n labels = [int(label.split(\"_\")[0]) for label in topic_model.generate_topic_labels()]\n assert labels == sorted(list(set(topic_model.topics_)))\n\n labels = topic_model.generate_topic_labels(nr_words=1, topic_prefix=False)\n assert all([True if len(label) < 15 else False for label in labels])\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_set_labels(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n\n labels = topic_model.generate_topic_labels()\n topic_model.set_topic_labels(labels)\n assert topic_model.custom_labels_ == labels\n\n if model != \"online_topic_model\":\n labels = {1: \"My label\", 2: \"Another label\"}\n topic_model.set_topic_labels(labels)\n assert topic_model.custom_labels_[1 + topic_model._outliers] == \"My label\"\n assert topic_model.custom_labels_[2 + topic_model._outliers] == \"Another label\"\n\n labels = {1: \"Change label\", 3: \"New label\"}\n topic_model.set_topic_labels(labels)\n assert topic_model.custom_labels_[1 + topic_model._outliers] == \"Change label\"\n assert topic_model.custom_labels_[3 + topic_model._outliers] == \"New label\"\n else:\n labels = {\n sorted(set(topic_model.topics_))[0]: \"My label\",\n sorted(set(topic_model.topics_))[1]: \"Another label\",\n }\n topic_model.set_topic_labels(labels)\n assert topic_model.custom_labels_[0] == \"My label\"\n assert topic_model.custom_labels_[1] == \"Another label\"\n\n labels = {\n sorted(set(topic_model.topics_))[0]: \"Change label\",\n sorted(set(topic_model.topics_))[2]: \"New label\",\n }\n topic_model.set_topic_labels(labels)\n assert topic_model.custom_labels_[0 + topic_model._outliers] == \"Change label\"\n assert topic_model.custom_labels_[2 + topic_model._outliers] == \"New label\"\n" }, { "path": "tests/test_representation/test_representations.py", "content": "import copy\nimport pytest\nimport numpy as np\nimport pandas as pd\nfrom sklearn.feature_extraction.text import CountVectorizer\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n ],\n)\ndef test_update_topics(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n old_ctfidf = topic_model.c_tf_idf_\n old_topics = topic_model.topics_\n\n topic_model.update_topics(documents, n_gram_range=(1, 3))\n\n assert old_ctfidf.shape[1] < topic_model.c_tf_idf_.shape[1]\n assert old_topics == topic_model.topics_\n\n updated_topics = [topic if topic != 1 else 0 for topic in old_topics]\n topic_model.update_topics(documents, topics=updated_topics, n_gram_range=(1, 3))\n\n assert len(set(old_topics)) - 1 == len(set(topic_model.topics_))\n\n old_topics = topic_model.topics_\n updated_topics = [topic if topic != 2 else 0 for topic in old_topics]\n topic_model.update_topics(documents, topics=updated_topics, n_gram_range=(1, 3))\n\n assert len(set(old_topics)) - 1 == len(set(topic_model.topics_))\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_extract_topics(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n nr_topics = 5\n documents = pd.DataFrame(\n {\n \"Document\": documents,\n \"ID\": range(len(documents)),\n \"Topic\": np.random.randint(-1, nr_topics - 1, len(documents)),\n }\n )\n topic_model._update_topic_size(documents)\n topic_model._extract_topics(documents)\n freq = topic_model.get_topic_freq()\n\n assert topic_model.c_tf_idf_.shape[0] == 5\n assert topic_model.c_tf_idf_.shape[1] > 100\n assert isinstance(freq, pd.DataFrame)\n assert nr_topics == len(freq.Topic.unique())\n assert freq.Count.sum() == len(documents)\n assert len(freq.Topic.unique()) == len(freq)\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_extract_topics_custom_cv(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n nr_topics = 5\n documents = pd.DataFrame(\n {\n \"Document\": documents,\n \"ID\": range(len(documents)),\n \"Topic\": np.random.randint(-1, nr_topics - 1, len(documents)),\n }\n )\n\n cv = CountVectorizer(ngram_range=(1, 2))\n topic_model.vectorizer_model = cv\n topic_model._update_topic_size(documents)\n topic_model._extract_topics(documents)\n freq = topic_model.get_topic_freq()\n\n assert topic_model.c_tf_idf_.shape[0] == 5\n assert topic_model.c_tf_idf_.shape[1] > 100\n assert isinstance(freq, pd.DataFrame)\n assert nr_topics == len(freq.Topic.unique())\n assert freq.Count.sum() == len(documents)\n assert len(freq.Topic.unique()) == len(freq)\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\n@pytest.mark.parametrize(\"reduced_topics\", [2, 4, 10])\ndef test_topic_reduction(model, reduced_topics, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n old_topics = copy.deepcopy(topic_model.topics_)\n old_freq = topic_model.get_topic_freq()\n\n topic_model.reduce_topics(documents, nr_topics=reduced_topics)\n\n new_freq = topic_model.get_topic_freq()\n\n if model != \"online_topic_model\":\n assert old_freq.Count.sum() == new_freq.Count.sum()\n assert len(old_freq.Topic.unique()) == len(old_freq)\n assert len(new_freq.Topic.unique()) == len(new_freq)\n assert len(topic_model.topics_) == len(old_topics)\n assert topic_model.topics_ != old_topics\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_topic_reduction_edge_cases(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topic_model.nr_topics = 100\n nr_topics = 5\n topics = np.random.randint(-1, nr_topics - 1, len(documents))\n old_documents = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": topics})\n topic_model._update_topic_size(old_documents)\n old_documents = topic_model._sort_mappings_by_frequency(old_documents)\n topic_model._extract_topics(old_documents)\n old_freq = topic_model.get_topic_freq()\n\n new_documents = topic_model._reduce_topics(old_documents)\n new_freq = topic_model.get_topic_freq()\n\n assert not set(old_documents.Topic).difference(set(new_documents.Topic))\n pd.testing.assert_frame_equal(old_documents, new_documents)\n pd.testing.assert_frame_equal(old_freq, new_freq)\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_find_topics(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n similar_topics, similarity = topic_model.find_topics(\"car\")\n\n assert np.mean(similarity) > 0.1\n assert len(similar_topics) > 0\n" }, { "path": "tests/test_sub_models/__init__.py", "content": "" }, { "path": "tests/test_sub_models/test_cluster.py", "content": "import pytest\nimport pandas as pd\n\nfrom sklearn.datasets import make_blobs\nfrom sklearn.cluster import KMeans\nfrom hdbscan import HDBSCAN\n\nfrom bertopic import BERTopic\n\n\n@pytest.mark.parametrize(\"cluster_model\", [\"hdbscan\", \"kmeans\"])\n@pytest.mark.parametrize(\n \"samples,features,centers\",\n [\n (200, 500, 1),\n (500, 200, 1),\n (200, 500, 2),\n (500, 200, 2),\n (200, 500, 4),\n (500, 200, 4),\n ],\n)\ndef test_hdbscan_cluster_embeddings(cluster_model, samples, features, centers):\n embeddings, _ = make_blobs(n_samples=samples, centers=centers, n_features=features, random_state=42)\n documents = [str(i + 1) for i in range(embeddings.shape[0])]\n old_df = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": None})\n\n if cluster_model == \"kmeans\":\n cluster_model = KMeans(n_clusters=centers)\n else:\n cluster_model = HDBSCAN(\n min_cluster_size=10,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n )\n\n model = BERTopic(hdbscan_model=cluster_model)\n new_df, _ = model._cluster_embeddings(embeddings, old_df)\n\n assert len(new_df.Topic.unique()) == centers\n assert \"Topic\" in new_df.columns\n pd.testing.assert_frame_equal(old_df.drop(\"Topic\", axis=1), new_df.drop(\"Topic\", axis=1))\n\n\n@pytest.mark.parametrize(\"cluster_model\", [\"hdbscan\", \"kmeans\"])\n@pytest.mark.parametrize(\n \"samples,features,centers\",\n [\n (200, 500, 1),\n (500, 200, 1),\n (200, 500, 2),\n (500, 200, 2),\n (200, 500, 4),\n (500, 200, 4),\n ],\n)\ndef test_custom_hdbscan_cluster_embeddings(cluster_model, samples, features, centers):\n embeddings, _ = make_blobs(n_samples=samples, centers=centers, n_features=features, random_state=42)\n documents = [str(i + 1) for i in range(embeddings.shape[0])]\n old_df = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": None})\n if cluster_model == \"kmeans\":\n cluster_model = KMeans(n_clusters=centers)\n else:\n cluster_model = HDBSCAN(\n min_cluster_size=10,\n metric=\"euclidean\",\n cluster_selection_method=\"eom\",\n prediction_data=True,\n )\n\n model = BERTopic(hdbscan_model=cluster_model)\n new_df, _ = model._cluster_embeddings(embeddings, old_df)\n\n assert len(new_df.Topic.unique()) == centers\n assert \"Topic\" in new_df.columns\n pd.testing.assert_frame_equal(old_df.drop(\"Topic\", axis=1), new_df.drop(\"Topic\", axis=1))\n" }, { "path": "tests/test_sub_models/test_dim_reduction.py", "content": "import copy\nimport pytest\nimport numpy as np\nfrom umap import UMAP\nfrom sklearn.decomposition import PCA\n\nfrom bertopic import BERTopic\n\n\n@pytest.mark.parametrize(\"dim_model\", [UMAP, PCA])\n@pytest.mark.parametrize(\n \"embeddings,shape,n_components\",\n [\n (np.random.rand(100, 128), 100, 5),\n (np.random.rand(10, 256), 10, 5),\n (np.random.rand(50, 15), 50, 10),\n ],\n)\ndef test_reduce_dimensionality(dim_model, embeddings, shape, n_components):\n model = BERTopic(umap_model=dim_model(n_components=n_components))\n umap_embeddings = model._reduce_dimensionality(embeddings)\n assert umap_embeddings.shape == (shape, n_components)\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_custom_reduce_dimensionality(model, request):\n embeddings = np.random.rand(500, 128)\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n umap_embeddings = topic_model._reduce_dimensionality(embeddings)\n assert umap_embeddings.shape[1] < embeddings.shape[1]\n" }, { "path": "tests/test_sub_models/test_embeddings.py", "content": "import copy\nimport pytest\nimport numpy as np\nfrom bertopic import BERTopic\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_extract_embeddings(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n single_embedding = topic_model._extract_embeddings(\"a document\")\n multiple_embeddings = topic_model._extract_embeddings([\"something different\", \"another document\"])\n sim_matrix = cosine_similarity(single_embedding, multiple_embeddings)[0]\n\n assert single_embedding.shape[0] == 1\n assert single_embedding.shape[1] == 384\n assert np.min(single_embedding) > -5\n assert np.max(single_embedding) < 5\n\n assert multiple_embeddings.shape[0] == 2\n assert multiple_embeddings.shape[1] == 384\n assert np.min(multiple_embeddings) > -5\n assert np.max(multiple_embeddings) < 5\n\n assert sim_matrix[0] < 0.5\n assert sim_matrix[1] > 0.5\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_extract_embeddings_compare(model, embedding_model, request):\n docs = [\"some document\"]\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n bertopic_embeddings = topic_model._extract_embeddings(docs)\n\n assert isinstance(bertopic_embeddings, np.ndarray)\n assert bertopic_embeddings.shape == (1, 384)\n\n sentence_embeddings = embedding_model.encode(docs, show_progress_bar=False)\n assert np.array_equal(bertopic_embeddings, sentence_embeddings)\n\n\ndef test_extract_incorrect_embeddings():\n with pytest.raises(ValueError):\n model = BERTopic(language=\"Unknown language\")\n model.fit([\"some document\"])\n" }, { "path": "tests/test_utils.py", "content": "import pytest\nimport logging\nimport numpy as np\nfrom typing import List\nfrom bertopic._utils import (\n check_documents_type,\n check_embeddings_shape,\n MyLogger,\n select_topic_representation,\n get_unique_distances,\n)\nfrom scipy.sparse import csr_matrix\n\n\ndef test_logger():\n logger = MyLogger()\n logger.configure(\"DEBUG\")\n assert isinstance(logger.logger, logging.Logger)\n assert logger.logger.level == 10\n\n logger = MyLogger()\n logger.configure(\"WARNING\")\n assert isinstance(logger.logger, logging.Logger)\n assert logger.logger.level == 30\n\n\n@pytest.mark.parametrize(\n \"docs\",\n [\"A document not in an iterable\", [None], 5],\n)\ndef test_check_documents_type(docs):\n with pytest.raises(TypeError):\n check_documents_type(docs)\n\n\ndef test_check_embeddings_shape():\n docs = [\"doc_one\", \"doc_two\"]\n embeddings = np.array([[1, 2, 3], [2, 3, 4]])\n check_embeddings_shape(embeddings, docs)\n\n\ndef test_make_unique_distances():\n def check_dists(dists: List[float], noise_max: float):\n unique_dists = get_unique_distances(np.array(dists, dtype=float), noise_max=noise_max)\n assert len(unique_dists) == len(dists), \"The number of elements must be the same\"\n assert len(dists) == len(np.unique(unique_dists)), \"The distances must be unique\"\n\n check_dists([0, 0, 0.5, 0.75, 1, 1], noise_max=1e-7)\n\n # testing whether the distances are sorted in ascending order when if the noise is extremely high\n check_dists([0, 0, 0, 0.5, 0.75, 1, 1], noise_max=20)\n\n # test whether the distances are sorted in ascending order when the distances are all the same\n check_dists([0, 0, 0, 0, 0, 0, 0], noise_max=1e-7)\n\n\ndef test_select_topic_representation():\n ctfidf_embeddings = np.array([[1, 1, 1]])\n ctfidf_embeddings_sparse = csr_matrix(\n (ctfidf_embeddings.reshape(-1).tolist(), ([0, 0, 0], [0, 1, 2])),\n shape=ctfidf_embeddings.shape,\n )\n topic_embeddings = np.array([[2, 2, 2]])\n\n # Use topic embeddings\n repr_, ctfidf_used = select_topic_representation(ctfidf_embeddings, topic_embeddings, use_ctfidf=False)\n np.testing.assert_array_equal(topic_embeddings, repr_)\n assert not ctfidf_used\n\n # Fallback to c-TF-IDF\n repr_, ctfidf_used = select_topic_representation(ctfidf_embeddings, None, use_ctfidf=False)\n np.testing.assert_array_equal(ctfidf_embeddings, repr_)\n assert ctfidf_used\n\n # Use c-TF-IDF\n repr_, ctfidf_used = select_topic_representation(ctfidf_embeddings, topic_embeddings, use_ctfidf=True)\n np.testing.assert_array_equal(ctfidf_embeddings, repr_)\n assert ctfidf_used\n\n # Fallback to topic embeddings\n repr_, ctfidf_used = select_topic_representation(None, topic_embeddings, use_ctfidf=True)\n np.testing.assert_array_equal(topic_embeddings, repr_)\n assert not ctfidf_used\n\n # `scipy.sparse.csr_matrix` can be used as c-TF-IDF embeddings\n np.testing.assert_array_equal(\n ctfidf_embeddings,\n select_topic_representation(ctfidf_embeddings_sparse, None, use_ctfidf=True, output_ndarray=True)[0],\n )\n\n # check that `csr_matrix` is not casted to `np.ndarray` when `ctfidf_as_ndarray` is False\n repr_ = select_topic_representation(ctfidf_embeddings_sparse, None, output_ndarray=False)[0]\n\n assert isinstance(repr_, csr_matrix)\n" }, { "path": "tests/test_variations/__init__.py", "content": "" }, { "path": "tests/test_variations/test_class.py", "content": "import copy\nimport pytest\nfrom sklearn.datasets import fetch_20newsgroups\n\ndata = fetch_20newsgroups(subset=\"all\", remove=(\"headers\", \"footers\", \"quotes\"))\nclasses = [data[\"target_names\"][i] for i in data[\"target\"]][:1000]\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_class(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics_per_class_global = topic_model.topics_per_class(documents, classes=classes, global_tuning=True)\n topics_per_class_local = topic_model.topics_per_class(documents, classes=classes, global_tuning=False)\n\n assert topics_per_class_global.Frequency.sum() == len(documents)\n assert topics_per_class_local.Frequency.sum() == len(documents)\n assert set(topics_per_class_global.Topic.unique()) == set(topic_model.topics_)\n assert set(topics_per_class_local.Topic.unique()) == set(topic_model.topics_)\n assert len(topics_per_class_global.Class.unique()) == len(set(classes))\n assert len(topics_per_class_local.Class.unique()) == len(set(classes))\n" }, { "path": "tests/test_variations/test_dynamic.py", "content": "import copy\nimport pytest\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_dynamic(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n timestamps = [i % 10 for i in range(len(documents))]\n topics_over_time = topic_model.topics_over_time(documents, timestamps)\n\n assert topics_over_time.Frequency.sum() == len(documents)\n assert set(topics_over_time.Topic.unique()) == set(topic_model.topics_)\n assert len(topics_over_time.Timestamp.unique()) == len(set(timestamps))\n" }, { "path": "tests/test_variations/test_hierarchy.py", "content": "import copy\nimport pytest\nfrom scipy.cluster import hierarchy as sch\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_hierarchy(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n hierarchical_topics = topic_model.hierarchical_topics(documents)\n\n merged_topics = set([v for vals in hierarchical_topics.Topics.values for v in vals])\n\n assert len(hierarchical_topics) > 0\n assert merged_topics == set(topic_model.topics_).difference({-1})\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_linkage(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n linkage_function = lambda x: sch.linkage(x, \"single\", optimal_ordering=True)\n hierarchical_topics = topic_model.hierarchical_topics(documents, linkage_function=linkage_function)\n merged_topics = set([v for vals in hierarchical_topics.Topics.values for v in vals])\n tree = topic_model.get_topic_tree(hierarchical_topics)\n\n assert len(hierarchical_topics) > 0\n assert len(tree) > 50\n assert len(tree.split(\"\\n\")) <= 2 * len(set(topic_model.topics_))\n assert merged_topics == set(topic_model.topics_).difference({-1})\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_tree(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n linkage_function = lambda x: sch.linkage(x, \"single\", optimal_ordering=True)\n hierarchical_topics = topic_model.hierarchical_topics(documents, linkage_function=linkage_function)\n merged_topics = set([v for vals in hierarchical_topics.Topics.values for v in vals])\n tree = topic_model.get_topic_tree(hierarchical_topics)\n\n assert len(hierarchical_topics) > 0\n assert len(tree) > 50\n assert len(tree.split(\"\\n\")) <= 2 * len(set(topic_model.topics_))\n assert merged_topics == set(topic_model.topics_).difference({-1})\n" }, { "path": "tests/test_vectorizers/__init__.py", "content": "" }, { "path": "tests/test_vectorizers/test_ctfidf.py", "content": "import copy\nimport pytest\nimport numpy as np\nimport pandas as pd\nfrom packaging import version\nfrom scipy.sparse import csr_matrix\nfrom sklearn import __version__ as sklearn_version\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom bertopic.vectorizers import ClassTfidfTransformer\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_ctfidf(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topics = topic_model.topics_\n documents = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": topics})\n documents_per_topic = documents.groupby([\"Topic\"], as_index=False).agg({\"Document\": \" \".join})\n documents = topic_model._preprocess_text(documents_per_topic.Document.values)\n count = topic_model.vectorizer_model.fit(documents)\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = count.get_feature_names_out()\n else:\n words = count.get_feature_names()\n\n X = count.transform(documents)\n transformer = ClassTfidfTransformer().fit(X)\n c_tf_idf = transformer.transform(X)\n\n assert len(words) > 1000\n assert all([isinstance(x, str) for x in words])\n\n assert isinstance(X, csr_matrix)\n assert isinstance(c_tf_idf, csr_matrix)\n\n assert X.shape[0] == len(set(topics))\n assert X.shape[1] == len(words)\n\n assert c_tf_idf.shape[0] == len(set(topics))\n assert c_tf_idf.shape[1] == len(words)\n\n assert np.min(X) == 0\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"base_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_ctfidf_custom_cv(model, documents, request):\n cv = CountVectorizer(ngram_range=(1, 3), stop_words=\"english\")\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n topic_model.vectorizer_model = cv\n topics = topic_model.topics_\n documents = pd.DataFrame({\"Document\": documents, \"ID\": range(len(documents)), \"Topic\": topics})\n documents_per_topic = documents.groupby([\"Topic\"], as_index=False).agg({\"Document\": \" \".join})\n documents = topic_model._preprocess_text(documents_per_topic.Document.values)\n count = topic_model.vectorizer_model.fit(documents)\n\n # Scikit-Learn Deprecation: get_feature_names is deprecated in 1.0\n # and will be removed in 1.2. Please use get_feature_names_out instead.\n if version.parse(sklearn_version) >= version.parse(\"1.0.0\"):\n words = count.get_feature_names_out()\n else:\n words = count.get_feature_names()\n\n X = count.transform(documents)\n transformer = ClassTfidfTransformer().fit(X)\n c_tf_idf = transformer.transform(X)\n\n assert len(words) > 1000\n assert all([isinstance(x, str) for x in words])\n\n assert isinstance(X, csr_matrix)\n assert isinstance(c_tf_idf, csr_matrix)\n\n assert X.shape[0] == len(set(topics))\n assert X.shape[1] == len(words)\n\n assert c_tf_idf.shape[0] == len(set(topics))\n assert c_tf_idf.shape[1] == len(words)\n\n assert np.min(X) == 0\n" }, { "path": "tests/test_vectorizers/test_online_cv.py", "content": "import copy\nimport pytest\nfrom bertopic.vectorizers import OnlineCountVectorizer\n\n\n@pytest.mark.parametrize(\n \"model\",\n [\n (\"kmeans_pca_topic_model\"),\n (\"custom_topic_model\"),\n (\"merged_topic_model\"),\n (\"reduced_topic_model\"),\n (\"online_topic_model\"),\n ],\n)\ndef test_online_cv(model, documents, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n vectorizer_model = OnlineCountVectorizer(stop_words=\"english\", ngram_range=(2, 2))\n\n topics = [topic_model.get_topic(topic) for topic in set(topic_model.topics_)]\n topic_model.update_topics(documents, vectorizer_model=vectorizer_model)\n new_topics = [topic_model.get_topic(topic) for topic in set(topic_model.topics_)]\n\n for old_topic, new_topic in zip(topics, new_topics):\n if old_topic[0][0] != \"\":\n assert old_topic != new_topic\n\n\n@pytest.mark.parametrize(\"model\", [(\"online_topic_model\")])\ndef test_clean_bow(model, request):\n topic_model = copy.deepcopy(request.getfixturevalue(model))\n\n original_shape = topic_model.vectorizer_model.X_.shape\n topic_model.vectorizer_model.delete_min_df = 2\n topic_model.vectorizer_model._clean_bow()\n\n assert original_shape[0] == topic_model.vectorizer_model.X_.shape[0]\n assert original_shape[1] > topic_model.vectorizer_model.X_.shape[1]\n" } ]