Repository: Mozilla-Ocho/Memory-Cache Branch: main Commit: 8b54e4e7daa8 Files: 115 Total size: 320.2 KB Directory structure: gitextract_ahfxg4dk/ ├── .gitignore ├── ATTRIBUTIONS.md ├── LICENSE ├── README.md ├── docs/ │ ├── .gitignore │ ├── 404.html │ ├── CNAME │ ├── Gemfile │ ├── _config.yml │ ├── _includes/ │ │ ├── footer.html │ │ └── header.html │ ├── _layouts/ │ │ ├── default.html │ │ ├── home.html │ │ ├── page.html │ │ └── post.html │ ├── _posts/ │ │ ├── 2023-11-06-introducing-memory-cache.markdown │ │ ├── 2023-11-30-we-have-a-website.markdown │ │ ├── 2024-03-01-memory-cache-and-ai-privacy.markdown │ │ ├── 2024-03-06-designlog-update.markdown │ │ ├── 2024-03-07-devlog.markdown │ │ ├── 2024-03-15-devlog.markdown │ │ └── 2024-04-19-memory-cache-hub.markdown │ ├── _sass/ │ │ ├── memorycache.scss │ │ └── minima.scss │ ├── about.markdown │ ├── assets/ │ │ └── main.scss │ ├── faq.md │ ├── index.markdown │ └── readme.md ├── extension/ │ ├── content-script.js │ ├── manifest.json │ └── popup/ │ ├── marked.esm.js │ ├── memory_cache.html │ ├── memory_cache.js │ └── styles.css ├── scratch/ │ ├── backend/ │ │ ├── hub/ │ │ │ ├── .gitignore │ │ │ ├── PLAN.md │ │ │ ├── README.md │ │ │ ├── docker/ │ │ │ │ ├── Dockerfile.hub-builder-gnu-linux │ │ │ │ ├── Dockerfile.hub-builder-old-gnu-linux │ │ │ │ ├── Dockerfile.hub-dev │ │ │ │ └── Dockerfile.hub-dev-cuda │ │ │ ├── requirements/ │ │ │ │ ├── hub-base.txt │ │ │ │ ├── hub-builder.txt │ │ │ │ └── hub-cpu.txt │ │ │ └── src/ │ │ │ ├── api/ │ │ │ │ ├── llamafile_api.py │ │ │ │ └── thread_api.py │ │ │ ├── async_utils.py │ │ │ ├── chat.py │ │ │ ├── chat2.py │ │ │ ├── chat3.py │ │ │ ├── fastapi_app.py │ │ │ ├── gradio_app.py │ │ │ ├── hub.py │ │ │ ├── hub_build_gnu_linux.py │ │ │ ├── hub_build_macos.py │ │ │ ├── hub_build_windows.py │ │ │ ├── llamafile_infos.json │ │ │ ├── llamafile_infos.py │ │ │ ├── llamafile_manager.py │ │ │ └── static/ │ │ │ └── index.html │ │ ├── langserve-demo/ │ │ │ ├── .gitignore │ │ │ ├── Dockerfile.cpu │ │ │ ├── README.md │ │ │ ├── client.py │ │ │ ├── requirements-cpu.txt │ │ │ ├── requirements.txt │ │ │ └── serve.py │ │ └── python-llamafile-manager/ │ │ ├── .gitignore │ │ ├── Dockerfile.plm │ │ ├── Dockerfile.plm-builder-gnu-linux │ │ ├── README.md │ │ ├── build_gnu_linux.py │ │ ├── manager.py │ │ └── requirements.txt │ ├── browser-client/ │ │ ├── .gitignore │ │ ├── README.md │ │ ├── package.json │ │ ├── src/ │ │ │ ├── index.html │ │ │ ├── main.js │ │ │ ├── styleguide.html │ │ │ └── styles.css │ │ └── webpack.config.js │ └── hub-browser-client/ │ ├── .gitignore │ ├── README.md │ ├── config/ │ │ ├── env.js │ │ ├── getHttpsConfig.js │ │ ├── jest/ │ │ │ ├── babelTransform.js │ │ │ ├── cssTransform.js │ │ │ └── fileTransform.js │ │ ├── modules.js │ │ ├── paths.js │ │ ├── webpack/ │ │ │ └── persistentCache/ │ │ │ └── createEnvironmentHash.js │ │ ├── webpack.config.js │ │ └── webpackDevServer.config.js │ ├── package.json │ ├── public/ │ │ ├── index.html │ │ ├── manifest.json │ │ └── robots.txt │ ├── scripts/ │ │ ├── build.js │ │ ├── start.js │ │ └── test.js │ ├── src/ │ │ ├── App.css │ │ ├── App.test.tsx │ │ ├── App.tsx │ │ ├── api/ │ │ │ └── llamafile_api.ts │ │ ├── components/ │ │ │ └── llamafile_details.tsx │ │ ├── index.css │ │ ├── index.tsx │ │ ├── react-app-env.d.ts │ │ ├── reportWebVitals.ts │ │ ├── setupTests.ts │ │ └── types.ts │ └── tsconfig.json └── scripts/ └── run_ingest.sh ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .DS_Store extension/.web-extension-id extension/web-ext-artifacts/ ================================================ FILE: ATTRIBUTIONS.md ================================================ ## Icons brain_24.png icon licensed under [CC-by 3.0 Unported](https://creativecommons.org/licenses/by/3.0/) from user 'Howcolour' on www.iconfinder.com save-icon-16.png icon licensed under [CC-by 3.0](https://creativecommons.org/licenses/by/3.0/) from user 'Bhuvan' from Noun Project file-icon-16.png icon licensed under [CC-by 3.0](https://creativecommons.org/licenses/by/3.0/) from user 'Mas Dhimas' from Noun Project ## Helpful Links CSS Gradient tool used: https://cssgradient.io/ Pastel Rainbow color palette by user allyasdf on color-hex: https://www.color-hex.com/color-palette/5361 CSS Trick - border-top-linear-gradient solution fromL https://michaelharley.net/posts/2021/01/12/how-to-create-a-border-top-linear-gradient/ ================================================ FILE: LICENSE ================================================ Mozilla Public License Version 2.0 ================================== 1. Definitions -------------- 1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. 1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. 1.3. "Contribution" means Covered Software of a particular Contributor. 1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. 1.5. "Incompatible With Secondary Licenses" means (a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or (b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. 1.6. "Executable Form" means any form of the work other than Source Code Form. 1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. "License" means this document. 1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. 1.10. "Modifications" means any of the following: (a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or (b) any new file in Source Code Form that contains any Covered Software. 1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. 1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. 1.13. "Source Code Form" means the form of the work preferred for making modifications. 1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. 2. License Grants and Conditions -------------------------------- 2.1. Grants Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: (a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and (b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. 2.2. Effective Date The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. 2.3. Limitations on Grant Scope The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a Contributor: (a) for any code that a Contributor has removed from Covered Software; or (b) for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its Contributions with other software (except as part of its Contributor Version); or (c) under Patent Claims infringed by Covered Software in the absence of its Contributions. This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with the notice requirements in Section 3.4). 2.4. Subsequent Licenses No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). 2.5. Representation Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. 2.6. Fair Use This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. 2.7. Conditions Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. 3. Responsibilities ------------------- 3.1. Distribution of Source Form All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. 3.2. Distribution of Executable Form If You distribute Covered Software in Executable Form then: (a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and (b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. 3.3. Distribution of a Larger Work You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). 3.4. Notices You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. 3.5. Application of Additional Terms You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional disclaimers of warranty and limitations of liability specific to any jurisdiction. 4. Inability to Comply Due to Statute or Regulation --------------------------------------------------- If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a recipient of ordinary skill to be able to understand it. 5. Termination -------------- 5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. 5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. ************************************************************************ * * * 6. Disclaimer of Warranty * * ------------------------- * * * * Covered Software is provided under this License on an "as is" * * basis, without warranty of any kind, either expressed, implied, or * * statutory, including, without limitation, warranties that the * * Covered Software is free of defects, merchantable, fit for a * * particular purpose or non-infringing. The entire risk as to the * * quality and performance of the Covered Software is with You. * * Should any Covered Software prove defective in any respect, You * * (not any Contributor) assume the cost of any necessary servicing, * * repair, or correction. This disclaimer of warranty constitutes an * * essential part of this License. No use of any Covered Software is * * authorized under this License except under this disclaimer. * * * ************************************************************************ ************************************************************************ * * * 7. Limitation of Liability * * -------------------------- * * * * Under no circumstances and under no legal theory, whether tort * * (including negligence), contract, or otherwise, shall any * * Contributor, or anyone who distributes Covered Software as * * permitted above, be liable to You for any direct, indirect, * * special, incidental, or consequential damages of any character * * including, without limitation, damages for lost profits, loss of * * goodwill, work stoppage, computer failure or malfunction, or any * * and all other commercial damages or losses, even if such party * * shall have been informed of the possibility of such damages. This * * limitation of liability shall not apply to liability for death or * * personal injury resulting from such party's negligence to the * * extent applicable law prohibits such limitation. Some * * jurisdictions do not allow the exclusion or limitation of * * incidental or consequential damages, so this exclusion and * * limitation may not apply to You. * * * ************************************************************************ 8. Litigation ------------- Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. 9. Miscellaneous ---------------- This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. 10. Versions of the License --------------------------- 10.1. New Versions Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. Each version will be given a distinguishing version number. 10.2. Effect of New Versions You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. 10.3. Modified Versions If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. Exhibit A - Source Code Form License Notice ------------------------------------------- This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE file in a relevant directory) where a recipient would be likely to look for such a notice. You may add additional accurate notices of copyright ownership. Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------- This Source Code Form is "Incompatible With Secondary Licenses", as defined by the Mozilla Public License, v. 2.0. ================================================ FILE: README.md ================================================ # Memory Cache Memory Cache is a project that allows you to save a webpage while you're browsing in Firefox as a PDF, and save it to a synchronized folder that can be used in conjunction with privateGPT to augment a local language model. | ⚠️: This setup uses the primordial version of privateGPT. I'm working from a fork that can be found [here](https://github.com/misslivirose/privateGPT). | | ---------------------------------------------------------------------------------------------------------------------- | ## Prerequisites 1. Set up [privateGPT](https://github.com/imartinez/privateGPT) - either using the primordial checkpoint, or from my fork. 2. Create a symlink between a subdirectory in your default Downloads folder called 'MemoryCache' and a 'MemoryCache' directory created inside of /PrivateGPT/source_documents/MemoryCache 3. Apply patch to Firefox to add the `printerSettings.silentMode` property to the Tabs API. [See wiki page for instructions](https://github.com/Mozilla-Ocho/Memory-Cache/wiki/Modifying-Firefox-to-Save-PDF-files-automagically-to-MemoryCache) 4. Copy /scripts/run_ingest.sh into your privateGPT directory and run it to start `inotifywait` watching your downloads directory for new content ## Setting up the Extension 1. Clone the Memory-Cache GitHub repository to your local machine 2. In Firefox, navigate to `about:debugging` and click on 'This Firefox' 3. Click 'Load Temporary Add-on" and open the `extension/manifest.json` file in the MemoryCacheExt directory ## Using the Extension 1. Under the 'Extensions' menu, add the Memory Cache extension to the toolbar 2. When you want to save a page to your Memory Cache, click the icon and select the 'Save' button. This will save the file silently as a PDF if you are using a Firefox build with the `printerSettings.silentMode` property addition. ================================================ FILE: docs/.gitignore ================================================ _site .sass-cache .jekyll-cache .jekyll-metadata vendor ================================================ FILE: docs/404.html ================================================ --- permalink: /404.html layout: default ---

404

Page not found :(

The requested page could not be found.

================================================ FILE: docs/CNAME ================================================ memorycache.ai ================================================ FILE: docs/Gemfile ================================================ source "https://rubygems.org" # Hello! This is where you manage which Jekyll version is used to run. # When you want to use a different version, change it below, save the # file and run `bundle install`. Run Jekyll with `bundle exec`, like so: # # bundle exec jekyll serve # # This will help ensure the proper Jekyll version is running. # Happy Jekylling! # This is the default theme for new Jekyll sites. You may change this to anything you like. gem "minima", "~> 2.5" # If you want to use GitHub Pages, remove the "gem "jekyll"" above and # uncomment the line below. To upgrade, run `bundle update github-pages`. gem "github-pages", "~> 228", group: :jekyll_plugins # If you have any plugins, put them here! group :jekyll_plugins do gem "jekyll-feed", "~> 0.12" end # Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem # and associated library. platforms :mingw, :x64_mingw, :mswin, :jruby do gem "tzinfo", ">= 1", "< 3" gem "tzinfo-data" end # Performance-booster for watching directories on Windows gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] # Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem # do not have a Java counterpart. gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] gem "webrick", "~> 1.8" ================================================ FILE: docs/_config.yml ================================================ # Welcome to Jekyll! # # This config file is meant for settings that affect your whole blog, values # which you are expected to set up once and rarely edit after that. If you find # yourself editing this file very often, consider using Jekyll's data files # feature for the data you need to update frequently. # # For technical reasons, this file is *NOT* reloaded automatically when you use # 'bundle exec jekyll serve'. If you change this file, please restart the server process. # # If you need help with YAML syntax, here are some quick references for you: # https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml # https://learnxinyminutes.com/docs/yaml/ # # Site settings # These are used to personalize your new site. If you look in the HTML files, # you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. # You can create any custom variable you would like, and they will be accessible # in the templates via {{ site.myvariable }}. title: MemoryCache email: oerickson@mozilla.com description: MemoryCache is an experimental developer project to turn a local desktop environment into an on-device AI agent. # baseurl: "/Memory-Cache" # the subpath of your site, e.g. /blog url: "https://memorycache.ai/" # the base hostname & protocol for your site, e.g. http://example.com github_username: Mozilla-Ocho # Build settings theme: minima plugins: - jekyll-feed # Exclude from processing. # The following items will not be processed, by default. # Any item listed under the `exclude:` key here will be automatically added to # the internal "default list". # # Excluded items can be processed by explicitly listing the directories or # their entries' file path in the `include:` list. # # exclude: # - .sass-cache/ # - .jekyll-cache/ # - gemfiles/ # - Gemfile # - Gemfile.lock # - node_modules/ # - vendor/bundle/ # - vendor/cache/ # - vendor/gems/ # - vendor/ruby/ ================================================ FILE: docs/_includes/footer.html ================================================ ================================================ FILE: docs/_includes/header.html ================================================ ================================================ FILE: docs/_layouts/default.html ================================================ {%- include head.html -%} {%- include header.html -%}
{{ content }}
{%- include footer.html -%} ================================================ FILE: docs/_layouts/home.html ================================================ --- layout: default ---
{%- if page.title -%}

{{ page.title }}

{%- endif -%} {{ content }}

MemoryCache is an experimental development project to turn a local desktop environment into an on-device AI agent.

Every human is unique. The original vision of the personal computer was as a companion tool for creating intelligence, and the internet was born as a way to connect people and data together around the world. Today, artificial intelligence is upending the way that we interact with data and information, but control of these systems is most often provided through an API endpoint, run in the cloud, and abstracting away deep personal agency in favor of productivity.

MemoryCache, a Mozilla Innovation Project, is an experimental AI Firefox add-on that partners with privateGPT to quickly save your browser history to your local machine and have a local AI model ingest these - and any other local files you give it - to augment responses to a chat interface that comes built-in with privateGPT. We have an ambition to use MemoryCache to move beyond the chat interface, and find a way to utilize idle compute time to generate net new insights that reflect what you've actually read and learned - not the entirety of the internet at scale.

Design mockup of a future interface idea for MemoryCache

We're not breaking ground on AI innovation (in fact, we're using an old, "deprecated" file format from a whole six months ago), by design. MemoryCache is a project that allows us to sow some seeds of exploration into creating a deeply personalized AI experience that returns to the original vision of the computer as a companion for our own thought. With MemoryCache, weirdness and unpredictability is part of the charm.

We're a small team working on MemoryCache as a part-time project within Mozilla's innovation group, looking at ways that our personal data and files are used to form insights and new neural connections for our own creative purpose. We're working in the open not because we have answers, but because we want to contribute our way of thinking to one another in a way where others can join in.

{%- if site.posts.size > 0 -%}

{{ page.list_title | default: "Updates" }}

subscribe via RSS

{%- endif -%}
================================================ FILE: docs/_layouts/page.html ================================================ --- layout: default ---

{{ page.title | escape }}

{{ content }}
================================================ FILE: docs/_layouts/post.html ================================================ --- layout: default ---

{{ page.title | escape }}

{%- if page.author -%} • {%- endif -%}

{{ content }}
{%- if site.disqus.shortname -%} {%- include disqus_comments.html -%} {%- endif -%}
================================================ FILE: docs/_posts/2023-11-06-introducing-memory-cache.markdown ================================================ --- layout: post title: "Introducing Memory Cache" date: 2023-11-06 14:47:57 -0500 categories: developer-blog --- Most AI development today is centered around services. Companies offer tailored insights and powerful agents that can replicate all aspects of the human experience. AI is supposedly "passing the bar exam", diagnosing medical issues, and everything in-between. What is the role of a human being in an increasingly online world? In practice, AI is a complex web of big data sources (e.g. the entirety of the internet). Pairing massive amounts of data with increasingly powerful cloud computing capabilities has resulted in unprecedented software development capabilities. Adopting naming practices and principles from science fiction stories, Silicon Valley is racing down a path towards a fictional idea of a "sentient computer" with AGI. Artificial intelligence is a field gives us more modalities and capabilities to use with computers, and what really matters is how we (as humans) use the technology at hand. Not that long ago, computing was grounded in the idea that digital literacy was a skill to be adopted and used in service of greater problems to be solved. We, as individuals, had control over our data, our files, our thoughts. Over the past several years, Big Tech has traded us systems of addictive social media sites for yottabytes of our personal data in the service of "personalization" - a.k.a targeted advertising. Memory Cache is an exploration into human-first artificial intelligence, starting with the actual idea of the personal computer. The project is an experiment in local, on-premise AI: what you can do with a standard gaming desktop that sits in your home, and actually works for you. It bridges your browser history with your local file system, so that you can use the power of openly licensed AI and open source code to inspect, query, and tinker with an AI that is under your own control. ================================================ FILE: docs/_posts/2023-11-30-we-have-a-website.markdown ================================================ --- layout: post title: "We have a website! And other MemoryCache Updates" date: 2023-11-30 11:47:00 -0800 categories: developer-blog --- We've been continuing our work on MemoryCache over the past several weeks, and are excited to have our [landing page](https://memorycache.ai) up and running. The updated design for MemoryCache has been something we've been iterating on, and it's been a fun process to talk about what sort of emotions we want to seed MemoryCache tools with. We've settled on an initial design style guide and have recently landed several contributions to enable a vanilla Firefox version of the extension. Our team is working on MemoryCache as a sandbox for exploring concepts related to small, local, and patient AI. We're a small project with big ambitions, and look forward to continuing down several areas of exploration in the coming weeks, including: * Building an app experience to automatically generate insights from newly added data, to act as a gentle reminder of places that can grow from your recently added content * Updating the project website to include more details about the philosophy, design, and thinking around the project and how we envision it growing * Competitive and secondary research reporting that we can publish that shares our insights and findings on how people think about recall and note-taking * Understanding how to evaluate and generate personal insights outside of the chat interface model * Exploring a social layer to easily distill and share insights within a trusted network of people Follow along with us on our [GitHub repo](https://github.com/misslivirose/Memory-Cache) - we'd love to see you there! ================================================ FILE: docs/_posts/2024-03-01-memory-cache-and-ai-privacy.markdown ================================================ --- layout: post title: "MemoryCache and AI Privacy" date: 2024-03-01 07:08:57 -0500 categories: developer-blog --- _Author: Liv Erickson_ It's been an exciting few months since we first shared [MemoryCache](https://memorycache.ai/developer-blog/2023/11/06/introducing-memory-cache.html), a home for experimenting with local-first AI that learns what you learn. While we'll be sharing more updates about what the team has been working on in the coming weeks by way of more regular development blogs, I wanted to share a podcast that I recently recorded with [Schalk Neethling for the Mechanical Ink Podcast](https://schalkneethling.substack.com/p/privacy-ai-and-an-ai-digital-memory) that goes in-depth about the principles behind MemoryCache. [![Watch the video](https://img.youtube.com/vi/CGdxLfcU9TU/0.jpg)](https://www.youtube.com/watch?v=CGdxLfcU9TU) In this podcast, we go into the motivations behind the project and how it was originally created, as well as the overall challenges that are presented with the growing creation of synthetic content and preserving authentic connections online in an area of unprecedented, generated personalization. At it's core, MemoryCache is a project exploring what it means to urgently, yet collaboratively, envision futures where AI technologies enable us to build a more authentic relationship with information and ourselves through small acts of insight and [embracing friction](https://foundation.mozilla.org/en/blog/speculative-friction-in-generative-ai/) as it presents novel outcomes. More coming soon! ================================================ FILE: docs/_posts/2024-03-06-designlog-update.markdown ================================================ --- layout: post title: "MemoryCache March Design Update" date: 2024-03-06 08 -0500 categories: developer-blog --- _Author: Kate Taylor_ Hi! My name is Kate and I am a designer working on MemoryCache in the Mozilla Innovation organization. My official title is Design Technologist, which describes the focus between humans and technology. Humans is an important word in this context because it is not specific to a group of people, but the recognition that the choices we make as technologists have effects that ripple on to humans who may or may not be aware of what happens with their information. Information is powerful in the world of AI and the handling of it deserves genuine respect, which in turn builds on an atmosphere of trust and safety. MemoryCache serves as an open and safe testbed to explore the ideas of what humans need to both benefit from an AI agent while maintaining control over their information and the technical processes involved. As a designer on this project, my work is intended to create an environment that feels like a true augmentation of your creative thought work.
Sketches and notes describing why establishing context is important for AI tasks and the feeling of security
Early concepts and notes exploring the idea of what safety in an AI Agent experience looks like
When we started the project last year, the world was in a different place. Reflecting back on the time along with the goings-on in the AI space is bringing to mind a lot of big personal feelings as well as acknowledgment of the generally fearful vibes. I tend to pay attention to feelings because they are what allow us (as people) to find the things that matter most. It’s difficult to not sense an amount of fear when people are faced with a lot of change. This especially becomes clear when comparing conversations with people in and out of the tech field. This fear has brought a lot of very meaningful interpersonal conversations about what this technology means -- what job does it do well, what jobs do we (as people) do well and want to keep, how did we get to this point in time. These conversations are the motivation for contributing to Open Source AI because the power of understanding the world around you is boundless, and the barriers to this knowledge are mysterious but significant. Awareness of the barriers to entry to an experience is where designers do their best work. We strive to find meaningful solutions to problems that will sustain. Acknowledging the emotional barrier is the foundation of how we are thinking about MemoryCache. Depending on who you are, interacting with a chatbot has baggage - in the same way that social situations differ across individuals. There are social aspects, language considerations, articulation differences, historical contexts, etc. This is A LOT to deal with as a user of a system. When approaching the design work for MemoryCache, our guiding light is to take into account the unique humanity of each person and allow for the ability to utilize the technology to create an environment that nurtures human needs rather than profit from them.
Design mockups and explorations for the interface of MemoryCache as a desktop application
Exploratory work for interactions that combine with chat interface to interact with the agent in personalized ways for various usecases
Design mockup with various color theme options
Design mockups and explorations for the UI exploring themes and personalization in combination with input methods for interaction
This philosophy is core to the work we are doing with MemoryCache. We believe that personalized experiences for interacting with your own information provides a safe space for working with your thought materials with a lot of flexibility and personalized modularity. When thinking about what safety means in relation to AI computing, in 2024 this is a complicated subject. We are not just speaking about access to information, but access to people and the very things that make us human. Painting, drawing, and tinkering have always been the safe space in my life, personally. The process of making things provides the opportunity to explore your thoughts and experiences without the judgment or unsolicited opinions of others. This time spent reflecting tends to be where the most valuable ideas come to light in other areas of life (similar to the idea of “shower thoughts”). We are iterating on this concept with the idea that the agent could work with the person in creating more of those shower-thought moments. Flipping the idea of asking the agent for a task with the agent providing insights that you could find valuable. The mockups below demonstrate our current thinking for an interface that we can start building and working with as needs evolve
Design mockup for the MemoryCache agent's UI
Latest design mockup for MemoryCache agent
![Image4](https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/881b41b1-4217-4fbe-85a0-cfdbe3732697) We are excited about the potential of where MemoryCache can go. Part of the magic of developing in the open is that we will learn along the way what matters most to people and evolve from there. The logo and visual design we are running with for MemoryCache visualizes our hope for celebration of individuality and personal empowerment through technology advancements. The world can be a messy place, but your individual context is yours to control. In our next round of work, we are building on this philosophy to expose the Agent’s capabilities in meaningful interactions that support the ability to augment your thought work in the way that your brain thinks. ================================================ FILE: docs/_posts/2024-03-07-devlog.markdown ================================================ --- layout: post title: "Memory Cache Dev Log March 7 2024" date: 2024-03-07 08 -0500 categories: developer-blog --- _Author: John Shaughnessy_ # Memory Cache Dev Log, March 7 2024 A couple months ago [we introduced Memory Cache](https://future.mozilla.org/blog/introducing-memorycache/): > Memory Cache, a Mozilla Innovation Project, is an early exploration project that augments an on-device, personal model with local files saved from the browser to reflect a more personalized and tailored experience through the lens of privacy and agency Since then we've been quiet.... _too quiet_. ## New phone, who dis? It's my first time writing on this blog, so I want to introduce myself. My name is John Shaughnessy. I'm a software engineer at Mozilla. I got involved in Memory Cache a few months ago by resolving an issue that was added to the github repo and a couple weeks ago I started building Memory Cache V2. ## Why V2? Memory Cache V1 was a browser extension and that made it convenient to collect and feed documents to an LLM-based program called `privateGPT`. PrivateGPT would break the documents into fragments, save those fragments in a vector database, and let you perform similarity search on those documents via a command line interface. We were running an old version of PrivateGPT based on LangChain. There were several big, obvious technical gaps between Memory Cache V1 and what we'd need in order to do the kind of investigative research and product development we wanted to do. It seemed to me that if we really wanted to explore the design space, we'd need to roll our own backend and ship a friendly client UI alongside it. We'd need to speed up inference and we'd need more control over how it ingested documents, inserted context, batched background tasks and presented information to you. We also needed to fix the "getting started" experience. Setting up V1 required users to be comfortable working on the command line, managing python environments, and in general understanding their file system. As far as I'm aware, there are only three of us who have gone through the steps to actually set up and run V1. We were inspired by [Llamafile](https://github.com/Mozilla-Ocho/llamafile/) and [cosmopolitan](https://justine.lol/ape.html), which create executables that you just download and run on many platforms. And lastly, we're excited about multiplayer opportunities. Could insights that my LLM generates become useful to my teammates? Under what circumstances would I want to share my data with others? How should I separate what's private, semi-private, or public? ### Running LLMs for Inference I wasn't very familiar with running LLMs, and I certainly hadn't written an application that did "Retrieval-Augmented Generation" (RAG), which was what we wanted Memory Cache to do. So I started down a long, winding path. Liv and I chatted with Iván Martínez who wrote `privateGPT`. He was super helpful! And it was exciting to talk to someone who'd built something that let us prototype what we wanted to do so quickly. Mozilla had just announced [Llamafile](https://github.com/Mozilla-Ocho/llamafile), which seemed like a great way to package an LLM and serve it on many platforms. I wasn't familiar with either [Llama.cpp](https://github.com/ggerganov/llama.cpp) or [cosmo](https://cosmo.zip/), so there was a lot to learn. [Justine](https://github.com/jart) and [Stephen](https://github.com/stlhood) were incredibly helpful and generous with their time. I didn't contribute much back to the project other than trying to write accurate reports of a couple issues ([#214](https://github.com/Mozilla-Ocho/llamafile/issues/214), [#232](https://github.com/Mozilla-Ocho/llamafile/issues/232)) I ran into along the way. Initially when I was looking into `Llamafile`, I wanted to repackage `privateGPT` as a `Llamafile` so that we could distribute it as a standalone executable. Eventually realized this wasn't a good idea. `Llamafile` bundles `Llama.cpp` programs as executables. `Cosmopolitan` _can_ also bundle things like a python interpreter, but tracking down platform-specific dependencies of `privateGPT` and handling them in a way that was compatible with cosmo was not going to be straightforward. It's just not what the project was designed to do.
Once I worked through the issues I was having with my GPU, I was amazed and excited to see how fast LLMs can run. I made a short comparison video that shows the difference: [llamafile CPU vs GPU](https://www.youtube.com/watch?v=G9wBw8jLJwU). I thought I might extend the simple HTTP server that's baked into `Llamafile` with all the capabilities we'd want in Memory Cache. Justine helped me get some "hello world" programs up and running, and I started reading some examples of C++ servers. I'm not much of a C++ programmer, and I was not feeling very confident that this was the direction I really wanted to go. I like working in Rust, and I knew that Rust had some kind of story for getting `C` and `C++` binding working, so I wrote a kind of LLM "hello world" program using [rustformers/llm](https://github.com/rustformers/llm). But after about a week of fiddling with Llamafile, Llama.cpp, and rustformers, I felt like I was going down a bit of a rabbit hole, and I wanted to pull myself back up to the problem at hand. ### Langchain and Langserve Ok. So if we weren't going to build out a C++ or Rust server, what _should_ we be doing? `PrivateGPT` was a python project, and the basic functionality was similar to what I'd done in some simple programs I'd written with hugging face's `transformers` library. (I mentioned these in a [blog post](https://johnshaughnessy.com/blog/posts/osai-kube) and [talk](https://www.youtube.com/watch?v=AHd3jCMQQLs) about upskilling in AI.) It seemed like `LangChain` and `LlamaIndex` were the two popular python libraries / frameworks for building RAG apps, so I wrote a "hello world" with LangChain. It was... fine. But it seemed like a _lot_ more functionality (and abstraction, complexity, and code) than I wanted. I ended up dropping the framework after reading the docs for `ChromaDB` and `FastAPI`. `ChromaDB` is a vector database for turning documents into fragments and then run similarity search (the fundamentals of a RAG system).
I needed to choose a database, and I chose this one arbitrarily. Langchain had an official "integration" for Chroma, but I felt like Chroma was so simple that I couldn't imagine an "integration" being helpful. `FastAPI` is a python library for setting up http servers, and is "batteries included" in some very convenient ways: - It's compatible with [pydantic](https://docs.pydantic.dev/) which lets you define types, validate user input against them, and generate `OpenAPI` spec from them. - It comes with [swagger-ui](https://github.com/swagger-api/swagger-ui) which gives an interactive browser interface to your APIs. - It's compatible with a bunch of other random helpful things like [python-multipart](https://github.com/Kludex/python-multipart). The other thing to know about `FastAPI` is that as far as http libraries go, it's very easy to use. I was reading documentation about `Langserve`, which seemed like a kind of fancy server for `Langchain` apps until I realized that actually `FastAPI`, `pydantic`, `swagger-ui` et. al were doing all the heavy lifting. So, I dropped LangChain and Langserve and decided I'd wait until I encountered an actually hard problem before picking up another framework. (And who knows -- such a problem might be right around the corner!) It helped to read LangChain docs and code to figure out what RAG even is. After that I was able to get basic rag app working (without the framework). I felt pretty good about it. ### Inference I still needed to decide how to run the LLM. I had explored Llamafiles and Hugging face's `transformers` library. The other popular option seemed to be `ollama`, so I gave that a shot. `Ollama` ended up being very easy to get up and running. I don't know very much about the project. So far I'm a fan. But I didn't want users of Memory Cache to have to download and run an LLM inference server/process by themselves. It just feels like a very clunky user experience. I want to distribute ONE executable that does everything. Maybe I'm out of the loop, but I didn't feel very good about any of the options. Like, what I really wanted was to write a python program that handled RAG, project files, and generating various artifacts by talking to an llm, and I also wanted it to run the LLM, and I also wanted it to be an HTTP server to serve a browser client. I suppose that's a complex list of requirements, but it seemed like a reasonable approach for Memory Cache. And I didn't find any examples of people doing this. I had the idea of using Llamafiles for inference and a python web server for the rest of the "brains", which could also serve a static client. That way, the python code stays simple (it doesn't bring with it any of the transformers / hugging face / llm / cuda code). ### Memory Cache Hub I did a series of short spikes to piece together exactly how such an app could work. I wrote a bit about each one in this [PR](https://github.com/Mozilla-Ocho/Memory-Cache/pull/58). In the end, I landed on a (technical) design that I'm pretty happy with. I'm putting the pieces together in a repo called [Memory Cache Hub](https://github.com/johnshaughnessy/Memory-Cache-Hub/) (which will graduate to the Mozilla-Ocho org when it's ready). The [README.md](https://github.com/johnshaughnessy/Memory-Cache-Hub/blob/main/README.md) has more details, but here's the high level: ```plaintext Memory Cache Hub is a core component of Memory Cache: - It exposes APIs used by the browser extension, browser client, and plugins. - It serves static files including the browser client and various project artifacts. - It downloads and runs llamafiles as subprocesses. - It ingests and retrieves document fragments with the help of a vector database. - It generates various artifacts using prompt templates and large language models. Memory Cache Hub is designed to run on your own machine. All of your data is stored locally and is never uploaded to any server. To use Memory Cache Hub: - Download the latest release for your platform (Windows, MacOS, or GNU/Linux) - Run the release executable. It will open a new tab in your browser showing the Memory Cache GUI. - If the GUI does not open automatically, you can navigate to http://localhost:4444 in your browser. Each release build of Memory Cache Hub is a standalone executable that includes the browser client and all necessary assets. By "standalone", we mean that you do not need to install any additional software to use Memory Cache. A Firefox browser extension for Memory Cache that extends its functionality is also available. More information can be found in the main Memory Cache repository. ``` There are two key ideas here: - Inference is provided by llamafiles that the hub downloads and runs. - We use `PyInstaller` to bundle the hub and the browser client into a single executable that we can release. The rest of the requirements are handled easily in python because of the great libraries and tools that are available (`fastapi`, `pydantic`, `chromadb`, etc). Getting the two novel ideas to work was challenging. I'm not a python expert, so figuring out the `asyncio` and `subprocess` stuff to download and run llamafiles was tricky. And `PyInstaller` has a long list of "gotchas" and "beware" warnings in its docs. I'm still not convinced I'm using it correctly, even though the executables I'm producing seem like they're doing the right thing. ## The Front End By this time I had built three measly, unimpressive browser clients for Memory Cache. The first was compatible with `privateGPT`, the second was compatible with some early versions of the Memory Cache Hub. I built the third with `gradio` but quickly decided that it did not spark joy. And none of these felt like good starting points for a designer to jump into the building process. I've started working on a kind of "hello world" dashboard for Memory Cache using `tailwindcss`. I want to avoid reinventing the wheel and make sure the basic interactions feel good. I've exposed most of the Hub's APIs in the client interface by now. It doesn't look or feel good yet, but it's good to have the basic capabilities working.
## What We're Aiming For The technical pieces have started to fall into place. We're aiming to have the next iteration of Memory Cache - one that you can easily download and run on your own machine - in a matter of weeks. In an ideal world, we'd ship by the end of Q1, which is a few weeks away. It won't be perfect, but it'll be far enough along that the feedback we get will be much more valuable, and will help shape the next steps. ================================================ FILE: docs/_posts/2024-03-15-devlog.markdown ================================================ --- layout: post title: "Memory Cache Dev Log March 15 2024" date: 2024-03-15 08 -0500 categories: developer-blog --- _Author: John Shaughnessy_ # Memory Cache Dev Log, March 15 2024 Last Friday, during a casual weekly engineering call, a colleague asked how LLM libraries (llama.cpp, llamafile, langchain, llamaindex, HF transformers, ollama, etc) handle the different chat templates and special tokens that models train on. It was a good question, and none of us seemed to have a complete answer. The subsequent discussion and research made me realize that a naive approach towards writing model-agnostic application would have unfortunate limitations. Insofar as the differences between models are actually important to the use case, application developers should write model-specific code. ## Text Summarization I thought about the capabilities that are important for an application like Memory Cache, and which models would be good at providing those capabilities. The first obvious one was text summarization. I had "hacked" summarization by asking an assistant-type model (llamafile) to summarize text, but a model trained specifically for text summarization would be a better fit. I tested a popular text summarization model with with HF transformers, since I didn't find any relevant llamafiles. (If they're out there, I don't know how to find them.) I wanted to make sure that HF code could be built and bundled to a native application with PyInstaller, since that's how we want to build and bundle Memory Cache as a standalone executable. I verified that it could with a [small test project](https://github.com/johnshaughnessy/summarization-test). Bundling HF dependencies like pytorch increases the complexity of the release process because we'd go from 3 build targets (MacOS, Windows, Linux) to 8 build targets (assuming support for every platform that pytorch supports): - `Linux` + `CUDA 11.8` - `Linux` + `CUDA 12.1` - `Linux` + `ROCm 5.7` - `Linux` + `CPU` - `Mac` + `CPU` - `Windows` + `CUDA 11.8` - `Windows` + `CUDA 12.1` - `Windows` + `CPU` It's good to know that this is possible, but since our near-term goal for Memory Cache is just to prove out the technical bits mentioned in the [previous dev log](https://memorycache.ai/developer-blog/2024/03/07/devlog.html), we'll likely stick with the text summarization "hack" for now. ## Training Agents Text summarization is still a simple task (in terms of inference inputs and outputs), so models trained to summarize are likely interchangable for the most part (modulo input/output lengths). However, once we start looking at more complicated types of tasks (like tool use / function calling / memory), the differences between models will be exaggerated. Consider an example like [this dataset](https://huggingface.co/datasets/smangrul/assistant_chatbot_dataset) meant to help train a model to with act with agentic intentions, beliefs (memory), actions, and chat: ``` Context: <|begincontext|><|beginlastuserutterance|>I am feeling hungry so I would like to find a place to eat.<|endlastuserutterance|><|endcontext|> ``` ``` Target: <|begintarget|><|begindsts|><|begindst|><|beginintent|>FindRestaurants<|endintent|><|beginbelief|><|endbelief|><|enddst|><|enddsts|><|beginuseraction|>INFORM_INTENT->Restaurants^intent~FindRestaurants<|enduseraction|><|beginaction|>REQUEST->Restaurants^city~<|endaction|><|beginresponse|>Do you have a specific which you want the eating place to be located at?<|endresponse|><|endtarget|> ``` Here, we can see that there are _many_ special tokens that the application developer would need to be aware of: ``` - - - - ``` Research on how to train these types of models is still rapidly evolving. I suspect attempting to abstract away these differences will lead to leaky or nerfed abstractions in libraries and toolkits. For now, my guess is that it's better to write application code targeting the specific models you want to use. ## Conclusion Even if a dedicated text summarization model doesn't make it into the upcoming release, this was a valuable excursion. These are the exact types of problems I hoped to stumble over along the way. ================================================ FILE: docs/_posts/2024-04-19-memory-cache-hub.markdown ================================================ --- layout: post title: "Memory Cache Hub" date: 2024-04-19 08 -0500 categories: developer-blog --- _Author: John Shaughnessy_ # Memory Cache Hub In a [dev log](https://memorycache.ai/developer-blog/2024/03/07/devlog.html) last month, I explained why we were building Memory Cache Hub. We wanted: - our own backend to learn about and play around with [`RAG`](https://python.langchain.com/docs/expression_language/cookbook/retrieval), - a friendly browser-based UI, - to experiment with `llamafile`s, - to experiment with bundling python files with `PyInstaller` The work outlined in that blog post is done. You can try Memory Cache Hub by following the [installation instructions in the README](https://github.com/Mozilla-Ocho/Memory-Cache-Hub?tab=readme-ov-file#installation). My goal for Memory Cache Hub was just to get the project to this point. It was useful to build and I learned a lot, but there are no plans to continue development. For the rest of this post, I would like to share some thoughts/takeaways from working on the project. - Client/Server architecture is convenient, especially with OpenAPI specs. - Browser clients are great until they're not. - Llamafiles are relatively painless. - Python and PyInstaller pros/cons. - Github Actions and large files. - There's a lot of regular (non-AI) work that needs doing. ## The Client / Server Architecture is convenient, especially with OpenAPI specs. This is probably a boring point to start with, and it's old news. Still, I thought it'd be worth mentioning a couple of ways that it turned out to be nice to have the main guts of the application implemented behind an HTTP server. When I was testing out `llamafiles`, I wanted to try enabling GPU acceleration, but my main development machine had some compatibility issues. Since Memory Cache was built as a separate client/server, I could just run the server on another machine (with a compatible GPU) and run the client on my main development machine. It was super painless. We built Memory Cache with on-device AI in mind, but another form that could make sense is to run AI workloads on a dedicated homelab server (e.g. "an Xbox for AI") or in a private cloud. If the AI apps expose everything over HTTP apis, it's easy to play around with these kinds of setups. Another time I was glad to have the server implemented separately from the client was when I wanted to build an emacs plugin that leveraged RAG + LLMs for my programming projects. I wrote about the project [on my blog](https://www.johnshaughnessy.com/blog/posts/acorn_pal_emacs). As I was building the plugin I realized that I could probably just plug in to Memory Cache Hub instead of building another RAG/LLM app. It ended up working great! Unfortunately, I stopped working on the emacs plugin and left it in an unfinished state, mostly because I couldn't generate `elisp` client code for Memory Cache Hub's [Open API](https://openapi-generator.tech/docs/generators) spec. By the way, if anyone wants to write an elisp generator for Open API, that would be really great! I ended up generating typescript code from the OpenAPI spec for use in the [Memory Cache Browser Client](https://github.com/Mozilla-Ocho/Memory-Cache-Browser-Client). The relevant bit of code was this: ```sh # Download the openapi.json spec from the server curl http://localhost:4444/openapi.json > $PROJECT_ROOT/openapi.json # Generate typescript code yarn openapi-generator-cli generate -i $PROJECT_ROOT/openapi.json -g typescript-fetch -o $PROJECT_ROOT/src/api/ ``` ## Browser Clients Are Great, Until They're Not I am familiar with web front end tools -- React, javascript, parcel, css, canvas, etc. So, I liked the idea of building a front end for Memory Cache in the browser. No need to bundle things with [electron](https://www.electronjs.org/), and no need to train other developers I might be working with (who were also mostly familiar with web development). For the most part, this worked out great. While the UI isn't "beautiful" or "breathtaking" -- it was painless and quick to build and it'd be easy for someone who really cared about it to come in and improve things. That said, there were a couple of areas where working in the browser was pretty frustrating: 1. You can't specify directories via a file picker. 2. You can't directly send the user to a file URL. ### No file picker for me The way Memory Cache works is that the user specifies files in their filesystem that they want to add to their "cache"s. The server will make its own copies of the files in those directories for ingestion and such. The problem is that while browsers have built-in support for a file upload window, there's no way to tell the browser that we want the user to specify full paths to directories on their hard drive. It's not surprising that browser's don't support this. This isn't really what they're made for. But it means that for this initial version of Memory Cache Hub, I settled for telling the user to type complete file paths into an input field rather than having a file picker UI. This feels really bad and particularly unpolished, even for a demo app.
### No file previews The browser acts as a file viewer if you specify the path of a file prefixed with `file://` in the address bar. This is convenient, because I wanted to let users easily view the files in their cache. Unfortunately, due to security concerns, the browser disallows redirects to `file://` links. This means that the best I could do for Memory Cache was provide a "copy" button that puts the `file://` URI onto the user's clipboard. Then, they can open a new tab, paste the URL and preview the file. This is a much worse experience.
My client could also have provided file previews (e.g. of PDF's) directly with the server sending the file contents to the client, but I didn't end up going down this route. Again, this isn't surprising because I'm mostly using the browser as an application UI toolkit and that's not really what it's for. Electron (or something like it) would have been a better choice here. ## Llamafiles are (relatively) painless Using `llamafiles` for inference turned out to be an easy win. The dependencies of my python application stayed pretty simple because I didn't need to bring in hugging face / pytorch dependencies (and further separate platforms along `CUDA`/`ROCm`/`CPU` boundaries).
There are some "gotchas" with using `llamafiles`, most of which are documented in the [`llamafile README`](https://github.com/Mozilla-Ocho/llamafile?tab=readme-ov-file). For example, I don't end up enabling GPU support because I didn't spend time on handling errors that can occur if `llamafile` fails to move model weights to GPU for whatever reason. There are also still some platform-specific troubleshooting tips you need to follow if the `llamafile` server fails to start for whatever reason. Still, my overall feeling was that this was a pretty nice way to bundle an inference server with an application, and I hope to see more models bundled as `llamafile`s in the future. ## Python and PyInstaller Pros and Cons I'm not very deeply embedded in the Python world, so figuring out how people built end-user programs with Python was new to me. For example, I know that [Blender](https://www.blender.org/) has a lot of python code, but as far as I can tell, the core is built with C and C++. I found `PyInstaller` and had success building standalone executables with it (as described in [this previous blog post](https://memorycache.ai/developer-blog/2024/03/07/devlog.html) and [this one too](https://memorycache.ai/developer-blog/2024/03/15/devlog.html)). It worked, which is great. But there were some hurdles and downsides. The first complaint is about the way `single-file` builds work. At startup, they need to unpack the supporting files. In our case, we had something like ~10,000 supporting files (which is probably our fault, not `PyInstaller`s) that get unpacked to a temporary directory. This takes ~30 seconds of basically just waiting around with no progress indicator or anything else of that nature. `PyInstaller` has an experimental feature for [adding a splash screen](https://pyinstaller.org/en/stable/usage.html#splash-screen-experimental), but I didn't end up trying it out because of the disclaimer at the top explaining that the feature doesn't work on MacOS. So, the single-file executable version of Memory Cache Hub appears as if it hangs for 30 seconds when you start it before eventually finishing the unpacking process. The second complaint is not really about `PyInstaller` and more about using Python at all, which is that in the end we're still running a python interpreter at runtime. There's no real "compile to bytecode/machine code" (except for those dependencies written in something like [Cython](https://cython.org/)). It seems like python is the most well-supported ecosystem for developer tools for ML / AI, and part of me wishes I were spending my time in C or Rust. Not that I'm excellent with those languages, but considering that I get better at whatever I spend time doing, I'd rather be getting better at things that give me more control over what the computer is actually doing.
Nothing is stopping me from choosing different tools for my next project, and after all - `llama.cpp` is pretty darn popular and I'm looking forward to trying the (rust-based) [burn](https://github.com/tracel-ai/burn) project. ## Github Actions and Large Files Ok, so here's another problem with my gigantic bundled python executables with 10,000 files... My build pipeline takes 2+ hours to finish!
Uploading the build artifacts from the runner to github takes a long time -- especially for the zips that have over 10,000 files in them. This feels pretty terrible. Again, I think the problem is not with Github or PyInstaller or anything like that -- The problem is thinking that shipping 10,000 files was a good idea. It wasn't -- I regret it. haha.
## There's a lot of non-AI work to be done 90% of the effort I put into this project was completely unrelated to AI, machine learning, rag, etc. It was all, "How do I build a python server", "How do we want this browser client to work?", "What's PyInstaller?", "How do we set up Github Actions?" etc. The idea was that once we had all of this ground work out of the way, we'd have a (user-facing) playground to experiment with whatever AI stuff we wanted. That's all fine and good, but I'm not sure how much time we're going to actually spend in that experiment phase, since in the meantime, there have been many other projects vying for our attention. My thoughts about this at the moment are two fold. First, if your main goal is to experiment and learn something about AI or ML -- Don't bother trying to wrap it in an end-user application. Just write your python program or Jupyter notebook or whatever and do the learning. Don't worry if it doesn't work on other platforms or only supports whatever kind of GPU you happen to be running -- none of that changes the math / AI / deep learning / ML stuff that you were actually interested in. All of that other stuff is a distraction if all you wanted was the core thing. However -- if your goal is to experiment with an AI or ML thing that you want people to use -- Then get those people on board using your thing as fast as possible, even if that means getting on a call with that, having them share their desktop and following your instructions to set up a python environment. Whatever you need to do to get them actually running your code and using your thing and giving you feedback -- that's the hurdle you should cross. That doesn't mean you have to ship out to the whole world. Maybe you know your thing is not ready for that. But if you have a particular user in mind and you want them involved and giving you constant feedback, it's good to bring them in early. ## What Now? Learning the build / deploy side of things was pretty helpful and useful. I'd never built a python application like this one before, and I enjoyed myself along the way. There's been some interest in connecting Memory Cache more directly with the browser history, with Slack, with emails, and other document/information sources. That direction is probably pretty useful -- and a lot of other people are exploring that space too. However, I'll likely leave that to others. My next projects will be unrelated to Memory Cache. There are a lot of simple ideas I want to play around with in the space just to deepen my understanding of LLMs, and there are a lot of projects external to Mozilla that I'd like to learn more about and maybe contribute to. ## Screenshots
Files
About Memory Cache
Vector Search
Chat depends on a model running
The model selection page
The model selection page
Retrieval augmented chat
================================================ FILE: docs/_sass/memorycache.scss ================================================ @import url('https://fonts.googleapis.com/css2?family=Work+Sans:wght@300;400;500;600&display=swap'); body { font-family: 'Work Sans'; } .site-title { font-family: 'Work Sans'; color: coral; } .introduction { background-image: url('../assets/images/header-background.png'); background-repeat:no-repeat; background-size: 675px 210.75px; background-position: right; width: 100%; min-height: 201px; border-bottom-width: 1px; border-bottom-style: solid; border-bottom-color: #F0EFEF; } .page-content { background-color: white; } a { color: #180AB8; } .site-logo { width: 166px; height: 36px; } .site-header { border-top-width: 5px; border-top-style: solid; border-image: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87EBDA 76.04%, #BAE1FF 100%) 1 0 0 0; } .callout-left { width: 75%; padding-top: 5%; } .page-content { border-top-width: 1px; border-top-style: solid; border-top-color: #F0EFEF; } .post-list-heading { font-size: 16px; padding-top: 5px; } .post-link { font-size: 16px; } .moz-logo { width: 128px; float: right; } .detailed-overview { padding-top: 5px; border-top-width: 1px; border-bottom-width: 1px; border-top-style: solid; border-bottom-style: solid; border-top-color: #F0EFEF; border-bottom-color: #F0EFEF; } figcaption { text-align: center; font-style: italic; margin: 1em 0 3em 0; } ================================================ FILE: docs/_sass/minima.scss ================================================ @charset "utf-8"; // Define defaults for each variable. $base-font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol" !default; $base-font-size: 16px !default; $base-font-weight: 400 !default; $small-font-size: $base-font-size * 0.875 !default; $base-line-height: 1.5 !default; $spacing-unit: 30px !default; $text-color: #111 !default; $background-color: #fdfdfd !default; $brand-color: #2a7ae2 !default; $grey-color: #828282 !default; $grey-color-light: lighten($grey-color, 40%) !default; $grey-color-dark: darken($grey-color, 25%) !default; $table-text-align: left !default; // Width of the content area $content-width: 800px !default; $on-palm: 600px !default; $on-laptop: 800px !default; // Use media queries like this: // @include media-query($on-palm) { // .wrapper { // padding-right: $spacing-unit / 2; // padding-left: $spacing-unit / 2; // } // } @mixin media-query($device) { @media screen and (max-width: $device) { @content; } } @mixin relative-font-size($ratio) { font-size: $base-font-size * $ratio; } // Import partials. @import "minima/base", "minima/layout", "minima/syntax-highlighting" ; ================================================ FILE: docs/about.markdown ================================================ --- layout: page title: About permalink: /about/ --- Memory Cache is an exploration into synthesis, discovery, and sharing of insights more effectively through the use of technology. Unlike most explorations into artificial intelligence, Memory Cache is designed to be completely personalized, on-device, and private. It is meant to explore the nuances of how individuals think, from the perspective of learning alongside us over time from the articles we read and save. ================================================ FILE: docs/assets/main.scss ================================================ --- --- @import "minima"; @import "memorycache"; ================================================ FILE: docs/faq.md ================================================ --- layout: default title: FAQ --- # Frequently Asked Questions **Q: How do I try MemoryCache?** Right now (as of December 7, 2023), MemoryCache requires a few manual steps to set up the end to end workflow. There are three components: a) a Firefox extension, b) a local instance of privateGPT, and c) a symlinked folder between privateGPT and your local Downloads folder. There is also an optional configuration that can be done to a private build of Firefox to save files to your local machine as PDF files instead of HTML files. Check out the [GitHub repository](https://github.com/Mozilla-Ocho/Memory-Cache) for more detailed instructions. We are looking into ways to streamline the deployment of MemoryCache to require less manual configuration, but if you're here at this stage, you're at the very earliest stages of our explorations. **Q: Does MemoryCache send my data anywhere?** No. One of the core principles of MemoryCache is that you have full control over the system, and that it all stays on your device. If you're a developer or someone who just likes to tinker with your computer applications, and you want to cloud-ify this, feel free! But we're looking to stay entirely local. **Q: Why is MemoryCache using an old language model and primordial privateGPT?** MemoryCache is using an old language model ([Nomic AI's gpt4all-j v1.3 groovy.ggml](https://huggingface.co/nomic-ai/gpt4all-j)) and primordial privateGPT because right now, this combo is the one that passes our criteria for the type of responses that it generates. This tech is almost a year old, and there have been many advancements in local AI that we'll be integrating in over time, but we're a small team exploring a lot of different subsets of this problem space and the quality of the insight generated is a sweet spot that we want to preserve. This is a temporary tradeoff, but we want to be careful to keep a consistent benchmark for insight generation. GPT-J was trained on the 'Pile' dataset, and the versions between the 1.0 release and 1.3 release also added the ShareGPT and Dolly datasettes. The Databricks Dolly dataset is licensed under the Creative Commons license with human contributions and wikipedia entries. The ShareGPT dataset is human prompts and ChatGPT output responses that were submitted by human users. **Q: What kind of tasks would I use MemoryCache for?** MemoryCache is ultimately leaning into the weird and creative parts of human insight. The goal with MemoryCache is to "learn what you learn", which is why you are in control of what you want files you want to augment the application with. This can be helpful for research, brainstorming, creative writing, and synthesis of new ideas to connect seemingly unrelated topics together to find new insights and learnings from the body of knowledge that matters most to you. **Q: Is this a Firefox project?** No. MemoryCache is a hackathon-style project by the Mozilla Innovation Ecosystem team, not a Firefox project.While the project uses a Firefox extension as a way of collecting information, this is a set of scripts and tools to augment privateGPT, a native AI application. It's meant to streamline the process of saving information that you might read in the browser to store it in your own personal library of information. ================================================ FILE: docs/index.markdown ================================================ --- # Feel free to add content and custom Front Matter to this file. # To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults layout: home --- ================================================ FILE: docs/readme.md ================================================ Placeholder ================================================ FILE: extension/content-script.js ================================================ function getPageText() { const head = document.head.innerHTML; const body = document.body.innerText; return `\n\n${head}\n\n\n${body}\n\n`; } browser.runtime.onMessage.addListener((message, _sender) => { console.log("[MemoryCache Extension] Received message:", message); if (message.action === "getPageText") { return Promise.resolve(getPageText()); } }); ================================================ FILE: extension/manifest.json ================================================ { "manifest_version" : 2, "name": "MemoryCache", "version" : "1.0", "description" : "Saves a copy of reader view of a tab to a specific directory", "icons" : { "48" : "icons/memwrite-48.png" }, "permissions" : [ "downloads", "", "tabs", "storage" ], "browser_action" : { "browser_style" : true, "default_icon" : "icons/memwrite-32.png", "default_title" : "Memory Cache", "default_popup" : "popup/memory_cache.html" }, "content_scripts": [ { "matches": [""], "js": ["content-script.js"] } ] } ================================================ FILE: extension/popup/marked.esm.js ================================================ /** * marked v10.0.0 - a markdown parser * Copyright (c) 2011-2023, Christopher Jeffrey. (MIT Licensed) * https://github.com/markedjs/marked */ /** * DO NOT EDIT THIS FILE * The code in this file is generated from files in ./src/ */ /** * Gets the original marked default options. */ function _getDefaults() { return { async: false, breaks: false, extensions: null, gfm: true, hooks: null, pedantic: false, renderer: null, silent: false, tokenizer: null, walkTokens: null }; } let _defaults = _getDefaults(); function changeDefaults(newDefaults) { _defaults = newDefaults; } /** * Helpers */ const escapeTest = /[&<>"']/; const escapeReplace = new RegExp(escapeTest.source, 'g'); const escapeTestNoEncode = /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/; const escapeReplaceNoEncode = new RegExp(escapeTestNoEncode.source, 'g'); const escapeReplacements = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; const getEscapeReplacement = (ch) => escapeReplacements[ch]; function escape(html, encode) { if (encode) { if (escapeTest.test(html)) { return html.replace(escapeReplace, getEscapeReplacement); } } else { if (escapeTestNoEncode.test(html)) { return html.replace(escapeReplaceNoEncode, getEscapeReplacement); } } return html; } const unescapeTest = /&(#(?:\d+)|(?:#x[0-9A-Fa-f]+)|(?:\w+));?/ig; function unescape(html) { // explicitly match decimal, hex, and named HTML entities return html.replace(unescapeTest, (_, n) => { n = n.toLowerCase(); if (n === 'colon') return ':'; if (n.charAt(0) === '#') { return n.charAt(1) === 'x' ? String.fromCharCode(parseInt(n.substring(2), 16)) : String.fromCharCode(+n.substring(1)); } return ''; }); } const caret = /(^|[^\[])\^/g; function edit(regex, opt) { regex = typeof regex === 'string' ? regex : regex.source; opt = opt || ''; const obj = { replace: (name, val) => { val = typeof val === 'object' && 'source' in val ? val.source : val; val = val.replace(caret, '$1'); regex = regex.replace(name, val); return obj; }, getRegex: () => { return new RegExp(regex, opt); } }; return obj; } function cleanUrl(href) { try { href = encodeURI(href).replace(/%25/g, '%'); } catch (e) { return null; } return href; } const noopTest = { exec: () => null }; function splitCells(tableRow, count) { // ensure that every cell-delimiting pipe has a space // before it to distinguish it from an escaped pipe const row = tableRow.replace(/\|/g, (match, offset, str) => { let escaped = false; let curr = offset; while (--curr >= 0 && str[curr] === '\\') escaped = !escaped; if (escaped) { // odd number of slashes means | is escaped // so we leave it alone return '|'; } else { // add space before unescaped | return ' |'; } }), cells = row.split(/ \|/); let i = 0; // First/last cell in a row cannot be empty if it has no leading/trailing pipe if (!cells[0].trim()) { cells.shift(); } if (cells.length > 0 && !cells[cells.length - 1].trim()) { cells.pop(); } if (count) { if (cells.length > count) { cells.splice(count); } else { while (cells.length < count) cells.push(''); } } for (; i < cells.length; i++) { // leading or trailing whitespace is ignored per the gfm spec cells[i] = cells[i].trim().replace(/\\\|/g, '|'); } return cells; } /** * Remove trailing 'c's. Equivalent to str.replace(/c*$/, ''). * /c*$/ is vulnerable to REDOS. * * @param str * @param c * @param invert Remove suffix of non-c chars instead. Default falsey. */ function rtrim(str, c, invert) { const l = str.length; if (l === 0) { return ''; } // Length of suffix matching the invert condition. let suffLen = 0; // Step left until we fail to match the invert condition. while (suffLen < l) { const currChar = str.charAt(l - suffLen - 1); if (currChar === c && !invert) { suffLen++; } else if (currChar !== c && invert) { suffLen++; } else { break; } } return str.slice(0, l - suffLen); } function findClosingBracket(str, b) { if (str.indexOf(b[1]) === -1) { return -1; } let level = 0; for (let i = 0; i < str.length; i++) { if (str[i] === '\\') { i++; } else if (str[i] === b[0]) { level++; } else if (str[i] === b[1]) { level--; if (level < 0) { return i; } } } return -1; } function outputLink(cap, link, raw, lexer) { const href = link.href; const title = link.title ? escape(link.title) : null; const text = cap[1].replace(/\\([\[\]])/g, '$1'); if (cap[0].charAt(0) !== '!') { lexer.state.inLink = true; const token = { type: 'link', raw, href, title, text, tokens: lexer.inlineTokens(text) }; lexer.state.inLink = false; return token; } return { type: 'image', raw, href, title, text: escape(text) }; } function indentCodeCompensation(raw, text) { const matchIndentToCode = raw.match(/^(\s+)(?:```)/); if (matchIndentToCode === null) { return text; } const indentToCode = matchIndentToCode[1]; return text .split('\n') .map(node => { const matchIndentInNode = node.match(/^\s+/); if (matchIndentInNode === null) { return node; } const [indentInNode] = matchIndentInNode; if (indentInNode.length >= indentToCode.length) { return node.slice(indentToCode.length); } return node; }) .join('\n'); } /** * Tokenizer */ class _Tokenizer { options; // TODO: Fix this rules type rules; lexer; constructor(options) { this.options = options || _defaults; } space(src) { const cap = this.rules.block.newline.exec(src); if (cap && cap[0].length > 0) { return { type: 'space', raw: cap[0] }; } } code(src) { const cap = this.rules.block.code.exec(src); if (cap) { const text = cap[0].replace(/^ {1,4}/gm, ''); return { type: 'code', raw: cap[0], codeBlockStyle: 'indented', text: !this.options.pedantic ? rtrim(text, '\n') : text }; } } fences(src) { const cap = this.rules.block.fences.exec(src); if (cap) { const raw = cap[0]; const text = indentCodeCompensation(raw, cap[3] || ''); return { type: 'code', raw, lang: cap[2] ? cap[2].trim().replace(this.rules.inline._escapes, '$1') : cap[2], text }; } } heading(src) { const cap = this.rules.block.heading.exec(src); if (cap) { let text = cap[2].trim(); // remove trailing #s if (/#$/.test(text)) { const trimmed = rtrim(text, '#'); if (this.options.pedantic) { text = trimmed.trim(); } else if (!trimmed || / $/.test(trimmed)) { // CommonMark requires space before trailing #s text = trimmed.trim(); } } return { type: 'heading', raw: cap[0], depth: cap[1].length, text, tokens: this.lexer.inline(text) }; } } hr(src) { const cap = this.rules.block.hr.exec(src); if (cap) { return { type: 'hr', raw: cap[0] }; } } blockquote(src) { const cap = this.rules.block.blockquote.exec(src); if (cap) { const text = rtrim(cap[0].replace(/^ *>[ \t]?/gm, ''), '\n'); const top = this.lexer.state.top; this.lexer.state.top = true; const tokens = this.lexer.blockTokens(text); this.lexer.state.top = top; return { type: 'blockquote', raw: cap[0], tokens, text }; } } list(src) { let cap = this.rules.block.list.exec(src); if (cap) { let bull = cap[1].trim(); const isordered = bull.length > 1; const list = { type: 'list', raw: '', ordered: isordered, start: isordered ? +bull.slice(0, -1) : '', loose: false, items: [] }; bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}`; if (this.options.pedantic) { bull = isordered ? bull : '[*+-]'; } // Get next list item const itemRegex = new RegExp(`^( {0,3}${bull})((?:[\t ][^\\n]*)?(?:\\n|$))`); let raw = ''; let itemContents = ''; let endsWithBlankLine = false; // Check if current bullet point can start a new List Item while (src) { let endEarly = false; if (!(cap = itemRegex.exec(src))) { break; } if (this.rules.block.hr.test(src)) { // End list if bullet was actually HR (possibly move into itemRegex?) break; } raw = cap[0]; src = src.substring(raw.length); let line = cap[2].split('\n', 1)[0].replace(/^\t+/, (t) => ' '.repeat(3 * t.length)); let nextLine = src.split('\n', 1)[0]; let indent = 0; if (this.options.pedantic) { indent = 2; itemContents = line.trimStart(); } else { indent = cap[2].search(/[^ ]/); // Find first non-space char indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent itemContents = line.slice(indent); indent += cap[1].length; } let blankLine = false; if (!line && /^ *$/.test(nextLine)) { // Items begin with at most one blank line raw += nextLine + '\n'; src = src.substring(nextLine.length + 1); endEarly = true; } if (!endEarly) { const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])((?:[ \t][^\\n]*)?(?:\\n|$))`); const hrRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`); const fencesBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:\`\`\`|~~~)`); const headingBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}#`); // Check if following lines should be included in List Item while (src) { const rawLine = src.split('\n', 1)[0]; nextLine = rawLine; // Re-align to follow commonmark nesting rules if (this.options.pedantic) { nextLine = nextLine.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' '); } // End list item if found code fences if (fencesBeginRegex.test(nextLine)) { break; } // End list item if found start of new heading if (headingBeginRegex.test(nextLine)) { break; } // End list item if found start of new bullet if (nextBulletRegex.test(nextLine)) { break; } // Horizontal rule found if (hrRegex.test(src)) { break; } if (nextLine.search(/[^ ]/) >= indent || !nextLine.trim()) { // Dedent if possible itemContents += '\n' + nextLine.slice(indent); } else { // not enough indentation if (blankLine) { break; } // paragraph continuation unless last line was a different block level element if (line.search(/[^ ]/) >= 4) { // indented code block break; } if (fencesBeginRegex.test(line)) { break; } if (headingBeginRegex.test(line)) { break; } if (hrRegex.test(line)) { break; } itemContents += '\n' + nextLine; } if (!blankLine && !nextLine.trim()) { // Check if current line is blank blankLine = true; } raw += rawLine + '\n'; src = src.substring(rawLine.length + 1); line = nextLine.slice(indent); } } if (!list.loose) { // If the previous item ended with a blank line, the list is loose if (endsWithBlankLine) { list.loose = true; } else if (/\n *\n *$/.test(raw)) { endsWithBlankLine = true; } } let istask = null; let ischecked; // Check for task list items if (this.options.gfm) { istask = /^\[[ xX]\] /.exec(itemContents); if (istask) { ischecked = istask[0] !== '[ ] '; itemContents = itemContents.replace(/^\[[ xX]\] +/, ''); } } list.items.push({ type: 'list_item', raw, task: !!istask, checked: ischecked, loose: false, text: itemContents, tokens: [] }); list.raw += raw; } // Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic list.items[list.items.length - 1].raw = raw.trimEnd(); list.items[list.items.length - 1].text = itemContents.trimEnd(); list.raw = list.raw.trimEnd(); // Item child tokens handled here at end because we needed to have the final item to trim it first for (let i = 0; i < list.items.length; i++) { this.lexer.state.top = false; list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []); if (!list.loose) { // Check if list should be loose const spacers = list.items[i].tokens.filter(t => t.type === 'space'); const hasMultipleLineBreaks = spacers.length > 0 && spacers.some(t => /\n.*\n/.test(t.raw)); list.loose = hasMultipleLineBreaks; } } // Set all items to loose if list is loose if (list.loose) { for (let i = 0; i < list.items.length; i++) { list.items[i].loose = true; } } return list; } } html(src) { const cap = this.rules.block.html.exec(src); if (cap) { const token = { type: 'html', block: true, raw: cap[0], pre: cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style', text: cap[0] }; return token; } } def(src) { const cap = this.rules.block.def.exec(src); if (cap) { const tag = cap[1].toLowerCase().replace(/\s+/g, ' '); const href = cap[2] ? cap[2].replace(/^<(.*)>$/, '$1').replace(this.rules.inline._escapes, '$1') : ''; const title = cap[3] ? cap[3].substring(1, cap[3].length - 1).replace(this.rules.inline._escapes, '$1') : cap[3]; return { type: 'def', tag, raw: cap[0], href, title }; } } table(src) { const cap = this.rules.block.table.exec(src); if (cap) { if (!/[:|]/.test(cap[2])) { // delimiter row must have a pipe (|) or colon (:) otherwise it is a setext heading return; } const item = { type: 'table', raw: cap[0], header: splitCells(cap[1]).map(c => { return { text: c, tokens: [] }; }), align: cap[2].replace(/^\||\| *$/g, '').split('|'), rows: cap[3] && cap[3].trim() ? cap[3].replace(/\n[ \t]*$/, '').split('\n') : [] }; if (item.header.length === item.align.length) { let l = item.align.length; let i, j, k, row; for (i = 0; i < l; i++) { const align = item.align[i]; if (align) { if (/^ *-+: *$/.test(align)) { item.align[i] = 'right'; } else if (/^ *:-+: *$/.test(align)) { item.align[i] = 'center'; } else if (/^ *:-+ *$/.test(align)) { item.align[i] = 'left'; } else { item.align[i] = null; } } } l = item.rows.length; for (i = 0; i < l; i++) { item.rows[i] = splitCells(item.rows[i], item.header.length).map(c => { return { text: c, tokens: [] }; }); } // parse child tokens inside headers and cells // header child tokens l = item.header.length; for (j = 0; j < l; j++) { item.header[j].tokens = this.lexer.inline(item.header[j].text); } // cell child tokens l = item.rows.length; for (j = 0; j < l; j++) { row = item.rows[j]; for (k = 0; k < row.length; k++) { row[k].tokens = this.lexer.inline(row[k].text); } } return item; } } } lheading(src) { const cap = this.rules.block.lheading.exec(src); if (cap) { return { type: 'heading', raw: cap[0], depth: cap[2].charAt(0) === '=' ? 1 : 2, text: cap[1], tokens: this.lexer.inline(cap[1]) }; } } paragraph(src) { const cap = this.rules.block.paragraph.exec(src); if (cap) { const text = cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1]; return { type: 'paragraph', raw: cap[0], text, tokens: this.lexer.inline(text) }; } } text(src) { const cap = this.rules.block.text.exec(src); if (cap) { return { type: 'text', raw: cap[0], text: cap[0], tokens: this.lexer.inline(cap[0]) }; } } escape(src) { const cap = this.rules.inline.escape.exec(src); if (cap) { return { type: 'escape', raw: cap[0], text: escape(cap[1]) }; } } tag(src) { const cap = this.rules.inline.tag.exec(src); if (cap) { if (!this.lexer.state.inLink && /^/i.test(cap[0])) { this.lexer.state.inLink = false; } if (!this.lexer.state.inRawBlock && /^<(pre|code|kbd|script)(\s|>)/i.test(cap[0])) { this.lexer.state.inRawBlock = true; } else if (this.lexer.state.inRawBlock && /^<\/(pre|code|kbd|script)(\s|>)/i.test(cap[0])) { this.lexer.state.inRawBlock = false; } return { type: 'html', raw: cap[0], inLink: this.lexer.state.inLink, inRawBlock: this.lexer.state.inRawBlock, block: false, text: cap[0] }; } } link(src) { const cap = this.rules.inline.link.exec(src); if (cap) { const trimmedUrl = cap[2].trim(); if (!this.options.pedantic && /^$/.test(trimmedUrl))) { return; } // ending angle bracket cannot be escaped const rtrimSlash = rtrim(trimmedUrl.slice(0, -1), '\\'); if ((trimmedUrl.length - rtrimSlash.length) % 2 === 0) { return; } } else { // find closing parenthesis const lastParenIndex = findClosingBracket(cap[2], '()'); if (lastParenIndex > -1) { const start = cap[0].indexOf('!') === 0 ? 5 : 4; const linkLen = start + cap[1].length + lastParenIndex; cap[2] = cap[2].substring(0, lastParenIndex); cap[0] = cap[0].substring(0, linkLen).trim(); cap[3] = ''; } } let href = cap[2]; let title = ''; if (this.options.pedantic) { // split pedantic href and title const link = /^([^'"]*[^\s])\s+(['"])(.*)\2/.exec(href); if (link) { href = link[1]; title = link[3]; } } else { title = cap[3] ? cap[3].slice(1, -1) : ''; } href = href.trim(); if (/^$/.test(trimmedUrl))) { // pedantic allows starting angle bracket without ending angle bracket href = href.slice(1); } else { href = href.slice(1, -1); } } return outputLink(cap, { href: href ? href.replace(this.rules.inline._escapes, '$1') : href, title: title ? title.replace(this.rules.inline._escapes, '$1') : title }, cap[0], this.lexer); } } reflink(src, links) { let cap; if ((cap = this.rules.inline.reflink.exec(src)) || (cap = this.rules.inline.nolink.exec(src))) { let link = (cap[2] || cap[1]).replace(/\s+/g, ' '); link = links[link.toLowerCase()]; if (!link) { const text = cap[0].charAt(0); return { type: 'text', raw: text, text }; } return outputLink(cap, link, cap[0], this.lexer); } } emStrong(src, maskedSrc, prevChar = '') { let match = this.rules.inline.emStrong.lDelim.exec(src); if (!match) return; // _ can't be between two alphanumerics. \p{L}\p{N} includes non-english alphabet/numbers as well if (match[3] && prevChar.match(/[\p{L}\p{N}]/u)) return; const nextChar = match[1] || match[2] || ''; if (!nextChar || !prevChar || this.rules.inline.punctuation.exec(prevChar)) { // unicode Regex counts emoji as 1 char; spread into array for proper count (used multiple times below) const lLength = [...match[0]].length - 1; let rDelim, rLength, delimTotal = lLength, midDelimTotal = 0; const endReg = match[0][0] === '*' ? this.rules.inline.emStrong.rDelimAst : this.rules.inline.emStrong.rDelimUnd; endReg.lastIndex = 0; // Clip maskedSrc to same section of string as src (move to lexer?) maskedSrc = maskedSrc.slice(-1 * src.length + lLength); while ((match = endReg.exec(maskedSrc)) != null) { rDelim = match[1] || match[2] || match[3] || match[4] || match[5] || match[6]; if (!rDelim) continue; // skip single * in __abc*abc__ rLength = [...rDelim].length; if (match[3] || match[4]) { // found another Left Delim delimTotal += rLength; continue; } else if (match[5] || match[6]) { // either Left or Right Delim if (lLength % 3 && !((lLength + rLength) % 3)) { midDelimTotal += rLength; continue; // CommonMark Emphasis Rules 9-10 } } delimTotal -= rLength; if (delimTotal > 0) continue; // Haven't found enough closing delimiters // Remove extra characters. *a*** -> *a* rLength = Math.min(rLength, rLength + delimTotal + midDelimTotal); // char length can be >1 for unicode characters; const lastCharLength = [...match[0]][0].length; const raw = src.slice(0, lLength + match.index + lastCharLength + rLength); // Create `em` if smallest delimiter has odd char count. *a*** if (Math.min(lLength, rLength) % 2) { const text = raw.slice(1, -1); return { type: 'em', raw, text, tokens: this.lexer.inlineTokens(text) }; } // Create 'strong' if smallest delimiter has even char count. **a*** const text = raw.slice(2, -2); return { type: 'strong', raw, text, tokens: this.lexer.inlineTokens(text) }; } } } codespan(src) { const cap = this.rules.inline.code.exec(src); if (cap) { let text = cap[2].replace(/\n/g, ' '); const hasNonSpaceChars = /[^ ]/.test(text); const hasSpaceCharsOnBothEnds = /^ /.test(text) && / $/.test(text); if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) { text = text.substring(1, text.length - 1); } text = escape(text, true); return { type: 'codespan', raw: cap[0], text }; } } br(src) { const cap = this.rules.inline.br.exec(src); if (cap) { return { type: 'br', raw: cap[0] }; } } del(src) { const cap = this.rules.inline.del.exec(src); if (cap) { return { type: 'del', raw: cap[0], text: cap[2], tokens: this.lexer.inlineTokens(cap[2]) }; } } autolink(src) { const cap = this.rules.inline.autolink.exec(src); if (cap) { let text, href; if (cap[2] === '@') { text = escape(cap[1]); href = 'mailto:' + text; } else { text = escape(cap[1]); href = text; } return { type: 'link', raw: cap[0], text, href, tokens: [ { type: 'text', raw: text, text } ] }; } } url(src) { let cap; if (cap = this.rules.inline.url.exec(src)) { let text, href; if (cap[2] === '@') { text = escape(cap[0]); href = 'mailto:' + text; } else { // do extended autolink path validation let prevCapZero; do { prevCapZero = cap[0]; cap[0] = this.rules.inline._backpedal.exec(cap[0])[0]; } while (prevCapZero !== cap[0]); text = escape(cap[0]); if (cap[1] === 'www.') { href = 'http://' + cap[0]; } else { href = cap[0]; } } return { type: 'link', raw: cap[0], text, href, tokens: [ { type: 'text', raw: text, text } ] }; } } inlineText(src) { const cap = this.rules.inline.text.exec(src); if (cap) { let text; if (this.lexer.state.inRawBlock) { text = cap[0]; } else { text = escape(cap[0]); } return { type: 'text', raw: cap[0], text }; } } } /** * Block-Level Grammar */ // Not all rules are defined in the object literal // @ts-expect-error const block = { newline: /^(?: *(?:\n|$))+/, code: /^( {4}[^\n]+(?:\n(?: *(?:\n|$))*)?)+/, fences: /^ {0,3}(`{3,}(?=[^`\n]*(?:\n|$))|~{3,})([^\n]*)(?:\n|$)(?:|([\s\S]*?)(?:\n|$))(?: {0,3}\1[~`]* *(?=\n|$)|$)/, hr: /^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/, heading: /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/, blockquote: /^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/, list: /^( {0,3}bull)([ \t][^\n]+?)?(?:\n|$)/, html: '^ {0,3}(?:' // optional indentation + '<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:[^\\n]*\\n+|$)' // (1) + '|comment[^\\n]*(\\n+|$)' // (2) + '|<\\?[\\s\\S]*?(?:\\?>\\n*|$)' // (3) + '|\\n*|$)' // (4) + '|\\n*|$)' // (5) + '|)[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (6) + '|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (7) open tag + '|(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (7) closing tag + ')', def: /^ {0,3}\[(label)\]: *(?:\n *)?([^<\s][^\s]*|<.*?>)(?:(?: +(?:\n *)?| *\n *)(title))? *(?:\n+|$)/, table: noopTest, lheading: /^(?!bull )((?:.|\n(?!\s*?\n|bull ))+?)\n {0,3}(=+|-+) *(?:\n+|$)/, // regex template, placeholders will be replaced according to different paragraph // interruption rules of commonmark and the original markdown spec: _paragraph: /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/, text: /^[^\n]+/ }; block._label = /(?!\s*\])(?:\\.|[^\[\]\\])+/; block._title = /(?:"(?:\\"?|[^"\\])*"|'[^'\n]*(?:\n[^'\n]+)*\n?'|\([^()]*\))/; block.def = edit(block.def) .replace('label', block._label) .replace('title', block._title) .getRegex(); block.bullet = /(?:[*+-]|\d{1,9}[.)])/; block.listItemStart = edit(/^( *)(bull) */) .replace('bull', block.bullet) .getRegex(); block.list = edit(block.list) .replace(/bull/g, block.bullet) .replace('hr', '\\n+(?=\\1?(?:(?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$))') .replace('def', '\\n+(?=' + block.def.source + ')') .getRegex(); block._tag = 'address|article|aside|base|basefont|blockquote|body|caption' + '|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption' + '|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe' + '|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option' + '|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr' + '|track|ul'; block._comment = /|$)/; block.html = edit(block.html, 'i') .replace('comment', block._comment) .replace('tag', block._tag) .replace('attribute', / +[a-zA-Z:_][\w.:-]*(?: *= *"[^"\n]*"| *= *'[^'\n]*'| *= *[^\s"'=<>`]+)?/) .getRegex(); block.lheading = edit(block.lheading) .replace(/bull/g, block.bullet) // lists can interrupt .getRegex(); block.paragraph = edit(block._paragraph) .replace('hr', block.hr) .replace('heading', ' {0,3}#{1,6}(?:\\s|$)') .replace('|lheading', '') // setex headings don't interrupt commonmark paragraphs .replace('|table', '') .replace('blockquote', ' {0,3}>') .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', block._tag) // pars can be interrupted by type (6) html blocks .getRegex(); block.blockquote = edit(block.blockquote) .replace('paragraph', block.paragraph) .getRegex(); /** * Normal Block Grammar */ block.normal = { ...block }; /** * GFM Block Grammar */ block.gfm = { ...block.normal, table: '^ *([^\\n ].*)\\n' // Header + ' {0,3}((?:\\| *)?:?-+:? *(?:\\| *:?-+:? *)*(?:\\| *)?)' // Align + '(?:\\n((?:(?! *\\n|hr|heading|blockquote|code|fences|list|html).*(?:\\n|$))*)\\n*|$)' // Cells }; block.gfm.table = edit(block.gfm.table) .replace('hr', block.hr) .replace('heading', ' {0,3}#{1,6}(?:\\s|$)') .replace('blockquote', ' {0,3}>') .replace('code', ' {4}[^\\n]') .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', block._tag) // tables can be interrupted by type (6) html blocks .getRegex(); block.gfm.paragraph = edit(block._paragraph) .replace('hr', block.hr) .replace('heading', ' {0,3}#{1,6}(?:\\s|$)') .replace('|lheading', '') // setex headings don't interrupt commonmark paragraphs .replace('table', block.gfm.table) // interrupt paragraphs with table .replace('blockquote', ' {0,3}>') .replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n') .replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt .replace('html', ')|<(?:script|pre|style|textarea|!--)') .replace('tag', block._tag) // pars can be interrupted by type (6) html blocks .getRegex(); /** * Pedantic grammar (original John Gruber's loose markdown specification) */ block.pedantic = { ...block.normal, html: edit('^ *(?:comment *(?:\\n|\\s*$)' + '|<(tag)[\\s\\S]+? *(?:\\n{2,}|\\s*$)' // closed tag + '|\\s]*)*?/?> *(?:\\n{2,}|\\s*$))') .replace('comment', block._comment) .replace(/tag/g, '(?!(?:' + 'a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub' + '|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)' + '\\b)\\w+(?!:|[^\\w\\s@]*@)\\b') .getRegex(), def: /^ *\[([^\]]+)\]: *]+)>?(?: +(["(][^\n]+[")]))? *(?:\n+|$)/, heading: /^(#{1,6})(.*)(?:\n+|$)/, fences: noopTest, lheading: /^(.+?)\n {0,3}(=+|-+) *(?:\n+|$)/, paragraph: edit(block.normal._paragraph) .replace('hr', block.hr) .replace('heading', ' *#{1,6} *[^\n]') .replace('lheading', block.lheading) .replace('blockquote', ' {0,3}>') .replace('|fences', '') .replace('|list', '') .replace('|html', '') .getRegex() }; /** * Inline-Level Grammar */ // Not all rules are defined in the object literal // @ts-expect-error const inline = { escape: /^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/, autolink: /^<(scheme:[^\s\x00-\x1f<>]*|email)>/, url: noopTest, tag: '^comment' + '|^' // self-closing tag + '|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>' // open tag + '|^<\\?[\\s\\S]*?\\?>' // processing instruction, e.g. + '|^' // declaration, e.g. + '|^', link: /^!?\[(label)\]\(\s*(href)(?:\s+(title))?\s*\)/, reflink: /^!?\[(label)\]\[(ref)\]/, nolink: /^!?\[(ref)\](?:\[\])?/, reflinkSearch: 'reflink|nolink(?!\\()', emStrong: { lDelim: /^(?:\*+(?:((?!\*)[punct])|[^\s*]))|^_+(?:((?!_)[punct])|([^\s_]))/, // (1) and (2) can only be a Right Delimiter. (3) and (4) can only be Left. (5) and (6) can be either Left or Right. // | Skip orphan inside strong | Consume to delim | (1) #*** | (2) a***#, a*** | (3) #***a, ***a | (4) ***# | (5) #***# | (6) a***a rDelimAst: /^[^_*]*?__[^_*]*?\*[^_*]*?(?=__)|[^*]+(?=[^*])|(?!\*)[punct](\*+)(?=[\s]|$)|[^punct\s](\*+)(?!\*)(?=[punct\s]|$)|(?!\*)[punct\s](\*+)(?=[^punct\s])|[\s](\*+)(?!\*)(?=[punct])|(?!\*)[punct](\*+)(?!\*)(?=[punct])|[^punct\s](\*+)(?=[^punct\s])/, rDelimUnd: /^[^_*]*?\*\*[^_*]*?_[^_*]*?(?=\*\*)|[^_]+(?=[^_])|(?!_)[punct](_+)(?=[\s]|$)|[^punct\s](_+)(?!_)(?=[punct\s]|$)|(?!_)[punct\s](_+)(?=[^punct\s])|[\s](_+)(?!_)(?=[punct])|(?!_)[punct](_+)(?!_)(?=[punct])/ // ^- Not allowed for _ }, code: /^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/, br: /^( {2,}|\\)\n(?!\s*$)/, del: noopTest, text: /^(`+|[^`])(?:(?= {2,}\n)|[\s\S]*?(?:(?=[\\`^|~'; inline.punctuation = edit(inline.punctuation, 'u').replace(/punctuation/g, inline._punctuation).getRegex(); // sequences em should skip over [title](link), `code`, inline.blockSkip = /\[[^[\]]*?\]\([^\(\)]*?\)|`[^`]*?`|<[^<>]*?>/g; inline.anyPunctuation = /\\[punct]/g; inline._escapes = /\\([punct])/g; inline._comment = edit(block._comment).replace('(?:-->|$)', '-->').getRegex(); inline.emStrong.lDelim = edit(inline.emStrong.lDelim, 'u') .replace(/punct/g, inline._punctuation) .getRegex(); inline.emStrong.rDelimAst = edit(inline.emStrong.rDelimAst, 'gu') .replace(/punct/g, inline._punctuation) .getRegex(); inline.emStrong.rDelimUnd = edit(inline.emStrong.rDelimUnd, 'gu') .replace(/punct/g, inline._punctuation) .getRegex(); inline.anyPunctuation = edit(inline.anyPunctuation, 'gu') .replace(/punct/g, inline._punctuation) .getRegex(); inline._escapes = edit(inline._escapes, 'gu') .replace(/punct/g, inline._punctuation) .getRegex(); inline._scheme = /[a-zA-Z][a-zA-Z0-9+.-]{1,31}/; inline._email = /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/; inline.autolink = edit(inline.autolink) .replace('scheme', inline._scheme) .replace('email', inline._email) .getRegex(); inline._attribute = /\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/; inline.tag = edit(inline.tag) .replace('comment', inline._comment) .replace('attribute', inline._attribute) .getRegex(); inline._label = /(?:\[(?:\\.|[^\[\]\\])*\]|\\.|`[^`]*`|[^\[\]\\`])*?/; inline._href = /<(?:\\.|[^\n<>\\])+>|[^\s\x00-\x1f]*/; inline._title = /"(?:\\"?|[^"\\])*"|'(?:\\'?|[^'\\])*'|\((?:\\\)?|[^)\\])*\)/; inline.link = edit(inline.link) .replace('label', inline._label) .replace('href', inline._href) .replace('title', inline._title) .getRegex(); inline.reflink = edit(inline.reflink) .replace('label', inline._label) .replace('ref', block._label) .getRegex(); inline.nolink = edit(inline.nolink) .replace('ref', block._label) .getRegex(); inline.reflinkSearch = edit(inline.reflinkSearch, 'g') .replace('reflink', inline.reflink) .replace('nolink', inline.nolink) .getRegex(); /** * Normal Inline Grammar */ inline.normal = { ...inline }; /** * Pedantic Inline Grammar */ inline.pedantic = { ...inline.normal, strong: { start: /^__|\*\*/, middle: /^__(?=\S)([\s\S]*?\S)__(?!_)|^\*\*(?=\S)([\s\S]*?\S)\*\*(?!\*)/, endAst: /\*\*(?!\*)/g, endUnd: /__(?!_)/g }, em: { start: /^_|\*/, middle: /^()\*(?=\S)([\s\S]*?\S)\*(?!\*)|^_(?=\S)([\s\S]*?\S)_(?!_)/, endAst: /\*(?!\*)/g, endUnd: /_(?!_)/g }, link: edit(/^!?\[(label)\]\((.*?)\)/) .replace('label', inline._label) .getRegex(), reflink: edit(/^!?\[(label)\]\s*\[([^\]]*)\]/) .replace('label', inline._label) .getRegex() }; /** * GFM Inline Grammar */ inline.gfm = { ...inline.normal, escape: edit(inline.escape).replace('])', '~|])').getRegex(), _extended_email: /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/, url: /^((?:ftp|https?):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.?)+[^\s<]*|^email/, _backpedal: /(?:[^?!.,:;*_'"~()&]+|\([^)]*\)|&(?![a-zA-Z0-9]+;$)|[?!.,:;*_'"~)]+(?!$))+/, del: /^(~~?)(?=[^\s~])([\s\S]*?[^\s~])\1(?=[^~]|$)/, text: /^([`~]+|[^`~])(?:(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\ { return leading + ' '.repeat(tabs.length); }); } let token; let lastToken; let cutSrc; let lastParagraphClipped; while (src) { if (this.options.extensions && this.options.extensions.block && this.options.extensions.block.some((extTokenizer) => { if (token = extTokenizer.call({ lexer: this }, src, tokens)) { src = src.substring(token.raw.length); tokens.push(token); return true; } return false; })) { continue; } // newline if (token = this.tokenizer.space(src)) { src = src.substring(token.raw.length); if (token.raw.length === 1 && tokens.length > 0) { // if there's a single \n as a spacer, it's terminating the last line, // so move it there so that we don't get unnecessary paragraph tags tokens[tokens.length - 1].raw += '\n'; } else { tokens.push(token); } continue; } // code if (token = this.tokenizer.code(src)) { src = src.substring(token.raw.length); lastToken = tokens[tokens.length - 1]; // An indented code block cannot interrupt a paragraph. if (lastToken && (lastToken.type === 'paragraph' || lastToken.type === 'text')) { lastToken.raw += '\n' + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text; } else { tokens.push(token); } continue; } // fences if (token = this.tokenizer.fences(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // heading if (token = this.tokenizer.heading(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // hr if (token = this.tokenizer.hr(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // blockquote if (token = this.tokenizer.blockquote(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // list if (token = this.tokenizer.list(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // html if (token = this.tokenizer.html(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // def if (token = this.tokenizer.def(src)) { src = src.substring(token.raw.length); lastToken = tokens[tokens.length - 1]; if (lastToken && (lastToken.type === 'paragraph' || lastToken.type === 'text')) { lastToken.raw += '\n' + token.raw; lastToken.text += '\n' + token.raw; this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text; } else if (!this.tokens.links[token.tag]) { this.tokens.links[token.tag] = { href: token.href, title: token.title }; } continue; } // table (gfm) if (token = this.tokenizer.table(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // lheading if (token = this.tokenizer.lheading(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // top-level paragraph // prevent paragraph consuming extensions by clipping 'src' to extension start cutSrc = src; if (this.options.extensions && this.options.extensions.startBlock) { let startIndex = Infinity; const tempSrc = src.slice(1); let tempStart; this.options.extensions.startBlock.forEach((getStartIndex) => { tempStart = getStartIndex.call({ lexer: this }, tempSrc); if (typeof tempStart === 'number' && tempStart >= 0) { startIndex = Math.min(startIndex, tempStart); } }); if (startIndex < Infinity && startIndex >= 0) { cutSrc = src.substring(0, startIndex + 1); } } if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) { lastToken = tokens[tokens.length - 1]; if (lastParagraphClipped && lastToken.type === 'paragraph') { lastToken.raw += '\n' + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue.pop(); this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text; } else { tokens.push(token); } lastParagraphClipped = (cutSrc.length !== src.length); src = src.substring(token.raw.length); continue; } // text if (token = this.tokenizer.text(src)) { src = src.substring(token.raw.length); lastToken = tokens[tokens.length - 1]; if (lastToken && lastToken.type === 'text') { lastToken.raw += '\n' + token.raw; lastToken.text += '\n' + token.text; this.inlineQueue.pop(); this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text; } else { tokens.push(token); } continue; } if (src) { const errMsg = 'Infinite loop on byte: ' + src.charCodeAt(0); if (this.options.silent) { console.error(errMsg); break; } else { throw new Error(errMsg); } } } this.state.top = true; return tokens; } inline(src, tokens = []) { this.inlineQueue.push({ src, tokens }); return tokens; } /** * Lexing/Compiling */ inlineTokens(src, tokens = []) { let token, lastToken, cutSrc; // String with links masked to avoid interference with em and strong let maskedSrc = src; let match; let keepPrevChar, prevChar; // Mask out reflinks if (this.tokens.links) { const links = Object.keys(this.tokens.links); if (links.length > 0) { while ((match = this.tokenizer.rules.inline.reflinkSearch.exec(maskedSrc)) != null) { if (links.includes(match[0].slice(match[0].lastIndexOf('[') + 1, -1))) { maskedSrc = maskedSrc.slice(0, match.index) + '[' + 'a'.repeat(match[0].length - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.reflinkSearch.lastIndex); } } } } // Mask out other blocks while ((match = this.tokenizer.rules.inline.blockSkip.exec(maskedSrc)) != null) { maskedSrc = maskedSrc.slice(0, match.index) + '[' + 'a'.repeat(match[0].length - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.blockSkip.lastIndex); } // Mask out escaped characters while ((match = this.tokenizer.rules.inline.anyPunctuation.exec(maskedSrc)) != null) { maskedSrc = maskedSrc.slice(0, match.index) + '++' + maskedSrc.slice(this.tokenizer.rules.inline.anyPunctuation.lastIndex); } while (src) { if (!keepPrevChar) { prevChar = ''; } keepPrevChar = false; // extensions if (this.options.extensions && this.options.extensions.inline && this.options.extensions.inline.some((extTokenizer) => { if (token = extTokenizer.call({ lexer: this }, src, tokens)) { src = src.substring(token.raw.length); tokens.push(token); return true; } return false; })) { continue; } // escape if (token = this.tokenizer.escape(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // tag if (token = this.tokenizer.tag(src)) { src = src.substring(token.raw.length); lastToken = tokens[tokens.length - 1]; if (lastToken && token.type === 'text' && lastToken.type === 'text') { lastToken.raw += token.raw; lastToken.text += token.text; } else { tokens.push(token); } continue; } // link if (token = this.tokenizer.link(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // reflink, nolink if (token = this.tokenizer.reflink(src, this.tokens.links)) { src = src.substring(token.raw.length); lastToken = tokens[tokens.length - 1]; if (lastToken && token.type === 'text' && lastToken.type === 'text') { lastToken.raw += token.raw; lastToken.text += token.text; } else { tokens.push(token); } continue; } // em & strong if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // code if (token = this.tokenizer.codespan(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // br if (token = this.tokenizer.br(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // del (gfm) if (token = this.tokenizer.del(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // autolink if (token = this.tokenizer.autolink(src)) { src = src.substring(token.raw.length); tokens.push(token); continue; } // url (gfm) if (!this.state.inLink && (token = this.tokenizer.url(src))) { src = src.substring(token.raw.length); tokens.push(token); continue; } // text // prevent inlineText consuming extensions by clipping 'src' to extension start cutSrc = src; if (this.options.extensions && this.options.extensions.startInline) { let startIndex = Infinity; const tempSrc = src.slice(1); let tempStart; this.options.extensions.startInline.forEach((getStartIndex) => { tempStart = getStartIndex.call({ lexer: this }, tempSrc); if (typeof tempStart === 'number' && tempStart >= 0) { startIndex = Math.min(startIndex, tempStart); } }); if (startIndex < Infinity && startIndex >= 0) { cutSrc = src.substring(0, startIndex + 1); } } if (token = this.tokenizer.inlineText(cutSrc)) { src = src.substring(token.raw.length); if (token.raw.slice(-1) !== '_') { // Track prevChar before string of ____ started prevChar = token.raw.slice(-1); } keepPrevChar = true; lastToken = tokens[tokens.length - 1]; if (lastToken && lastToken.type === 'text') { lastToken.raw += token.raw; lastToken.text += token.text; } else { tokens.push(token); } continue; } if (src) { const errMsg = 'Infinite loop on byte: ' + src.charCodeAt(0); if (this.options.silent) { console.error(errMsg); break; } else { throw new Error(errMsg); } } } return tokens; } } /** * Renderer */ class _Renderer { options; constructor(options) { this.options = options || _defaults; } code(code, infostring, escaped) { const lang = (infostring || '').match(/^\S*/)?.[0]; code = code.replace(/\n$/, '') + '\n'; if (!lang) { return '
'
                + (escaped ? code : escape(code, true))
                + '
\n'; } return '
'
            + (escaped ? code : escape(code, true))
            + '
\n'; } blockquote(quote) { return `
\n${quote}
\n`; } html(html, block) { return html; } heading(text, level, raw) { // ignore IDs return `${text}\n`; } hr() { return '
\n'; } list(body, ordered, start) { const type = ordered ? 'ol' : 'ul'; const startatt = (ordered && start !== 1) ? (' start="' + start + '"') : ''; return '<' + type + startatt + '>\n' + body + '\n'; } listitem(text, task, checked) { return `
  • ${text}
    • \n`; } checkbox(checked) { return ''; } paragraph(text) { return `

    ${text}

    \n`; } table(header, body) { if (body) body = `${body}`; return '\n' + '\n' + header + '\n' + body + '
    \n'; } tablerow(content) { return `\n${content}\n`; } tablecell(content, flags) { const type = flags.header ? 'th' : 'td'; const tag = flags.align ? `<${type} align="${flags.align}">` : `<${type}>`; return tag + content + `\n`; } /** * span level renderer */ strong(text) { return `${text}`; } em(text) { return `${text}`; } codespan(text) { return `${text}`; } br() { return '
    '; } del(text) { return `${text}`; } link(href, title, text) { const cleanHref = cleanUrl(href); if (cleanHref === null) { return text; } href = cleanHref; let out = '    '; return out; } image(href, title, text) { const cleanHref = cleanUrl(href); if (cleanHref === null) { return text; } href = cleanHref; let out = `${text} 0 && item.tokens[0].type === 'paragraph') { item.tokens[0].text = checkbox + ' ' + item.tokens[0].text; if (item.tokens[0].tokens && item.tokens[0].tokens.length > 0 && item.tokens[0].tokens[0].type === 'text') { item.tokens[0].tokens[0].text = checkbox + ' ' + item.tokens[0].tokens[0].text; } } else { item.tokens.unshift({ type: 'text', text: checkbox + ' ' }); } } else { itemBody += checkbox + ' '; } } itemBody += this.parse(item.tokens, loose); body += this.renderer.listitem(itemBody, task, !!checked); } out += this.renderer.list(body, ordered, start); continue; } case 'html': { const htmlToken = token; out += this.renderer.html(htmlToken.text, htmlToken.block); continue; } case 'paragraph': { const paragraphToken = token; out += this.renderer.paragraph(this.parseInline(paragraphToken.tokens)); continue; } case 'text': { let textToken = token; let body = textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text; while (i + 1 < tokens.length && tokens[i + 1].type === 'text') { textToken = tokens[++i]; body += '\n' + (textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text); } out += top ? this.renderer.paragraph(body) : body; continue; } default: { const errMsg = 'Token with "' + token.type + '" type was not found.'; if (this.options.silent) { console.error(errMsg); return ''; } else { throw new Error(errMsg); } } } } return out; } /** * Parse Inline Tokens */ parseInline(tokens, renderer) { renderer = renderer || this.renderer; let out = ''; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; // Run any renderer extensions if (this.options.extensions && this.options.extensions.renderers && this.options.extensions.renderers[token.type]) { const ret = this.options.extensions.renderers[token.type].call({ parser: this }, token); if (ret !== false || !['escape', 'html', 'link', 'image', 'strong', 'em', 'codespan', 'br', 'del', 'text'].includes(token.type)) { out += ret || ''; continue; } } switch (token.type) { case 'escape': { const escapeToken = token; out += renderer.text(escapeToken.text); break; } case 'html': { const tagToken = token; out += renderer.html(tagToken.text); break; } case 'link': { const linkToken = token; out += renderer.link(linkToken.href, linkToken.title, this.parseInline(linkToken.tokens, renderer)); break; } case 'image': { const imageToken = token; out += renderer.image(imageToken.href, imageToken.title, imageToken.text); break; } case 'strong': { const strongToken = token; out += renderer.strong(this.parseInline(strongToken.tokens, renderer)); break; } case 'em': { const emToken = token; out += renderer.em(this.parseInline(emToken.tokens, renderer)); break; } case 'codespan': { const codespanToken = token; out += renderer.codespan(codespanToken.text); break; } case 'br': { out += renderer.br(); break; } case 'del': { const delToken = token; out += renderer.del(this.parseInline(delToken.tokens, renderer)); break; } case 'text': { const textToken = token; out += renderer.text(textToken.text); break; } default: { const errMsg = 'Token with "' + token.type + '" type was not found.'; if (this.options.silent) { console.error(errMsg); return ''; } else { throw new Error(errMsg); } } } } return out; } } class _Hooks { options; constructor(options) { this.options = options || _defaults; } static passThroughHooks = new Set([ 'preprocess', 'postprocess' ]); /** * Process markdown before marked */ preprocess(markdown) { return markdown; } /** * Process HTML after marked is finished */ postprocess(html) { return html; } } class Marked { defaults = _getDefaults(); options = this.setOptions; parse = this.#parseMarkdown(_Lexer.lex, _Parser.parse); parseInline = this.#parseMarkdown(_Lexer.lexInline, _Parser.parseInline); Parser = _Parser; Renderer = _Renderer; TextRenderer = _TextRenderer; Lexer = _Lexer; Tokenizer = _Tokenizer; Hooks = _Hooks; constructor(...args) { this.use(...args); } /** * Run callback for every token */ walkTokens(tokens, callback) { let values = []; for (const token of tokens) { values = values.concat(callback.call(this, token)); switch (token.type) { case 'table': { const tableToken = token; for (const cell of tableToken.header) { values = values.concat(this.walkTokens(cell.tokens, callback)); } for (const row of tableToken.rows) { for (const cell of row) { values = values.concat(this.walkTokens(cell.tokens, callback)); } } break; } case 'list': { const listToken = token; values = values.concat(this.walkTokens(listToken.items, callback)); break; } default: { const genericToken = token; if (this.defaults.extensions?.childTokens?.[genericToken.type]) { this.defaults.extensions.childTokens[genericToken.type].forEach((childTokens) => { values = values.concat(this.walkTokens(genericToken[childTokens], callback)); }); } else if (genericToken.tokens) { values = values.concat(this.walkTokens(genericToken.tokens, callback)); } } } } return values; } use(...args) { const extensions = this.defaults.extensions || { renderers: {}, childTokens: {} }; args.forEach((pack) => { // copy options to new object const opts = { ...pack }; // set async to true if it was set to true before opts.async = this.defaults.async || opts.async || false; // ==-- Parse "addon" extensions --== // if (pack.extensions) { pack.extensions.forEach((ext) => { if (!ext.name) { throw new Error('extension name required'); } if ('renderer' in ext) { // Renderer extensions const prevRenderer = extensions.renderers[ext.name]; if (prevRenderer) { // Replace extension with func to run new extension but fall back if false extensions.renderers[ext.name] = function (...args) { let ret = ext.renderer.apply(this, args); if (ret === false) { ret = prevRenderer.apply(this, args); } return ret; }; } else { extensions.renderers[ext.name] = ext.renderer; } } if ('tokenizer' in ext) { // Tokenizer Extensions if (!ext.level || (ext.level !== 'block' && ext.level !== 'inline')) { throw new Error("extension level must be 'block' or 'inline'"); } const extLevel = extensions[ext.level]; if (extLevel) { extLevel.unshift(ext.tokenizer); } else { extensions[ext.level] = [ext.tokenizer]; } if (ext.start) { // Function to check for start of token if (ext.level === 'block') { if (extensions.startBlock) { extensions.startBlock.push(ext.start); } else { extensions.startBlock = [ext.start]; } } else if (ext.level === 'inline') { if (extensions.startInline) { extensions.startInline.push(ext.start); } else { extensions.startInline = [ext.start]; } } } } if ('childTokens' in ext && ext.childTokens) { // Child tokens to be visited by walkTokens extensions.childTokens[ext.name] = ext.childTokens; } }); opts.extensions = extensions; } // ==-- Parse "overwrite" extensions --== // if (pack.renderer) { const renderer = this.defaults.renderer || new _Renderer(this.defaults); for (const prop in pack.renderer) { const rendererFunc = pack.renderer[prop]; const rendererKey = prop; const prevRenderer = renderer[rendererKey]; // Replace renderer with func to run extension, but fall back if false renderer[rendererKey] = (...args) => { let ret = rendererFunc.apply(renderer, args); if (ret === false) { ret = prevRenderer.apply(renderer, args); } return ret || ''; }; } opts.renderer = renderer; } if (pack.tokenizer) { const tokenizer = this.defaults.tokenizer || new _Tokenizer(this.defaults); for (const prop in pack.tokenizer) { const tokenizerFunc = pack.tokenizer[prop]; const tokenizerKey = prop; const prevTokenizer = tokenizer[tokenizerKey]; // Replace tokenizer with func to run extension, but fall back if false tokenizer[tokenizerKey] = (...args) => { let ret = tokenizerFunc.apply(tokenizer, args); if (ret === false) { ret = prevTokenizer.apply(tokenizer, args); } return ret; }; } opts.tokenizer = tokenizer; } // ==-- Parse Hooks extensions --== // if (pack.hooks) { const hooks = this.defaults.hooks || new _Hooks(); for (const prop in pack.hooks) { const hooksFunc = pack.hooks[prop]; const hooksKey = prop; const prevHook = hooks[hooksKey]; if (_Hooks.passThroughHooks.has(prop)) { hooks[hooksKey] = (arg) => { if (this.defaults.async) { return Promise.resolve(hooksFunc.call(hooks, arg)).then(ret => { return prevHook.call(hooks, ret); }); } const ret = hooksFunc.call(hooks, arg); return prevHook.call(hooks, ret); }; } else { hooks[hooksKey] = (...args) => { let ret = hooksFunc.apply(hooks, args); if (ret === false) { ret = prevHook.apply(hooks, args); } return ret; }; } } opts.hooks = hooks; } // ==-- Parse WalkTokens extensions --== // if (pack.walkTokens) { const walkTokens = this.defaults.walkTokens; const packWalktokens = pack.walkTokens; opts.walkTokens = function (token) { let values = []; values.push(packWalktokens.call(this, token)); if (walkTokens) { values = values.concat(walkTokens.call(this, token)); } return values; }; } this.defaults = { ...this.defaults, ...opts }; }); return this; } setOptions(opt) { this.defaults = { ...this.defaults, ...opt }; return this; } lexer(src, options) { return _Lexer.lex(src, options ?? this.defaults); } parser(tokens, options) { return _Parser.parse(tokens, options ?? this.defaults); } #parseMarkdown(lexer, parser) { return (src, options) => { const origOpt = { ...options }; const opt = { ...this.defaults, ...origOpt }; // Show warning if an extension set async to true but the parse was called with async: false if (this.defaults.async === true && origOpt.async === false) { if (!opt.silent) { console.warn('marked(): The async option was set to true by an extension. The async: false option sent to parse will be ignored.'); } opt.async = true; } const throwError = this.#onError(!!opt.silent, !!opt.async); // throw error in case of non string input if (typeof src === 'undefined' || src === null) { return throwError(new Error('marked(): input parameter is undefined or null')); } if (typeof src !== 'string') { return throwError(new Error('marked(): input parameter is of type ' + Object.prototype.toString.call(src) + ', string expected')); } if (opt.hooks) { opt.hooks.options = opt; } if (opt.async) { return Promise.resolve(opt.hooks ? opt.hooks.preprocess(src) : src) .then(src => lexer(src, opt)) .then(tokens => opt.walkTokens ? Promise.all(this.walkTokens(tokens, opt.walkTokens)).then(() => tokens) : tokens) .then(tokens => parser(tokens, opt)) .then(html => opt.hooks ? opt.hooks.postprocess(html) : html) .catch(throwError); } try { if (opt.hooks) { src = opt.hooks.preprocess(src); } const tokens = lexer(src, opt); if (opt.walkTokens) { this.walkTokens(tokens, opt.walkTokens); } let html = parser(tokens, opt); if (opt.hooks) { html = opt.hooks.postprocess(html); } return html; } catch (e) { return throwError(e); } }; } #onError(silent, async) { return (e) => { e.message += '\nPlease report this to https://github.com/markedjs/marked.'; if (silent) { const msg = '

    An error occurred:

    '
                    + escape(e.message + '', true)
                    + '
    '; if (async) { return Promise.resolve(msg); } return msg; } if (async) { return Promise.reject(e); } throw e; }; } } const markedInstance = new Marked(); function marked(src, opt) { return markedInstance.parse(src, opt); } /** * Sets the default options. * * @param options Hash of options */ marked.options = marked.setOptions = function (options) { markedInstance.setOptions(options); marked.defaults = markedInstance.defaults; changeDefaults(marked.defaults); return marked; }; /** * Gets the original marked default options. */ marked.getDefaults = _getDefaults; marked.defaults = _defaults; /** * Use Extension */ marked.use = function (...args) { markedInstance.use(...args); marked.defaults = markedInstance.defaults; changeDefaults(marked.defaults); return marked; }; /** * Run callback for every token */ marked.walkTokens = function (tokens, callback) { return markedInstance.walkTokens(tokens, callback); }; /** * Compiles markdown to HTML without enclosing `p` tag. * * @param src String of markdown source to be compiled * @param options Hash of options * @return String of compiled HTML */ marked.parseInline = markedInstance.parseInline; /** * Expose */ marked.Parser = _Parser; marked.parser = _Parser.parse; marked.Renderer = _Renderer; marked.TextRenderer = _TextRenderer; marked.Lexer = _Lexer; marked.lexer = _Lexer.lex; marked.Tokenizer = _Tokenizer; marked.Hooks = _Hooks; marked.parse = marked; const options = marked.options; const setOptions = marked.setOptions; const use = marked.use; const walkTokens = marked.walkTokens; const parseInline = marked.parseInline; const parse = marked; const parser = _Parser.parse; const lexer = _Lexer.lex; export { _Hooks as Hooks, _Lexer as Lexer, Marked, _Parser as Parser, _Renderer as Renderer, _TextRenderer as TextRenderer, _Tokenizer as Tokenizer, _defaults as defaults, _getDefaults as getDefaults, lexer, marked, options, parse, parseInline, parser, setOptions, use, walkTokens }; //# sourceMappingURL=marked.esm.js.map ================================================ FILE: extension/popup/memory_cache.html ================================================
    Save page as PDF
    Save page as HTML
    Add text note
    View on GitHub memorycache.ai/
    ================================================ FILE: extension/popup/memory_cache.js ================================================ import { marked } from "./marked.esm.js"; const DOWNLOAD_SUBDIRECTORY = "MemoryCache"; /* Generate a file name based on date and time */ function generateFileName(ext) { return ( new Date().toISOString().concat(0, 19).replaceAll(":", ".") + "." + ext ); } async function savePDF() { try { await browser.tabs.saveAsPDF({ toFileName: `${DOWNLOAD_SUBDIRECTORY}/PAGE${generateFileName("pdf")}`, silentMode: true, // silentMode requires a custom build of Firefox }); } catch (_e) { // Fallback to non-silent mode. await browser.tabs.saveAsPDF({ // Omit the DOWNLOAD_SUBDIRECTORY prefix because saveAsPDF will not respect it. toFileName: `PAGE${generateFileName("pdf")}`, }); } } // Send a message to the content script. // // We need code to run in the content script context for anything // that accesses the DOM or needs to outlive the popup window. function send(message) { return new Promise((resolve, _reject) => { browser.tabs.query({ active: true, currentWindow: true }, (tabs) => { resolve(browser.tabs.sendMessage(tabs[0].id, message)); }); }); } async function saveHtml() { const text = await send({ action: "getPageText" }); const filename = `${DOWNLOAD_SUBDIRECTORY}/PAGE${generateFileName("html")}`; const file = new File([text], filename, { type: "text/plain" }); const url = URL.createObjectURL(file); browser.downloads.download({ url, filename, saveAs: false }); } function saveNote() { const text = document.querySelector("#text-note").value; const filename = `${DOWNLOAD_SUBDIRECTORY}/NOTE${generateFileName("md")}`; const file = new File([text], filename, { type: "text/plain" }); const url = URL.createObjectURL(file); browser.downloads.download({ url, filename, saveAs: false }); document.querySelector("#text-note").value = ""; browser.storage.local.set({ noteDraft: "" }); } function debounce(func, delay) { let debounceTimer; return function () { const context = this; const args = arguments; clearTimeout(debounceTimer); debounceTimer = setTimeout(() => func.apply(context, args), delay); }; } function saveNoteDraft() { const noteDraft = document.querySelector("#text-note").value; browser.storage.local.set({ noteDraft }); } document.getElementById("save-pdf-button").addEventListener("click", savePDF); document.getElementById("save-html-button").addEventListener("click", saveHtml); document.getElementById("save-pdf-button").addEventListener("click", savePDF); document.getElementById("save-note-button").addEventListener("click", saveNote); document .getElementById("text-note") .addEventListener("input", debounce(saveNoteDraft, 250)); browser.storage.local.get("noteDraft").then((res) => { if (res.noteDraft) { document.querySelector("#text-note").value = res.noteDraft; } }); function setTextView(showPreview) { var textArea = document.getElementById("text-note"); var previewDiv = document.getElementById("preview-note"); if (showPreview) { textArea.style.display = "none"; previewDiv.style.display = "block"; previewDiv.innerHTML = marked(textArea.value); } else { // Switch to editing mode previewDiv.style.display = "none"; textArea.style.display = "block"; } } document.getElementById("edit-button").addEventListener("click", () => { setTextView(false); }); document.getElementById("preview-button").addEventListener("click", () => { setTextView(true); }); ================================================ FILE: extension/popup/styles.css ================================================ :root { --border-radius: 8px; --primary: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87EBDA 76.04%, #BAE1FF 100%); --primary-hover:linear-gradient(90deg, #FFA8B1 0%, #FFD9AD 26.56%, #FFFFAD 50.52%, #80EAD8 76.04%, #ADDCFF 100%); --primary-active: linear-gradient(90deg, #FF9EA8 0%, #FFD29E 26.56%, #FFFF9E 50.52%, #73E8D4 76.04%, #99D3FF 100%); --primary-content: #000000; --secondary: #180AB8; --secondary-hover: #1609A6; --secondary-active: #130893; --secondary-content: #fff; --inactive-content:#F9F9F9; --interaction-inactive:#B6B9BF; --base-100: #F6F6F6; --base-200: #fff; --base-content: #2d3d46; --base-content-subtle: #565D6D; --info: #3ac0f8; --info-content: #000; --warning: #fcbc23; --warning-content: #000; --success: #37d399; --success-content: #000; --error: #f87272; --error-content: #000; --border-1: #EDEDED; --xxs:4px; --xs:8px; --sm:12px; --md:16px; --lg:24px; --xl:40px; } body { background: var(--base-100); font-weight: 500; font-size: .9em; font-family: 'Work Sans'; width: 290px; } #header { line-height: 24px; font-size: 16px; color: var(--base-content); background-color: var(--base-100); border-bottom-color: var(--border-1); border-bottom-width: 1px; padding:8px; } .body-container { padding:0 8px; display:flex; flex-direction: column; } .button { cursor:pointer; border-radius: var(--border-radius); line-height: 21px; letter-spacing: 0em; text-align: center; color: #565D6D; padding:8px; } .primary-btn { background: var(--primary); margin-bottom:.8em; } .primary-btn:hover { background:var(--primary-hover); } .primary-btn:active { background:var(--primary-active); } .secondary-btn { background:var(--base-100); border: 2px solid var(--border-1); } .secondary-btn:hover { background:var(--base-200); border: 2px solid var(--border-1); } .secondary-btn:active { background:var(--base-200); border: 2px solid var(--border-1); } .text-field { margin-bottom:8px; } .text-field textarea { width: 96%; } .text-field label { color: var(--base-content-subtle); } #text-note { height: 100px; border-radius: var(--border-radius); } .border { margin:16px 0; height:1px; background-color: #C6C6C6; border-radius: var(--border-radius); } .header { } .footer { background-color:var(--base-200); padding:8px; display:flex; justify-content: space-between; } a { text-decoration: none; color:var(--secondary); font-size: 14px; font-weight: 400; } a:hover, a:focus, a:visited { color: var(--secondary-hover); } ================================================ FILE: scratch/backend/hub/.gitignore ================================================ *.log *.spec dist/ build/ venv/ sqlite.db __pycache__/ *.pyc ================================================ FILE: scratch/backend/hub/PLAN.md ================================================ # Memory Cache Hub The `hub` is a central component of Memory Cache: - It exposes APIs used by `browser-extension`, `browser-client`, and plugins. - It serves the static `browser-client` files over HTTP. - It downloads `llamafile`s and runs them as subprocesses. - It interacts with a vector database to ingest and retrieve document fragments. - It synthesizes queries and prompts for backend `llm`s on behalf of the user. ## Control Flow When `memory-cache-hub` starts, it should: - Check whether another instance of `memory-cache-hub` is already running. If it is, log an error and exit. - Start the FastAPI server. If the port is already in use, log an error and exit. From then on, everything is driven by API requests. ## API Endpoints | Route | Method | Summary | |:-----------------------------------------|:-------|:--------------------------------------------| | `/api/llamafile/list` | GET | List available llamafiles | | `/api/llamafile/run` | POST | Run a llamafile | | `/api/llamafile/stop` | POST | Stop a running llamafile | | `/api/llamafile/get` | GET | Get information about a llamafile | | `/api/llamafile/download` | POST | Initiate download of a llamafile | | `/api/llamafile/delete` | POST | Delete a llamafile | | `/api/llamafile/check_download_progress` | POST | Check download progress of a llamafile | | `/api/threads/list` | GET | List chat threads | | `/api/threads/get` | GET | Get information about a chat thread | | `/api/threads/create` | POST | Create a new chat thread | | `/api/threads/delete` | POST | Delete a chat thread | | `/api/threads/send_message` | POST | Send a message to a chat thread | | `/api/threads/rag_send_message` | POST | Send a message to a chat thread using RAG | | `/api/threads/get_messages` | GET | Get messages from a chat thread | | `/api/threads/config` | POST | Configure a chat thread | | `/api/datastore/ingest` | POST | Ingest document fragments in the data store | | `/api/datastore/status` | GET | Get status of the data store | | `/api/datastore/config` | POST | Configure the data store | | `/api/datastore/config` | GET | Get configuration of the data store | ================================================ FILE: scratch/backend/hub/README.md ================================================ # Memory Cache Hub A backend for Memory Cache built on [langchain](https://python.langchain.com/), bundled as an executable with [PyInstaller](https://pyinstaller.org/). ## Overview The `hub` is a central component of Memory Cache: - It exposes APIs used by `browser-extension`, `browser-client`, and plugins. - It serves the static `browser-client` files over HTTP. - It downloads `llamafile`s and runs them as subprocesses. - It interacts with a vector database to ingest and retrieve document fragments. - It synthesizes queries and prompts for backend `llm`s on behalf of the user. ## Usage ```sh LLAMAFILES_DIR=~/media/llamafile ./dist/memory-cache-hub-gnu-linux ``` ## Development You can develop `hub` on your local machine or using the provided Docker development environment. If you are developing on your local machine, you will need to install the dependencies listed in the `requirements/` files. We recommend using a virtual environment to manage these dependencies, as per the instructions in the various "Building..." sections below. ### Development with virtual environment Create a virtual environment: ```bash python3.11 -m venv venv ``` Activate it: ``` source venv/bin/activate ``` Install the dependencies: ```bash pip install -r requirements/hub-base.txt \ -r requirements/hub-cpu.txt \ -r requirements/hub-builder.txt ``` Run the program: ```bash LLAMAFILES_DIR=~/media/llamafile python3 src/hub.py ``` Or build with: ``` sh python src/hub_build_gnu_linux.py ``` ### Docker Development Environment A development environment for working on `hub` is provided by the Dockerfile `docker/Dockerfile.hub-dev`. The basic workflow is to build this image and then bind mount the source code when you run the container. You will also want to bind mount a `LLAMAFILES_DIR` pointing to a directory where you'll store `llamafile`s. These can be quite large, so we avoid re-downloading them every time we start the container. When you are satisfied with development, you will want to package the `hub` as an executable with `PyInstaller`. Since `PyInstaller` does not support cross-compilation, you will need to run the build commands on the platform you are targeting. For example, to build a MacOS executable, you will need to run the build commands on a MacOS machine. Examples of how to build and use the development and builder images are provided in the sections below. #### Using the Docker Development Environment Build the development image: ```bash docker build -f docker/Dockerfile.hub-dev -t memory-cache/hub-dev . ``` Run the development container: ```bash docker run -it --rm \ -v $(pwd):/hub \ -v ~/media/llamafile:/llamafiles \ -e LLAMAFILES_DIR=/llamafiles \ -p 8800:8800 \ memory-cache/hub-dev \ python3 src/hub.py ``` Replace `~/media/llamafile` with the path to the directory where you want to store `llamafile`s. #### Using the Docker Development Environment with NVIDIA GPUs If you have an NVIDIA GPU, you'll need to make sure that you have the NVIDIA Container Toolkit installed and that you have the appropriate drivers and libraries installed on your host machine. A script for configuring an Ubuntu 22.04 machine can be found in the [OSAI-Ubuntu](https://github.com/johnshaughnessy/osai-ubuntu) repository. Once you've set up your host machine, build the development image with CUDA support: ```sh docker build -f docker/Dockerfile.hub-dev-cuda -t memory-cache/hub-dev-cuda . ``` Then run the development container with CUDA support: ```sh docker run -it --rm \ --gpus all \ -v $(pwd):/hub \ -v ~/media/llamafile:/llamafiles \ -e LLAMAFILES_DIR=/llamafiles \ -e CUDA_VISIBLE_DEVICES=1 \ -p 8800:8800 \ memory-cache/hub-dev-cuda \ python3 src/hub.py ``` ## Building for GNU/Linux Build the builder image: ```bash docker build -f docker/Dockerfile.hub-builder-gnu-linux -t memory-cache/hub-builder-gnu-linux . docker build -f docker/Dockerfile.hub-builder-old-gnu-linux -t memory-cache/hub-builder-old-gnu-linux . ``` Run the builder container: ```bash docker run -it --rm \ -v $(pwd):/hub \ memory-cache/hub-builder-gnu-linux docker run -it --rm \ -v $(pwd):/hub \ memory-cache/hub-builder-old-gnu-linux ``` The builder will generate `memory-cache-hub-gnu-linux` in the `dist` directory. ## Building for MacOS On MacOS, we use a python virtual environment to install the dependencies and run the build commands. Create the virtual environment: ```bash python3.11 -m venv venv source venv/bin/activate ``` Install the dependencies: ```bash pip install -r requirements/hub-base.txt \ -r requirements/hub-cpu.txt \ -r requirements/hub-builder.txt ``` Build the executable: ```bash python3.11 src/hub_build_macos.py ``` The builder will generate `memory-cache-hub-macos` in the `dist` directory. When you are done, deactivate the virtual environment: ``` sh deactivate ``` If you want to remove the virtual environment, just delete the `venv` directory. ## Building on Windows On Windows, we use a python virtual environment to install the dependencies and run the build commands. Install `python 3.11` from the [official website](https://www.python.org/downloads/). Create the virtual environment: ```bash py -3.11 -m venv venv venv\Scripts\activate ``` Install the dependencies: ```bash pip install -r requirements\hub-base.txt -r requirements\hub-cpu.txt -r requirements\hub-builder.txt ``` Build the executable: ```bash python src\hub_build_windows.py ``` The builder will generate `memory-cache-hub-windows.exe` in the `dist` directory. When you are done, deactivate the virtual environment: ``` sh deactivate ``` If you want to remove the virtual environment, just delete the `venv` directory. ## Plan/TODO - [ ] Write Hello World server - [ ] Bundle with PyInstaller on Linux - [ ] Bundle with PyInstaller on MacOS - [ ] Bundle with PyInstaller on Windows - [ ] Test NVIDIA w/ CUDA - [ ] Test AMD w/ HIP/Rocm - [ ] Test x86-64 - [ ] Test Apple silicon - [ ] Add llamafile management - [ ] Add ingestion - [ ] Add retrieval - [ ] Connect to browser client ## Miscellaneous Notes ### `python 3.11` We use `python 3.11` (not `3.12` or later) because `faiss-cpu` only supports up to `3.11` at the time of this writing: https://pypi.org/project/faiss-cpu/ ================================================ FILE: scratch/backend/hub/docker/Dockerfile.hub-builder-gnu-linux ================================================ from ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update RUN apt-get install -y python3.11 python3.11-distutils python3.11-dev && \ apt-get install -y python3-pip RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ update-alternatives --set python3 /usr/bin/python3.11 RUN python3.11 -m pip install --upgrade pip setuptools wheel WORKDIR /hub COPY requirements/hub-base.txt ./ RUN pip install --no-cache-dir -r hub-base.txt COPY requirements/hub-cpu.txt ./ RUN pip install --no-cache-dir -r hub-cpu.txt # GNU/Linux # # PyInstaller requires the ldd terminal application to discover the shared libraries required by each program or shared library. It is typically found in the distribution-package glibc or libc-bin. # # It also requires the objdump terminal application to extract information from object files and the objcopy terminal application to append data to the bootloader. These are typically found in the distribution-package binutils. RUN apt-get install -y binutils RUN apt-get install -y libc-bin COPY requirements/hub-builder.txt ./ RUN pip install --no-cache-dir -r hub-builder.txt COPY . . CMD [ "python3", "./src/hub_build_gnu_linux.py" ] ================================================ FILE: scratch/backend/hub/docker/Dockerfile.hub-builder-old-gnu-linux ================================================ # Use an old version of ubuntu FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update RUN apt-get install -y python3.11 python3.11-distutils python3.11-dev && \ apt-get install -y python3-pip RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ update-alternatives --set python3 /usr/bin/python3.11 #RUN python3.11 -m pip install --upgrade pip setuptools wheel COPY requirements/hub-base.txt ./ RUN pip install --no-cache-dir -r hub-base.txt COPY requirements/hub-cpu.txt ./ RUN pip install --no-cache-dir -r hub-cpu.txt COPY requirements/hub-builder.txt ./ RUN pip install --no-cache-dir -r hub-builder.txt COPY . . CMD [ "python3", "./src/hub_build_gnu_linux.py" ] ================================================ FILE: scratch/backend/hub/docker/Dockerfile.hub-dev ================================================ from ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive # Install software-properties-common to add PPAs RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update # Install Python 3.11 and pip RUN apt-get install -y python3.11 python3.11-distutils && \ apt-get install -y python3-pip # Update alternatives to use Python 3.11 as the default python3 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ update-alternatives --set python3 /usr/bin/python3.11 # Ensure pip is updated and set to use the correct Python version RUN python3.11 -m pip install --upgrade pip setuptools wheel RUN apt-get install -y wget RUN wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf RUN chmod +x /usr/bin/ape # RUN sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" # RUN sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" WORKDIR /hub COPY requirements/hub-base.txt ./ RUN pip install --no-cache-dir -r hub-base.txt COPY requirements/hub-cpu.txt ./ RUN pip install --no-cache-dir -r hub-cpu.txt COPY . . CMD [ "python3", "./src/hub.py" ] ================================================ FILE: scratch/backend/hub/docker/Dockerfile.hub-dev-cuda ================================================ FROM nvidia/cuda:12.3.1-devel-ubuntu22.04 ENV DEBIAN_FRONTEND=noninteractive # Install software-properties-common to add PPAs RUN apt-get update && \ apt-get install -y software-properties-common && \ add-apt-repository ppa:deadsnakes/ppa && \ apt-get update # Install Python 3.11 and pip RUN apt-get install -y python3.11 python3.11-distutils && \ apt-get install -y python3-pip # Update alternatives to use Python 3.11 as the default python3 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ update-alternatives --set python3 /usr/bin/python3.11 # Ensure pip is updated and set to use the correct Python version RUN python3.11 -m pip install --upgrade pip setuptools wheel RUN apt-get install -y wget RUN wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf RUN chmod +x /usr/bin/ape # RUN sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" # RUN sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" WORKDIR /hub COPY requirements/hub-base.txt ./ RUN pip install --no-cache-dir -r hub-base.txt COPY requirements/hub-cpu.txt ./ RUN pip install --no-cache-dir -r hub-cpu.txt COPY . . CMD [ "python3", "./src/hub.py" ] ================================================ FILE: scratch/backend/hub/requirements/hub-base.txt ================================================ aiofiles bs4 certifi fastapi langchain langchainhub langchain-openai langchain-cli langserve[all] requests ~= 2.31 tqdm uvicorn psutil ================================================ FILE: scratch/backend/hub/requirements/hub-builder.txt ================================================ pyinstaller ================================================ FILE: scratch/backend/hub/requirements/hub-cpu.txt ================================================ faiss-cpu ================================================ FILE: scratch/backend/hub/src/api/llamafile_api.py ================================================ from fastapi import APIRouter from pydantic import BaseModel from llamafile_manager import get_llamafile_manager from typing import Optional router = APIRouter() manager = get_llamafile_manager() class LlamafileInfo(BaseModel): name: str url: str downloaded: bool running: bool download_progress: Optional[int] class ListLlamafilesResponse(BaseModel): llamafiles: list[LlamafileInfo] @router.get("/list_llamafiles") async def list_llamafiles(): """List all llamafiles, including those that have not been downloaded.""" llamafiles = manager.list_all_llamafiles() llamafile_infos = [] for info in llamafiles: llamafile_infos.append(LlamafileInfo(name=info.name, url=info.url, downloaded=manager.has_llamafile(info.name), running=manager.is_llamafile_running(info.name), download_progress=manager.llamafile_download_progress(info.name))) return ListLlamafilesResponse(llamafiles=llamafile_infos) class GetLlamafileRequest(BaseModel): name: str class GetLlamafileResponse(BaseModel): # Respond with the llamafile info or None if the llamafile is not found llamafile: Optional[LlamafileInfo] @router.post("/get_llamafile") async def get_llamafile(request: GetLlamafileRequest): """Get the llamafile info for the llamafile of the given name.""" all_llamafile_infos = manager.list_all_llamafiles() llamafile = next((l for l in all_llamafile_infos if l.name == request.name), None) if llamafile is None: return GetLlamafileResponse(llamafile=None) return GetLlamafileResponse( llamafile=LlamafileInfo(name=llamafile.name, url=llamafile.url, downloaded=manager.has_llamafile(llamafile.name), running=manager.is_llamafile_running(llamafile.name), download_progress=manager.llamafile_download_progress(llamafile.name))) class DownloadLlamafileRequest(BaseModel): name: str class DownloadLlamafileResponse(BaseModel): success: bool @router.post("/download_llamafile") async def download_llamafile(request: DownloadLlamafileRequest): """Download the llamafile of the given name.""" result = manager.download_llamafile_by_name(request.name) return DownloadLlamafileResponse(success=result is not None) class LlamafileDownloadProgressRequest(BaseModel): name: str class LlamafileDownloadProgressResponse(BaseModel): progress: Optional[int] @router.post("/llamafile_download_progress") async def llamafile_download_progress(request: LlamafileDownloadProgressRequest): """Get the download progress of the llamafile of the given name.""" progress = manager.llamafile_download_progress(request.name) return LlamafileDownloadProgressResponse(progress=progress) class RunLlamafileRequest(BaseModel): name: str class RunLlamafileResponse(BaseModel): success: bool @router.post("/run_llamafile") async def run_llamafile(request: RunLlamafileRequest): """Download the llamafile of the given name.""" # The given name might not be valid, in which case the manager will throw. # If the manager throws, return success: false try: result = manager.run_llamafile(request.name, ["--host", "0.0.0.0", "--port", "8800", "--nobrowser"]) #"-ngl", "999"]) return RunLlamafileResponse(success=True) except ValueError: return RunLlamafileResponse(success=False) class StopLlamafileRequest(BaseModel): name: str class StopLlamafileResponse(BaseModel): success: bool @router.post("/stop_llamafile") async def stop_llamafile(request: StopLlamafileRequest): """Stop the llamafile of the given name.""" # The given name might not be valid, in which case the manager will throw. # If the manager throws, return success: false try: result = manager.stop_llamafile_by_name(request.name) return StopLlamafileResponse(success=result) except ValueError: return StopLlamafileResponse(success=False) ================================================ FILE: scratch/backend/hub/src/api/thread_api.py ================================================ from fastapi import APIRouter from pydantic import BaseModel router = APIRouter() class ListThreadsResponse(BaseModel): threads: list[str] @router.get("/list_threads") async def list_threads(): return ListThreadsResponse(threads=["thread1", "thread2", "thread3"]) class GetThreadResponse(BaseModel): messages: list[str] @router.get("/get_thread") async def get_thread(): return GetThreadResponse(messages=["message1", "message2", "message3"]) class AppendToThreadRequest(BaseModel): message: str class AppendToThreadResponse(BaseModel): success: bool @router.post("/append_to_thread") async def append_to_thread(request: AppendToThreadRequest): return AppendToThreadResponse(success=True) ================================================ FILE: scratch/backend/hub/src/async_utils.py ================================================ import threading import asyncio def start_async_loop(loop): asyncio.set_event_loop(loop) loop.run_forever() async def wait_for(coroutines, finish_event): await asyncio.gather(*coroutines) finish_event.set() def run(coroutines, loop): finish_event = threading.Event() asyncio.run_coroutine_threadsafe(wait_for(coroutines, finish_event), loop) return finish_event def run_async(coroutines): finish_event = threading.Event() asyncio.run_coroutine_threadsafe(wait_for(coroutines, finish_event), get_my_loop()) return finish_event loop = None def set_my_loop(l): global loop loop = l def get_my_loop(): global loop return loop ================================================ FILE: scratch/backend/hub/src/chat.py ================================================ from langchain_core.messages import HumanMessage, SystemMessage from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser chat = ChatOpenAI(temperature=0, openai_api_key="KEY", base_url="http://localhost:8080/v1") messages = [ SystemMessage( content="You are a helpful assistant. You keep answers short and to the point. You do not add any extra information. For example, if you are asked how to accomplish a command line task, you reply with the command to use with not additional comments." ), HumanMessage( content="Who wrote Linux?" ), SystemMessage( content="Linus Torvalds" ), HumanMessage( content="When?" ), SystemMessage( content="1991" ), HumanMessage( content="What is the capital of France?" ), SystemMessage( content="Paris" ), HumanMessage( content="How do I add a git origin?" ), SystemMessage( content="git remote add " ), HumanMessage( content="How do I find the process on a port (in Linux)?" ), SystemMessage( content="lsof -i :" ), ] while True: user_input = input("> ") if user_input.lower() == "exit": break message = HumanMessage(content=f"{user_input}") messages.append(message) prompt = ChatPromptTemplate.from_messages(messages) chain = prompt | chat | StrOutputParser() response = chain.invoke({}) print(response) messages.append(SystemMessage(content=response)) ================================================ FILE: scratch/backend/hub/src/chat2.py ================================================ from langchain_community.chat_models import ChatOllama from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate llm = ChatOllama(model="mixtral:8x7b-instruct-v0.1-fp16") llm.base_url = "http://localhost:11434" prompt = ChatPromptTemplate.from_template("Tell me a short joke about {topic}") chain = prompt | llm | StrOutputParser() print(chain.invoke({"topic": "Space travel"})) ================================================ FILE: scratch/backend/hub/src/chat3.py ================================================ from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_core.runnables.history import RunnableWithMessageHistory from langchain_community.chat_message_histories import SQLChatMessageHistory from langchain_openai import ChatOpenAI # This is where we configure the session id config = {"configurable": {"session_id": "test_session_id2"}} prompt = ChatPromptTemplate.from_messages( [ ("system", "You are a helpful assistant."), MessagesPlaceholder(variable_name="history"), ("human", "{question}"), ] ) chat = ChatOpenAI(temperature=0, openai_api_key="KEY", base_url="http://localhost:8080/v1") chain = prompt | chat chain_with_history = RunnableWithMessageHistory( chain, lambda session_id: SQLChatMessageHistory( session_id=session_id, connection_string="sqlite:///sqlite.db" ), input_messages_key="question", history_messages_key="history", ) print(chain_with_history.invoke({"question": "Whats my name"}, config=config)) ================================================ FILE: scratch/backend/hub/src/fastapi_app.py ================================================ from fastapi import FastAPI from fastapi.staticfiles import StaticFiles #from langchain_community.llms.llamafile import Llamafile #from langserve import add_routes import os import sys from api.thread_api import router as thread_router from api.llamafile_api import router as llamafile_router from fastapi.middleware.cors import CORSMiddleware app = FastAPI( title="Memory Cache Hub", version="1.0", description="Manage llamafiles, document store, and vector database.", ) origins = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://192.168.0.141", "http://192.168.0.141:8080", "http://192.168.0.141:3000", ] app.add_middleware( CORSMiddleware, allow_origins=origins, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) app.include_router(thread_router, prefix="/api/thread") app.include_router(llamafile_router, prefix="/api/llamafile") #llm = Llamafile(streaming=True) #llm.base_url = "http://localhost:8800" #add_routes(app, llm, path="/llm") if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): # The application is frozen by PyInstaller bundle_dir = sys._MEIPASS static_files_dir = os.path.join(bundle_dir, 'browser-client') else: # The application is running in a normal Python environment bundle_dir = os.path.dirname(os.path.abspath(__file__)) static_files_dir = os.path.join(bundle_dir, '..', '..', '..', 'hub-browser-client', 'build') app.mount("/", StaticFiles(directory=static_files_dir, html=True), name="static") ================================================ FILE: scratch/backend/hub/src/gradio_app.py ================================================ import gradio as gr import requests from time import sleep # Define functions that will interact with the FastAPI endpoints def list_llamafiles(): # If the request fails, it throws an error. We do not want the app to crash, so we catch the error and return an empty string. try: response = requests.get("http://localhost:8001/api/llamafile_manager/list_llamafiles") if response.status_code == 200: return "\n".join(response.json()) return "" except: return "" def has_llamafile(name): response = requests.get(f"http://localhost:8001/api/llamafile_manager/has_llamafile/{name}") if response.status_code == 200: return response.json() return "Error checking llamafile." def download_llamafile(url, name): response = requests.post("http://localhost:8001/api/llamafile_manager/download_llamafile", json={"url": url, "name": name}) if response.status_code == 200: return "Download initiated." return "Failed to initiate download." def download_progress(url, name): response = requests.post("http://localhost:8001/api/llamafile_manager/download_progress", json={"url": url, "name": name}) if response.status_code == 200: return response.json() return "Failed to get download progress." def run_llamafile(name, args): response = requests.post("http://localhost:8001/api/llamafile_manager/run_llamafile", json={"name": name, "args": args.split()}) if response.status_code == 200: return response.text return "Failed to run llamafile." num = 0 def increment(): global num while True: num += 1 yield num sleep(1) def my_inc(): global num def inner(): global num num += 1 return num return inner # Create the Gradio interface with gr.Blocks() as app: with gr.Tab("Llamafile Manager"): gr.Markdown("# Llamafile Manager") gr.Textbox(value = my_inc(), label = "Seconds", interactive=False, every=1) with gr.Tab("List Llamafiles"): gr.Markdown("List all Llamafiles") gr.Button("List Llamafiles").click(list_llamafiles, [], gr.Textbox(label="Llamafiles")) with gr.Tab("Check Llamafile"): gr.Markdown("Check if a Llamafile exists") name_input = gr.Textbox(label="Llamafile Name") gr.Button("Check").click(has_llamafile, [name_input], gr.Textbox(label="Exists")) with gr.Tab("Download Llamafile"): gr.Markdown("Download a Llamafile") url_input = gr.Textbox(label="URL") name_input_download = gr.Textbox(label="Name") gr.Button("Download").click(download_llamafile, [url_input, name_input_download], gr.Textbox(label="Status")) with gr.Tab("Download Progress"): gr.Markdown("Check Download Progress") url_input_progress = gr.Textbox(label="URL") name_input_progress = gr.Textbox(label="Name") gr.Button("Check Progress").click(download_progress, [url_input_progress, name_input_progress], gr.Textbox(label="Progress")) with gr.Tab("Run Llamafile"): gr.Markdown("Run a Llamafile") name_input_run = gr.Textbox(label="Name") args_input_run = gr.Textbox(label="Args (space-separated)") gr.Button("Run").click(run_llamafile, [name_input_run, args_input_run], gr.Textbox(label="Run Status")) iface = app ================================================ FILE: scratch/backend/hub/src/hub.py ================================================ from llamafile_manager import get_llamafile_manager from async_utils import start_async_loop, set_my_loop import asyncio import threading import os import uvicorn import webbrowser from fastapi_app import app # from gradio_app import iface def run_api_server(): uvicorn.run(app, host="0.0.0.0", port=8001) # def run_gradio_interface(): # iface.launch() if __name__ == "__main__": llamafiles_dir = os.environ.get('LLAMAFILES_DIR') if not llamafiles_dir: raise ValueError("LLAMAFILES_DIR environment variable is not set") manager = get_llamafile_manager(llamafiles_dir) loop = asyncio.new_event_loop() set_my_loop(loop) t = threading.Thread(target=start_async_loop, args=(loop,), daemon=True) t.start() t2 = threading.Thread(target=run_api_server, daemon=True) t2.start() # t3 = threading.Thread(target=run_gradio_interface, daemon=True) # t3.start() webbrowser.open("http://localhost:8001/", new=0) #webbrowser.open("http://localhost:7860/", new=0) t.join() manager.stop_all_llamafiles() ================================================ FILE: scratch/backend/hub/src/hub_build_gnu_linux.py ================================================ import PyInstaller.__main__ import os current_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) additional_files = [ (os.path.join(current_directory, 'requirements', 'hub-base.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-cpu.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-builder.txt'), '.'), (os.path.join(current_directory, '..', '..', 'hub-browser-client', 'build'), 'browser-client'), ] entry_point = os.path.join(current_directory, "src", "hub.py") PyInstaller.__main__.run([ entry_point, '--onefile', # Bundle everything into a single executable # '--hidden-import=module_name', # Uncomment and replace with actual module names if there are hidden imports '--clean', # Clean PyInstaller build folder before building '--name=memory-cache-hub-gnu-linux', ] + [f'--add-data={src}{os.pathsep}{dst}' for src, dst in additional_files]) ================================================ FILE: scratch/backend/hub/src/hub_build_macos.py ================================================ import PyInstaller.__main__ import os current_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) additional_files = [ (os.path.join(current_directory, 'requirements', 'hub-base.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-cpu.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-builder.txt'), '.'), (os.path.join(current_directory, '..', '..', 'hub-browser-client', 'build'), 'browser-client'), ] entry_point = os.path.join(current_directory, "src", "hub.py") PyInstaller.__main__.run([ entry_point, '--onefile', # Bundle everything into a single executable # '--hidden-import=module_name', # Uncomment and replace with actual module names if there are hidden imports '--clean', # Clean PyInstaller build folder before building '--name=memory-cache-hub-macos', ] + [f'--add-data={src}{os.pathsep}{dst}' for src, dst in additional_files]) ================================================ FILE: scratch/backend/hub/src/hub_build_windows.py ================================================ import PyInstaller.__main__ import os current_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) additional_files = [ (os.path.join(current_directory, 'requirements', 'hub-base.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-cpu.txt'), '.'), (os.path.join(current_directory, 'requirements', 'hub-builder.txt'), '.'), (os.path.join(current_directory, '..', '..', 'hub-browser-client', 'build'), 'browser-client'), ] entry_point = os.path.join(current_directory, "src", "hub.py") PyInstaller.__main__.run([ entry_point, '--onefile', # Bundle everything into a single executable # '--hidden-import=module_name', # Uncomment and replace with actual module names if there are hidden imports '--clean', # Clean PyInstaller build folder before building '--name=memory-cache-hub-windows', ] + [f'--add-data={src}{os.pathsep}{dst}' for src, dst in additional_files]) ================================================ FILE: scratch/backend/hub/src/llamafile_infos.json ================================================ [ { "Model": "LLaVA 1.5", "Size": "3.97 GB", "License": "LLaMA 2", "License URL": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", "filename": "llava-v1.5-7b-q4.llamafile", "url": "https://huggingface.co/jartine/llava-v1.5-7B-GGUF/resolve/main/llava-v1.5-7b-q4.llamafile?download=true" }, { "Model": "Mistral-7B-Instruct", "Size": "5.15 GB", "License": "Apache 2.0", "License URL": "https://choosealicense.com/licenses/apache-2.0/", "filename": "mistral-7b-instruct-v0.2.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q5_K_M.llamafile?download=true" }, { "Model": "Mixtral-8x7B-Instruct", "Size": "30.03 GB", "License": "Apache 2.0", "License URL": "https://choosealicense.com/licenses/apache-2.0/", "filename": "mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/Mixtral-8x7B-Instruct-v0.1-llamafile/resolve/main/mixtral-8x7b-instruct-v0.1.Q5_K_M.llamafile?download=true" }, { "Model": "WizardCoder-Python-34B", "Size": "22.23 GB", "License": "LLaMA 2", "License URL": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", "filename": "wizardcoder-python-34b-v1.0.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/WizardCoder-Python-34B-V1.0-llamafile/resolve/main/wizardcoder-python-34b-v1.0.Q5_K_M.llamafile?download=true" }, { "Model": "WizardCoder-Python-13B", "Size": "7.33 GB", "License": "LLaMA 2", "License URL": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", "filename": "wizardcoder-python-13b.llamafile", "url": "https://huggingface.co/jartine/wizardcoder-13b-python/resolve/main/wizardcoder-python-13b.llamafile?download=true" }, { "Model": "TinyLlama-1.1B", "Size": "0.76 GB", "License": "Apache 2.0", "License URL": "https://choosealicense.com/licenses/apache-2.0/", "filename": "TinyLlama-1.1B-Chat-v1.0.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/TinyLlama-1.1B-Chat-v1.0-GGUF/resolve/main/TinyLlama-1.1B-Chat-v1.0.Q5_K_M.llamafile?download=true" }, { "Model": "Rocket-3B", "Size": "1.89 GB", "License": "cc-by-sa-4.0", "License URL": "https://creativecommons.org/licenses/by-sa/4.0/deed.en", "filename": "rocket-3b.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/rocket-3B-llamafile/resolve/main/rocket-3b.Q5_K_M.llamafile?download=true" }, { "Model": "Phi-2", "Size": "1.96 GB", "License": "MIT", "License URL": "https://huggingface.co/microsoft/phi-2/resolve/main/LICENSE", "filename": "phi-2.Q5_K_M.llamafile", "url": "https://huggingface.co/jartine/phi-2-llamafile/resolve/main/phi-2.Q5_K_M.llamafile?download=true" } ] ================================================ FILE: scratch/backend/hub/src/llamafile_infos.py ================================================ # llamafile_name_llava_v1_5_7b_q4 = "llava-v1.5-7b-q4.llamafile" # llamafile_url_llava_v1_5_7b_q4 = "https://huggingface.co/jartine/llava-v1.5-7B-GGUF/resolve/main/llava-v1.5-7b-q4.llamafile?download=true" # llamafile_name_mistral_7b_instruct_v0_2_q5_k_m = "mistral-7b-instruct-v0.2.Q5_K_M.llamafile" # llamafile_url_mistral_7b_instruct_v0_2_q5_k_m = "https://huggingface.co/jartine/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q5_K_M.llamafile?download=true" # Parse the json in llamafile_infos.json import json import os class LlamafileInfo: def __init__(self, info_dict): self.model = info_dict['Model'] self.size = info_dict['Size'] self.license = info_dict['License'] self.license_url = info_dict['License URL'] self.name = info_dict['filename'] self.url = info_dict['url'] def get_llamafile_infos(): llamafile_infos_path = os.path.join(os.path.dirname(__file__), "llamafile_infos.json") with open(llamafile_infos_path, "r") as f: llamafile_infos_dicts = json.load(f) # Convert each dictionary to a LlamafileInfo object llamafile_infos = [LlamafileInfo(info_dict) for info_dict in llamafile_infos_dicts] return llamafile_infos ================================================ FILE: scratch/backend/hub/src/llamafile_manager.py ================================================ import os import asyncio import aiohttp import aiofiles #import certifi import asyncio import subprocess import psutil from llamafile_infos import get_llamafile_infos from async_utils import run_async class DownloadHandle: def __init__(self): self.url = None self.filename = None self.llamafile_name = None self.content_length = 0 self.written = 0 self.coroutine = None def progress(self): return int(100 * self.written / self.content_length if self.content_length > 0 else 0) def __repr__(self): return f"DownloadHandle(url={self.url}, filename={self.filename}, content_length={self.content_length}, written={self.written})" async def download(handle: DownloadHandle): # BUG On MacOS, https requests failed unless I disabled ssl checking. # TODO Fix ssl issue on MacOS # This github issue may be related: # https://github.com/aio-libs/aiohttp/issues/955 #async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: async with session.get(handle.url) as response: handle.content_length = int(response.headers.get('content-length', 0)) handle.written = 0 async with aiofiles.open(handle.filename, 'wb') as file: async for data in response.content.iter_chunked(1024): await file.write(data) handle.written += len(data) async def update_tqdm(pbar, handle: DownloadHandle): while handle.progress() < 100: # We don't know the total size until the download starts, so we update it here pbar.total = handle.content_length / 1024 pbar.update(handle.written / 1024 - pbar.n) await asyncio.sleep(0.1) class RunHandle: def __init__(self): self.llamafile_name = None self.filename = None self.args = [] self.process = None def __repr__(self): return f"RunHandle(filename={self.filename}, args={self.args}, process={self.process})" _instance = None def get_llamafile_manager(llamafiles_dir: str = None): global _instance if _instance is None: _instance = LlamafileManager(llamafiles_dir) if _instance.llamafiles_dir is None and llamafiles_dir is not None: _instance.llamafiles_dir = llamafiles_dir if _instance.llamafiles_dir != None and llamafiles_dir != _instance.llamafiles_dir: raise ValueError("LlamafileManager already created with a different llamafiles_dir") return _instance class LlamafileManager: def __init__(self, llamafiles_dir: str): self.llamafiles_dir = llamafiles_dir self.download_handles = [] self.run_handles = [] def list_all_llamafiles(self): return get_llamafile_infos() def list_llamafiles(self): return [f for f in os.listdir(self.llamafiles_dir) if f.endswith('.llamafile')] def has_llamafile(self, name): return name in self.list_llamafiles() def download_llamafile_by_name(self, name): for info in self.list_all_llamafiles(): if info.name == name: return self.download_llamafile(info.url, info.name) return None def download_llamafile(self, url, name): # If we already have a download handle for this file, delete that other handle for handle in self.download_handles: if handle.llamafile_name == name: self.download_handles.remove(handle) break handle = DownloadHandle() self.download_handles.append(handle) handle.url = url handle.llamafile_name = name handle.filename = os.path.join(self.llamafiles_dir, name) handle.coroutine = download(handle) handle.finish_event = run_async([handle.coroutine]) return handle def run_llamafile(self, name: str, args: list): if not self.has_llamafile(name): raise ValueError(f"llamafile {name} is not available") handle = RunHandle() self.run_handles.append(handle) handle.llamafile_name = name handle.filename = os.path.join(self.llamafiles_dir, name) # Print the file path, and check if the file exists print(handle.filename) if not os.path.isfile(handle.filename): raise FileNotFoundError(f"{name} not found in {self.llamafiles_dir}") if os.name == 'posix' or os.name == 'darwin': if not os.access(handle.filename, os.X_OK): os.chmod(handle.filename, 0o755) handle.args = args cmd = f"{handle.filename} {' '.join(args)}" handle.process = subprocess.Popen(["sh", "-c", cmd]) return handle def is_llamafile_running(self, name: str): return any(h for h in self.run_handles if h.llamafile_name == name) def stop_llamafile_by_name(self, name: str): for handle in self.run_handles: if handle.llamafile_name == name: return self.stop_llamafile(handle) return False def stop_llamafile(self, handle: RunHandle): print(f"Stopping process {handle.process.pid}") if handle.process.poll() is None: try: parent = psutil.Process(handle.process.pid) children = parent.children(recursive=True) # Get all child processes for child in children: print(f"Terminating child process {child.pid}, {child.name()}") child.terminate() # Terminate each child gone, still_alive = psutil.wait_procs(children, timeout=3, callback=None) for p in still_alive: p.kill() # Force kill if still alive after timeout print(f"Terminating parent process {parent.pid}, {parent.name()}") handle.process.terminate() # Terminate the parent process handle.process.wait() # Wait for the parent process to terminate except psutil.NoSuchProcess: print(f"Process {handle.process.pid} does not exist anymore.") else: print(f"Process {handle.process.pid} is not running") self.run_handles.remove(handle) return True def stop_all_llamafiles(self): for handle in self.run_handles: self.stop_llamafile(handle) self.run_handles.clear() def llamafile_download_progress(self, name: str): for handle in self.download_handles: if handle.llamafile_name == name: return handle.progress() return None ================================================ FILE: scratch/backend/hub/src/static/index.html ================================================ hello ================================================ FILE: scratch/backend/langserve-demo/.gitignore ================================================ openai-api-key ================================================ FILE: scratch/backend/langserve-demo/Dockerfile.cpu ================================================ from python:3.11 WORKDIR /usr/src/app COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY requirements-cpu.txt ./ RUN pip install --no-cache-dir -r requirements-cpu.txt COPY . . CMD [ "python3", "./serve.py" ] ================================================ FILE: scratch/backend/langserve-demo/README.md ================================================ # Lang Serve Demo A demo app built with `langchain` and `langserve`. ## Dockerfiles | Filename | Purpose | |:-------------------------|:-------------------------------| | `Dockerfile.cpu` | Basic setup. CPU support only. | ## Usage From within this directory, build with: ``` sh docker build -f Dockerfile.cpu -t memory-cache/lang-serve-demo-cpu . ``` Save an API key to a file called `openai-api-key` and run: ``` sh docker run \ --rm \ --name lang-serve-demo-cpu \ -p 8800:8800 \ -e OPENAI_API_KEY="$(cat openai-api-key)" \ -v ./:/usr/src/app/ \ memory-cache/lang-serve-demo-cpu ``` (Note: I'll remove the OpenAI dependency shortly. It's only here because I'm starting with langchain's demo server.) Then run a client to interact with the server: ``` sh docker exec lang-serve-demo-cpu python client.py ``` ## Miscellaneous Notes ### Dockerfile.cpu: `python 3.11` We use `python 3.11` (not `3.12` or later) in `Dockerfile.cpu` because `faiss-cpu` only supports up to `3.11` at the time of this writing: https://pypi.org/project/faiss-cpu/ ================================================ FILE: scratch/backend/langserve-demo/client.py ================================================ #!/usr/bin/env python3 from langserve import RemoteRunnable remote_chain = RemoteRunnable("http://localhost:8800/agent/") response = remote_chain.invoke({ "input": "Hello, how are you?", "chat_history": [] # Providing an empty list as this is the first call }) # Parse json response, then get the 'output' key print(response["output"]) ================================================ FILE: scratch/backend/langserve-demo/requirements-cpu.txt ================================================ faiss-cpu ================================================ FILE: scratch/backend/langserve-demo/requirements.txt ================================================ bs4 fastapi langchain langchainhub langchain-openai langchain-cli langserve[all] uvicorn ================================================ FILE: scratch/backend/langserve-demo/serve.py ================================================ #!/usr/bin/env python from typing import List from fastapi import FastAPI from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI from langchain_community.document_loaders import WebBaseLoader from langchain_openai import OpenAIEmbeddings from langchain_community.vectorstores import FAISS from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.tools.retriever import create_retriever_tool # JFS: Remove Tavily Search #from langchain_community.tools.tavily_search import TavilySearchResults from langchain_openai import ChatOpenAI from langchain import hub from langchain.agents import create_openai_functions_agent from langchain.agents import AgentExecutor from langchain.pydantic_v1 import BaseModel, Field from langchain_core.messages import BaseMessage from langserve import add_routes # 1. Load Retriever loader = WebBaseLoader("https://docs.smith.langchain.com/overview") docs = loader.load() text_splitter = RecursiveCharacterTextSplitter() documents = text_splitter.split_documents(docs) embeddings = OpenAIEmbeddings() vector = FAISS.from_documents(documents, embeddings) retriever = vector.as_retriever() # 2. Create Tools retriever_tool = create_retriever_tool( retriever, "langsmith_search", "Search for information about LangSmith. For any questions about LangSmith, you must use this tool!", ) # JFS : Remove Tavily Search #search = TavilySearchResults() #tools = [retriever_tool, search] tools = [retriever_tool] # 3. Create Agent prompt = hub.pull("hwchase17/openai-functions-agent") llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0) agent = create_openai_functions_agent(llm, tools, prompt) agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True) # 4. App definition app = FastAPI( title="LangChain Server", version="1.0", description="A simple API server using LangChain's Runnable interfaces", ) # 5. Adding chain route # We need to add these input/output schemas because the current AgentExecutor # is lacking in schemas. class Input(BaseModel): input: str chat_history: List[BaseMessage] = Field( ..., extra={"widget": {"type": "chat", "input": "location"}}, ) class Output(BaseModel): output: str add_routes( app, agent_executor.with_types(input_type=Input, output_type=Output), path="/agent", ) if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8800) ================================================ FILE: scratch/backend/python-llamafile-manager/.gitignore ================================================ llama.log main.log build/ dist/ bin/ python-llamafile-manager-gnu-linux.spec ================================================ FILE: scratch/backend/python-llamafile-manager/Dockerfile.plm ================================================ from ubuntu:22.04 RUN apt-get update RUN apt-get install -y python3 python3-pip WORKDIR /usr/src/app COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt RUN apt-get install wget -y RUN wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf RUN chmod +x /usr/bin/ape # RUN sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" # RUN sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register" COPY . . CMD [ "python3", "./manager.py" ] ================================================ FILE: scratch/backend/python-llamafile-manager/Dockerfile.plm-builder-gnu-linux ================================================ # GNU/Linux # # PyInstaller requires the ldd terminal application to discover the shared libraries required by each program or shared library. It is typically found in the distribution-package glibc or libc-bin. # # It also requires the objdump terminal application to extract information from object files and the objcopy terminal application to append data to the bootloader. These are typically found in the distribution-package binutils. FROM ubuntu:22.04 RUN apt-get update RUN apt-get install -y binutils RUN apt-get install -y libc-bin RUN apt-get install -y python3 python3-pip RUN pip install pyinstaller WORKDIR /usr/src/app COPY requirements.txt ./ RUN pip install --no-cache-dir -r requirements.txt COPY . . CMD [ "python3", "./build_gnu_linux.py" ] ================================================ FILE: scratch/backend/python-llamafile-manager/README.md ================================================ # Python Llamafile Manager A python program that downloads and executes [llamafiles](https://github.com/Mozilla-Ocho/llamafile), bundled with [PyInstaller](https://pyinstaller.org/en/stable/). ## Why? I wrote a python program using `langchain` that assumes an LLM model is exposed somewhere via HTTP. `Llamafile`s are portable executable that expose an HTTP interface to LLMs. Rather than asking users to download and run `Llamafiles`, I want my python program to manage this on their behalf. I plan to bundle my python program with `PyInstaller`, so I will make sure that `python-llamafile-manager` can be bundled with `PyInstaller` too. ## Usage From within this directory, build with: ``` sh docker build -f Dockerfile.plm -t memory-cache/python-llamafile-manager . ``` Run with: ``` sh docker run \ --name python-llamafile-manager \ -it \ --rm \ -e LLAMAFILE_BIN_DIR=/usr/src/app/bin \ -v ~/media/llamafile/:/usr/src/app/bin/ \ -v ./:/usr/src/app/ \ -p 8800:8800 \ memory-cache/python-llamafile-manager \ python3 manager.py ``` ## Packaging with PyInstaller ### GNU/Linux > GNU/Linux > > PyInstaller requires the ldd terminal application to discover the shared libraries required by each program or shared library. It is typically found in the distribution-package glibc or libc-bin. > > It also requires the objdump terminal application to extract information from object files and the objcopy terminal application to append data to the bootloader. These are typically found in the distribution-package binutils. ``` sh docker build -f Dockerfile.plm-builder-gnu-linux -t memory-cache/plm-builder-gnu-linux . ``` ``` sh docker run \ --name plm-builder-gnu-linux \ -it \ --rm \ -v ./:/usr/src/app/ \ memory-cache/plm-builder-gnu-linux ``` ================================================ FILE: scratch/backend/python-llamafile-manager/build_gnu_linux.py ================================================ import PyInstaller.__main__ import os # Define the path to the directory containing the llamafiles and other necessary files. # Assuming these files are in the same directory as the script for simplicity. current_directory = os.path.dirname(os.path.abspath(__file__)) # List of tuples specifying additional files/directories and their destination in the distribution. # Adjust paths as necessary. additional_files = [ (os.path.join(current_directory, 'requirements.txt'), '.'), # Add other necessary files or directories here. # Example: (os.path.join(current_directory, 'data_folder'), 'data_folder') ] # Entry point of the application entry_point = os.path.join(current_directory, 'manager.py') # Build the application with PyInstaller PyInstaller.__main__.run([ entry_point, '--onefile', # Bundle everything into a single executable '--add-data=' + ';'.join([f'{src}{os.pathsep}{dst}' for src, dst in additional_files]), # Add additional files/directories # '--hidden-import=module_name', # Uncomment and replace with actual module names if there are hidden imports '--clean', # Clean PyInstaller build folder before building '--name=python-llamafile-manager-gnu-linux', # Name of the generated executable # Additional flags can be added as needed. ]) ================================================ FILE: scratch/backend/python-llamafile-manager/manager.py ================================================ import subprocess import os import stat import requests import sys from tqdm import tqdm from time import sleep url_llava_v1_5_7b_q4 = "https://huggingface.co/jartine/llava-v1.5-7B-GGUF/resolve/main/llava-v1.5-7b-q4.llamafile?download=true" def find_llamafiles(directory: str): """Check for files with .llamafile extension in the specified directory.""" return [file for file in os.listdir(directory) if file.endswith('.llamafile')] process = None def execute_llamafile(directory: str, filename: str, args: list): """Execute a .llamafile as a subprocess with optional arguments.""" global process filepath = os.path.join(directory, filename) # print the file path print(filepath) if not os.path.isfile(filepath): raise FileNotFoundError(f"{filename} not found in {directory}") if not os.access(filepath, os.X_OK): raise PermissionError(f"{filename} is not executable") print(args) print([filepath] + args) process = subprocess.Popen([filepath] + args) def is_process_alive(): """Check if the subprocess is alive.""" global process return process is not None and process.poll() is None def stop_process(): """Stop the subprocess if it is running.""" global process if is_process_alive(): process.terminate() process.wait() def restart_process(directory: str, filename: str, args: list): """Restart the .llamafile subprocess with optional arguments.""" stop_process() execute_llamafile(directory, filename, args) def download_file_with_tqdm(url: str, destination: str): """Download a file from a URL with a progress bar.""" response = requests.get(url, stream=True) total_size = int(response.headers.get('content-length', 0)) block_size = 1024 with open(destination, 'wb') as file: for data in tqdm(response.iter_content(block_size), total=total_size/block_size, unit='KB', unit_scale=True): file.write(data) def download_file(url: str, destination: str): """Download a file from a URL.""" response = requests.get(url) with open(destination, 'wb') as file: file.write(response.content) def make_executable_unix(destination: str): """Mark a file as executable (Unix-like systems).""" os.chmod(destination, os.stat(destination).st_mode | stat.S_IEXEC) def make_executable_windows(destination: str): """Windows-specific handling to 'mark' a file as executable is not applicable.""" pass # Windows uses file associations to execute files, so no action needed here. def download_and_make_executable(url: str, destination: str): """Download a file from a URL and mark it as executable.""" #download_file(url, destination) download_file_with_tqdm(url, destination) if os.name != 'nt': make_executable_unix(destination) else: make_executable_windows(destination) # Example usage: if __name__ == "__main__": # Get directory from environment variable or use default directory = os.environ.get('LLAMAFILE_BIN_DIR') if directory is None: # Print error and exit print("Error: LLAMAFILE_BIN_DIR environment variable not set") sys.exit(1) llamafiles = find_llamafiles(directory) print("Found llamafiles:", llamafiles) if 'foo.llamafile' not in llamafiles: response = input("foo.llamafile not found. Do you want to download foo.llamafile? (y/n): ") if response.lower() == 'y': download_and_make_executable(url_llava_v1_5_7b_q4, os.path.join(directory, 'foo.llamafile')) else: print("foo.llamafile not found and not downloaded") sys.exit(1) llamafiles = find_llamafiles(directory) if 'foo.llamafile' not in llamafiles: print("Error: foo.llamafile not found") sys.exit(1) execute_llamafile(directory, 'foo.llamafile', ['--host', '0.0.0.0', '--port', '8800']) # Check if the process is running every 5 seconds while is_process_alive(): print("foo.llamafile is running") sleep(5) ================================================ FILE: scratch/backend/python-llamafile-manager/requirements.txt ================================================ requests ~= 2.31 tqdm ================================================ FILE: scratch/browser-client/.gitignore ================================================ build/ node_modules/ ================================================ FILE: scratch/browser-client/README.md ================================================ # Memory Cache Browser Client A browser client for memory cache. Tested with: - `node v18.18.2` - `npm 10.2.5` To install dependencies, run `npm ci`. To build the client, run `npm run build`. This directory only contains the client-side code. The server is implemented in [a separate `privateGPT` repo](https://github.com/johnshaughnessy/privateGPT). Design mockup for phase 1 of interface: ![phase1-memorycacheui](https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/9e0c37fb-9c96-4edd-8db1-1c0ab7f29acb) ================================================ FILE: scratch/browser-client/package.json ================================================ { "name": "memory-cache-browser-client", "version": "1.0.0", "description": "A browser client for memory cache.", "main": "main.js", "scripts": { "build": "webpack --mode production" }, "author": "", "license": "MPL-2.0", "devDependencies": { "css-loader": "^6.8.1", "html-webpack-plugin": "^5.5.4", "style-loader": "^3.3.3", "webpack": "^5.89.0", "webpack-cli": "^5.1.4" }, "dependencies": { "socket.io-client": "^4.7.2" } } ================================================ FILE: scratch/browser-client/src/index.html ================================================ Kate's Memory Cache

    • Kate's Work Cache

    ================================================ FILE: scratch/browser-client/src/main.js ================================================ import { io } from "socket.io-client"; import "./styles.css"; const socket = io(`http://${window.location.hostname}:5001`); socket.on("connect", () => { console.log("connected"); }); socket.on("disconnect", () => { console.log("disconnected"); }); socket.on("message", (message) => { console.log(message); }); socket.on("error", (error) => { console.error(error); }); // socket.emit("message", JSON.stringify({ text: "hello world!" })); const chatHistory = document.getElementById("chat-history"); const chatTextArea = document.getElementById("chat-textarea"); const sendButton = document.getElementById("send-button"); function ChatHistoryMessage(message) { const messageElement = document.createElement("div"); messageElement.classList.add("chat-history-message"); messageElement.innerText = message; return messageElement; } let nextChatMessage = ChatHistoryMessage("Hello!"); // chatHistory.appendChild(nextChatMessage); const replies = new Map(); socket.on("message", (raw) => { console.log("Received message from server:", raw); const message = JSON.parse(raw); console.log("message", message); if (message.kind === "first_reply") { nextChatMessage = ChatHistoryMessage(""); chatHistory.appendChild(nextChatMessage); replies.set(message.message_sid, { first: message, chatMessage: nextChatMessage, }); nextChatMessage.innerText += message.text; } else if (message.kind === "second_reply") { const reply = replies.get(message.message_sid); reply.second = message; // reply.chatMessage.innerText += "\n"; // reply.chatMessage.innerText += `\n${message.text}\n`; // Send reply and text reply.chatMessage.innerText += `\n[Replied in ${message.time} seconds.]\n${message.text}\n`; } else if (message.kind === "source_document") { const { message_sid, kind, text, source } = message; const reply = replies.get(message.message_sid); reply.sourceDocuments = reply.sourceDocuments || []; reply.sourceDocuments.push(message); reply.chatMessage.innerText += `\n\n[${source}]\n${text}\n`; } // We don't scroll to the bottom while the server is sending us messages. }); function sendChatMessage() { const text = chatTextArea.value; chatTextArea.value = ""; chatHistory.appendChild(ChatHistoryMessage(text)); // Scroll to the bottom of the chatHistory chatHistory.scrollTop = chatHistory.scrollHeight; // Send the message to the server socket.send( JSON.stringify({ text, }), ); // This will be sent as a JSON string console.log("Message sent to server: " + text); } let isShiftPressed = false; chatTextArea.addEventListener("keydown", (event) => { if (event.key === "Shift") { isShiftPressed = true; } }); chatTextArea.addEventListener("keyup", (event) => { if (event.key === "Shift") { isShiftPressed = false; } }); chatTextArea.addEventListener("input", (event) => { if (event.inputType === "insertLineBreak" && !isShiftPressed) { sendChatMessage(); } }); sendButton.addEventListener("click", sendChatMessage); ================================================ FILE: scratch/browser-client/src/styleguide.html ================================================

    Color variables

    Interactions

    Primary
    Primary Hover
    Primary Focus
    Secondary
    Secondary Hover
    Secondary Focus
    Interaction Inactive

    Base / Backgrounds

    Base-100
    Base-200

    Messaging

    Info
    Success
    Warning
    Error
    ================================================ FILE: scratch/browser-client/src/styles.css ================================================ :root { /* Font */ --font-family: "Work Sans", sans-serif; --font-size-sm: .8em; --font-size-md: 1em; --font-size-lg: 1.6em; --font-weight-normal: 400; --font-weight-bold: 500; /* Avoiding using representational numbers for the styling system at this point to keep things simple */ --primary: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87EBDA 76.04%, #BAE1FF 100%); --primary-hover:linear-gradient(90deg, #FFA8B1 0%, #FFD9AD 26.56%, #FFFFAD 50.52%, #80EAD8 76.04%, #ADDCFF 100%); --primary-focus: linear-gradient(90deg, #FF9EA8 0%, #FFD29E 26.56%, #FFFF9E 50.52%, #73E8D4 76.04%, #99D3FF 100%); --primary-content: #000000; --secondary: #FFBF76; --secondary-hover: #E5AC6A; --secondary-focus: #CC995E; --secondary-content: #000000; --inactive-content:#F9F9F9; --interaction-inactive:#B6B9BF; --base-100: #fff; --base-200: #F0EFEF; --base-content: #2d3d46; --base-content-subtle: #9296a0; --info: #3ac0f8; --info-content: #000; --warning: #fcbc23; --warning-content: #000; --success: #37d399; --success-content: #000; --error: #f87272; --error-content: #000; --border-1: #DDD; --border-radius: 4px; /* Layout spacer utilities */ --xxs:4px; --xs:8px; --sm:12px; --md:16px; --lg:24px; --xl:40px; } /* Resets */ html { padding:0; font-family:var(--font-family); } body { margin:0; font-family:var(--font-family); font-size: 1em; background: var(--base-200); } /* Layout */ .mc-main-container { margin:auto; /* background-color: var(--base-100); */ max-width: 1024px; display: flex; flex-direction: column; } .mc-body-container { margin:auto; width: 800px; display: flex; flex-direction: column; padding-top: var(--md); } .mc-body-section { margin-bottom: var(--md); display: flex; margin: var(--sm); justify-content: space-between; } .mc-col { display:flex; flex-direction: column; flex:1; } .mc-row { display:flex; flex-direction: row; vertical-align: middle; flex:1; } .mc-header { width:100%; border-bottom: .1em solid var(--border-1); padding: var(--md) var(--sm); font-size:var(--font-size-md); } .mc-header p { padding: 0; margin:auto 0; } .mc-logo { width: 150px; height: fit-content; margin-right: var(--sm); } /* Buttons */ .mc-button { cursor:pointer; border-radius: var(--border-radius); line-height: 21px; letter-spacing: 0em; text-align: center; color: #565D6D; padding:8px; border:none; } .mc-primary-btn { background: var(--primary); margin-bottom:.8em; display: flex; padding: var(--md) var(--lg); line-height: 24px; } .mc-primary-btn:hover { background:var(--primary-hover); } .mc-primary-btn:active { background:var(--primary-active); } /* Chat + Text */ .mc-query { margin-right:var(--lg); margin-bottom:var(--lg); } label { margin-bottom: var(--xs); } textarea { flex:1; max-width: 100%; padding: var(--sm); font-family: var(--font-family); font-size: var(--font-size-md); border: 1px solid var(--border-1); resize: none; box-sizing: border-box; min-height: 90px; line-height: 150%; } #chat-textarea:focus-visible { outline:2px solid var(--secondary); } .mc-query-btn { background: var(--primary); display: flex; padding: var(--md) var(--lg); line-height: 24px; max-height: 100%; max-width: 100%; margin: auto 0; font-weight: var(--font-weight-bold); font-size: .9em; font-family: var(--font-family); } .mc-query-btn img { margin-right:var(--sm); } .mc-chat-history { width: 800px; overflow-y: scroll; display: flex; flex-direction: column; max-height: 65vh; padding: var(--sm); } .mc-chat-history-message { padding: 10px; margin-bottom: 10px; border-radius: 5px; align-self: flex-start; border-radius: 5px; background: var(--base-100); } ================================================ FILE: scratch/browser-client/webpack.config.js ================================================ const path = require("path"); const HtmlWebpackPlugin = require("html-webpack-plugin"); module.exports = { entry: "./src/main.js", output: { path: path.resolve(__dirname, "build"), filename: "bundle.js", }, module: { rules: [ { test: /\.css$/, use: ["style-loader", "css-loader"], }, ], }, plugins: [ new HtmlWebpackPlugin({ template: "./src/index.html", }), ], }; ================================================ FILE: scratch/hub-browser-client/.gitignore ================================================ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. # dependencies /node_modules /.pnp .pnp.js # testing /coverage # production /build # misc .DS_Store .env.local .env.development.local .env.test.local .env.production.local npm-debug.log* yarn-debug.log* yarn-error.log* ================================================ FILE: scratch/hub-browser-client/README.md ================================================ # Memory Cache Hub Browser Client A browser client for interacting with the memory cache hub. ================================================ FILE: scratch/hub-browser-client/config/env.js ================================================ 'use strict'; const fs = require('fs'); const path = require('path'); const paths = require('./paths'); // Make sure that including paths.js after env.js will read .env variables. delete require.cache[require.resolve('./paths')]; const NODE_ENV = process.env.NODE_ENV; if (!NODE_ENV) { throw new Error( 'The NODE_ENV environment variable is required but was not specified.' ); } // https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use const dotenvFiles = [ `${paths.dotenv}.${NODE_ENV}.local`, // Don't include `.env.local` for `test` environment // since normally you expect tests to produce the same // results for everyone NODE_ENV !== 'test' && `${paths.dotenv}.local`, `${paths.dotenv}.${NODE_ENV}`, paths.dotenv, ].filter(Boolean); // Load environment variables from .env* files. Suppress warnings using silent // if this file is missing. dotenv will never modify any environment variables // that have already been set. Variable expansion is supported in .env files. // https://github.com/motdotla/dotenv // https://github.com/motdotla/dotenv-expand dotenvFiles.forEach(dotenvFile => { if (fs.existsSync(dotenvFile)) { require('dotenv-expand')( require('dotenv').config({ path: dotenvFile, }) ); } }); // We support resolving modules according to `NODE_PATH`. // This lets you use absolute paths in imports inside large monorepos: // https://github.com/facebook/create-react-app/issues/253. // It works similar to `NODE_PATH` in Node itself: // https://nodejs.org/api/modules.html#modules_loading_from_the_global_folders // Note that unlike in Node, only *relative* paths from `NODE_PATH` are honored. // Otherwise, we risk importing Node.js core modules into an app instead of webpack shims. // https://github.com/facebook/create-react-app/issues/1023#issuecomment-265344421 // We also resolve them to make sure all tools using them work consistently. const appDirectory = fs.realpathSync(process.cwd()); process.env.NODE_PATH = (process.env.NODE_PATH || '') .split(path.delimiter) .filter(folder => folder && !path.isAbsolute(folder)) .map(folder => path.resolve(appDirectory, folder)) .join(path.delimiter); // Grab NODE_ENV and REACT_APP_* environment variables and prepare them to be // injected into the application via DefinePlugin in webpack configuration. const REACT_APP = /^REACT_APP_/i; function getClientEnvironment(publicUrl) { const raw = Object.keys(process.env) .filter(key => REACT_APP.test(key)) .reduce( (env, key) => { env[key] = process.env[key]; return env; }, { // Useful for determining whether we’re running in production mode. // Most importantly, it switches React into the correct mode. NODE_ENV: process.env.NODE_ENV || 'development', // Useful for resolving the correct path to static assets in `public`. // For example, . // This should only be used as an escape hatch. Normally you would put // images into the `src` and `import` them in code to get their paths. PUBLIC_URL: publicUrl, // We support configuring the sockjs pathname during development. // These settings let a developer run multiple simultaneous projects. // They are used as the connection `hostname`, `pathname` and `port` // in webpackHotDevClient. They are used as the `sockHost`, `sockPath` // and `sockPort` options in webpack-dev-server. WDS_SOCKET_HOST: process.env.WDS_SOCKET_HOST, WDS_SOCKET_PATH: process.env.WDS_SOCKET_PATH, WDS_SOCKET_PORT: process.env.WDS_SOCKET_PORT, // Whether or not react-refresh is enabled. // It is defined here so it is available in the webpackHotDevClient. FAST_REFRESH: process.env.FAST_REFRESH !== 'false', } ); // Stringify all values so we can feed into webpack DefinePlugin const stringified = { 'process.env': Object.keys(raw).reduce((env, key) => { env[key] = JSON.stringify(raw[key]); return env; }, {}), }; return { raw, stringified }; } module.exports = getClientEnvironment; ================================================ FILE: scratch/hub-browser-client/config/getHttpsConfig.js ================================================ 'use strict'; const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); const chalk = require('react-dev-utils/chalk'); const paths = require('./paths'); // Ensure the certificate and key provided are valid and if not // throw an easy to debug error function validateKeyAndCerts({ cert, key, keyFile, crtFile }) { let encrypted; try { // publicEncrypt will throw an error with an invalid cert encrypted = crypto.publicEncrypt(cert, Buffer.from('test')); } catch (err) { throw new Error( `The certificate "${chalk.yellow(crtFile)}" is invalid.\n${err.message}` ); } try { // privateDecrypt will throw an error with an invalid key crypto.privateDecrypt(key, encrypted); } catch (err) { throw new Error( `The certificate key "${chalk.yellow(keyFile)}" is invalid.\n${ err.message }` ); } } // Read file and throw an error if it doesn't exist function readEnvFile(file, type) { if (!fs.existsSync(file)) { throw new Error( `You specified ${chalk.cyan( type )} in your env, but the file "${chalk.yellow(file)}" can't be found.` ); } return fs.readFileSync(file); } // Get the https config // Return cert files if provided in env, otherwise just true or false function getHttpsConfig() { const { SSL_CRT_FILE, SSL_KEY_FILE, HTTPS } = process.env; const isHttps = HTTPS === 'true'; if (isHttps && SSL_CRT_FILE && SSL_KEY_FILE) { const crtFile = path.resolve(paths.appPath, SSL_CRT_FILE); const keyFile = path.resolve(paths.appPath, SSL_KEY_FILE); const config = { cert: readEnvFile(crtFile, 'SSL_CRT_FILE'), key: readEnvFile(keyFile, 'SSL_KEY_FILE'), }; validateKeyAndCerts({ ...config, keyFile, crtFile }); return config; } return isHttps; } module.exports = getHttpsConfig; ================================================ FILE: scratch/hub-browser-client/config/jest/babelTransform.js ================================================ 'use strict'; const babelJest = require('babel-jest').default; const hasJsxRuntime = (() => { if (process.env.DISABLE_NEW_JSX_TRANSFORM === 'true') { return false; } try { require.resolve('react/jsx-runtime'); return true; } catch (e) { return false; } })(); module.exports = babelJest.createTransformer({ presets: [ [ require.resolve('babel-preset-react-app'), { runtime: hasJsxRuntime ? 'automatic' : 'classic', }, ], ], babelrc: false, configFile: false, }); ================================================ FILE: scratch/hub-browser-client/config/jest/cssTransform.js ================================================ 'use strict'; // This is a custom Jest transformer turning style imports into empty objects. // http://facebook.github.io/jest/docs/en/webpack.html module.exports = { process() { return 'module.exports = {};'; }, getCacheKey() { // The output is always the same. return 'cssTransform'; }, }; ================================================ FILE: scratch/hub-browser-client/config/jest/fileTransform.js ================================================ 'use strict'; const path = require('path'); const camelcase = require('camelcase'); // This is a custom Jest transformer turning file imports into filenames. // http://facebook.github.io/jest/docs/en/webpack.html module.exports = { process(src, filename) { const assetFilename = JSON.stringify(path.basename(filename)); if (filename.match(/\.svg$/)) { // Based on how SVGR generates a component name: // https://github.com/smooth-code/svgr/blob/01b194cf967347d43d4cbe6b434404731b87cf27/packages/core/src/state.js#L6 const pascalCaseFilename = camelcase(path.parse(filename).name, { pascalCase: true, }); const componentName = `Svg${pascalCaseFilename}`; return `const React = require('react'); module.exports = { __esModule: true, default: ${assetFilename}, ReactComponent: React.forwardRef(function ${componentName}(props, ref) { return { $$typeof: Symbol.for('react.element'), type: 'svg', ref: ref, key: null, props: Object.assign({}, props, { children: ${assetFilename} }) }; }), };`; } return `module.exports = ${assetFilename};`; }, }; ================================================ FILE: scratch/hub-browser-client/config/modules.js ================================================ 'use strict'; const fs = require('fs'); const path = require('path'); const paths = require('./paths'); const chalk = require('react-dev-utils/chalk'); const resolve = require('resolve'); /** * Get additional module paths based on the baseUrl of a compilerOptions object. * * @param {Object} options */ function getAdditionalModulePaths(options = {}) { const baseUrl = options.baseUrl; if (!baseUrl) { return ''; } const baseUrlResolved = path.resolve(paths.appPath, baseUrl); // We don't need to do anything if `baseUrl` is set to `node_modules`. This is // the default behavior. if (path.relative(paths.appNodeModules, baseUrlResolved) === '') { return null; } // Allow the user set the `baseUrl` to `appSrc`. if (path.relative(paths.appSrc, baseUrlResolved) === '') { return [paths.appSrc]; } // If the path is equal to the root directory we ignore it here. // We don't want to allow importing from the root directly as source files are // not transpiled outside of `src`. We do allow importing them with the // absolute path (e.g. `src/Components/Button.js`) but we set that up with // an alias. if (path.relative(paths.appPath, baseUrlResolved) === '') { return null; } // Otherwise, throw an error. throw new Error( chalk.red.bold( "Your project's `baseUrl` can only be set to `src` or `node_modules`." + ' Create React App does not support other values at this time.' ) ); } /** * Get webpack aliases based on the baseUrl of a compilerOptions object. * * @param {*} options */ function getWebpackAliases(options = {}) { const baseUrl = options.baseUrl; if (!baseUrl) { return {}; } const baseUrlResolved = path.resolve(paths.appPath, baseUrl); if (path.relative(paths.appPath, baseUrlResolved) === '') { return { src: paths.appSrc, }; } } /** * Get jest aliases based on the baseUrl of a compilerOptions object. * * @param {*} options */ function getJestAliases(options = {}) { const baseUrl = options.baseUrl; if (!baseUrl) { return {}; } const baseUrlResolved = path.resolve(paths.appPath, baseUrl); if (path.relative(paths.appPath, baseUrlResolved) === '') { return { '^src/(.*)$': '/src/$1', }; } } function getModules() { // Check if TypeScript is setup const hasTsConfig = fs.existsSync(paths.appTsConfig); const hasJsConfig = fs.existsSync(paths.appJsConfig); if (hasTsConfig && hasJsConfig) { throw new Error( 'You have both a tsconfig.json and a jsconfig.json. If you are using TypeScript please remove your jsconfig.json file.' ); } let config; // If there's a tsconfig.json we assume it's a // TypeScript project and set up the config // based on tsconfig.json if (hasTsConfig) { const ts = require(resolve.sync('typescript', { basedir: paths.appNodeModules, })); config = ts.readConfigFile(paths.appTsConfig, ts.sys.readFile).config; // Otherwise we'll check if there is jsconfig.json // for non TS projects. } else if (hasJsConfig) { config = require(paths.appJsConfig); } config = config || {}; const options = config.compilerOptions || {}; const additionalModulePaths = getAdditionalModulePaths(options); return { additionalModulePaths: additionalModulePaths, webpackAliases: getWebpackAliases(options), jestAliases: getJestAliases(options), hasTsConfig, }; } module.exports = getModules(); ================================================ FILE: scratch/hub-browser-client/config/paths.js ================================================ 'use strict'; const path = require('path'); const fs = require('fs'); const getPublicUrlOrPath = require('react-dev-utils/getPublicUrlOrPath'); // Make sure any symlinks in the project folder are resolved: // https://github.com/facebook/create-react-app/issues/637 const appDirectory = fs.realpathSync(process.cwd()); const resolveApp = relativePath => path.resolve(appDirectory, relativePath); // We use `PUBLIC_URL` environment variable or "homepage" field to infer // "public path" at which the app is served. // webpack needs to know it to put the right