Showing preview only (350K chars total). Download the full file or copy to clipboard to get everything.
Repository: Mozilla-Ocho/Memory-Cache
Branch: main
Commit: 8b54e4e7daa8
Files: 115
Total size: 320.2 KB
Directory structure:
gitextract_ahfxg4dk/
├── .gitignore
├── ATTRIBUTIONS.md
├── LICENSE
├── README.md
├── docs/
│ ├── .gitignore
│ ├── 404.html
│ ├── CNAME
│ ├── Gemfile
│ ├── _config.yml
│ ├── _includes/
│ │ ├── footer.html
│ │ └── header.html
│ ├── _layouts/
│ │ ├── default.html
│ │ ├── home.html
│ │ ├── page.html
│ │ └── post.html
│ ├── _posts/
│ │ ├── 2023-11-06-introducing-memory-cache.markdown
│ │ ├── 2023-11-30-we-have-a-website.markdown
│ │ ├── 2024-03-01-memory-cache-and-ai-privacy.markdown
│ │ ├── 2024-03-06-designlog-update.markdown
│ │ ├── 2024-03-07-devlog.markdown
│ │ ├── 2024-03-15-devlog.markdown
│ │ └── 2024-04-19-memory-cache-hub.markdown
│ ├── _sass/
│ │ ├── memorycache.scss
│ │ └── minima.scss
│ ├── about.markdown
│ ├── assets/
│ │ └── main.scss
│ ├── faq.md
│ ├── index.markdown
│ └── readme.md
├── extension/
│ ├── content-script.js
│ ├── manifest.json
│ └── popup/
│ ├── marked.esm.js
│ ├── memory_cache.html
│ ├── memory_cache.js
│ └── styles.css
├── scratch/
│ ├── backend/
│ │ ├── hub/
│ │ │ ├── .gitignore
│ │ │ ├── PLAN.md
│ │ │ ├── README.md
│ │ │ ├── docker/
│ │ │ │ ├── Dockerfile.hub-builder-gnu-linux
│ │ │ │ ├── Dockerfile.hub-builder-old-gnu-linux
│ │ │ │ ├── Dockerfile.hub-dev
│ │ │ │ └── Dockerfile.hub-dev-cuda
│ │ │ ├── requirements/
│ │ │ │ ├── hub-base.txt
│ │ │ │ ├── hub-builder.txt
│ │ │ │ └── hub-cpu.txt
│ │ │ └── src/
│ │ │ ├── api/
│ │ │ │ ├── llamafile_api.py
│ │ │ │ └── thread_api.py
│ │ │ ├── async_utils.py
│ │ │ ├── chat.py
│ │ │ ├── chat2.py
│ │ │ ├── chat3.py
│ │ │ ├── fastapi_app.py
│ │ │ ├── gradio_app.py
│ │ │ ├── hub.py
│ │ │ ├── hub_build_gnu_linux.py
│ │ │ ├── hub_build_macos.py
│ │ │ ├── hub_build_windows.py
│ │ │ ├── llamafile_infos.json
│ │ │ ├── llamafile_infos.py
│ │ │ ├── llamafile_manager.py
│ │ │ └── static/
│ │ │ └── index.html
│ │ ├── langserve-demo/
│ │ │ ├── .gitignore
│ │ │ ├── Dockerfile.cpu
│ │ │ ├── README.md
│ │ │ ├── client.py
│ │ │ ├── requirements-cpu.txt
│ │ │ ├── requirements.txt
│ │ │ └── serve.py
│ │ └── python-llamafile-manager/
│ │ ├── .gitignore
│ │ ├── Dockerfile.plm
│ │ ├── Dockerfile.plm-builder-gnu-linux
│ │ ├── README.md
│ │ ├── build_gnu_linux.py
│ │ ├── manager.py
│ │ └── requirements.txt
│ ├── browser-client/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── index.html
│ │ │ ├── main.js
│ │ │ ├── styleguide.html
│ │ │ └── styles.css
│ │ └── webpack.config.js
│ └── hub-browser-client/
│ ├── .gitignore
│ ├── README.md
│ ├── config/
│ │ ├── env.js
│ │ ├── getHttpsConfig.js
│ │ ├── jest/
│ │ │ ├── babelTransform.js
│ │ │ ├── cssTransform.js
│ │ │ └── fileTransform.js
│ │ ├── modules.js
│ │ ├── paths.js
│ │ ├── webpack/
│ │ │ └── persistentCache/
│ │ │ └── createEnvironmentHash.js
│ │ ├── webpack.config.js
│ │ └── webpackDevServer.config.js
│ ├── package.json
│ ├── public/
│ │ ├── index.html
│ │ ├── manifest.json
│ │ └── robots.txt
│ ├── scripts/
│ │ ├── build.js
│ │ ├── start.js
│ │ └── test.js
│ ├── src/
│ │ ├── App.css
│ │ ├── App.test.tsx
│ │ ├── App.tsx
│ │ ├── api/
│ │ │ └── llamafile_api.ts
│ │ ├── components/
│ │ │ └── llamafile_details.tsx
│ │ ├── index.css
│ │ ├── index.tsx
│ │ ├── react-app-env.d.ts
│ │ ├── reportWebVitals.ts
│ │ ├── setupTests.ts
│ │ └── types.ts
│ └── tsconfig.json
└── scripts/
└── run_ingest.sh
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
.DS_Store
extension/.web-extension-id
extension/web-ext-artifacts/
================================================
FILE: ATTRIBUTIONS.md
================================================
## Icons
brain_24.png icon licensed under [CC-by 3.0 Unported](https://creativecommons.org/licenses/by/3.0/) from user 'Howcolour' on www.iconfinder.com
save-icon-16.png icon licensed under [CC-by 3.0](https://creativecommons.org/licenses/by/3.0/) from user 'Bhuvan' from Noun Project
file-icon-16.png icon licensed under [CC-by 3.0](https://creativecommons.org/licenses/by/3.0/) from user 'Mas Dhimas' from Noun Project
## Helpful Links
CSS Gradient tool used: https://cssgradient.io/
Pastel Rainbow color palette by user allyasdf on color-hex: https://www.color-hex.com/color-palette/5361
CSS Trick - border-top-linear-gradient solution fromL https://michaelharley.net/posts/2021/01/12/how-to-create-a-border-top-linear-gradient/
================================================
FILE: LICENSE
================================================
Mozilla Public License Version 2.0
==================================
1. Definitions
--------------
1.1. "Contributor"
means each individual or legal entity that creates, contributes to
the creation of, or owns Covered Software.
1.2. "Contributor Version"
means the combination of the Contributions of others (if any) used
by a Contributor and that particular Contributor's Contribution.
1.3. "Contribution"
means Covered Software of a particular Contributor.
1.4. "Covered Software"
means Source Code Form to which the initial Contributor has attached
the notice in Exhibit A, the Executable Form of such Source Code
Form, and Modifications of such Source Code Form, in each case
including portions thereof.
1.5. "Incompatible With Secondary Licenses"
means
(a) that the initial Contributor has attached the notice described
in Exhibit B to the Covered Software; or
(b) that the Covered Software was made available under the terms of
version 1.1 or earlier of the License, but not also under the
terms of a Secondary License.
1.6. "Executable Form"
means any form of the work other than Source Code Form.
1.7. "Larger Work"
means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
means this document.
1.9. "Licensable"
means having the right to grant, to the maximum extent possible,
whether at the time of the initial grant or subsequently, any and
all of the rights conveyed by this License.
1.10. "Modifications"
means any of the following:
(a) any file in Source Code Form that results from an addition to,
deletion from, or modification of the contents of Covered
Software; or
(b) any new file in Source Code Form that contains any Covered
Software.
1.11. "Patent Claims" of a Contributor
means any patent claim(s), including without limitation, method,
process, and apparatus claims, in any patent Licensable by such
Contributor that would be infringed, but for the grant of the
License, by the making, using, selling, offering for sale, having
made, import, or transfer of either its Contributions or its
Contributor Version.
1.12. "Secondary License"
means either the GNU General Public License, Version 2.0, the GNU
Lesser General Public License, Version 2.1, the GNU Affero General
Public License, Version 3.0, or any later versions of those
licenses.
1.13. "Source Code Form"
means the form of the work preferred for making modifications.
1.14. "You" (or "Your")
means an individual or a legal entity exercising rights under this
License. For legal entities, "You" includes any entity that
controls, is controlled by, or is under common control with You. For
purposes of this definition, "control" means (a) the power, direct
or indirect, to cause the direction or management of such entity,
whether by contract or otherwise, or (b) ownership of more than
fifty percent (50%) of the outstanding shares or beneficial
ownership of such entity.
2. License Grants and Conditions
--------------------------------
2.1. Grants
Each Contributor hereby grants You a world-wide, royalty-free,
non-exclusive license:
(a) under intellectual property rights (other than patent or trademark)
Licensable by such Contributor to use, reproduce, make available,
modify, display, perform, distribute, and otherwise exploit its
Contributions, either on an unmodified basis, with Modifications, or
as part of a Larger Work; and
(b) under Patent Claims of such Contributor to make, use, sell, offer
for sale, have made, import, and otherwise transfer either its
Contributions or its Contributor Version.
2.2. Effective Date
The licenses granted in Section 2.1 with respect to any Contribution
become effective for each Contribution on the date the Contributor first
distributes such Contribution.
2.3. Limitations on Grant Scope
The licenses granted in this Section 2 are the only rights granted under
this License. No additional rights or licenses will be implied from the
distribution or licensing of Covered Software under this License.
Notwithstanding Section 2.1(b) above, no patent license is granted by a
Contributor:
(a) for any code that a Contributor has removed from Covered Software;
or
(b) for infringements caused by: (i) Your and any other third party's
modifications of Covered Software, or (ii) the combination of its
Contributions with other software (except as part of its Contributor
Version); or
(c) under Patent Claims infringed by Covered Software in the absence of
its Contributions.
This License does not grant any rights in the trademarks, service marks,
or logos of any Contributor (except as may be necessary to comply with
the notice requirements in Section 3.4).
2.4. Subsequent Licenses
No Contributor makes additional grants as a result of Your choice to
distribute the Covered Software under a subsequent version of this
License (see Section 10.2) or under the terms of a Secondary License (if
permitted under the terms of Section 3.3).
2.5. Representation
Each Contributor represents that the Contributor believes its
Contributions are its original creation(s) or it has sufficient rights
to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use
This License is not intended to limit any rights You have under
applicable copyright doctrines of fair use, fair dealing, or other
equivalents.
2.7. Conditions
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
in Section 2.1.
3. Responsibilities
-------------------
3.1. Distribution of Source Form
All distribution of Covered Software in Source Code Form, including any
Modifications that You create or to which You contribute, must be under
the terms of this License. You must inform recipients that the Source
Code Form of the Covered Software is governed by the terms of this
License, and how they can obtain a copy of this License. You may not
attempt to alter or restrict the recipients' rights in the Source Code
Form.
3.2. Distribution of Executable Form
If You distribute Covered Software in Executable Form then:
(a) such Covered Software must also be made available in Source Code
Form, as described in Section 3.1, and You must inform recipients of
the Executable Form how they can obtain a copy of such Source Code
Form by reasonable means in a timely manner, at a charge no more
than the cost of distribution to the recipient; and
(b) You may distribute such Executable Form under the terms of this
License, or sublicense it under different terms, provided that the
license for the Executable Form does not attempt to limit or alter
the recipients' rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work
You may create and distribute a Larger Work under terms of Your choice,
provided that You also comply with the requirements of this License for
the Covered Software. If the Larger Work is a combination of Covered
Software with a work governed by one or more Secondary Licenses, and the
Covered Software is not Incompatible With Secondary Licenses, this
License permits You to additionally distribute such Covered Software
under the terms of such Secondary License(s), so that the recipient of
the Larger Work may, at their option, further distribute the Covered
Software under the terms of either this License or such Secondary
License(s).
3.4. Notices
You may not remove or alter the substance of any license notices
(including copyright notices, patent notices, disclaimers of warranty,
or limitations of liability) contained within the Source Code Form of
the Covered Software, except that You may alter any license notices to
the extent required to remedy known factual inaccuracies.
3.5. Application of Additional Terms
You may choose to offer, and to charge a fee for, warranty, support,
indemnity or liability obligations to one or more recipients of Covered
Software. However, You may do so only on Your own behalf, and not on
behalf of any Contributor. You must make it absolutely clear that any
such warranty, support, indemnity, or liability obligation is offered by
You alone, and You hereby agree to indemnify every Contributor for any
liability incurred by such Contributor as a result of warranty, support,
indemnity or liability terms You offer. You may include additional
disclaimers of warranty and limitations of liability specific to any
jurisdiction.
4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Software due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description must
be placed in a text file included with all distributions of the Covered
Software under this License. Except to the extent prohibited by statute
or regulation, such description must be sufficiently detailed for a
recipient of ordinary skill to be able to understand it.
5. Termination
--------------
5.1. The rights granted under this License will terminate automatically
if You fail to comply with any of its terms. However, if You become
compliant, then the rights granted under this License from a particular
Contributor are reinstated (a) provisionally, unless and until such
Contributor explicitly and finally terminates Your grants, and (b) on an
ongoing basis, if such Contributor fails to notify You of the
non-compliance by some reasonable means prior to 60 days after You have
come back into compliance. Moreover, Your grants from a particular
Contributor are reinstated on an ongoing basis if such Contributor
notifies You of the non-compliance by some reasonable means, this is the
first time You have received notice of non-compliance with this License
from such Contributor, and You become compliant prior to 30 days after
Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent
infringement claim (excluding declaratory judgment actions,
counter-claims, and cross-claims) alleging that a Contributor Version
directly or indirectly infringes any patent, then the rights granted to
You by any and all Contributors for the Covered Software under Section
2.1 of this License shall terminate.
5.3. In the event of termination under Sections 5.1 or 5.2 above, all
end user license agreements (excluding distributors and resellers) which
have been validly granted by You or Your distributors under this License
prior to termination shall survive termination.
************************************************************************
* *
* 6. Disclaimer of Warranty *
* ------------------------- *
* *
* Covered Software is provided under this License on an "as is" *
* basis, without warranty of any kind, either expressed, implied, or *
* statutory, including, without limitation, warranties that the *
* Covered Software is free of defects, merchantable, fit for a *
* particular purpose or non-infringing. The entire risk as to the *
* quality and performance of the Covered Software is with You. *
* Should any Covered Software prove defective in any respect, You *
* (not any Contributor) assume the cost of any necessary servicing, *
* repair, or correction. This disclaimer of warranty constitutes an *
* essential part of this License. No use of any Covered Software is *
* authorized under this License except under this disclaimer. *
* *
************************************************************************
************************************************************************
* *
* 7. Limitation of Liability *
* -------------------------- *
* *
* Under no circumstances and under no legal theory, whether tort *
* (including negligence), contract, or otherwise, shall any *
* Contributor, or anyone who distributes Covered Software as *
* permitted above, be liable to You for any direct, indirect, *
* special, incidental, or consequential damages of any character *
* including, without limitation, damages for lost profits, loss of *
* goodwill, work stoppage, computer failure or malfunction, or any *
* and all other commercial damages or losses, even if such party *
* shall have been informed of the possibility of such damages. This *
* limitation of liability shall not apply to liability for death or *
* personal injury resulting from such party's negligence to the *
* extent applicable law prohibits such limitation. Some *
* jurisdictions do not allow the exclusion or limitation of *
* incidental or consequential damages, so this exclusion and *
* limitation may not apply to You. *
* *
************************************************************************
8. Litigation
-------------
Any litigation relating to this License may be brought only in the
courts of a jurisdiction where the defendant maintains its principal
place of business and such litigation shall be governed by laws of that
jurisdiction, without reference to its conflict-of-law provisions.
Nothing in this Section shall prevent a party's ability to bring
cross-claims or counter-claims.
9. Miscellaneous
----------------
This License represents the complete agreement concerning the subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. Any law or regulation which provides
that the language of a contract shall be construed against the drafter
shall not be used to construe this License against a Contributor.
10. Versions of the License
---------------------------
10.1. New Versions
Mozilla Foundation is the license steward. Except as provided in Section
10.3, no one other than the license steward has the right to modify or
publish new versions of this License. Each version will be given a
distinguishing version number.
10.2. Effect of New Versions
You may distribute the Covered Software under the terms of the version
of the License under which You originally received the Covered Software,
or under the terms of any subsequent version published by the license
steward.
10.3. Modified Versions
If you create software not governed by this License, and you want to
create a new license for such software, you may create and use a
modified version of this License if you rename the license and remove
any references to the name of the license steward (except to note that
such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary
Licenses
If You choose to distribute Source Code Form that is Incompatible With
Secondary Licenses under the terms of this version of the License, the
notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
-------------------------------------------
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
If it is not possible or desirable to put the notice in a particular
file, then You may include the notice in a location (such as a LICENSE
file in a relevant directory) where a recipient would be likely to look
for such a notice.
You may add additional accurate notices of copyright ownership.
Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.
================================================
FILE: README.md
================================================
# Memory Cache
Memory Cache is a project that allows you to save a webpage while you're browsing in Firefox as a PDF, and save it to a synchronized folder that can be used in conjunction with privateGPT to augment a local language model.
| ⚠️: This setup uses the primordial version of privateGPT. I'm working from a fork that can be found [here](https://github.com/misslivirose/privateGPT). |
| ---------------------------------------------------------------------------------------------------------------------- |
## Prerequisites
1. Set up [privateGPT](https://github.com/imartinez/privateGPT) - either using the primordial checkpoint, or from my fork.
2. Create a symlink between a subdirectory in your default Downloads folder called 'MemoryCache' and a 'MemoryCache' directory created inside of /PrivateGPT/source_documents/MemoryCache
3. Apply patch to Firefox to add the `printerSettings.silentMode` property to the Tabs API. [See wiki page for instructions](https://github.com/Mozilla-Ocho/Memory-Cache/wiki/Modifying-Firefox-to-Save-PDF-files-automagically-to-MemoryCache)
4. Copy /scripts/run_ingest.sh into your privateGPT directory and run it to start `inotifywait` watching your downloads directory for new content
## Setting up the Extension
1. Clone the Memory-Cache GitHub repository to your local machine
2. In Firefox, navigate to `about:debugging` and click on 'This Firefox'
3. Click 'Load Temporary Add-on" and open the `extension/manifest.json` file in the MemoryCacheExt directory
## Using the Extension
1. Under the 'Extensions' menu, add the Memory Cache extension to the toolbar
2. When you want to save a page to your Memory Cache, click the icon and select the 'Save' button. This will save the file silently as a PDF if you are using a Firefox build with the `printerSettings.silentMode` property addition.
================================================
FILE: docs/.gitignore
================================================
_site
.sass-cache
.jekyll-cache
.jekyll-metadata
vendor
================================================
FILE: docs/404.html
================================================
---
permalink: /404.html
layout: default
---
<style type="text/css" media="screen">
.container {
margin: 10px auto;
max-width: 600px;
text-align: center;
}
h1 {
margin: 30px 0;
font-size: 4em;
line-height: 1;
letter-spacing: -1px;
}
</style>
<div class="container">
<h1>404</h1>
<p><strong>Page not found :(</strong></p>
<p>The requested page could not be found.</p>
</div>
================================================
FILE: docs/CNAME
================================================
memorycache.ai
================================================
FILE: docs/Gemfile
================================================
source "https://rubygems.org"
# Hello! This is where you manage which Jekyll version is used to run.
# When you want to use a different version, change it below, save the
# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
#
# bundle exec jekyll serve
#
# This will help ensure the proper Jekyll version is running.
# Happy Jekylling!
# This is the default theme for new Jekyll sites. You may change this to anything you like.
gem "minima", "~> 2.5"
# If you want to use GitHub Pages, remove the "gem "jekyll"" above and
# uncomment the line below. To upgrade, run `bundle update github-pages`.
gem "github-pages", "~> 228", group: :jekyll_plugins
# If you have any plugins, put them here!
group :jekyll_plugins do
gem "jekyll-feed", "~> 0.12"
end
# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem
# and associated library.
platforms :mingw, :x64_mingw, :mswin, :jruby do
gem "tzinfo", ">= 1", "< 3"
gem "tzinfo-data"
end
# Performance-booster for watching directories on Windows
gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin]
# Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem
# do not have a Java counterpart.
gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby]
gem "webrick", "~> 1.8"
================================================
FILE: docs/_config.yml
================================================
# Welcome to Jekyll!
#
# This config file is meant for settings that affect your whole blog, values
# which you are expected to set up once and rarely edit after that. If you find
# yourself editing this file very often, consider using Jekyll's data files
# feature for the data you need to update frequently.
#
# For technical reasons, this file is *NOT* reloaded automatically when you use
# 'bundle exec jekyll serve'. If you change this file, please restart the server process.
#
# If you need help with YAML syntax, here are some quick references for you:
# https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml
# https://learnxinyminutes.com/docs/yaml/
#
# Site settings
# These are used to personalize your new site. If you look in the HTML files,
# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on.
# You can create any custom variable you would like, and they will be accessible
# in the templates via {{ site.myvariable }}.
title: MemoryCache
email: oerickson@mozilla.com
description: MemoryCache is an experimental developer project to turn a local desktop environment into an on-device AI agent.
# baseurl: "/Memory-Cache" # the subpath of your site, e.g. /blog
url: "https://memorycache.ai/" # the base hostname & protocol for your site, e.g. http://example.com
github_username: Mozilla-Ocho
# Build settings
theme: minima
plugins:
- jekyll-feed
# Exclude from processing.
# The following items will not be processed, by default.
# Any item listed under the `exclude:` key here will be automatically added to
# the internal "default list".
#
# Excluded items can be processed by explicitly listing the directories or
# their entries' file path in the `include:` list.
#
# exclude:
# - .sass-cache/
# - .jekyll-cache/
# - gemfiles/
# - Gemfile
# - Gemfile.lock
# - node_modules/
# - vendor/bundle/
# - vendor/cache/
# - vendor/gems/
# - vendor/ruby/
================================================
FILE: docs/_includes/footer.html
================================================
<footer class="site-footer h-card">
<data class="u-url" href="{{ "/" | relative_url }}"></data>
<div class="wrapper">
<h2 class="footer-heading">{{ site.title | escape }}</h2>
<div class="footer-col-wrapper">
<div class="footer-col footer-col-1">
<ul class="contact-list">
<li class="p-name">
{%- if site.author -%}
{{ site.author | escape }}
{%- else -%}
{{ site.title | escape }}
{%- endif -%}
</li>
{%- if site.email -%}
<li><a class="u-email" href="mailto:{{ site.email }}">{{ site.email }}</a></li>
{%- endif -%}
</ul>
</div>
<div class="footer-col footer-col-2">
{%- include social.html -%}
</div>
<div class="footer-col footer-col-3">
<a href="https://future.mozilla.org">
<img class="moz-logo" src="/../assets/images/mozilla-logo-bw-rgb.png" />
</a>
</div>
</div>
</div>
</footer>
================================================
FILE: docs/_includes/header.html
================================================
<header class="site-header" role="banner">
<div class="wrapper">
{%- assign default_paths = site.pages | map: "path" -%}
{%- assign page_paths = site.header_pages | default: default_paths -%}
<a class="site-title" rel="author" href="{{ "/" | relative_url }}">
<img class="site-logo" src="/assets/images/MC-LogoNov23.svg"/>
</a>
{%- if page_paths -%}
<nav class="site-nav">
<input type="checkbox" id="nav-trigger" class="nav-trigger" />
<label for="nav-trigger">
<span class="menu-icon">
<svg viewBox="0 0 18 15" width="18px" height="15px">
<path d="M18,1.484c0,0.82-0.665,1.484-1.484,1.484H1.484C0.665,2.969,0,2.304,0,1.484l0,0C0,0.665,0.665,0,1.484,0 h15.032C17.335,0,18,0.665,18,1.484L18,1.484z M18,7.516C18,8.335,17.335,9,16.516,9H1.484C0.665,9,0,8.335,0,7.516l0,0 c0-0.82,0.665-1.484,1.484-1.484h15.032C17.335,6.031,18,6.696,18,7.516L18,7.516z M18,13.516C18,14.335,17.335,15,16.516,15H1.484 C0.665,15,0,14.335,0,13.516l0,0c0-0.82,0.665-1.483,1.484-1.483h15.032C17.335,12.031,18,12.695,18,13.516L18,13.516z"/>
</svg>
</span>
</label>
<div class="trigger">
<a class="page-link" target="_blank" href="https://github.com/Mozilla-Ocho/Memory-Cache/wiki">Docs</a>
{%- for path in page_paths -%}
{%- assign my_page = site.pages | where: "path", path | first -%}
{%- if my_page.title -%}
<a class="page-link" href="{{ my_page.url | relative_url }}">{{ my_page.title | escape }}</a>
{%- endif -%}
{%- endfor -%}
</div>
</nav>
{%- endif -%}
</div>
</header>
================================================
FILE: docs/_layouts/default.html
================================================
<!DOCTYPE html>
<html lang="{{ page.lang | default: site.lang | default: "en" }}">
{%- include head.html -%}
<body>
{%- include header.html -%}
<main class="page-content" aria-label="Content">
<div class="wrapper">
{{ content }}
</div>
</main>
{%- include footer.html -%}
</body>
</html>
================================================
FILE: docs/_layouts/home.html
================================================
---
layout: default
---
<div class="home">
{%- if page.title -%}
<h1 class="page-heading">{{ page.title }}</h1>
{%- endif -%}
{{ content }}
<div class="introduction">
<h2 class="callout-left">MemoryCache is an experimental development project to turn a local desktop environment into an on-device AI agent.</h2>
</div>
<div class="detailed-overview">
<p> Every human is unique. The original vision of the personal computer was as a companion tool for creating intelligence, and the internet was born as a way to connect people and data together around the world. Today, artificial intelligence is upending the way that we interact with data and information, but control of these systems is most often provided through an API endpoint, run in the cloud, and abstracting away deep personal agency in favor of productivity. </p>
<p> MemoryCache, <a href="https://future.mozilla.org" target="_blank"> a Mozilla Innovation Project,</a> is an experimental AI Firefox add-on that partners with privateGPT to quickly save your browser history to your local machine and have a local AI model ingest these - and any other local files you give it - to augment responses to a chat interface that comes built-in with privateGPT. We have an ambition to use MemoryCache to move beyond the chat interface, and find a way to utilize idle compute time to generate net new insights that reflect what you've actually read and learned - not the entirety of the internet at scale. </p>
<figure>
<img src ="/../assets/images/DesktopApplication.png" />
<figcaption> Design mockup of a future interface idea for MemoryCache</figcaption>
</figure>
<p> We're not breaking ground on AI innovation (in fact, we're using an old, "deprecated" file format from a whole six months ago), by design. MemoryCache is a project that allows us to sow some seeds of exploration into creating a deeply personalized AI experience that returns to the original vision of the computer as a companion for our own thought. With MemoryCache, weirdness and unpredictability is part of the charm. </p>
<p> We're a small team working on MemoryCache as a part-time project within Mozilla's innovation group, looking at ways that our personal data and files are used to form insights and new neural connections for our own creative purpose. We're working in the open not because we have answers, but because we want to contribute our way of thinking to one another in a way where others can join in. </p>
</div>
{%- if site.posts.size > 0 -%}
<h2 class="post-list-heading">{{ page.list_title | default: "Updates" }}</h2>
<ul class="post-list">
{%- for post in site.posts -%}
<li>
{%- assign date_format = site.minima.date_format | default: "%b %-d, %Y" -%}
<span class="post-meta">{{ post.date | date: date_format }}</span>
<h3>
<a class="post-link" href="{{ post.url | relative_url }}">
{{ post.title | escape }}
</a>
</h3>
{%- if site.show_excerpts -%}
{{ post.excerpt }}
{%- endif -%}
</li>
{%- endfor -%}
</ul>
<p class="rss-subscribe">subscribe <a href="{{ "/feed.xml" | relative_url }}">via RSS</a></p>
{%- endif -%}
</div>
================================================
FILE: docs/_layouts/page.html
================================================
---
layout: default
---
<article class="post">
<header class="post-header">
<h1 class="post-title">{{ page.title | escape }}</h1>
</header>
<div class="post-content">
{{ content }}
</div>
</article>
================================================
FILE: docs/_layouts/post.html
================================================
---
layout: default
---
<article class="post h-entry" itemscope itemtype="http://schema.org/BlogPosting">
<header class="post-header">
<h1 class="post-title p-name" itemprop="name headline">{{ page.title | escape }}</h1>
<p class="post-meta">
<time class="dt-published" datetime="{{ page.date | date_to_xmlschema }}" itemprop="datePublished">
{%- assign date_format = site.minima.date_format | default: "%b %-d, %Y" -%}
{{ page.date | date: date_format }}
</time>
{%- if page.author -%}
• <span itemprop="author" itemscope itemtype="http://schema.org/Person"><span class="p-author h-card" itemprop="name">{{ page.author }}</span></span>
{%- endif -%}</p>
</header>
<div class="post-content e-content" itemprop="articleBody">
{{ content }}
</div>
{%- if site.disqus.shortname -%}
{%- include disqus_comments.html -%}
{%- endif -%}
<a class="u-url" href="{{ page.url | relative_url }}" hidden></a>
</article>
================================================
FILE: docs/_posts/2023-11-06-introducing-memory-cache.markdown
================================================
---
layout: post
title: "Introducing Memory Cache"
date: 2023-11-06 14:47:57 -0500
categories: developer-blog
---
Most AI development today is centered around services. Companies offer tailored insights and powerful agents that can replicate all aspects of the human experience. AI is supposedly "passing the bar exam", diagnosing medical issues, and everything in-between. What is the role of a human being in an increasingly online world?
In practice, AI is a complex web of big data sources (e.g. the entirety of the internet). Pairing massive amounts of data with increasingly powerful cloud computing capabilities has resulted in unprecedented software development capabilities. Adopting naming practices and principles from science fiction stories, Silicon Valley is racing down a path towards a fictional idea of a "sentient computer" with AGI. Artificial intelligence is a field gives us more modalities and capabilities to use with computers, and what really matters is how we (as humans) use the technology at hand.
Not that long ago, computing was grounded in the idea that digital literacy was a skill to be adopted and used in service of greater problems to be solved. We, as individuals, had control over our data, our files, our thoughts. Over the past several years, Big Tech has traded us systems of addictive social media sites for yottabytes of our personal data in the service of "personalization" - a.k.a targeted advertising.
Memory Cache is an exploration into human-first artificial intelligence, starting with the actual idea of the personal computer. The project is an experiment in local, on-premise AI: what you can do with a standard gaming desktop that sits in your home, and actually works for you. It bridges your browser history with your local file system, so that you can use the power of openly licensed AI and open source code to inspect, query, and tinker with an AI that is under your own control.
================================================
FILE: docs/_posts/2023-11-30-we-have-a-website.markdown
================================================
---
layout: post
title: "We have a website! And other MemoryCache Updates"
date: 2023-11-30 11:47:00 -0800
categories: developer-blog
---
We've been continuing our work on MemoryCache over the past several weeks, and are excited to have our [landing page](https://memorycache.ai) up and running. The updated design for MemoryCache has been something we've been iterating on, and it's been a fun process to talk about what sort of emotions we want to seed MemoryCache tools with. We've settled on an initial design style guide and have recently landed several contributions to enable a vanilla Firefox version of the extension.
Our team is working on MemoryCache as a sandbox for exploring concepts related to small, local, and patient AI. We're a small project with big ambitions, and look forward to continuing down several areas of exploration in the coming weeks, including:
* Building an app experience to automatically generate insights from newly added data, to act as a gentle reminder of places that can grow from your recently added content
* Updating the project website to include more details about the philosophy, design, and thinking around the project and how we envision it growing
* Competitive and secondary research reporting that we can publish that shares our insights and findings on how people think about recall and note-taking
* Understanding how to evaluate and generate personal insights outside of the chat interface model
* Exploring a social layer to easily distill and share insights within a trusted network of people
Follow along with us on our [GitHub repo](https://github.com/misslivirose/Memory-Cache) - we'd love to see you there!
================================================
FILE: docs/_posts/2024-03-01-memory-cache-and-ai-privacy.markdown
================================================
---
layout: post
title: "MemoryCache and AI Privacy"
date: 2024-03-01 07:08:57 -0500
categories: developer-blog
---
_Author: Liv Erickson_
It's been an exciting few months since we first shared [MemoryCache](https://memorycache.ai/developer-blog/2023/11/06/introducing-memory-cache.html), a home for experimenting with local-first AI that learns what you learn. While we'll be sharing more updates about what the team has been working on in the coming weeks by way of more regular development blogs, I wanted to share a podcast that I recently recorded with [Schalk Neethling for the Mechanical Ink Podcast](https://schalkneethling.substack.com/p/privacy-ai-and-an-ai-digital-memory) that goes in-depth about the principles behind MemoryCache.
[](https://www.youtube.com/watch?v=CGdxLfcU9TU)
In this podcast, we go into the motivations behind the project and how it was originally created, as well as the overall challenges that are presented with the growing creation of synthetic content and preserving authentic connections online in an area of unprecedented, generated personalization.
At it's core, MemoryCache is a project exploring what it means to urgently, yet collaboratively, envision futures where AI technologies enable us to build a more authentic relationship with information and ourselves through small acts of insight and [embracing friction](https://foundation.mozilla.org/en/blog/speculative-friction-in-generative-ai/) as it presents novel outcomes.
More coming soon!
================================================
FILE: docs/_posts/2024-03-06-designlog-update.markdown
================================================
---
layout: post
title: "MemoryCache March Design Update"
date: 2024-03-06 08 -0500
categories: developer-blog
---
_Author: Kate Taylor_
Hi! My name is Kate and I am a designer working on MemoryCache in the Mozilla Innovation organization. My official title is Design Technologist, which describes the focus between humans and technology. Humans is an important word in this context because it is not specific to a group of people, but the recognition that the choices we make as technologists have effects that ripple on to humans who may or may not be aware of what happens with their information. Information is powerful in the world of AI and the handling of it deserves genuine respect, which in turn builds on an atmosphere of trust and safety.
MemoryCache serves as an open and safe testbed to explore the ideas of what humans need to both benefit from an AI agent while maintaining control over their information and the technical processes involved. As a designer on this project, my work is intended to create an environment that feels like a true augmentation of your creative thought work.
<figure>
<img src="https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/1a3bb64a-dc65-4ff0-928c-4a61756bd8e6" alt="Sketches and notes describing why establishing context is important for AI tasks and the feeling of security">
<figcaption>Early concepts and notes exploring the idea of what safety in an AI Agent experience looks like</figcaption>
</figure>
When we started the project last year, the world was in a different place. Reflecting back on the time along with the goings-on in the AI space is bringing to mind a lot of big personal feelings as well as acknowledgment of the generally fearful vibes. I tend to pay attention to feelings because they are what allow us (as people) to find the things that matter most. It’s difficult to not sense an amount of fear when people are faced with a lot of change. This especially becomes clear when comparing conversations with people in and out of the tech field. This fear has brought a lot of very meaningful interpersonal conversations about what this technology means -- what job does it do well, what jobs do we (as people) do well and want to keep, how did we get to this point in time. These conversations are the motivation for contributing to Open Source AI because the power of understanding the world around you is boundless, and the barriers to this knowledge are mysterious but significant.
Awareness of the barriers to entry to an experience is where designers do their best work. We strive to find meaningful solutions to problems that will sustain. Acknowledging the emotional barrier is the foundation of how we are thinking about MemoryCache. Depending on who you are, interacting with a chatbot has baggage - in the same way that social situations differ across individuals. There are social aspects, language considerations, articulation differences, historical contexts, etc. This is A LOT to deal with as a user of a system. When approaching the design work for MemoryCache, our guiding light is to take into account the unique humanity of each person and allow for the ability to utilize the technology to create an environment that nurtures human needs rather than profit from them.
<figure>
<img src="https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/4dc37c30-ba29-45ff-b20d-94d6ed212ce7" alt="Design mockups and explorations for the interface of MemoryCache as a desktop application">
<figcaption>Exploratory work for interactions that combine with chat interface to interact with the agent in personalized ways for various usecases</figcaption>
</figure>
<figure>
<img src="https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/416bb481-0a73-409d-b464-5a5369f04c00" alt="Design mockup with various color theme options">
<figcaption>Design mockups and explorations for the UI exploring themes and personalization in combination with input methods for interaction</figcaption>
</figure>
This philosophy is core to the work we are doing with MemoryCache. We believe that personalized experiences for interacting with your own information provides a safe space for working with your thought materials with a lot of flexibility and personalized modularity.
When thinking about what safety means in relation to AI computing, in 2024 this is a complicated subject. We are not just speaking about access to information, but access to people and the very things that make us human. Painting, drawing, and tinkering have always been the safe space in my life, personally. The process of making things provides the opportunity to explore your thoughts and experiences without the judgment or unsolicited opinions of others. This time spent reflecting tends to be where the most valuable ideas come to light in other areas of life (similar to the idea of “shower thoughts”). We are iterating on this concept with the idea that the agent could work with the person in creating more of those shower-thought moments. Flipping the idea of asking the agent for a task with the agent providing insights that you could find valuable. The mockups below demonstrate our current thinking for an interface that we can start building and working with as needs evolve
<figure>
<img src="https://github.com/Mozilla-Ocho/Memory-Cache/assets/100849201/7ce4b868-a9cb-406c-a3da-44ce91277f58" alt="Design mockup for the MemoryCache agent's UI">
<figcaption>Latest design mockup for MemoryCache agent</figcaption>
</figure>
We are excited about the potential of where MemoryCache can go. Part of the magic of developing in the open is that we will learn along the way what matters most to people and evolve from there. The logo and visual design we are running with for MemoryCache visualizes our hope for celebration of individuality and personal empowerment through technology advancements. The world can be a messy place, but your individual context is yours to control. In our next round of work, we are building on this philosophy to expose the Agent’s capabilities in meaningful interactions that support the ability to augment your thought work in the way that your brain thinks.
================================================
FILE: docs/_posts/2024-03-07-devlog.markdown
================================================
---
layout: post
title: "Memory Cache Dev Log March 7 2024"
date: 2024-03-07 08 -0500
categories: developer-blog
---
_Author: John Shaughnessy_
# Memory Cache Dev Log, March 7 2024
A couple months ago [we introduced Memory Cache](https://future.mozilla.org/blog/introducing-memorycache/):
> Memory Cache, a Mozilla Innovation Project, is an early exploration project that augments an on-device, personal model with local files saved from the browser to reflect a more personalized and tailored experience through the lens of privacy and agency
Since then we've been quiet.... _too quiet_.
## New phone, who dis?
It's my first time writing on this blog, so I want to introduce myself. My name is John Shaughnessy. I'm a software engineer at Mozilla.
I got involved in Memory Cache a few months ago by resolving an issue that was added to the github repo and a couple weeks ago I started building Memory Cache V2.
## Why V2?
Memory Cache V1 was a browser extension and that made it convenient to collect and feed documents to an LLM-based program called `privateGPT`. PrivateGPT would break the documents into fragments, save those fragments in a vector database, and let you perform similarity search on those documents via a command line interface. We were running an old version of PrivateGPT based on LangChain.
There were several big, obvious technical gaps between Memory Cache V1 and what we'd need in order to do the kind of investigative research and product development we wanted to do.
It seemed to me that if we really wanted to explore the design space, we'd need to roll our own backend and ship a friendly client UI alongside it. We'd need to speed up inference and we'd need more control over how it ingested documents, inserted context, batched background tasks and presented information to you.
We also needed to fix the "getting started" experience. Setting up V1 required users to be comfortable working on the command line, managing python environments, and in general understanding their file system. As far as I'm aware, there are only three of us who have gone through the steps to actually set up and run V1. We were inspired by [Llamafile](https://github.com/Mozilla-Ocho/llamafile/) and [cosmopolitan](https://justine.lol/ape.html), which create executables that you just download and run on many platforms.
And lastly, we're excited about multiplayer opportunities. Could insights that my LLM generates become useful to my teammates? Under what circumstances would I want to share my data with others? How should I separate what's private, semi-private, or public?
### Running LLMs for Inference
I wasn't very familiar with running LLMs, and I certainly hadn't written an application that did "Retrieval-Augmented Generation" (RAG), which was what we wanted Memory Cache to do. So I started down a long, winding path.
Liv and I chatted with Iván Martínez who wrote `privateGPT`. He was super helpful! And it was exciting to talk to someone who'd built something that let us prototype what we wanted to do so quickly.
Mozilla had just announced [Llamafile](https://github.com/Mozilla-Ocho/llamafile), which seemed like a great way to package an LLM and serve it on many platforms. I wasn't familiar with either [Llama.cpp](https://github.com/ggerganov/llama.cpp) or [cosmo](https://cosmo.zip/), so there was a lot to learn. [Justine](https://github.com/jart) and [Stephen](https://github.com/stlhood) were incredibly helpful and generous with their time. I didn't contribute much back to the project other than trying to write accurate reports of a couple issues ([#214](https://github.com/Mozilla-Ocho/llamafile/issues/214), [#232](https://github.com/Mozilla-Ocho/llamafile/issues/232)) I ran into along the way.
Initially when I was looking into `Llamafile`, I wanted to repackage `privateGPT` as a `Llamafile` so that we could distribute it as a standalone executable. Eventually realized this wasn't a good idea. `Llamafile` bundles `Llama.cpp` programs as executables. `Cosmopolitan` _can_ also bundle things like a python interpreter, but tracking down platform-specific dependencies of `privateGPT` and handling them in a way that was compatible with cosmo was not going to be straightforward. It's just not what the project was designed to do.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/march_2024_dev_log/screenshot_026.png">
<figcaption></figcaption>
</figure>
Once I worked through the issues I was having with my GPU, I was amazed and excited to see how fast LLMs can run. I made a short comparison video that shows the difference: [llamafile CPU vs GPU](https://www.youtube.com/watch?v=G9wBw8jLJwU).
I thought I might extend the simple HTTP server that's baked into `Llamafile` with all the capabilities we'd want in Memory Cache. Justine helped me get some "hello world" programs up and running, and I started reading some examples of C++ servers. I'm not much of a C++ programmer, and I was not feeling very confident that this was the direction I really wanted to go.
I like working in Rust, and I knew that Rust had some kind of story for getting `C` and `C++` binding working, so I wrote a kind of LLM "hello world" program using [rustformers/llm](https://github.com/rustformers/llm). But after about a week of fiddling with Llamafile, Llama.cpp, and rustformers, I felt like I was going down a bit of a rabbit hole, and I wanted to pull myself back up to the problem at hand.
### Langchain and Langserve
Ok. So if we weren't going to build out a C++ or Rust server, what _should_ we be doing? `PrivateGPT` was a python project, and the basic functionality was similar to what I'd done in some simple programs I'd written with hugging face's `transformers` library. (I mentioned these in a [blog post](https://johnshaughnessy.com/blog/posts/osai-kube) and [talk](https://www.youtube.com/watch?v=AHd3jCMQQLs) about upskilling in AI.)
It seemed like `LangChain` and `LlamaIndex` were the two popular python libraries / frameworks for building RAG apps, so I wrote a "hello world" with LangChain. It was... fine. But it seemed like a _lot_ more functionality (and abstraction, complexity, and code) than I wanted.
I ended up dropping the framework after reading the docs for `ChromaDB` and `FastAPI`.
`ChromaDB` is a vector database for turning documents into fragments and then run similarity search (the fundamentals of a RAG system).
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/march_2024_dev_log/screenshot_024.png">
<figcaption></figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/march_2024_dev_log/screenshot_025.png">
<figcaption></figcaption>
</figure>
I needed to choose a database, and I chose this one arbitrarily. Langchain had an official "integration" for Chroma, but I felt like Chroma was so simple that I couldn't imagine an "integration" being helpful.
`FastAPI` is a python library for setting up http servers, and is "batteries included" in some very convenient ways:
- It's compatible with [pydantic](https://docs.pydantic.dev/) which lets you define types, validate user input against them, and generate `OpenAPI` spec from them.
- It comes with [swagger-ui](https://github.com/swagger-api/swagger-ui) which gives an interactive browser interface to your APIs.
- It's compatible with a bunch of other random helpful things like [python-multipart](https://github.com/Kludex/python-multipart).
The other thing to know about `FastAPI` is that as far as http libraries go, it's very easy to use. I was reading documentation about `Langserve`, which seemed like a kind of fancy server for `Langchain` apps until I realized that actually `FastAPI`, `pydantic`, `swagger-ui` et. al were doing all the heavy lifting.
So, I dropped LangChain and Langserve and decided I'd wait until I encountered an actually hard problem before picking up another framework. (And who knows -- such a problem might be right around the corner!)
It helped to read LangChain docs and code to figure out what RAG even is. After that I was able to get basic rag app working (without the framework). I felt pretty good about it.
### Inference
I still needed to decide how to run the LLM. I had explored Llamafiles and Hugging face's `transformers` library. The other popular option seemed to be `ollama`, so I gave that a shot.
`Ollama` ended up being very easy to get up and running. I don't know very much about the project. So far I'm a fan. But I didn't want users of Memory Cache to have to download and run an LLM inference server/process by themselves. It just feels like a very clunky user experience. I want to distribute ONE executable that does everything.
Maybe I'm out of the loop, but I didn't feel very good about any of the options. Like, what I really wanted was to write a python program that handled RAG, project files, and generating various artifacts by talking to an llm, and I also wanted it to run the LLM, and I also wanted it to be an HTTP server to serve a browser client. I suppose that's a complex list of requirements, but it seemed like a reasonable approach for Memory Cache. And I didn't find any examples of people doing this.
I had the idea of using Llamafiles for inference and a python web server for the rest of the "brains", which could also serve a static client. That way, the python code stays simple (it doesn't bring with it any of the transformers / hugging face / llm / cuda code).
### Memory Cache Hub
I did a series of short spikes to piece together exactly how such an app could work. I wrote a bit about each one in this [PR](https://github.com/Mozilla-Ocho/Memory-Cache/pull/58).
In the end, I landed on a (technical) design that I'm pretty happy with. I'm putting the pieces together in a repo called [Memory Cache Hub](https://github.com/johnshaughnessy/Memory-Cache-Hub/) (which will graduate to the Mozilla-Ocho org when it's ready). The [README.md](https://github.com/johnshaughnessy/Memory-Cache-Hub/blob/main/README.md) has more details, but here's the high level:
```plaintext
Memory Cache Hub is a core component of Memory Cache:
- It exposes APIs used by the browser extension, browser client, and plugins.
- It serves static files including the browser client and various project artifacts.
- It downloads and runs llamafiles as subprocesses.
- It ingests and retrieves document fragments with the help of a vector database.
- It generates various artifacts using prompt templates and large language models.
Memory Cache Hub is designed to run on your own machine. All of your data is stored locally and is never uploaded to any server.
To use Memory Cache Hub:
- Download the latest release for your platform (Windows, MacOS, or GNU/Linux)
- Run the release executable. It will open a new tab in your browser showing the Memory Cache GUI.
- If the GUI does not open automatically, you can navigate to http://localhost:4444 in your browser.
Each release build of Memory Cache Hub is a standalone executable that includes the browser client and all necessary assets. By "standalone", we mean that you do not need to install any additional software to use Memory Cache.
A Firefox browser extension for Memory Cache that extends its functionality is also available. More information can be found in the main Memory Cache repository.
```
There are two key ideas here:
- Inference is provided by llamafiles that the hub downloads and runs.
- We use `PyInstaller` to bundle the hub and the browser client into a single executable that we can release.
The rest of the requirements are handled easily in python because of the great libraries and tools that are available (`fastapi`, `pydantic`, `chromadb`, etc).
Getting the two novel ideas to work was challenging. I'm not a python expert, so figuring out the `asyncio` and `subprocess` stuff to download and run llamafiles was tricky. And `PyInstaller` has a long list of "gotchas" and "beware" warnings in its docs. I'm still not convinced I'm using it correctly, even though the executables I'm producing seem like they're doing the right thing.
## The Front End
By this time I had built three measly, unimpressive browser clients for Memory Cache. The first was compatible with `privateGPT`, the second was compatible with some early versions of the Memory Cache Hub. I built the third with `gradio` but quickly decided that it did not spark joy.
And none of these felt like good starting points for a designer to jump into the building process.
I've started working on a kind of "hello world" dashboard for Memory Cache using `tailwindcss`. I want to avoid reinventing the wheel and make sure the basic interactions feel good.
I've exposed most of the Hub's APIs in the client interface by now. It doesn't look or feel good yet, but it's good to have the basic capabilities working.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/march_2024_dev_log/screenshot_032.png">
<figcaption></figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/march_2024_dev_log/screenshot_039.png">
<figcaption></figcaption>
</figure>
## What We're Aiming For
The technical pieces have started to fall into place. We're aiming to have the next iteration of Memory Cache - one that you can easily download and run on your own machine - in a matter of weeks. In an ideal world, we'd ship by the end of Q1, which is a few weeks away.
It won't be perfect, but it'll be far enough along that the feedback we get will be much more valuable, and will help shape the next steps.
================================================
FILE: docs/_posts/2024-03-15-devlog.markdown
================================================
---
layout: post
title: "Memory Cache Dev Log March 15 2024"
date: 2024-03-15 08 -0500
categories: developer-blog
---
_Author: John Shaughnessy_
# Memory Cache Dev Log, March 15 2024
Last Friday, during a casual weekly engineering call, a colleague asked how LLM libraries (llama.cpp, llamafile, langchain, llamaindex, HF transformers, ollama, etc) handle the different chat templates and special tokens that models train on. It was a good question, and none of us seemed to have a complete answer.
The subsequent discussion and research made me realize that a naive approach towards writing model-agnostic application would have unfortunate limitations. Insofar as the differences between models are actually important to the use case, application developers should write model-specific code.
## Text Summarization
I thought about the capabilities that are important for an application like Memory Cache, and which models would be good at providing those capabilities. The first obvious one was text summarization. I had "hacked" summarization by asking an assistant-type model (llamafile) to summarize text, but a model trained specifically for text summarization would be a better fit.
I tested a popular text summarization model with with HF transformers, since I didn't find any relevant llamafiles. (If they're out there, I don't know how to find them.) I wanted to make sure that HF code could be built and bundled to a native application with PyInstaller, since that's how we want to build and bundle Memory Cache as a standalone executable. I verified that it could with a [small test project](https://github.com/johnshaughnessy/summarization-test).
Bundling HF dependencies like pytorch increases the complexity of the release process because we'd go from 3 build targets (MacOS, Windows, Linux) to 8 build targets (assuming support for every platform that pytorch supports):
- `Linux` + `CUDA 11.8`
- `Linux` + `CUDA 12.1`
- `Linux` + `ROCm 5.7`
- `Linux` + `CPU`
- `Mac` + `CPU`
- `Windows` + `CUDA 11.8`
- `Windows` + `CUDA 12.1`
- `Windows` + `CPU`
It's good to know that this is possible, but since our near-term goal for Memory Cache is just to prove out the technical bits mentioned in the [previous dev log](https://memorycache.ai/developer-blog/2024/03/07/devlog.html), we'll likely stick with the text summarization "hack" for now.
## Training Agents
Text summarization is still a simple task (in terms of inference inputs and outputs), so models trained to summarize are likely interchangable for the most part (modulo input/output lengths). However, once we start looking at more complicated types of tasks (like tool use / function calling / memory), the differences between models will be exaggerated.
Consider an example like [this dataset](https://huggingface.co/datasets/smangrul/assistant_chatbot_dataset) meant to help train a model to with act with agentic intentions, beliefs (memory), actions, and chat:
```
Context:
<|begincontext|><|beginlastuserutterance|>I am feeling hungry so I would like to find a place to eat.<|endlastuserutterance|><|endcontext|>
```
```
Target:
<|begintarget|><|begindsts|><|begindst|><|beginintent|>FindRestaurants<|endintent|><|beginbelief|><|endbelief|><|enddst|><|enddsts|><|beginuseraction|>INFORM_INTENT->Restaurants^intent~FindRestaurants<|enduseraction|><|beginaction|>REQUEST->Restaurants^city~<|endaction|><|beginresponse|>Do you have a specific which you want the eating place to be located at?<|endresponse|><|endtarget|>
```
Here, we can see that there are _many_ special tokens that the application developer would need to be aware of:
```
- <beginintent></endintent>
- <beginbelief></endbelief>
- <beginaction></endaction>
- <beginresponse></endresponse>
```
Research on how to train these types of models is still rapidly evolving. I suspect attempting to abstract away these differences will lead to leaky or nerfed abstractions in libraries and toolkits. For now, my guess is that it's better to write application code targeting the specific models you want to use.
## Conclusion
Even if a dedicated text summarization model doesn't make it into the upcoming release, this was a valuable excursion. These are the exact types of problems I hoped to stumble over along the way.
================================================
FILE: docs/_posts/2024-04-19-memory-cache-hub.markdown
================================================
---
layout: post
title: "Memory Cache Hub"
date: 2024-04-19 08 -0500
categories: developer-blog
---
_Author: John Shaughnessy_
# Memory Cache Hub
In a [dev log](https://memorycache.ai/developer-blog/2024/03/07/devlog.html) last month, I explained why we were building Memory Cache Hub. We wanted:
- our own backend to learn about and play around with [`RAG`](https://python.langchain.com/docs/expression_language/cookbook/retrieval),
- a friendly browser-based UI,
- to experiment with `llamafile`s,
- to experiment with bundling python files with `PyInstaller`
The work outlined in that blog post is done. You can try Memory Cache Hub by following the [installation instructions in the README](https://github.com/Mozilla-Ocho/Memory-Cache-Hub?tab=readme-ov-file#installation).
My goal for Memory Cache Hub was just to get the project to this point. It was useful to build and I learned a lot, but there are no plans to continue development.
For the rest of this post, I would like to share some thoughts/takeaways from working on the project.
- Client/Server architecture is convenient, especially with OpenAPI specs.
- Browser clients are great until they're not.
- Llamafiles are relatively painless.
- Python and PyInstaller pros/cons.
- Github Actions and large files.
- There's a lot of regular (non-AI) work that needs doing.
## The Client / Server Architecture is convenient, especially with OpenAPI specs.
This is probably a boring point to start with, and it's old news. Still, I thought it'd be worth mentioning a couple of ways that it turned out to be nice to have the main guts of the application implemented behind an HTTP server.
When I was testing out `llamafiles`, I wanted to try enabling GPU acceleration, but my main development machine had some compatibility issues. Since Memory Cache was built as a separate client/server, I could just run the server on another machine (with a compatible GPU) and run the client on my main development machine. It was super painless.
We built Memory Cache with on-device AI in mind, but another form that could make sense is to run AI workloads on a dedicated homelab server (e.g. "an Xbox for AI") or in a private cloud. If the AI apps expose everything over HTTP apis, it's easy to play around with these kinds of setups.
Another time I was glad to have the server implemented separately from the client was when I wanted to build an emacs plugin that leveraged RAG + LLMs for my programming projects. I wrote about the project [on my blog](https://www.johnshaughnessy.com/blog/posts/acorn_pal_emacs). As I was building the plugin I realized that I could probably just plug in to Memory Cache Hub instead of building another RAG/LLM app. It ended up working great!
Unfortunately, I stopped working on the emacs plugin and left it in an unfinished state, mostly because I couldn't generate `elisp` client code for Memory Cache Hub's [Open API](https://openapi-generator.tech/docs/generators) spec. By the way, if anyone wants to write an elisp generator for Open API, that would be really great!
I ended up generating typescript code from the OpenAPI spec for use in the [Memory Cache Browser Client](https://github.com/Mozilla-Ocho/Memory-Cache-Browser-Client). The relevant bit of code was this:
```sh
# Download the openapi.json spec from the server
curl http://localhost:4444/openapi.json > $PROJECT_ROOT/openapi.json
# Generate typescript code
yarn openapi-generator-cli generate -i $PROJECT_ROOT/openapi.json -g typescript-fetch -o $PROJECT_ROOT/src/api/
```
## Browser Clients Are Great, Until They're Not
I am familiar with web front end tools -- React, javascript, parcel, css, canvas, etc. So, I liked the idea of building a front end for Memory Cache in the browser. No need to bundle things with [electron](https://www.electronjs.org/), and no need to train other developers I might be working with (who were also mostly familiar with web development).
For the most part, this worked out great. While the UI isn't "beautiful" or "breathtaking" -- it was painless and quick to build and it'd be easy for someone who really cared about it to come in and improve things.
That said, there were a couple of areas where working in the browser was pretty frustrating:
1. You can't specify directories via a file picker.
2. You can't directly send the user to a file URL.
### No file picker for me
The way Memory Cache works is that the user specifies files in their filesystem that they want to add to their "cache"s. The server will make its own copies of the files in those directories for ingestion and such. The problem is that while browsers have built-in support for a file upload window, there's no way to tell the browser that we want the user to specify full paths to directories on their hard drive.
It's not surprising that browser's don't support this. This isn't really what they're made for. But it means that for this initial version of Memory Cache Hub, I settled for telling the user to type complete file paths into an input field rather than having a file picker UI. This feels really bad and particularly unpolished, even for a demo app.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_003.png">
<figcaption></figcaption>
</figure>
### No file previews
The browser acts as a file viewer if you specify the path of a file prefixed with `file://` in the address bar. This is convenient, because I wanted to let users easily view the files in their cache.
Unfortunately, due to security concerns, the browser disallows redirects to `file://` links. This means that the best I could do for Memory Cache was provide a "copy" button that puts the `file://` URI onto the user's clipboard. Then, they can open a new tab, paste the URL and preview the file. This is a much worse experience.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_016.png">
<figcaption></figcaption>
</figure>
My client could also have provided file previews (e.g. of PDF's) directly with the server sending the file contents to the client, but I didn't end up going down this route.
Again, this isn't surprising because I'm mostly using the browser as an application UI toolkit and that's not really what it's for. Electron (or something like it) would have been a better choice here.
## Llamafiles are (relatively) painless
Using `llamafiles` for inference turned out to be an easy win. The dependencies of my python application stayed pretty simple because I didn't need to bring in hugging face / pytorch dependencies (and further separate platforms along `CUDA`/`ROCm`/`CPU` boundaries).
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_009.png">
<figcaption></figcaption>
</figure>
There are some "gotchas" with using `llamafiles`, most of which are documented in the [`llamafile README`](https://github.com/Mozilla-Ocho/llamafile?tab=readme-ov-file). For example, I don't end up enabling GPU support because I didn't spend time on handling errors that can occur if `llamafile` fails to move model weights to GPU for whatever reason. There are also still some platform-specific troubleshooting tips you need to follow if the `llamafile` server fails to start for whatever reason.
Still, my overall feeling was that this was a pretty nice way to bundle an inference server with an application, and I hope to see more models bundled as `llamafile`s in the future.
## Python and PyInstaller Pros and Cons
I'm not very deeply embedded in the Python world, so figuring out how people built end-user programs with Python was new to me. For example, I know that [Blender](https://www.blender.org/) has a lot of python code, but as far as I can tell, the core is built with C and C++.
I found `PyInstaller` and had success building standalone executables with it (as described in [this previous blog post](https://memorycache.ai/developer-blog/2024/03/07/devlog.html) and [this one too](https://memorycache.ai/developer-blog/2024/03/15/devlog.html)).
It worked, which is great. But there were some hurdles and downsides.
The first complaint is about the way `single-file` builds work. At startup, they need to unpack the supporting files. In our case, we had something like ~10,000 supporting files (which is probably our fault, not `PyInstaller`s) that get unpacked to a temporary directory. This takes ~30 seconds of basically just waiting around with no progress indicator or anything else of that nature. `PyInstaller` has an experimental feature for [adding a splash screen](https://pyinstaller.org/en/stable/usage.html#splash-screen-experimental), but I didn't end up trying it out because of the disclaimer at the top explaining that the feature doesn't work on MacOS. So, the single-file executable version of Memory Cache Hub appears as if it hangs for 30 seconds when you start it before eventually finishing the unpacking process.
The second complaint is not really about `PyInstaller` and more about using Python at all, which is that in the end we're still running a python interpreter at runtime. There's no real "compile to bytecode/machine code" (except for those dependencies written in something like [Cython](https://cython.org/)). It seems like python is the most well-supported ecosystem for developer tools for ML / AI, and part of me wishes I were spending my time in C or Rust. Not that I'm excellent with those languages, but considering that I get better at whatever I spend time doing, I'd rather be getting better at things that give me more control over what the computer is actually doing.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_018.png">
<figcaption></figcaption>
</figure>
Nothing is stopping me from choosing different tools for my next project, and after all - `llama.cpp` is pretty darn popular and I'm looking forward to trying the (rust-based) [burn](https://github.com/tracel-ai/burn) project.
## Github Actions and Large Files
Ok, so here's another problem with my gigantic bundled python executables with 10,000 files... My build pipeline takes 2+ hours to finish!
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_021.png">
<figcaption></figcaption>
</figure>
Uploading the build artifacts from the runner to github takes a long time -- especially for the zips that have over 10,000 files in them. This feels pretty terrible. Again, I think the problem is not with Github or PyInstaller or anything like that -- The problem is thinking that shipping 10,000 files was a good idea. It wasn't -- I regret it. haha.
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_020.png">
<figcaption></figcaption>
</figure>
## There's a lot of non-AI work to be done
90% of the effort I put into this project was completely unrelated to AI, machine learning, rag, etc. It was all, "How do I build a python server", "How do we want this browser client to work?", "What's PyInstaller?", "How do we set up Github Actions?" etc.
The idea was that once we had all of this ground work out of the way, we'd have a (user-facing) playground to experiment with whatever AI stuff we wanted. That's all fine and good, but I'm not sure how much time we're going to actually spend in that experiment phase, since in the meantime, there have been many other projects vying for our attention.
My thoughts about this at the moment are two fold.
First, if your main goal is to experiment and learn something about AI or ML -- Don't bother trying to wrap it in an end-user application. Just write your python program or Jupyter notebook or whatever and do the learning. Don't worry if it doesn't work on other platforms or only supports whatever kind of GPU you happen to be running -- none of that changes the math / AI / deep learning / ML stuff that you were actually interested in. All of that other stuff is a distraction if all you wanted was the core thing.
However -- if your goal is to experiment with an AI or ML thing that you want people to use -- Then get those people on board using your thing as fast as possible, even if that means getting on a call with that, having them share their desktop and following your instructions to set up a python environment. Whatever you need to do to get them actually running your code and using your thing and giving you feedback -- that's the hurdle you should cross. That doesn't mean you have to ship out to the whole world. Maybe you know your thing is not ready for that. But if you have a particular user in mind and you want them involved and giving you constant feedback, it's good to bring them in early.
## What Now?
Learning the build / deploy side of things was pretty helpful and useful. I'd never built a python application like this one before, and I enjoyed myself along the way.
There's been some interest in connecting Memory Cache more directly with the browser history, with Slack, with emails, and other document/information sources. That direction is probably pretty useful -- and a lot of other people are exploring that space too.
However, I'll likely leave that to others. My next projects will be unrelated to Memory Cache. There are a lot of simple ideas I want to play around with in the space just to deepen my understanding of LLMs, and there are a lot of projects external to Mozilla that I'd like to learn more about and maybe contribute to.
## Screenshots
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_001.png">
<figcaption>Files</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_022.png">
<figcaption>About Memory Cache</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_004.png">
<figcaption>Vector Search</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_005.png">
<figcaption>Chat depends on a model running</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_007.png">
<figcaption>The model selection page</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_009.png">
<figcaption>The model selection page</figcaption>
</figure>
<figure>
<img class="rounded-rect" src="https://memorycache.ai/assets/images/2024-04-19-memory-cache-hub/screenshot_010.png">
<figcaption>Retrieval augmented chat</figcaption>
</figure>
================================================
FILE: docs/_sass/memorycache.scss
================================================
@import url('https://fonts.googleapis.com/css2?family=Work+Sans:wght@300;400;500;600&display=swap');
body {
font-family: 'Work Sans';
}
.site-title {
font-family: 'Work Sans';
color: coral;
}
.introduction {
background-image: url('../assets/images/header-background.png');
background-repeat:no-repeat;
background-size: 675px 210.75px;
background-position: right;
width: 100%;
min-height: 201px;
border-bottom-width: 1px;
border-bottom-style: solid;
border-bottom-color: #F0EFEF;
}
.page-content {
background-color: white;
}
a {
color: #180AB8;
}
.site-logo {
width: 166px;
height: 36px;
}
.site-header {
border-top-width: 5px;
border-top-style: solid;
border-image: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87EBDA 76.04%, #BAE1FF 100%) 1 0 0 0;
}
.callout-left {
width: 75%;
padding-top: 5%;
}
.page-content {
border-top-width: 1px;
border-top-style: solid;
border-top-color: #F0EFEF;
}
.post-list-heading {
font-size: 16px;
padding-top: 5px;
}
.post-link {
font-size: 16px;
}
.moz-logo {
width: 128px;
float: right;
}
.detailed-overview {
padding-top: 5px;
border-top-width: 1px;
border-bottom-width: 1px;
border-top-style: solid;
border-bottom-style: solid;
border-top-color: #F0EFEF;
border-bottom-color: #F0EFEF;
}
figcaption {
text-align: center;
font-style: italic;
margin: 1em 0 3em 0;
}
================================================
FILE: docs/_sass/minima.scss
================================================
@charset "utf-8";
// Define defaults for each variable.
$base-font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol" !default;
$base-font-size: 16px !default;
$base-font-weight: 400 !default;
$small-font-size: $base-font-size * 0.875 !default;
$base-line-height: 1.5 !default;
$spacing-unit: 30px !default;
$text-color: #111 !default;
$background-color: #fdfdfd !default;
$brand-color: #2a7ae2 !default;
$grey-color: #828282 !default;
$grey-color-light: lighten($grey-color, 40%) !default;
$grey-color-dark: darken($grey-color, 25%) !default;
$table-text-align: left !default;
// Width of the content area
$content-width: 800px !default;
$on-palm: 600px !default;
$on-laptop: 800px !default;
// Use media queries like this:
// @include media-query($on-palm) {
// .wrapper {
// padding-right: $spacing-unit / 2;
// padding-left: $spacing-unit / 2;
// }
// }
@mixin media-query($device) {
@media screen and (max-width: $device) {
@content;
}
}
@mixin relative-font-size($ratio) {
font-size: $base-font-size * $ratio;
}
// Import partials.
@import
"minima/base",
"minima/layout",
"minima/syntax-highlighting"
;
================================================
FILE: docs/about.markdown
================================================
---
layout: page
title: About
permalink: /about/
---
Memory Cache is an exploration into synthesis, discovery, and sharing of insights more effectively through the use of technology.
Unlike most explorations into artificial intelligence, Memory Cache is designed to be completely personalized, on-device, and private. It is meant to explore the nuances of how individuals think, from the perspective of learning alongside us over time from the articles we read and save.
================================================
FILE: docs/assets/main.scss
================================================
---
---
@import "minima";
@import "memorycache";
================================================
FILE: docs/faq.md
================================================
---
layout: default
title: FAQ
---
# Frequently Asked Questions
**Q: How do I try MemoryCache?**
Right now (as of December 7, 2023), MemoryCache requires a few manual steps to set up the end to end workflow. There are three components: a) a Firefox extension, b) a local instance of privateGPT, and c) a symlinked folder between privateGPT and your local Downloads folder. There is also an optional configuration that can be done to a private build of Firefox to save files to your local machine as PDF files instead of HTML files. Check out the [GitHub repository](https://github.com/Mozilla-Ocho/Memory-Cache) for more detailed instructions. We are looking into ways to streamline the deployment of MemoryCache to require less manual configuration, but if you're here at this stage, you're at the very earliest stages of our explorations.
**Q: Does MemoryCache send my data anywhere?**
No. One of the core principles of MemoryCache is that you have full control over the system, and that it all stays on your device. If you're a developer or someone who just likes to tinker with your computer applications, and you want to cloud-ify this, feel free! But we're looking to stay entirely local.
**Q: Why is MemoryCache using an old language model and primordial privateGPT?**
MemoryCache is using an old language model ([Nomic AI's gpt4all-j v1.3 groovy.ggml](https://huggingface.co/nomic-ai/gpt4all-j)) and primordial privateGPT because right now, this combo is the one that passes our criteria for the type of responses that it generates. This tech is almost a year old, and there have been many advancements in local AI that we'll be integrating in over time, but we're a small team exploring a lot of different subsets of this problem space and the quality of the insight generated is a sweet spot that we want to preserve. This is a temporary tradeoff, but we want to be careful to keep a consistent benchmark for insight generation.
GPT-J was trained on the 'Pile' dataset, and the versions between the 1.0 release and 1.3 release also added the ShareGPT and Dolly datasettes. The Databricks Dolly dataset is licensed under the Creative Commons license with human contributions and wikipedia entries. The ShareGPT dataset is human prompts and ChatGPT output responses that were submitted by human users.
**Q: What kind of tasks would I use MemoryCache for?**
MemoryCache is ultimately leaning into the weird and creative parts of human insight. The goal with MemoryCache is to "learn what you learn", which is why you are in control of what you want files you want to augment the application with. This can be helpful for research, brainstorming, creative writing, and synthesis of new ideas to connect seemingly unrelated topics together to find new insights and learnings from the body of knowledge that matters most to you.
**Q: Is this a Firefox project?**
No. MemoryCache is a hackathon-style project by the Mozilla Innovation Ecosystem team, not a Firefox project.While the project uses a Firefox extension as a way of collecting information, this is a set of scripts and tools to augment privateGPT, a native AI application. It's meant to streamline the process of saving information that you might read in the browser to store it in your own personal library of information.
================================================
FILE: docs/index.markdown
================================================
---
# Feel free to add content and custom Front Matter to this file.
# To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults
layout: home
---
================================================
FILE: docs/readme.md
================================================
Placeholder
================================================
FILE: extension/content-script.js
================================================
function getPageText() {
const head = document.head.innerHTML;
const body = document.body.innerText;
return `<!DOCTYPE html>\n<head>\n${head}\n</head>\n<body>\n${body}\n</body>\n</html>`;
}
browser.runtime.onMessage.addListener((message, _sender) => {
console.log("[MemoryCache Extension] Received message:", message);
if (message.action === "getPageText") {
return Promise.resolve(getPageText());
}
});
================================================
FILE: extension/manifest.json
================================================
{
"manifest_version" : 2,
"name": "MemoryCache",
"version" : "1.0",
"description" : "Saves a copy of reader view of a tab to a specific directory",
"icons" : {
"48" : "icons/memwrite-48.png"
},
"permissions" : [
"downloads",
"<all_urls>",
"tabs",
"storage"
],
"browser_action" : {
"browser_style" : true,
"default_icon" : "icons/memwrite-32.png",
"default_title" : "Memory Cache",
"default_popup" : "popup/memory_cache.html"
},
"content_scripts": [
{
"matches": ["<all_urls>"],
"js": ["content-script.js"]
}
]
}
================================================
FILE: extension/popup/marked.esm.js
================================================
/**
* marked v10.0.0 - a markdown parser
* Copyright (c) 2011-2023, Christopher Jeffrey. (MIT Licensed)
* https://github.com/markedjs/marked
*/
/**
* DO NOT EDIT THIS FILE
* The code in this file is generated from files in ./src/
*/
/**
* Gets the original marked default options.
*/
function _getDefaults() {
return {
async: false,
breaks: false,
extensions: null,
gfm: true,
hooks: null,
pedantic: false,
renderer: null,
silent: false,
tokenizer: null,
walkTokens: null
};
}
let _defaults = _getDefaults();
function changeDefaults(newDefaults) {
_defaults = newDefaults;
}
/**
* Helpers
*/
const escapeTest = /[&<>"']/;
const escapeReplace = new RegExp(escapeTest.source, 'g');
const escapeTestNoEncode = /[<>"']|&(?!(#\d{1,7}|#[Xx][a-fA-F0-9]{1,6}|\w+);)/;
const escapeReplaceNoEncode = new RegExp(escapeTestNoEncode.source, 'g');
const escapeReplacements = {
'&': '&',
'<': '<',
'>': '>',
'"': '"',
"'": '''
};
const getEscapeReplacement = (ch) => escapeReplacements[ch];
function escape(html, encode) {
if (encode) {
if (escapeTest.test(html)) {
return html.replace(escapeReplace, getEscapeReplacement);
}
}
else {
if (escapeTestNoEncode.test(html)) {
return html.replace(escapeReplaceNoEncode, getEscapeReplacement);
}
}
return html;
}
const unescapeTest = /&(#(?:\d+)|(?:#x[0-9A-Fa-f]+)|(?:\w+));?/ig;
function unescape(html) {
// explicitly match decimal, hex, and named HTML entities
return html.replace(unescapeTest, (_, n) => {
n = n.toLowerCase();
if (n === 'colon')
return ':';
if (n.charAt(0) === '#') {
return n.charAt(1) === 'x'
? String.fromCharCode(parseInt(n.substring(2), 16))
: String.fromCharCode(+n.substring(1));
}
return '';
});
}
const caret = /(^|[^\[])\^/g;
function edit(regex, opt) {
regex = typeof regex === 'string' ? regex : regex.source;
opt = opt || '';
const obj = {
replace: (name, val) => {
val = typeof val === 'object' && 'source' in val ? val.source : val;
val = val.replace(caret, '$1');
regex = regex.replace(name, val);
return obj;
},
getRegex: () => {
return new RegExp(regex, opt);
}
};
return obj;
}
function cleanUrl(href) {
try {
href = encodeURI(href).replace(/%25/g, '%');
}
catch (e) {
return null;
}
return href;
}
const noopTest = { exec: () => null };
function splitCells(tableRow, count) {
// ensure that every cell-delimiting pipe has a space
// before it to distinguish it from an escaped pipe
const row = tableRow.replace(/\|/g, (match, offset, str) => {
let escaped = false;
let curr = offset;
while (--curr >= 0 && str[curr] === '\\')
escaped = !escaped;
if (escaped) {
// odd number of slashes means | is escaped
// so we leave it alone
return '|';
}
else {
// add space before unescaped |
return ' |';
}
}), cells = row.split(/ \|/);
let i = 0;
// First/last cell in a row cannot be empty if it has no leading/trailing pipe
if (!cells[0].trim()) {
cells.shift();
}
if (cells.length > 0 && !cells[cells.length - 1].trim()) {
cells.pop();
}
if (count) {
if (cells.length > count) {
cells.splice(count);
}
else {
while (cells.length < count)
cells.push('');
}
}
for (; i < cells.length; i++) {
// leading or trailing whitespace is ignored per the gfm spec
cells[i] = cells[i].trim().replace(/\\\|/g, '|');
}
return cells;
}
/**
* Remove trailing 'c's. Equivalent to str.replace(/c*$/, '').
* /c*$/ is vulnerable to REDOS.
*
* @param str
* @param c
* @param invert Remove suffix of non-c chars instead. Default falsey.
*/
function rtrim(str, c, invert) {
const l = str.length;
if (l === 0) {
return '';
}
// Length of suffix matching the invert condition.
let suffLen = 0;
// Step left until we fail to match the invert condition.
while (suffLen < l) {
const currChar = str.charAt(l - suffLen - 1);
if (currChar === c && !invert) {
suffLen++;
}
else if (currChar !== c && invert) {
suffLen++;
}
else {
break;
}
}
return str.slice(0, l - suffLen);
}
function findClosingBracket(str, b) {
if (str.indexOf(b[1]) === -1) {
return -1;
}
let level = 0;
for (let i = 0; i < str.length; i++) {
if (str[i] === '\\') {
i++;
}
else if (str[i] === b[0]) {
level++;
}
else if (str[i] === b[1]) {
level--;
if (level < 0) {
return i;
}
}
}
return -1;
}
function outputLink(cap, link, raw, lexer) {
const href = link.href;
const title = link.title ? escape(link.title) : null;
const text = cap[1].replace(/\\([\[\]])/g, '$1');
if (cap[0].charAt(0) !== '!') {
lexer.state.inLink = true;
const token = {
type: 'link',
raw,
href,
title,
text,
tokens: lexer.inlineTokens(text)
};
lexer.state.inLink = false;
return token;
}
return {
type: 'image',
raw,
href,
title,
text: escape(text)
};
}
function indentCodeCompensation(raw, text) {
const matchIndentToCode = raw.match(/^(\s+)(?:```)/);
if (matchIndentToCode === null) {
return text;
}
const indentToCode = matchIndentToCode[1];
return text
.split('\n')
.map(node => {
const matchIndentInNode = node.match(/^\s+/);
if (matchIndentInNode === null) {
return node;
}
const [indentInNode] = matchIndentInNode;
if (indentInNode.length >= indentToCode.length) {
return node.slice(indentToCode.length);
}
return node;
})
.join('\n');
}
/**
* Tokenizer
*/
class _Tokenizer {
options;
// TODO: Fix this rules type
rules;
lexer;
constructor(options) {
this.options = options || _defaults;
}
space(src) {
const cap = this.rules.block.newline.exec(src);
if (cap && cap[0].length > 0) {
return {
type: 'space',
raw: cap[0]
};
}
}
code(src) {
const cap = this.rules.block.code.exec(src);
if (cap) {
const text = cap[0].replace(/^ {1,4}/gm, '');
return {
type: 'code',
raw: cap[0],
codeBlockStyle: 'indented',
text: !this.options.pedantic
? rtrim(text, '\n')
: text
};
}
}
fences(src) {
const cap = this.rules.block.fences.exec(src);
if (cap) {
const raw = cap[0];
const text = indentCodeCompensation(raw, cap[3] || '');
return {
type: 'code',
raw,
lang: cap[2] ? cap[2].trim().replace(this.rules.inline._escapes, '$1') : cap[2],
text
};
}
}
heading(src) {
const cap = this.rules.block.heading.exec(src);
if (cap) {
let text = cap[2].trim();
// remove trailing #s
if (/#$/.test(text)) {
const trimmed = rtrim(text, '#');
if (this.options.pedantic) {
text = trimmed.trim();
}
else if (!trimmed || / $/.test(trimmed)) {
// CommonMark requires space before trailing #s
text = trimmed.trim();
}
}
return {
type: 'heading',
raw: cap[0],
depth: cap[1].length,
text,
tokens: this.lexer.inline(text)
};
}
}
hr(src) {
const cap = this.rules.block.hr.exec(src);
if (cap) {
return {
type: 'hr',
raw: cap[0]
};
}
}
blockquote(src) {
const cap = this.rules.block.blockquote.exec(src);
if (cap) {
const text = rtrim(cap[0].replace(/^ *>[ \t]?/gm, ''), '\n');
const top = this.lexer.state.top;
this.lexer.state.top = true;
const tokens = this.lexer.blockTokens(text);
this.lexer.state.top = top;
return {
type: 'blockquote',
raw: cap[0],
tokens,
text
};
}
}
list(src) {
let cap = this.rules.block.list.exec(src);
if (cap) {
let bull = cap[1].trim();
const isordered = bull.length > 1;
const list = {
type: 'list',
raw: '',
ordered: isordered,
start: isordered ? +bull.slice(0, -1) : '',
loose: false,
items: []
};
bull = isordered ? `\\d{1,9}\\${bull.slice(-1)}` : `\\${bull}`;
if (this.options.pedantic) {
bull = isordered ? bull : '[*+-]';
}
// Get next list item
const itemRegex = new RegExp(`^( {0,3}${bull})((?:[\t ][^\\n]*)?(?:\\n|$))`);
let raw = '';
let itemContents = '';
let endsWithBlankLine = false;
// Check if current bullet point can start a new List Item
while (src) {
let endEarly = false;
if (!(cap = itemRegex.exec(src))) {
break;
}
if (this.rules.block.hr.test(src)) { // End list if bullet was actually HR (possibly move into itemRegex?)
break;
}
raw = cap[0];
src = src.substring(raw.length);
let line = cap[2].split('\n', 1)[0].replace(/^\t+/, (t) => ' '.repeat(3 * t.length));
let nextLine = src.split('\n', 1)[0];
let indent = 0;
if (this.options.pedantic) {
indent = 2;
itemContents = line.trimStart();
}
else {
indent = cap[2].search(/[^ ]/); // Find first non-space char
indent = indent > 4 ? 1 : indent; // Treat indented code blocks (> 4 spaces) as having only 1 indent
itemContents = line.slice(indent);
indent += cap[1].length;
}
let blankLine = false;
if (!line && /^ *$/.test(nextLine)) { // Items begin with at most one blank line
raw += nextLine + '\n';
src = src.substring(nextLine.length + 1);
endEarly = true;
}
if (!endEarly) {
const nextBulletRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:[*+-]|\\d{1,9}[.)])((?:[ \t][^\\n]*)?(?:\\n|$))`);
const hrRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}((?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$)`);
const fencesBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}(?:\`\`\`|~~~)`);
const headingBeginRegex = new RegExp(`^ {0,${Math.min(3, indent - 1)}}#`);
// Check if following lines should be included in List Item
while (src) {
const rawLine = src.split('\n', 1)[0];
nextLine = rawLine;
// Re-align to follow commonmark nesting rules
if (this.options.pedantic) {
nextLine = nextLine.replace(/^ {1,4}(?=( {4})*[^ ])/g, ' ');
}
// End list item if found code fences
if (fencesBeginRegex.test(nextLine)) {
break;
}
// End list item if found start of new heading
if (headingBeginRegex.test(nextLine)) {
break;
}
// End list item if found start of new bullet
if (nextBulletRegex.test(nextLine)) {
break;
}
// Horizontal rule found
if (hrRegex.test(src)) {
break;
}
if (nextLine.search(/[^ ]/) >= indent || !nextLine.trim()) { // Dedent if possible
itemContents += '\n' + nextLine.slice(indent);
}
else {
// not enough indentation
if (blankLine) {
break;
}
// paragraph continuation unless last line was a different block level element
if (line.search(/[^ ]/) >= 4) { // indented code block
break;
}
if (fencesBeginRegex.test(line)) {
break;
}
if (headingBeginRegex.test(line)) {
break;
}
if (hrRegex.test(line)) {
break;
}
itemContents += '\n' + nextLine;
}
if (!blankLine && !nextLine.trim()) { // Check if current line is blank
blankLine = true;
}
raw += rawLine + '\n';
src = src.substring(rawLine.length + 1);
line = nextLine.slice(indent);
}
}
if (!list.loose) {
// If the previous item ended with a blank line, the list is loose
if (endsWithBlankLine) {
list.loose = true;
}
else if (/\n *\n *$/.test(raw)) {
endsWithBlankLine = true;
}
}
let istask = null;
let ischecked;
// Check for task list items
if (this.options.gfm) {
istask = /^\[[ xX]\] /.exec(itemContents);
if (istask) {
ischecked = istask[0] !== '[ ] ';
itemContents = itemContents.replace(/^\[[ xX]\] +/, '');
}
}
list.items.push({
type: 'list_item',
raw,
task: !!istask,
checked: ischecked,
loose: false,
text: itemContents,
tokens: []
});
list.raw += raw;
}
// Do not consume newlines at end of final item. Alternatively, make itemRegex *start* with any newlines to simplify/speed up endsWithBlankLine logic
list.items[list.items.length - 1].raw = raw.trimEnd();
list.items[list.items.length - 1].text = itemContents.trimEnd();
list.raw = list.raw.trimEnd();
// Item child tokens handled here at end because we needed to have the final item to trim it first
for (let i = 0; i < list.items.length; i++) {
this.lexer.state.top = false;
list.items[i].tokens = this.lexer.blockTokens(list.items[i].text, []);
if (!list.loose) {
// Check if list should be loose
const spacers = list.items[i].tokens.filter(t => t.type === 'space');
const hasMultipleLineBreaks = spacers.length > 0 && spacers.some(t => /\n.*\n/.test(t.raw));
list.loose = hasMultipleLineBreaks;
}
}
// Set all items to loose if list is loose
if (list.loose) {
for (let i = 0; i < list.items.length; i++) {
list.items[i].loose = true;
}
}
return list;
}
}
html(src) {
const cap = this.rules.block.html.exec(src);
if (cap) {
const token = {
type: 'html',
block: true,
raw: cap[0],
pre: cap[1] === 'pre' || cap[1] === 'script' || cap[1] === 'style',
text: cap[0]
};
return token;
}
}
def(src) {
const cap = this.rules.block.def.exec(src);
if (cap) {
const tag = cap[1].toLowerCase().replace(/\s+/g, ' ');
const href = cap[2] ? cap[2].replace(/^<(.*)>$/, '$1').replace(this.rules.inline._escapes, '$1') : '';
const title = cap[3] ? cap[3].substring(1, cap[3].length - 1).replace(this.rules.inline._escapes, '$1') : cap[3];
return {
type: 'def',
tag,
raw: cap[0],
href,
title
};
}
}
table(src) {
const cap = this.rules.block.table.exec(src);
if (cap) {
if (!/[:|]/.test(cap[2])) {
// delimiter row must have a pipe (|) or colon (:) otherwise it is a setext heading
return;
}
const item = {
type: 'table',
raw: cap[0],
header: splitCells(cap[1]).map(c => {
return { text: c, tokens: [] };
}),
align: cap[2].replace(/^\||\| *$/g, '').split('|'),
rows: cap[3] && cap[3].trim() ? cap[3].replace(/\n[ \t]*$/, '').split('\n') : []
};
if (item.header.length === item.align.length) {
let l = item.align.length;
let i, j, k, row;
for (i = 0; i < l; i++) {
const align = item.align[i];
if (align) {
if (/^ *-+: *$/.test(align)) {
item.align[i] = 'right';
}
else if (/^ *:-+: *$/.test(align)) {
item.align[i] = 'center';
}
else if (/^ *:-+ *$/.test(align)) {
item.align[i] = 'left';
}
else {
item.align[i] = null;
}
}
}
l = item.rows.length;
for (i = 0; i < l; i++) {
item.rows[i] = splitCells(item.rows[i], item.header.length).map(c => {
return { text: c, tokens: [] };
});
}
// parse child tokens inside headers and cells
// header child tokens
l = item.header.length;
for (j = 0; j < l; j++) {
item.header[j].tokens = this.lexer.inline(item.header[j].text);
}
// cell child tokens
l = item.rows.length;
for (j = 0; j < l; j++) {
row = item.rows[j];
for (k = 0; k < row.length; k++) {
row[k].tokens = this.lexer.inline(row[k].text);
}
}
return item;
}
}
}
lheading(src) {
const cap = this.rules.block.lheading.exec(src);
if (cap) {
return {
type: 'heading',
raw: cap[0],
depth: cap[2].charAt(0) === '=' ? 1 : 2,
text: cap[1],
tokens: this.lexer.inline(cap[1])
};
}
}
paragraph(src) {
const cap = this.rules.block.paragraph.exec(src);
if (cap) {
const text = cap[1].charAt(cap[1].length - 1) === '\n'
? cap[1].slice(0, -1)
: cap[1];
return {
type: 'paragraph',
raw: cap[0],
text,
tokens: this.lexer.inline(text)
};
}
}
text(src) {
const cap = this.rules.block.text.exec(src);
if (cap) {
return {
type: 'text',
raw: cap[0],
text: cap[0],
tokens: this.lexer.inline(cap[0])
};
}
}
escape(src) {
const cap = this.rules.inline.escape.exec(src);
if (cap) {
return {
type: 'escape',
raw: cap[0],
text: escape(cap[1])
};
}
}
tag(src) {
const cap = this.rules.inline.tag.exec(src);
if (cap) {
if (!this.lexer.state.inLink && /^<a /i.test(cap[0])) {
this.lexer.state.inLink = true;
}
else if (this.lexer.state.inLink && /^<\/a>/i.test(cap[0])) {
this.lexer.state.inLink = false;
}
if (!this.lexer.state.inRawBlock && /^<(pre|code|kbd|script)(\s|>)/i.test(cap[0])) {
this.lexer.state.inRawBlock = true;
}
else if (this.lexer.state.inRawBlock && /^<\/(pre|code|kbd|script)(\s|>)/i.test(cap[0])) {
this.lexer.state.inRawBlock = false;
}
return {
type: 'html',
raw: cap[0],
inLink: this.lexer.state.inLink,
inRawBlock: this.lexer.state.inRawBlock,
block: false,
text: cap[0]
};
}
}
link(src) {
const cap = this.rules.inline.link.exec(src);
if (cap) {
const trimmedUrl = cap[2].trim();
if (!this.options.pedantic && /^</.test(trimmedUrl)) {
// commonmark requires matching angle brackets
if (!(/>$/.test(trimmedUrl))) {
return;
}
// ending angle bracket cannot be escaped
const rtrimSlash = rtrim(trimmedUrl.slice(0, -1), '\\');
if ((trimmedUrl.length - rtrimSlash.length) % 2 === 0) {
return;
}
}
else {
// find closing parenthesis
const lastParenIndex = findClosingBracket(cap[2], '()');
if (lastParenIndex > -1) {
const start = cap[0].indexOf('!') === 0 ? 5 : 4;
const linkLen = start + cap[1].length + lastParenIndex;
cap[2] = cap[2].substring(0, lastParenIndex);
cap[0] = cap[0].substring(0, linkLen).trim();
cap[3] = '';
}
}
let href = cap[2];
let title = '';
if (this.options.pedantic) {
// split pedantic href and title
const link = /^([^'"]*[^\s])\s+(['"])(.*)\2/.exec(href);
if (link) {
href = link[1];
title = link[3];
}
}
else {
title = cap[3] ? cap[3].slice(1, -1) : '';
}
href = href.trim();
if (/^</.test(href)) {
if (this.options.pedantic && !(/>$/.test(trimmedUrl))) {
// pedantic allows starting angle bracket without ending angle bracket
href = href.slice(1);
}
else {
href = href.slice(1, -1);
}
}
return outputLink(cap, {
href: href ? href.replace(this.rules.inline._escapes, '$1') : href,
title: title ? title.replace(this.rules.inline._escapes, '$1') : title
}, cap[0], this.lexer);
}
}
reflink(src, links) {
let cap;
if ((cap = this.rules.inline.reflink.exec(src))
|| (cap = this.rules.inline.nolink.exec(src))) {
let link = (cap[2] || cap[1]).replace(/\s+/g, ' ');
link = links[link.toLowerCase()];
if (!link) {
const text = cap[0].charAt(0);
return {
type: 'text',
raw: text,
text
};
}
return outputLink(cap, link, cap[0], this.lexer);
}
}
emStrong(src, maskedSrc, prevChar = '') {
let match = this.rules.inline.emStrong.lDelim.exec(src);
if (!match)
return;
// _ can't be between two alphanumerics. \p{L}\p{N} includes non-english alphabet/numbers as well
if (match[3] && prevChar.match(/[\p{L}\p{N}]/u))
return;
const nextChar = match[1] || match[2] || '';
if (!nextChar || !prevChar || this.rules.inline.punctuation.exec(prevChar)) {
// unicode Regex counts emoji as 1 char; spread into array for proper count (used multiple times below)
const lLength = [...match[0]].length - 1;
let rDelim, rLength, delimTotal = lLength, midDelimTotal = 0;
const endReg = match[0][0] === '*' ? this.rules.inline.emStrong.rDelimAst : this.rules.inline.emStrong.rDelimUnd;
endReg.lastIndex = 0;
// Clip maskedSrc to same section of string as src (move to lexer?)
maskedSrc = maskedSrc.slice(-1 * src.length + lLength);
while ((match = endReg.exec(maskedSrc)) != null) {
rDelim = match[1] || match[2] || match[3] || match[4] || match[5] || match[6];
if (!rDelim)
continue; // skip single * in __abc*abc__
rLength = [...rDelim].length;
if (match[3] || match[4]) { // found another Left Delim
delimTotal += rLength;
continue;
}
else if (match[5] || match[6]) { // either Left or Right Delim
if (lLength % 3 && !((lLength + rLength) % 3)) {
midDelimTotal += rLength;
continue; // CommonMark Emphasis Rules 9-10
}
}
delimTotal -= rLength;
if (delimTotal > 0)
continue; // Haven't found enough closing delimiters
// Remove extra characters. *a*** -> *a*
rLength = Math.min(rLength, rLength + delimTotal + midDelimTotal);
// char length can be >1 for unicode characters;
const lastCharLength = [...match[0]][0].length;
const raw = src.slice(0, lLength + match.index + lastCharLength + rLength);
// Create `em` if smallest delimiter has odd char count. *a***
if (Math.min(lLength, rLength) % 2) {
const text = raw.slice(1, -1);
return {
type: 'em',
raw,
text,
tokens: this.lexer.inlineTokens(text)
};
}
// Create 'strong' if smallest delimiter has even char count. **a***
const text = raw.slice(2, -2);
return {
type: 'strong',
raw,
text,
tokens: this.lexer.inlineTokens(text)
};
}
}
}
codespan(src) {
const cap = this.rules.inline.code.exec(src);
if (cap) {
let text = cap[2].replace(/\n/g, ' ');
const hasNonSpaceChars = /[^ ]/.test(text);
const hasSpaceCharsOnBothEnds = /^ /.test(text) && / $/.test(text);
if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) {
text = text.substring(1, text.length - 1);
}
text = escape(text, true);
return {
type: 'codespan',
raw: cap[0],
text
};
}
}
br(src) {
const cap = this.rules.inline.br.exec(src);
if (cap) {
return {
type: 'br',
raw: cap[0]
};
}
}
del(src) {
const cap = this.rules.inline.del.exec(src);
if (cap) {
return {
type: 'del',
raw: cap[0],
text: cap[2],
tokens: this.lexer.inlineTokens(cap[2])
};
}
}
autolink(src) {
const cap = this.rules.inline.autolink.exec(src);
if (cap) {
let text, href;
if (cap[2] === '@') {
text = escape(cap[1]);
href = 'mailto:' + text;
}
else {
text = escape(cap[1]);
href = text;
}
return {
type: 'link',
raw: cap[0],
text,
href,
tokens: [
{
type: 'text',
raw: text,
text
}
]
};
}
}
url(src) {
let cap;
if (cap = this.rules.inline.url.exec(src)) {
let text, href;
if (cap[2] === '@') {
text = escape(cap[0]);
href = 'mailto:' + text;
}
else {
// do extended autolink path validation
let prevCapZero;
do {
prevCapZero = cap[0];
cap[0] = this.rules.inline._backpedal.exec(cap[0])[0];
} while (prevCapZero !== cap[0]);
text = escape(cap[0]);
if (cap[1] === 'www.') {
href = 'http://' + cap[0];
}
else {
href = cap[0];
}
}
return {
type: 'link',
raw: cap[0],
text,
href,
tokens: [
{
type: 'text',
raw: text,
text
}
]
};
}
}
inlineText(src) {
const cap = this.rules.inline.text.exec(src);
if (cap) {
let text;
if (this.lexer.state.inRawBlock) {
text = cap[0];
}
else {
text = escape(cap[0]);
}
return {
type: 'text',
raw: cap[0],
text
};
}
}
}
/**
* Block-Level Grammar
*/
// Not all rules are defined in the object literal
// @ts-expect-error
const block = {
newline: /^(?: *(?:\n|$))+/,
code: /^( {4}[^\n]+(?:\n(?: *(?:\n|$))*)?)+/,
fences: /^ {0,3}(`{3,}(?=[^`\n]*(?:\n|$))|~{3,})([^\n]*)(?:\n|$)(?:|([\s\S]*?)(?:\n|$))(?: {0,3}\1[~`]* *(?=\n|$)|$)/,
hr: /^ {0,3}((?:-[\t ]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})(?:\n+|$)/,
heading: /^ {0,3}(#{1,6})(?=\s|$)(.*)(?:\n+|$)/,
blockquote: /^( {0,3}> ?(paragraph|[^\n]*)(?:\n|$))+/,
list: /^( {0,3}bull)([ \t][^\n]+?)?(?:\n|$)/,
html: '^ {0,3}(?:' // optional indentation
+ '<(script|pre|style|textarea)[\\s>][\\s\\S]*?(?:</\\1>[^\\n]*\\n+|$)' // (1)
+ '|comment[^\\n]*(\\n+|$)' // (2)
+ '|<\\?[\\s\\S]*?(?:\\?>\\n*|$)' // (3)
+ '|<![A-Z][\\s\\S]*?(?:>\\n*|$)' // (4)
+ '|<!\\[CDATA\\[[\\s\\S]*?(?:\\]\\]>\\n*|$)' // (5)
+ '|</?(tag)(?: +|\\n|/?>)[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (6)
+ '|<(?!script|pre|style|textarea)([a-z][\\w-]*)(?:attribute)*? */?>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (7) open tag
+ '|</(?!script|pre|style|textarea)[a-z][\\w-]*\\s*>(?=[ \\t]*(?:\\n|$))[\\s\\S]*?(?:(?:\\n *)+\\n|$)' // (7) closing tag
+ ')',
def: /^ {0,3}\[(label)\]: *(?:\n *)?([^<\s][^\s]*|<.*?>)(?:(?: +(?:\n *)?| *\n *)(title))? *(?:\n+|$)/,
table: noopTest,
lheading: /^(?!bull )((?:.|\n(?!\s*?\n|bull ))+?)\n {0,3}(=+|-+) *(?:\n+|$)/,
// regex template, placeholders will be replaced according to different paragraph
// interruption rules of commonmark and the original markdown spec:
_paragraph: /^([^\n]+(?:\n(?!hr|heading|lheading|blockquote|fences|list|html|table| +\n)[^\n]+)*)/,
text: /^[^\n]+/
};
block._label = /(?!\s*\])(?:\\.|[^\[\]\\])+/;
block._title = /(?:"(?:\\"?|[^"\\])*"|'[^'\n]*(?:\n[^'\n]+)*\n?'|\([^()]*\))/;
block.def = edit(block.def)
.replace('label', block._label)
.replace('title', block._title)
.getRegex();
block.bullet = /(?:[*+-]|\d{1,9}[.)])/;
block.listItemStart = edit(/^( *)(bull) */)
.replace('bull', block.bullet)
.getRegex();
block.list = edit(block.list)
.replace(/bull/g, block.bullet)
.replace('hr', '\\n+(?=\\1?(?:(?:- *){3,}|(?:_ *){3,}|(?:\\* *){3,})(?:\\n+|$))')
.replace('def', '\\n+(?=' + block.def.source + ')')
.getRegex();
block._tag = 'address|article|aside|base|basefont|blockquote|body|caption'
+ '|center|col|colgroup|dd|details|dialog|dir|div|dl|dt|fieldset|figcaption'
+ '|figure|footer|form|frame|frameset|h[1-6]|head|header|hr|html|iframe'
+ '|legend|li|link|main|menu|menuitem|meta|nav|noframes|ol|optgroup|option'
+ '|p|param|section|source|summary|table|tbody|td|tfoot|th|thead|title|tr'
+ '|track|ul';
block._comment = /<!--(?!-?>)[\s\S]*?(?:-->|$)/;
block.html = edit(block.html, 'i')
.replace('comment', block._comment)
.replace('tag', block._tag)
.replace('attribute', / +[a-zA-Z:_][\w.:-]*(?: *= *"[^"\n]*"| *= *'[^'\n]*'| *= *[^\s"'=<>`]+)?/)
.getRegex();
block.lheading = edit(block.lheading)
.replace(/bull/g, block.bullet) // lists can interrupt
.getRegex();
block.paragraph = edit(block._paragraph)
.replace('hr', block.hr)
.replace('heading', ' {0,3}#{1,6}(?:\\s|$)')
.replace('|lheading', '') // setex headings don't interrupt commonmark paragraphs
.replace('|table', '')
.replace('blockquote', ' {0,3}>')
.replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
.replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
.replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
.replace('tag', block._tag) // pars can be interrupted by type (6) html blocks
.getRegex();
block.blockquote = edit(block.blockquote)
.replace('paragraph', block.paragraph)
.getRegex();
/**
* Normal Block Grammar
*/
block.normal = { ...block };
/**
* GFM Block Grammar
*/
block.gfm = {
...block.normal,
table: '^ *([^\\n ].*)\\n' // Header
+ ' {0,3}((?:\\| *)?:?-+:? *(?:\\| *:?-+:? *)*(?:\\| *)?)' // Align
+ '(?:\\n((?:(?! *\\n|hr|heading|blockquote|code|fences|list|html).*(?:\\n|$))*)\\n*|$)' // Cells
};
block.gfm.table = edit(block.gfm.table)
.replace('hr', block.hr)
.replace('heading', ' {0,3}#{1,6}(?:\\s|$)')
.replace('blockquote', ' {0,3}>')
.replace('code', ' {4}[^\\n]')
.replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
.replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
.replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
.replace('tag', block._tag) // tables can be interrupted by type (6) html blocks
.getRegex();
block.gfm.paragraph = edit(block._paragraph)
.replace('hr', block.hr)
.replace('heading', ' {0,3}#{1,6}(?:\\s|$)')
.replace('|lheading', '') // setex headings don't interrupt commonmark paragraphs
.replace('table', block.gfm.table) // interrupt paragraphs with table
.replace('blockquote', ' {0,3}>')
.replace('fences', ' {0,3}(?:`{3,}(?=[^`\\n]*\\n)|~{3,})[^\\n]*\\n')
.replace('list', ' {0,3}(?:[*+-]|1[.)]) ') // only lists starting from 1 can interrupt
.replace('html', '</?(?:tag)(?: +|\\n|/?>)|<(?:script|pre|style|textarea|!--)')
.replace('tag', block._tag) // pars can be interrupted by type (6) html blocks
.getRegex();
/**
* Pedantic grammar (original John Gruber's loose markdown specification)
*/
block.pedantic = {
...block.normal,
html: edit('^ *(?:comment *(?:\\n|\\s*$)'
+ '|<(tag)[\\s\\S]+?</\\1> *(?:\\n{2,}|\\s*$)' // closed tag
+ '|<tag(?:"[^"]*"|\'[^\']*\'|\\s[^\'"/>\\s]*)*?/?> *(?:\\n{2,}|\\s*$))')
.replace('comment', block._comment)
.replace(/tag/g, '(?!(?:'
+ 'a|em|strong|small|s|cite|q|dfn|abbr|data|time|code|var|samp|kbd|sub'
+ '|sup|i|b|u|mark|ruby|rt|rp|bdi|bdo|span|br|wbr|ins|del|img)'
+ '\\b)\\w+(?!:|[^\\w\\s@]*@)\\b')
.getRegex(),
def: /^ *\[([^\]]+)\]: *<?([^\s>]+)>?(?: +(["(][^\n]+[")]))? *(?:\n+|$)/,
heading: /^(#{1,6})(.*)(?:\n+|$)/,
fences: noopTest,
lheading: /^(.+?)\n {0,3}(=+|-+) *(?:\n+|$)/,
paragraph: edit(block.normal._paragraph)
.replace('hr', block.hr)
.replace('heading', ' *#{1,6} *[^\n]')
.replace('lheading', block.lheading)
.replace('blockquote', ' {0,3}>')
.replace('|fences', '')
.replace('|list', '')
.replace('|html', '')
.getRegex()
};
/**
* Inline-Level Grammar
*/
// Not all rules are defined in the object literal
// @ts-expect-error
const inline = {
escape: /^\\([!"#$%&'()*+,\-./:;<=>?@\[\]\\^_`{|}~])/,
autolink: /^<(scheme:[^\s\x00-\x1f<>]*|email)>/,
url: noopTest,
tag: '^comment'
+ '|^</[a-zA-Z][\\w:-]*\\s*>' // self-closing tag
+ '|^<[a-zA-Z][\\w-]*(?:attribute)*?\\s*/?>' // open tag
+ '|^<\\?[\\s\\S]*?\\?>' // processing instruction, e.g. <?php ?>
+ '|^<![a-zA-Z]+\\s[\\s\\S]*?>' // declaration, e.g. <!DOCTYPE html>
+ '|^<!\\[CDATA\\[[\\s\\S]*?\\]\\]>',
link: /^!?\[(label)\]\(\s*(href)(?:\s+(title))?\s*\)/,
reflink: /^!?\[(label)\]\[(ref)\]/,
nolink: /^!?\[(ref)\](?:\[\])?/,
reflinkSearch: 'reflink|nolink(?!\\()',
emStrong: {
lDelim: /^(?:\*+(?:((?!\*)[punct])|[^\s*]))|^_+(?:((?!_)[punct])|([^\s_]))/,
// (1) and (2) can only be a Right Delimiter. (3) and (4) can only be Left. (5) and (6) can be either Left or Right.
// | Skip orphan inside strong | Consume to delim | (1) #*** | (2) a***#, a*** | (3) #***a, ***a | (4) ***# | (5) #***# | (6) a***a
rDelimAst: /^[^_*]*?__[^_*]*?\*[^_*]*?(?=__)|[^*]+(?=[^*])|(?!\*)[punct](\*+)(?=[\s]|$)|[^punct\s](\*+)(?!\*)(?=[punct\s]|$)|(?!\*)[punct\s](\*+)(?=[^punct\s])|[\s](\*+)(?!\*)(?=[punct])|(?!\*)[punct](\*+)(?!\*)(?=[punct])|[^punct\s](\*+)(?=[^punct\s])/,
rDelimUnd: /^[^_*]*?\*\*[^_*]*?_[^_*]*?(?=\*\*)|[^_]+(?=[^_])|(?!_)[punct](_+)(?=[\s]|$)|[^punct\s](_+)(?!_)(?=[punct\s]|$)|(?!_)[punct\s](_+)(?=[^punct\s])|[\s](_+)(?!_)(?=[punct])|(?!_)[punct](_+)(?!_)(?=[punct])/ // ^- Not allowed for _
},
code: /^(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/,
br: /^( {2,}|\\)\n(?!\s*$)/,
del: noopTest,
text: /^(`+|[^`])(?:(?= {2,}\n)|[\s\S]*?(?:(?=[\\<!\[`*_]|\b_|$)|[^ ](?= {2,}\n)))/,
punctuation: /^((?![*_])[\spunctuation])/
};
// list of unicode punctuation marks, plus any missing characters from CommonMark spec
inline._punctuation = '\\p{P}$+<=>`^|~';
inline.punctuation = edit(inline.punctuation, 'u').replace(/punctuation/g, inline._punctuation).getRegex();
// sequences em should skip over [title](link), `code`, <html>
inline.blockSkip = /\[[^[\]]*?\]\([^\(\)]*?\)|`[^`]*?`|<[^<>]*?>/g;
inline.anyPunctuation = /\\[punct]/g;
inline._escapes = /\\([punct])/g;
inline._comment = edit(block._comment).replace('(?:-->|$)', '-->').getRegex();
inline.emStrong.lDelim = edit(inline.emStrong.lDelim, 'u')
.replace(/punct/g, inline._punctuation)
.getRegex();
inline.emStrong.rDelimAst = edit(inline.emStrong.rDelimAst, 'gu')
.replace(/punct/g, inline._punctuation)
.getRegex();
inline.emStrong.rDelimUnd = edit(inline.emStrong.rDelimUnd, 'gu')
.replace(/punct/g, inline._punctuation)
.getRegex();
inline.anyPunctuation = edit(inline.anyPunctuation, 'gu')
.replace(/punct/g, inline._punctuation)
.getRegex();
inline._escapes = edit(inline._escapes, 'gu')
.replace(/punct/g, inline._punctuation)
.getRegex();
inline._scheme = /[a-zA-Z][a-zA-Z0-9+.-]{1,31}/;
inline._email = /[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+(@)[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(?![-_])/;
inline.autolink = edit(inline.autolink)
.replace('scheme', inline._scheme)
.replace('email', inline._email)
.getRegex();
inline._attribute = /\s+[a-zA-Z:_][\w.:-]*(?:\s*=\s*"[^"]*"|\s*=\s*'[^']*'|\s*=\s*[^\s"'=<>`]+)?/;
inline.tag = edit(inline.tag)
.replace('comment', inline._comment)
.replace('attribute', inline._attribute)
.getRegex();
inline._label = /(?:\[(?:\\.|[^\[\]\\])*\]|\\.|`[^`]*`|[^\[\]\\`])*?/;
inline._href = /<(?:\\.|[^\n<>\\])+>|[^\s\x00-\x1f]*/;
inline._title = /"(?:\\"?|[^"\\])*"|'(?:\\'?|[^'\\])*'|\((?:\\\)?|[^)\\])*\)/;
inline.link = edit(inline.link)
.replace('label', inline._label)
.replace('href', inline._href)
.replace('title', inline._title)
.getRegex();
inline.reflink = edit(inline.reflink)
.replace('label', inline._label)
.replace('ref', block._label)
.getRegex();
inline.nolink = edit(inline.nolink)
.replace('ref', block._label)
.getRegex();
inline.reflinkSearch = edit(inline.reflinkSearch, 'g')
.replace('reflink', inline.reflink)
.replace('nolink', inline.nolink)
.getRegex();
/**
* Normal Inline Grammar
*/
inline.normal = { ...inline };
/**
* Pedantic Inline Grammar
*/
inline.pedantic = {
...inline.normal,
strong: {
start: /^__|\*\*/,
middle: /^__(?=\S)([\s\S]*?\S)__(?!_)|^\*\*(?=\S)([\s\S]*?\S)\*\*(?!\*)/,
endAst: /\*\*(?!\*)/g,
endUnd: /__(?!_)/g
},
em: {
start: /^_|\*/,
middle: /^()\*(?=\S)([\s\S]*?\S)\*(?!\*)|^_(?=\S)([\s\S]*?\S)_(?!_)/,
endAst: /\*(?!\*)/g,
endUnd: /_(?!_)/g
},
link: edit(/^!?\[(label)\]\((.*?)\)/)
.replace('label', inline._label)
.getRegex(),
reflink: edit(/^!?\[(label)\]\s*\[([^\]]*)\]/)
.replace('label', inline._label)
.getRegex()
};
/**
* GFM Inline Grammar
*/
inline.gfm = {
...inline.normal,
escape: edit(inline.escape).replace('])', '~|])').getRegex(),
_extended_email: /[A-Za-z0-9._+-]+(@)[a-zA-Z0-9-_]+(?:\.[a-zA-Z0-9-_]*[a-zA-Z0-9])+(?![-_])/,
url: /^((?:ftp|https?):\/\/|www\.)(?:[a-zA-Z0-9\-]+\.?)+[^\s<]*|^email/,
_backpedal: /(?:[^?!.,:;*_'"~()&]+|\([^)]*\)|&(?![a-zA-Z0-9]+;$)|[?!.,:;*_'"~)]+(?!$))+/,
del: /^(~~?)(?=[^\s~])([\s\S]*?[^\s~])\1(?=[^~]|$)/,
text: /^([`~]+|[^`~])(?:(?= {2,}\n)|(?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)|[\s\S]*?(?:(?=[\\<!\[`*~_]|\b_|https?:\/\/|ftp:\/\/|www\.|$)|[^ ](?= {2,}\n)|[^a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-](?=[a-zA-Z0-9.!#$%&'*+\/=?_`{\|}~-]+@)))/
};
inline.gfm.url = edit(inline.gfm.url, 'i')
.replace('email', inline.gfm._extended_email)
.getRegex();
/**
* GFM + Line Breaks Inline Grammar
*/
inline.breaks = {
...inline.gfm,
br: edit(inline.br).replace('{2,}', '*').getRegex(),
text: edit(inline.gfm.text)
.replace('\\b_', '\\b_| {2,}\\n')
.replace(/\{2,\}/g, '*')
.getRegex()
};
/**
* Block Lexer
*/
class _Lexer {
tokens;
options;
state;
tokenizer;
inlineQueue;
constructor(options) {
// TokenList cannot be created in one go
// @ts-expect-error
this.tokens = [];
this.tokens.links = Object.create(null);
this.options = options || _defaults;
this.options.tokenizer = this.options.tokenizer || new _Tokenizer();
this.tokenizer = this.options.tokenizer;
this.tokenizer.options = this.options;
this.tokenizer.lexer = this;
this.inlineQueue = [];
this.state = {
inLink: false,
inRawBlock: false,
top: true
};
const rules = {
block: block.normal,
inline: inline.normal
};
if (this.options.pedantic) {
rules.block = block.pedantic;
rules.inline = inline.pedantic;
}
else if (this.options.gfm) {
rules.block = block.gfm;
if (this.options.breaks) {
rules.inline = inline.breaks;
}
else {
rules.inline = inline.gfm;
}
}
this.tokenizer.rules = rules;
}
/**
* Expose Rules
*/
static get rules() {
return {
block,
inline
};
}
/**
* Static Lex Method
*/
static lex(src, options) {
const lexer = new _Lexer(options);
return lexer.lex(src);
}
/**
* Static Lex Inline Method
*/
static lexInline(src, options) {
const lexer = new _Lexer(options);
return lexer.inlineTokens(src);
}
/**
* Preprocessing
*/
lex(src) {
src = src
.replace(/\r\n|\r/g, '\n');
this.blockTokens(src, this.tokens);
let next;
while (next = this.inlineQueue.shift()) {
this.inlineTokens(next.src, next.tokens);
}
return this.tokens;
}
blockTokens(src, tokens = []) {
if (this.options.pedantic) {
src = src.replace(/\t/g, ' ').replace(/^ +$/gm, '');
}
else {
src = src.replace(/^( *)(\t+)/gm, (_, leading, tabs) => {
return leading + ' '.repeat(tabs.length);
});
}
let token;
let lastToken;
let cutSrc;
let lastParagraphClipped;
while (src) {
if (this.options.extensions
&& this.options.extensions.block
&& this.options.extensions.block.some((extTokenizer) => {
if (token = extTokenizer.call({ lexer: this }, src, tokens)) {
src = src.substring(token.raw.length);
tokens.push(token);
return true;
}
return false;
})) {
continue;
}
// newline
if (token = this.tokenizer.space(src)) {
src = src.substring(token.raw.length);
if (token.raw.length === 1 && tokens.length > 0) {
// if there's a single \n as a spacer, it's terminating the last line,
// so move it there so that we don't get unnecessary paragraph tags
tokens[tokens.length - 1].raw += '\n';
}
else {
tokens.push(token);
}
continue;
}
// code
if (token = this.tokenizer.code(src)) {
src = src.substring(token.raw.length);
lastToken = tokens[tokens.length - 1];
// An indented code block cannot interrupt a paragraph.
if (lastToken && (lastToken.type === 'paragraph' || lastToken.type === 'text')) {
lastToken.raw += '\n' + token.raw;
lastToken.text += '\n' + token.text;
this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text;
}
else {
tokens.push(token);
}
continue;
}
// fences
if (token = this.tokenizer.fences(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// heading
if (token = this.tokenizer.heading(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// hr
if (token = this.tokenizer.hr(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// blockquote
if (token = this.tokenizer.blockquote(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// list
if (token = this.tokenizer.list(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// html
if (token = this.tokenizer.html(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// def
if (token = this.tokenizer.def(src)) {
src = src.substring(token.raw.length);
lastToken = tokens[tokens.length - 1];
if (lastToken && (lastToken.type === 'paragraph' || lastToken.type === 'text')) {
lastToken.raw += '\n' + token.raw;
lastToken.text += '\n' + token.raw;
this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text;
}
else if (!this.tokens.links[token.tag]) {
this.tokens.links[token.tag] = {
href: token.href,
title: token.title
};
}
continue;
}
// table (gfm)
if (token = this.tokenizer.table(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// lheading
if (token = this.tokenizer.lheading(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// top-level paragraph
// prevent paragraph consuming extensions by clipping 'src' to extension start
cutSrc = src;
if (this.options.extensions && this.options.extensions.startBlock) {
let startIndex = Infinity;
const tempSrc = src.slice(1);
let tempStart;
this.options.extensions.startBlock.forEach((getStartIndex) => {
tempStart = getStartIndex.call({ lexer: this }, tempSrc);
if (typeof tempStart === 'number' && tempStart >= 0) {
startIndex = Math.min(startIndex, tempStart);
}
});
if (startIndex < Infinity && startIndex >= 0) {
cutSrc = src.substring(0, startIndex + 1);
}
}
if (this.state.top && (token = this.tokenizer.paragraph(cutSrc))) {
lastToken = tokens[tokens.length - 1];
if (lastParagraphClipped && lastToken.type === 'paragraph') {
lastToken.raw += '\n' + token.raw;
lastToken.text += '\n' + token.text;
this.inlineQueue.pop();
this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text;
}
else {
tokens.push(token);
}
lastParagraphClipped = (cutSrc.length !== src.length);
src = src.substring(token.raw.length);
continue;
}
// text
if (token = this.tokenizer.text(src)) {
src = src.substring(token.raw.length);
lastToken = tokens[tokens.length - 1];
if (lastToken && lastToken.type === 'text') {
lastToken.raw += '\n' + token.raw;
lastToken.text += '\n' + token.text;
this.inlineQueue.pop();
this.inlineQueue[this.inlineQueue.length - 1].src = lastToken.text;
}
else {
tokens.push(token);
}
continue;
}
if (src) {
const errMsg = 'Infinite loop on byte: ' + src.charCodeAt(0);
if (this.options.silent) {
console.error(errMsg);
break;
}
else {
throw new Error(errMsg);
}
}
}
this.state.top = true;
return tokens;
}
inline(src, tokens = []) {
this.inlineQueue.push({ src, tokens });
return tokens;
}
/**
* Lexing/Compiling
*/
inlineTokens(src, tokens = []) {
let token, lastToken, cutSrc;
// String with links masked to avoid interference with em and strong
let maskedSrc = src;
let match;
let keepPrevChar, prevChar;
// Mask out reflinks
if (this.tokens.links) {
const links = Object.keys(this.tokens.links);
if (links.length > 0) {
while ((match = this.tokenizer.rules.inline.reflinkSearch.exec(maskedSrc)) != null) {
if (links.includes(match[0].slice(match[0].lastIndexOf('[') + 1, -1))) {
maskedSrc = maskedSrc.slice(0, match.index) + '[' + 'a'.repeat(match[0].length - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.reflinkSearch.lastIndex);
}
}
}
}
// Mask out other blocks
while ((match = this.tokenizer.rules.inline.blockSkip.exec(maskedSrc)) != null) {
maskedSrc = maskedSrc.slice(0, match.index) + '[' + 'a'.repeat(match[0].length - 2) + ']' + maskedSrc.slice(this.tokenizer.rules.inline.blockSkip.lastIndex);
}
// Mask out escaped characters
while ((match = this.tokenizer.rules.inline.anyPunctuation.exec(maskedSrc)) != null) {
maskedSrc = maskedSrc.slice(0, match.index) + '++' + maskedSrc.slice(this.tokenizer.rules.inline.anyPunctuation.lastIndex);
}
while (src) {
if (!keepPrevChar) {
prevChar = '';
}
keepPrevChar = false;
// extensions
if (this.options.extensions
&& this.options.extensions.inline
&& this.options.extensions.inline.some((extTokenizer) => {
if (token = extTokenizer.call({ lexer: this }, src, tokens)) {
src = src.substring(token.raw.length);
tokens.push(token);
return true;
}
return false;
})) {
continue;
}
// escape
if (token = this.tokenizer.escape(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// tag
if (token = this.tokenizer.tag(src)) {
src = src.substring(token.raw.length);
lastToken = tokens[tokens.length - 1];
if (lastToken && token.type === 'text' && lastToken.type === 'text') {
lastToken.raw += token.raw;
lastToken.text += token.text;
}
else {
tokens.push(token);
}
continue;
}
// link
if (token = this.tokenizer.link(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// reflink, nolink
if (token = this.tokenizer.reflink(src, this.tokens.links)) {
src = src.substring(token.raw.length);
lastToken = tokens[tokens.length - 1];
if (lastToken && token.type === 'text' && lastToken.type === 'text') {
lastToken.raw += token.raw;
lastToken.text += token.text;
}
else {
tokens.push(token);
}
continue;
}
// em & strong
if (token = this.tokenizer.emStrong(src, maskedSrc, prevChar)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// code
if (token = this.tokenizer.codespan(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// br
if (token = this.tokenizer.br(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// del (gfm)
if (token = this.tokenizer.del(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// autolink
if (token = this.tokenizer.autolink(src)) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// url (gfm)
if (!this.state.inLink && (token = this.tokenizer.url(src))) {
src = src.substring(token.raw.length);
tokens.push(token);
continue;
}
// text
// prevent inlineText consuming extensions by clipping 'src' to extension start
cutSrc = src;
if (this.options.extensions && this.options.extensions.startInline) {
let startIndex = Infinity;
const tempSrc = src.slice(1);
let tempStart;
this.options.extensions.startInline.forEach((getStartIndex) => {
tempStart = getStartIndex.call({ lexer: this }, tempSrc);
if (typeof tempStart === 'number' && tempStart >= 0) {
startIndex = Math.min(startIndex, tempStart);
}
});
if (startIndex < Infinity && startIndex >= 0) {
cutSrc = src.substring(0, startIndex + 1);
}
}
if (token = this.tokenizer.inlineText(cutSrc)) {
src = src.substring(token.raw.length);
if (token.raw.slice(-1) !== '_') { // Track prevChar before string of ____ started
prevChar = token.raw.slice(-1);
}
keepPrevChar = true;
lastToken = tokens[tokens.length - 1];
if (lastToken && lastToken.type === 'text') {
lastToken.raw += token.raw;
lastToken.text += token.text;
}
else {
tokens.push(token);
}
continue;
}
if (src) {
const errMsg = 'Infinite loop on byte: ' + src.charCodeAt(0);
if (this.options.silent) {
console.error(errMsg);
break;
}
else {
throw new Error(errMsg);
}
}
}
return tokens;
}
}
/**
* Renderer
*/
class _Renderer {
options;
constructor(options) {
this.options = options || _defaults;
}
code(code, infostring, escaped) {
const lang = (infostring || '').match(/^\S*/)?.[0];
code = code.replace(/\n$/, '') + '\n';
if (!lang) {
return '<pre><code>'
+ (escaped ? code : escape(code, true))
+ '</code></pre>\n';
}
return '<pre><code class="language-'
+ escape(lang)
+ '">'
+ (escaped ? code : escape(code, true))
+ '</code></pre>\n';
}
blockquote(quote) {
return `<blockquote>\n${quote}</blockquote>\n`;
}
html(html, block) {
return html;
}
heading(text, level, raw) {
// ignore IDs
return `<h${level}>${text}</h${level}>\n`;
}
hr() {
return '<hr>\n';
}
list(body, ordered, start) {
const type = ordered ? 'ol' : 'ul';
const startatt = (ordered && start !== 1) ? (' start="' + start + '"') : '';
return '<' + type + startatt + '>\n' + body + '</' + type + '>\n';
}
listitem(text, task, checked) {
return `<li>${text}</li>\n`;
}
checkbox(checked) {
return '<input '
+ (checked ? 'checked="" ' : '')
+ 'disabled="" type="checkbox">';
}
paragraph(text) {
return `<p>${text}</p>\n`;
}
table(header, body) {
if (body)
body = `<tbody>${body}</tbody>`;
return '<table>\n'
+ '<thead>\n'
+ header
+ '</thead>\n'
+ body
+ '</table>\n';
}
tablerow(content) {
return `<tr>\n${content}</tr>\n`;
}
tablecell(content, flags) {
const type = flags.header ? 'th' : 'td';
const tag = flags.align
? `<${type} align="${flags.align}">`
: `<${type}>`;
return tag + content + `</${type}>\n`;
}
/**
* span level renderer
*/
strong(text) {
return `<strong>${text}</strong>`;
}
em(text) {
return `<em>${text}</em>`;
}
codespan(text) {
return `<code>${text}</code>`;
}
br() {
return '<br>';
}
del(text) {
return `<del>${text}</del>`;
}
link(href, title, text) {
const cleanHref = cleanUrl(href);
if (cleanHref === null) {
return text;
}
href = cleanHref;
let out = '<a href="' + href + '"';
if (title) {
out += ' title="' + title + '"';
}
out += '>' + text + '</a>';
return out;
}
image(href, title, text) {
const cleanHref = cleanUrl(href);
if (cleanHref === null) {
return text;
}
href = cleanHref;
let out = `<img src="${href}" alt="${text}"`;
if (title) {
out += ` title="${title}"`;
}
out += '>';
return out;
}
text(text) {
return text;
}
}
/**
* TextRenderer
* returns only the textual part of the token
*/
class _TextRenderer {
// no need for block level renderers
strong(text) {
return text;
}
em(text) {
return text;
}
codespan(text) {
return text;
}
del(text) {
return text;
}
html(text) {
return text;
}
text(text) {
return text;
}
link(href, title, text) {
return '' + text;
}
image(href, title, text) {
return '' + text;
}
br() {
return '';
}
}
/**
* Parsing & Compiling
*/
class _Parser {
options;
renderer;
textRenderer;
constructor(options) {
this.options = options || _defaults;
this.options.renderer = this.options.renderer || new _Renderer();
this.renderer = this.options.renderer;
this.renderer.options = this.options;
this.textRenderer = new _TextRenderer();
}
/**
* Static Parse Method
*/
static parse(tokens, options) {
const parser = new _Parser(options);
return parser.parse(tokens);
}
/**
* Static Parse Inline Method
*/
static parseInline(tokens, options) {
const parser = new _Parser(options);
return parser.parseInline(tokens);
}
/**
* Parse Loop
*/
parse(tokens, top = true) {
let out = '';
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
// Run any renderer extensions
if (this.options.extensions && this.options.extensions.renderers && this.options.extensions.renderers[token.type]) {
const genericToken = token;
const ret = this.options.extensions.renderers[genericToken.type].call({ parser: this }, genericToken);
if (ret !== false || !['space', 'hr', 'heading', 'code', 'table', 'blockquote', 'list', 'html', 'paragraph', 'text'].includes(genericToken.type)) {
out += ret || '';
continue;
}
}
switch (token.type) {
case 'space': {
continue;
}
case 'hr': {
out += this.renderer.hr();
continue;
}
case 'heading': {
const headingToken = token;
out += this.renderer.heading(this.parseInline(headingToken.tokens), headingToken.depth, unescape(this.parseInline(headingToken.tokens, this.textRenderer)));
continue;
}
case 'code': {
const codeToken = token;
out += this.renderer.code(codeToken.text, codeToken.lang, !!codeToken.escaped);
continue;
}
case 'table': {
const tableToken = token;
let header = '';
// header
let cell = '';
for (let j = 0; j < tableToken.header.length; j++) {
cell += this.renderer.tablecell(this.parseInline(tableToken.header[j].tokens), { header: true, align: tableToken.align[j] });
}
header += this.renderer.tablerow(cell);
let body = '';
for (let j = 0; j < tableToken.rows.length; j++) {
const row = tableToken.rows[j];
cell = '';
for (let k = 0; k < row.length; k++) {
cell += this.renderer.tablecell(this.parseInline(row[k].tokens), { header: false, align: tableToken.align[k] });
}
body += this.renderer.tablerow(cell);
}
out += this.renderer.table(header, body);
continue;
}
case 'blockquote': {
const blockquoteToken = token;
const body = this.parse(blockquoteToken.tokens);
out += this.renderer.blockquote(body);
continue;
}
case 'list': {
const listToken = token;
const ordered = listToken.ordered;
const start = listToken.start;
const loose = listToken.loose;
let body = '';
for (let j = 0; j < listToken.items.length; j++) {
const item = listToken.items[j];
const checked = item.checked;
const task = item.task;
let itemBody = '';
if (item.task) {
const checkbox = this.renderer.checkbox(!!checked);
if (loose) {
if (item.tokens.length > 0 && item.tokens[0].type === 'paragraph') {
item.tokens[0].text = checkbox + ' ' + item.tokens[0].text;
if (item.tokens[0].tokens && item.tokens[0].tokens.length > 0 && item.tokens[0].tokens[0].type === 'text') {
item.tokens[0].tokens[0].text = checkbox + ' ' + item.tokens[0].tokens[0].text;
}
}
else {
item.tokens.unshift({
type: 'text',
text: checkbox + ' '
});
}
}
else {
itemBody += checkbox + ' ';
}
}
itemBody += this.parse(item.tokens, loose);
body += this.renderer.listitem(itemBody, task, !!checked);
}
out += this.renderer.list(body, ordered, start);
continue;
}
case 'html': {
const htmlToken = token;
out += this.renderer.html(htmlToken.text, htmlToken.block);
continue;
}
case 'paragraph': {
const paragraphToken = token;
out += this.renderer.paragraph(this.parseInline(paragraphToken.tokens));
continue;
}
case 'text': {
let textToken = token;
let body = textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text;
while (i + 1 < tokens.length && tokens[i + 1].type === 'text') {
textToken = tokens[++i];
body += '\n' + (textToken.tokens ? this.parseInline(textToken.tokens) : textToken.text);
}
out += top ? this.renderer.paragraph(body) : body;
continue;
}
default: {
const errMsg = 'Token with "' + token.type + '" type was not found.';
if (this.options.silent) {
console.error(errMsg);
return '';
}
else {
throw new Error(errMsg);
}
}
}
}
return out;
}
/**
* Parse Inline Tokens
*/
parseInline(tokens, renderer) {
renderer = renderer || this.renderer;
let out = '';
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
// Run any renderer extensions
if (this.options.extensions && this.options.extensions.renderers && this.options.extensions.renderers[token.type]) {
const ret = this.options.extensions.renderers[token.type].call({ parser: this }, token);
if (ret !== false || !['escape', 'html', 'link', 'image', 'strong', 'em', 'codespan', 'br', 'del', 'text'].includes(token.type)) {
out += ret || '';
continue;
}
}
switch (token.type) {
case 'escape': {
const escapeToken = token;
out += renderer.text(escapeToken.text);
break;
}
case 'html': {
const tagToken = token;
out += renderer.html(tagToken.text);
break;
}
case 'link': {
const linkToken = token;
out += renderer.link(linkToken.href, linkToken.title, this.parseInline(linkToken.tokens, renderer));
break;
}
case 'image': {
const imageToken = token;
out += renderer.image(imageToken.href, imageToken.title, imageToken.text);
break;
}
case 'strong': {
const strongToken = token;
out += renderer.strong(this.parseInline(strongToken.tokens, renderer));
break;
}
case 'em': {
const emToken = token;
out += renderer.em(this.parseInline(emToken.tokens, renderer));
break;
}
case 'codespan': {
const codespanToken = token;
out += renderer.codespan(codespanToken.text);
break;
}
case 'br': {
out += renderer.br();
break;
}
case 'del': {
const delToken = token;
out += renderer.del(this.parseInline(delToken.tokens, renderer));
break;
}
case 'text': {
const textToken = token;
out += renderer.text(textToken.text);
break;
}
default: {
const errMsg = 'Token with "' + token.type + '" type was not found.';
if (this.options.silent) {
console.error(errMsg);
return '';
}
else {
throw new Error(errMsg);
}
}
}
}
return out;
}
}
class _Hooks {
options;
constructor(options) {
this.options = options || _defaults;
}
static passThroughHooks = new Set([
'preprocess',
'postprocess'
]);
/**
* Process markdown before marked
*/
preprocess(markdown) {
return markdown;
}
/**
* Process HTML after marked is finished
*/
postprocess(html) {
return html;
}
}
class Marked {
defaults = _getDefaults();
options = this.setOptions;
parse = this.#parseMarkdown(_Lexer.lex, _Parser.parse);
parseInline = this.#parseMarkdown(_Lexer.lexInline, _Parser.parseInline);
Parser = _Parser;
Renderer = _Renderer;
TextRenderer = _TextRenderer;
Lexer = _Lexer;
Tokenizer = _Tokenizer;
Hooks = _Hooks;
constructor(...args) {
this.use(...args);
}
/**
* Run callback for every token
*/
walkTokens(tokens, callback) {
let values = [];
for (const token of tokens) {
values = values.concat(callback.call(this, token));
switch (token.type) {
case 'table': {
const tableToken = token;
for (const cell of tableToken.header) {
values = values.concat(this.walkTokens(cell.tokens, callback));
}
for (const row of tableToken.rows) {
for (const cell of row) {
values = values.concat(this.walkTokens(cell.tokens, callback));
}
}
break;
}
case 'list': {
const listToken = token;
values = values.concat(this.walkTokens(listToken.items, callback));
break;
}
default: {
const genericToken = token;
if (this.defaults.extensions?.childTokens?.[genericToken.type]) {
this.defaults.extensions.childTokens[genericToken.type].forEach((childTokens) => {
values = values.concat(this.walkTokens(genericToken[childTokens], callback));
});
}
else if (genericToken.tokens) {
values = values.concat(this.walkTokens(genericToken.tokens, callback));
}
}
}
}
return values;
}
use(...args) {
const extensions = this.defaults.extensions || { renderers: {}, childTokens: {} };
args.forEach((pack) => {
// copy options to new object
const opts = { ...pack };
// set async to true if it was set to true before
opts.async = this.defaults.async || opts.async || false;
// ==-- Parse "addon" extensions --== //
if (pack.extensions) {
pack.extensions.forEach((ext) => {
if (!ext.name) {
throw new Error('extension name required');
}
if ('renderer' in ext) { // Renderer extensions
const prevRenderer = extensions.renderers[ext.name];
if (prevRenderer) {
// Replace extension with func to run new extension but fall back if false
extensions.renderers[ext.name] = function (...args) {
let ret = ext.renderer.apply(this, args);
if (ret === false) {
ret = prevRenderer.apply(this, args);
}
return ret;
};
}
else {
extensions.renderers[ext.name] = ext.renderer;
}
}
if ('tokenizer' in ext) { // Tokenizer Extensions
if (!ext.level || (ext.level !== 'block' && ext.level !== 'inline')) {
throw new Error("extension level must be 'block' or 'inline'");
}
const extLevel = extensions[ext.level];
if (extLevel) {
extLevel.unshift(ext.tokenizer);
}
else {
extensions[ext.level] = [ext.tokenizer];
}
if (ext.start) { // Function to check for start of token
if (ext.level === 'block') {
if (extensions.startBlock) {
extensions.startBlock.push(ext.start);
}
else {
extensions.startBlock = [ext.start];
}
}
else if (ext.level === 'inline') {
if (extensions.startInline) {
extensions.startInline.push(ext.start);
}
else {
extensions.startInline = [ext.start];
}
}
}
}
if ('childTokens' in ext && ext.childTokens) { // Child tokens to be visited by walkTokens
extensions.childTokens[ext.name] = ext.childTokens;
}
});
opts.extensions = extensions;
}
// ==-- Parse "overwrite" extensions --== //
if (pack.renderer) {
const renderer = this.defaults.renderer || new _Renderer(this.defaults);
for (const prop in pack.renderer) {
const rendererFunc = pack.renderer[prop];
const rendererKey = prop;
const prevRenderer = renderer[rendererKey];
// Replace renderer with func to run extension, but fall back if false
renderer[rendererKey] = (...args) => {
let ret = rendererFunc.apply(renderer, args);
if (ret === false) {
ret = prevRenderer.apply(renderer, args);
}
return ret || '';
};
}
opts.renderer = renderer;
}
if (pack.tokenizer) {
const tokenizer = this.defaults.tokenizer || new _Tokenizer(this.defaults);
for (const prop in pack.tokenizer) {
const tokenizerFunc = pack.tokenizer[prop];
const tokenizerKey = prop;
const prevTokenizer = tokenizer[tokenizerKey];
// Replace tokenizer with func to run extension, but fall back if false
tokenizer[tokenizerKey] = (...args) => {
let ret = tokenizerFunc.apply(tokenizer, args);
if (ret === false) {
ret = prevTokenizer.apply(tokenizer, args);
}
return ret;
};
}
opts.tokenizer = tokenizer;
}
// ==-- Parse Hooks extensions --== //
if (pack.hooks) {
const hooks = this.defaults.hooks || new _Hooks();
for (const prop in pack.hooks) {
const hooksFunc = pack.hooks[prop];
const hooksKey = prop;
const prevHook = hooks[hooksKey];
if (_Hooks.passThroughHooks.has(prop)) {
hooks[hooksKey] = (arg) => {
if (this.defaults.async) {
return Promise.resolve(hooksFunc.call(hooks, arg)).then(ret => {
return prevHook.call(hooks, ret);
});
}
const ret = hooksFunc.call(hooks, arg);
return prevHook.call(hooks, ret);
};
}
else {
hooks[hooksKey] = (...args) => {
let ret = hooksFunc.apply(hooks, args);
if (ret === false) {
ret = prevHook.apply(hooks, args);
}
return ret;
};
}
}
opts.hooks = hooks;
}
// ==-- Parse WalkTokens extensions --== //
if (pack.walkTokens) {
const walkTokens = this.defaults.walkTokens;
const packWalktokens = pack.walkTokens;
opts.walkTokens = function (token) {
let values = [];
values.push(packWalktokens.call(this, token));
if (walkTokens) {
values = values.concat(walkTokens.call(this, token));
}
return values;
};
}
this.defaults = { ...this.defaults, ...opts };
});
return this;
}
setOptions(opt) {
this.defaults = { ...this.defaults, ...opt };
return this;
}
lexer(src, options) {
return _Lexer.lex(src, options ?? this.defaults);
}
parser(tokens, options) {
return _Parser.parse(tokens, options ?? this.defaults);
}
#parseMarkdown(lexer, parser) {
return (src, options) => {
const origOpt = { ...options };
const opt = { ...this.defaults, ...origOpt };
// Show warning if an extension set async to true but the parse was called with async: false
if (this.defaults.async === true && origOpt.async === false) {
if (!opt.silent) {
console.warn('marked(): The async option was set to true by an extension. The async: false option sent to parse will be ignored.');
}
opt.async = true;
}
const throwError = this.#onError(!!opt.silent, !!opt.async);
// throw error in case of non string input
if (typeof src === 'undefined' || src === null) {
return throwError(new Error('marked(): input parameter is undefined or null'));
}
if (typeof src !== 'string') {
return throwError(new Error('marked(): input parameter is of type '
+ Object.prototype.toString.call(src) + ', string expected'));
}
if (opt.hooks) {
opt.hooks.options = opt;
}
if (opt.async) {
return Promise.resolve(opt.hooks ? opt.hooks.preprocess(src) : src)
.then(src => lexer(src, opt))
.then(tokens => opt.walkTokens ? Promise.all(this.walkTokens(tokens, opt.walkTokens)).then(() => tokens) : tokens)
.then(tokens => parser(tokens, opt))
.then(html => opt.hooks ? opt.hooks.postprocess(html) : html)
.catch(throwError);
}
try {
if (opt.hooks) {
src = opt.hooks.preprocess(src);
}
const tokens = lexer(src, opt);
if (opt.walkTokens) {
this.walkTokens(tokens, opt.walkTokens);
}
let html = parser(tokens, opt);
if (opt.hooks) {
html = opt.hooks.postprocess(html);
}
return html;
}
catch (e) {
return throwError(e);
}
};
}
#onError(silent, async) {
return (e) => {
e.message += '\nPlease report this to https://github.com/markedjs/marked.';
if (silent) {
const msg = '<p>An error occurred:</p><pre>'
+ escape(e.message + '', true)
+ '</pre>';
if (async) {
return Promise.resolve(msg);
}
return msg;
}
if (async) {
return Promise.reject(e);
}
throw e;
};
}
}
const markedInstance = new Marked();
function marked(src, opt) {
return markedInstance.parse(src, opt);
}
/**
* Sets the default options.
*
* @param options Hash of options
*/
marked.options =
marked.setOptions = function (options) {
markedInstance.setOptions(options);
marked.defaults = markedInstance.defaults;
changeDefaults(marked.defaults);
return marked;
};
/**
* Gets the original marked default options.
*/
marked.getDefaults = _getDefaults;
marked.defaults = _defaults;
/**
* Use Extension
*/
marked.use = function (...args) {
markedInstance.use(...args);
marked.defaults = markedInstance.defaults;
changeDefaults(marked.defaults);
return marked;
};
/**
* Run callback for every token
*/
marked.walkTokens = function (tokens, callback) {
return markedInstance.walkTokens(tokens, callback);
};
/**
* Compiles markdown to HTML without enclosing `p` tag.
*
* @param src String of markdown source to be compiled
* @param options Hash of options
* @return String of compiled HTML
*/
marked.parseInline = markedInstance.parseInline;
/**
* Expose
*/
marked.Parser = _Parser;
marked.parser = _Parser.parse;
marked.Renderer = _Renderer;
marked.TextRenderer = _TextRenderer;
marked.Lexer = _Lexer;
marked.lexer = _Lexer.lex;
marked.Tokenizer = _Tokenizer;
marked.Hooks = _Hooks;
marked.parse = marked;
const options = marked.options;
const setOptions = marked.setOptions;
const use = marked.use;
const walkTokens = marked.walkTokens;
const parseInline = marked.parseInline;
const parse = marked;
const parser = _Parser.parse;
const lexer = _Lexer.lex;
export { _Hooks as Hooks, _Lexer as Lexer, Marked, _Parser as Parser, _Renderer as Renderer, _TextRenderer as TextRenderer, _Tokenizer as Tokenizer, _defaults as defaults, _getDefaults as getDefaults, lexer, marked, options, parse, parseInline, parser, setOptions, use, walkTokens };
//# sourceMappingURL=marked.esm.js.map
================================================
FILE: extension/popup/memory_cache.html
================================================
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Work+Sans:wght@400;500&display=swap" rel="stylesheet">
<link rel="stylesheet" href="styles.css">
</head>
<body>
<div id="header">
<img id="headericon" src="../icons/MC-LogoNov23.svg"/>
</div>
<div class="body-container">
<div id="save-pdf-button" class="button primary-btn">Save page as PDF</div>
<div id="save-html-button" class="button primary-btn">Save page as HTML</div>
<div class="border"></div>
<div class="text-field">
<label for="text-note">Add quick note (markdown supported)</label>
<textarea id="text-note" style="margin-bottom:4px;"></textarea>
<!-- <div id="preview-note" style="display:none;"></div>
<button id="edit-button" class="button secondary-btn">Edit</button>
<button id="preview-button" class="button secondary-btn">Preview note</button> -->
</div>
<div id="save-note-button" class="button primary-btn"> Add text note</div>
</div>
<div class="footer">
<a href="https://github.com/misslivirose/Memory-Cache">View on GitHub</a>
<a href="https://memorycache.ai/"> memorycache.ai/</a>
</div>
<script type="module" src="marked.esm.js"></script>
<script type="module" src="memory_cache.js"></script>
</body>
</html>
================================================
FILE: extension/popup/memory_cache.js
================================================
import { marked } from "./marked.esm.js";
const DOWNLOAD_SUBDIRECTORY = "MemoryCache";
/*
Generate a file name based on date and time
*/
function generateFileName(ext) {
return (
new Date().toISOString().concat(0, 19).replaceAll(":", ".") + "." + ext
);
}
async function savePDF() {
try {
await browser.tabs.saveAsPDF({
toFileName: `${DOWNLOAD_SUBDIRECTORY}/PAGE${generateFileName("pdf")}`,
silentMode: true, // silentMode requires a custom build of Firefox
});
} catch (_e) {
// Fallback to non-silent mode.
await browser.tabs.saveAsPDF({
// Omit the DOWNLOAD_SUBDIRECTORY prefix because saveAsPDF will not respect it.
toFileName: `PAGE${generateFileName("pdf")}`,
});
}
}
// Send a message to the content script.
//
// We need code to run in the content script context for anything
// that accesses the DOM or needs to outlive the popup window.
function send(message) {
return new Promise((resolve, _reject) => {
browser.tabs.query({ active: true, currentWindow: true }, (tabs) => {
resolve(browser.tabs.sendMessage(tabs[0].id, message));
});
});
}
async function saveHtml() {
const text = await send({ action: "getPageText" });
const filename = `${DOWNLOAD_SUBDIRECTORY}/PAGE${generateFileName("html")}`;
const file = new File([text], filename, { type: "text/plain" });
const url = URL.createObjectURL(file);
browser.downloads.download({ url, filename, saveAs: false });
}
function saveNote() {
const text = document.querySelector("#text-note").value;
const filename = `${DOWNLOAD_SUBDIRECTORY}/NOTE${generateFileName("md")}`;
const file = new File([text], filename, { type: "text/plain" });
const url = URL.createObjectURL(file);
browser.downloads.download({ url, filename, saveAs: false });
document.querySelector("#text-note").value = "";
browser.storage.local.set({ noteDraft: "" });
}
function debounce(func, delay) {
let debounceTimer;
return function () {
const context = this;
const args = arguments;
clearTimeout(debounceTimer);
debounceTimer = setTimeout(() => func.apply(context, args), delay);
};
}
function saveNoteDraft() {
const noteDraft = document.querySelector("#text-note").value;
browser.storage.local.set({ noteDraft });
}
document.getElementById("save-pdf-button").addEventListener("click", savePDF);
document.getElementById("save-html-button").addEventListener("click", saveHtml);
document.getElementById("save-pdf-button").addEventListener("click", savePDF);
document.getElementById("save-note-button").addEventListener("click", saveNote);
document
.getElementById("text-note")
.addEventListener("input", debounce(saveNoteDraft, 250));
browser.storage.local.get("noteDraft").then((res) => {
if (res.noteDraft) {
document.querySelector("#text-note").value = res.noteDraft;
}
});
function setTextView(showPreview) {
var textArea = document.getElementById("text-note");
var previewDiv = document.getElementById("preview-note");
if (showPreview) {
textArea.style.display = "none";
previewDiv.style.display = "block";
previewDiv.innerHTML = marked(textArea.value);
} else {
// Switch to editing mode
previewDiv.style.display = "none";
textArea.style.display = "block";
}
}
document.getElementById("edit-button").addEventListener("click", () => {
setTextView(false);
});
document.getElementById("preview-button").addEventListener("click", () => {
setTextView(true);
});
================================================
FILE: extension/popup/styles.css
================================================
:root {
--border-radius: 8px;
--primary: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87EBDA 76.04%, #BAE1FF 100%);
--primary-hover:linear-gradient(90deg, #FFA8B1 0%, #FFD9AD 26.56%, #FFFFAD 50.52%, #80EAD8 76.04%, #ADDCFF 100%);
--primary-active: linear-gradient(90deg, #FF9EA8 0%, #FFD29E 26.56%, #FFFF9E 50.52%, #73E8D4 76.04%, #99D3FF 100%);
--primary-content: #000000;
--secondary: #180AB8;
--secondary-hover: #1609A6;
--secondary-active: #130893;
--secondary-content: #fff;
--inactive-content:#F9F9F9;
--interaction-inactive:#B6B9BF;
--base-100: #F6F6F6;
--base-200: #fff;
--base-content: #2d3d46;
--base-content-subtle: #565D6D;
--info: #3ac0f8;
--info-content: #000;
--warning: #fcbc23;
--warning-content: #000;
--success: #37d399;
--success-content: #000;
--error: #f87272;
--error-content: #000;
--border-1: #EDEDED;
--xxs:4px;
--xs:8px;
--sm:12px;
--md:16px;
--lg:24px;
--xl:40px;
}
body {
background: var(--base-100);
font-weight: 500;
font-size: .9em;
font-family: 'Work Sans';
width: 290px;
}
#header {
line-height: 24px;
font-size: 16px;
color: var(--base-content);
background-color: var(--base-100);
border-bottom-color: var(--border-1);
border-bottom-width: 1px;
padding:8px;
}
.body-container {
padding:0 8px;
display:flex;
flex-direction: column;
}
.button {
cursor:pointer;
border-radius: var(--border-radius);
line-height: 21px;
letter-spacing: 0em;
text-align: center;
color: #565D6D;
padding:8px;
}
.primary-btn {
background: var(--primary);
margin-bottom:.8em;
}
.primary-btn:hover {
background:var(--primary-hover);
}
.primary-btn:active {
background:var(--primary-active);
}
.secondary-btn {
background:var(--base-100);
border: 2px solid var(--border-1);
}
.secondary-btn:hover {
background:var(--base-200);
border: 2px solid var(--border-1);
}
.secondary-btn:active {
background:var(--base-200);
border: 2px solid var(--border-1);
}
.text-field {
margin-bottom:8px;
}
.text-field textarea {
width: 96%;
}
.text-field label {
color: var(--base-content-subtle);
}
#text-note {
height: 100px;
border-radius: var(--border-radius);
}
.border {
margin:16px 0;
height:1px;
background-color: #C6C6C6;
border-radius: var(--border-radius);
}
.header {
}
.footer {
background-color:var(--base-200);
padding:8px;
display:flex;
justify-content: space-between;
}
a {
text-decoration: none;
color:var(--secondary);
font-size: 14px;
font-weight: 400;
}
a:hover, a:focus, a:visited {
color: var(--secondary-hover);
}
================================================
FILE: scratch/backend/hub/.gitignore
================================================
*.log
*.spec
dist/
build/
venv/
sqlite.db
__pycache__/
*.pyc
================================================
FILE: scratch/backend/hub/PLAN.md
================================================
# Memory Cache Hub
The `hub` is a central component of Memory Cache:
- It exposes APIs used by `browser-extension`, `browser-client`, and plugins.
- It serves the static `browser-client` files over HTTP.
- It downloads `llamafile`s and runs them as subprocesses.
- It interacts with a vector database to ingest and retrieve document fragments.
- It synthesizes queries and prompts for backend `llm`s on behalf of the user.
## Control Flow
When `memory-cache-hub` starts, it should:
- Check whether another instance of `memory-cache-hub` is already running. If it is, log an error and exit.
- Start the FastAPI server. If the port is already in use, log an error and exit.
From then on, everything is driven by API requests.
## API Endpoints
| Route | Method | Summary |
|:-----------------------------------------|:-------|:--------------------------------------------|
| `/api/llamafile/list` | GET | List available llamafiles |
| `/api/llamafile/run` | POST | Run a llamafile |
| `/api/llamafile/stop` | POST | Stop a running llamafile |
| `/api/llamafile/get` | GET | Get information about a llamafile |
| `/api/llamafile/download` | POST | Initiate download of a llamafile |
| `/api/llamafile/delete` | POST | Delete a llamafile |
| `/api/llamafile/check_download_progress` | POST | Check download progress of a llamafile |
| `/api/threads/list` | GET | List chat threads |
| `/api/threads/get` | GET | Get information about a chat thread |
| `/api/threads/create` | POST | Create a new chat thread |
| `/api/threads/delete` | POST | Delete a chat thread |
| `/api/threads/send_message` | POST | Send a message to a chat thread |
| `/api/threads/rag_send_message` | POST | Send a message to a chat thread using RAG |
| `/api/threads/get_messages` | GET | Get messages from a chat thread |
| `/api/threads/config` | POST | Configure a chat thread |
| `/api/datastore/ingest` | POST | Ingest document fragments in the data store |
| `/api/datastore/status` | GET | Get status of the data store |
| `/api/datastore/config` | POST | Configure the data store |
| `/api/datastore/config` | GET | Get configuration of the data store |
================================================
FILE: scratch/backend/hub/README.md
================================================
# Memory Cache Hub
A backend for Memory Cache built on [langchain](https://python.langchain.com/), bundled as an executable with [PyInstaller](https://pyinstaller.org/).
## Overview
The `hub` is a central component of Memory Cache:
- It exposes APIs used by `browser-extension`, `browser-client`, and plugins.
- It serves the static `browser-client` files over HTTP.
- It downloads `llamafile`s and runs them as subprocesses.
- It interacts with a vector database to ingest and retrieve document fragments.
- It synthesizes queries and prompts for backend `llm`s on behalf of the user.
## Usage
```sh
LLAMAFILES_DIR=~/media/llamafile ./dist/memory-cache-hub-gnu-linux
```
## Development
You can develop `hub` on your local machine or using the provided Docker development environment.
If you are developing on your local machine, you will need to install the dependencies listed in the `requirements/` files. We recommend using a virtual environment to manage these dependencies, as per the instructions in the various "Building..." sections below.
### Development with virtual environment
Create a virtual environment:
```bash
python3.11 -m venv venv
```
Activate it:
```
source venv/bin/activate
```
Install the dependencies:
```bash
pip install -r requirements/hub-base.txt \
-r requirements/hub-cpu.txt \
-r requirements/hub-builder.txt
```
Run the program:
```bash
LLAMAFILES_DIR=~/media/llamafile python3 src/hub.py
```
Or build with:
``` sh
python src/hub_build_gnu_linux.py
```
### Docker Development Environment
A development environment for working on `hub` is provided by the Dockerfile `docker/Dockerfile.hub-dev`.
The basic workflow is to build this image and then bind mount the source code when you run the container. You will also want to bind mount a `LLAMAFILES_DIR` pointing to a directory where you'll store `llamafile`s. These can be quite large, so we avoid re-downloading them every time we start the container.
When you are satisfied with development, you will want to package the `hub` as an executable with `PyInstaller`. Since `PyInstaller` does not support cross-compilation, you will need to run the build commands on the platform you are targeting. For example, to build a MacOS executable, you will need to run the build commands on a MacOS machine.
Examples of how to build and use the development and builder images are provided in the sections below.
#### Using the Docker Development Environment
Build the development image:
```bash
docker build -f docker/Dockerfile.hub-dev -t memory-cache/hub-dev .
```
Run the development container:
```bash
docker run -it --rm \
-v $(pwd):/hub \
-v ~/media/llamafile:/llamafiles \
-e LLAMAFILES_DIR=/llamafiles \
-p 8800:8800 \
memory-cache/hub-dev \
python3 src/hub.py
```
Replace `~/media/llamafile` with the path to the directory where you want to store `llamafile`s.
#### Using the Docker Development Environment with NVIDIA GPUs
If you have an NVIDIA GPU, you'll need to make sure that you have the NVIDIA Container Toolkit installed and that you have the appropriate drivers and libraries installed on your host machine. A script for configuring an Ubuntu 22.04 machine can be found in the [OSAI-Ubuntu](https://github.com/johnshaughnessy/osai-ubuntu) repository.
Once you've set up your host machine, build the development image with CUDA support:
```sh
docker build -f docker/Dockerfile.hub-dev-cuda -t memory-cache/hub-dev-cuda .
```
Then run the development container with CUDA support:
```sh
docker run -it --rm \
--gpus all \
-v $(pwd):/hub \
-v ~/media/llamafile:/llamafiles \
-e LLAMAFILES_DIR=/llamafiles \
-e CUDA_VISIBLE_DEVICES=1 \
-p 8800:8800 \
memory-cache/hub-dev-cuda \
python3 src/hub.py
```
## Building for GNU/Linux
Build the builder image:
```bash
docker build -f docker/Dockerfile.hub-builder-gnu-linux -t memory-cache/hub-builder-gnu-linux .
docker build -f docker/Dockerfile.hub-builder-old-gnu-linux -t memory-cache/hub-builder-old-gnu-linux .
```
Run the builder container:
```bash
docker run -it --rm \
-v $(pwd):/hub \
memory-cache/hub-builder-gnu-linux
docker run -it --rm \
-v $(pwd):/hub \
memory-cache/hub-builder-old-gnu-linux
```
The builder will generate `memory-cache-hub-gnu-linux` in the `dist` directory.
## Building for MacOS
On MacOS, we use a python virtual environment to install the dependencies and run the build commands.
Create the virtual environment:
```bash
python3.11 -m venv venv
source venv/bin/activate
```
Install the dependencies:
```bash
pip install -r requirements/hub-base.txt \
-r requirements/hub-cpu.txt \
-r requirements/hub-builder.txt
```
Build the executable:
```bash
python3.11 src/hub_build_macos.py
```
The builder will generate `memory-cache-hub-macos` in the `dist` directory.
When you are done, deactivate the virtual environment:
``` sh
deactivate
```
If you want to remove the virtual environment, just delete the `venv` directory.
## Building on Windows
On Windows, we use a python virtual environment to install the dependencies and run the build commands.
Install `python 3.11` from the [official website](https://www.python.org/downloads/).
Create the virtual environment:
```bash
py -3.11 -m venv venv
venv\Scripts\activate
```
Install the dependencies:
```bash
pip install -r requirements\hub-base.txt -r requirements\hub-cpu.txt -r requirements\hub-builder.txt
```
Build the executable:
```bash
python src\hub_build_windows.py
```
The builder will generate `memory-cache-hub-windows.exe` in the `dist` directory.
When you are done, deactivate the virtual environment:
``` sh
deactivate
```
If you want to remove the virtual environment, just delete the `venv` directory.
## Plan/TODO
- [ ] Write Hello World server
- [ ] Bundle with PyInstaller on Linux
- [ ] Bundle with PyInstaller on MacOS
- [ ] Bundle with PyInstaller on Windows
- [ ] Test NVIDIA w/ CUDA
- [ ] Test AMD w/ HIP/Rocm
- [ ] Test x86-64
- [ ] Test Apple silicon
- [ ] Add llamafile management
- [ ] Add ingestion
- [ ] Add retrieval
- [ ] Connect to browser client
## Miscellaneous Notes
### `python 3.11`
We use `python 3.11` (not `3.12` or later) because `faiss-cpu` only supports up to `3.11` at the time of this writing: https://pypi.org/project/faiss-cpu/
================================================
FILE: scratch/backend/hub/docker/Dockerfile.hub-builder-gnu-linux
================================================
from ubuntu:22.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update
RUN apt-get install -y python3.11 python3.11-distutils python3.11-dev && \
apt-get install -y python3-pip
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
update-alternatives --set python3 /usr/bin/python3.11
RUN python3.11 -m pip install --upgrade pip setuptools wheel
WORKDIR /hub
COPY requirements/hub-base.txt ./
RUN pip install --no-cache-dir -r hub-base.txt
COPY requirements/hub-cpu.txt ./
RUN pip install --no-cache-dir -r hub-cpu.txt
# GNU/Linux
#
# PyInstaller requires the ldd terminal application to discover the shared libraries required by each program or shared library. It is typically found in the distribution-package glibc or libc-bin.
#
# It also requires the objdump terminal application to extract information from object files and the objcopy terminal application to append data to the bootloader. These are typically found in the distribution-package binutils.
RUN apt-get install -y binutils
RUN apt-get install -y libc-bin
COPY requirements/hub-builder.txt ./
RUN pip install --no-cache-dir -r hub-builder.txt
COPY . .
CMD [ "python3", "./src/hub_build_gnu_linux.py" ]
================================================
FILE: scratch/backend/hub/docker/Dockerfile.hub-builder-old-gnu-linux
================================================
# Use an old version of ubuntu
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update
RUN apt-get install -y python3.11 python3.11-distutils python3.11-dev && \
apt-get install -y python3-pip
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
update-alternatives --set python3 /usr/bin/python3.11
#RUN python3.11 -m pip install --upgrade pip setuptools wheel
COPY requirements/hub-base.txt ./
RUN pip install --no-cache-dir -r hub-base.txt
COPY requirements/hub-cpu.txt ./
RUN pip install --no-cache-dir -r hub-cpu.txt
COPY requirements/hub-builder.txt ./
RUN pip install --no-cache-dir -r hub-builder.txt
COPY . .
CMD [ "python3", "./src/hub_build_gnu_linux.py" ]
================================================
FILE: scratch/backend/hub/docker/Dockerfile.hub-dev
================================================
from ubuntu:22.04
ENV DEBIAN_FRONTEND=noninteractive
# Install software-properties-common to add PPAs
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update
# Install Python 3.11 and pip
RUN apt-get install -y python3.11 python3.11-distutils && \
apt-get install -y python3-pip
# Update alternatives to use Python 3.11 as the default python3
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
update-alternatives --set python3 /usr/bin/python3.11
# Ensure pip is updated and set to use the correct Python version
RUN python3.11 -m pip install --upgrade pip setuptools wheel
RUN apt-get install -y wget
RUN wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf
RUN chmod +x /usr/bin/ape
# RUN sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
# RUN sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
WORKDIR /hub
COPY requirements/hub-base.txt ./
RUN pip install --no-cache-dir -r hub-base.txt
COPY requirements/hub-cpu.txt ./
RUN pip install --no-cache-dir -r hub-cpu.txt
COPY . .
CMD [ "python3", "./src/hub.py" ]
================================================
FILE: scratch/backend/hub/docker/Dockerfile.hub-dev-cuda
================================================
FROM nvidia/cuda:12.3.1-devel-ubuntu22.04
ENV DEBIAN_FRONTEND=noninteractive
# Install software-properties-common to add PPAs
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update
# Install Python 3.11 and pip
RUN apt-get install -y python3.11 python3.11-distutils && \
apt-get install -y python3-pip
# Update alternatives to use Python 3.11 as the default python3
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
update-alternatives --set python3 /usr/bin/python3.11
# Ensure pip is updated and set to use the correct Python version
RUN python3.11 -m pip install --upgrade pip setuptools wheel
RUN apt-get install -y wget
RUN wget -O /usr/bin/ape https://cosmo.zip/pub/cosmos/bin/ape-$(uname -m).elf
RUN chmod +x /usr/bin/ape
# RUN sh -c "echo ':APE:M::MZqFpD::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
# RUN sh -c "echo ':APE-jart:M::jartsr::/usr/bin/ape:' >/proc/sys/fs/binfmt_misc/register"
WORKDIR /hub
COPY requirements/hub-base.txt ./
RUN pip install --no-cache-dir -r hub-base.txt
COPY requirements/hub-cpu.txt ./
RUN pip install --no-cache-dir -r hub-cpu.txt
COPY . .
CMD [ "python3", "./src/hub.py" ]
================================================
FILE: scratch/backend/hub/requirements/hub-base.txt
================================================
aiofiles
bs4
certifi
fastapi
langchain
langchainhub
langchain-openai
langchain-cli
langserve[all]
requests ~= 2.31
tqdm
uvicorn
psutil
=================================
gitextract_ahfxg4dk/
├── .gitignore
├── ATTRIBUTIONS.md
├── LICENSE
├── README.md
├── docs/
│ ├── .gitignore
│ ├── 404.html
│ ├── CNAME
│ ├── Gemfile
│ ├── _config.yml
│ ├── _includes/
│ │ ├── footer.html
│ │ └── header.html
│ ├── _layouts/
│ │ ├── default.html
│ │ ├── home.html
│ │ ├── page.html
│ │ └── post.html
│ ├── _posts/
│ │ ├── 2023-11-06-introducing-memory-cache.markdown
│ │ ├── 2023-11-30-we-have-a-website.markdown
│ │ ├── 2024-03-01-memory-cache-and-ai-privacy.markdown
│ │ ├── 2024-03-06-designlog-update.markdown
│ │ ├── 2024-03-07-devlog.markdown
│ │ ├── 2024-03-15-devlog.markdown
│ │ └── 2024-04-19-memory-cache-hub.markdown
│ ├── _sass/
│ │ ├── memorycache.scss
│ │ └── minima.scss
│ ├── about.markdown
│ ├── assets/
│ │ └── main.scss
│ ├── faq.md
│ ├── index.markdown
│ └── readme.md
├── extension/
│ ├── content-script.js
│ ├── manifest.json
│ └── popup/
│ ├── marked.esm.js
│ ├── memory_cache.html
│ ├── memory_cache.js
│ └── styles.css
├── scratch/
│ ├── backend/
│ │ ├── hub/
│ │ │ ├── .gitignore
│ │ │ ├── PLAN.md
│ │ │ ├── README.md
│ │ │ ├── docker/
│ │ │ │ ├── Dockerfile.hub-builder-gnu-linux
│ │ │ │ ├── Dockerfile.hub-builder-old-gnu-linux
│ │ │ │ ├── Dockerfile.hub-dev
│ │ │ │ └── Dockerfile.hub-dev-cuda
│ │ │ ├── requirements/
│ │ │ │ ├── hub-base.txt
│ │ │ │ ├── hub-builder.txt
│ │ │ │ └── hub-cpu.txt
│ │ │ └── src/
│ │ │ ├── api/
│ │ │ │ ├── llamafile_api.py
│ │ │ │ └── thread_api.py
│ │ │ ├── async_utils.py
│ │ │ ├── chat.py
│ │ │ ├── chat2.py
│ │ │ ├── chat3.py
│ │ │ ├── fastapi_app.py
│ │ │ ├── gradio_app.py
│ │ │ ├── hub.py
│ │ │ ├── hub_build_gnu_linux.py
│ │ │ ├── hub_build_macos.py
│ │ │ ├── hub_build_windows.py
│ │ │ ├── llamafile_infos.json
│ │ │ ├── llamafile_infos.py
│ │ │ ├── llamafile_manager.py
│ │ │ └── static/
│ │ │ └── index.html
│ │ ├── langserve-demo/
│ │ │ ├── .gitignore
│ │ │ ├── Dockerfile.cpu
│ │ │ ├── README.md
│ │ │ ├── client.py
│ │ │ ├── requirements-cpu.txt
│ │ │ ├── requirements.txt
│ │ │ └── serve.py
│ │ └── python-llamafile-manager/
│ │ ├── .gitignore
│ │ ├── Dockerfile.plm
│ │ ├── Dockerfile.plm-builder-gnu-linux
│ │ ├── README.md
│ │ ├── build_gnu_linux.py
│ │ ├── manager.py
│ │ └── requirements.txt
│ ├── browser-client/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── package.json
│ │ ├── src/
│ │ │ ├── index.html
│ │ │ ├── main.js
│ │ │ ├── styleguide.html
│ │ │ └── styles.css
│ │ └── webpack.config.js
│ └── hub-browser-client/
│ ├── .gitignore
│ ├── README.md
│ ├── config/
│ │ ├── env.js
│ │ ├── getHttpsConfig.js
│ │ ├── jest/
│ │ │ ├── babelTransform.js
│ │ │ ├── cssTransform.js
│ │ │ └── fileTransform.js
│ │ ├── modules.js
│ │ ├── paths.js
│ │ ├── webpack/
│ │ │ └── persistentCache/
│ │ │ └── createEnvironmentHash.js
│ │ ├── webpack.config.js
│ │ └── webpackDevServer.config.js
│ ├── package.json
│ ├── public/
│ │ ├── index.html
│ │ ├── manifest.json
│ │ └── robots.txt
│ ├── scripts/
│ │ ├── build.js
│ │ ├── start.js
│ │ └── test.js
│ ├── src/
│ │ ├── App.css
│ │ ├── App.test.tsx
│ │ ├── App.tsx
│ │ ├── api/
│ │ │ └── llamafile_api.ts
│ │ ├── components/
│ │ │ └── llamafile_details.tsx
│ │ ├── index.css
│ │ ├── index.tsx
│ │ ├── react-app-env.d.ts
│ │ ├── reportWebVitals.ts
│ │ ├── setupTests.ts
│ │ └── types.ts
│ └── tsconfig.json
└── scripts/
└── run_ingest.sh
SYMBOL INDEX (223 symbols across 26 files)
FILE: extension/content-script.js
function getPageText (line 1) | function getPageText() {
FILE: extension/popup/marked.esm.js
function _getDefaults (line 15) | function _getDefaults() {
function changeDefaults (line 30) | function changeDefaults(newDefaults) {
function escape (line 49) | function escape(html, encode) {
function unescape (line 63) | function unescape(html) {
function edit (line 78) | function edit(regex, opt) {
function cleanUrl (line 94) | function cleanUrl(href) {
function splitCells (line 104) | function splitCells(tableRow, count) {
function rtrim (line 153) | function rtrim(str, c, invert) {
function findClosingBracket (line 175) | function findClosingBracket(str, b) {
function outputLink (line 197) | function outputLink(cap, link, raw, lexer) {
function indentCodeCompensation (line 222) | function indentCodeCompensation(raw, text) {
class _Tokenizer (line 246) | class _Tokenizer {
method constructor (line 251) | constructor(options) {
method space (line 254) | space(src) {
method code (line 263) | code(src) {
method fences (line 277) | fences(src) {
method heading (line 290) | heading(src) {
method hr (line 314) | hr(src) {
method blockquote (line 323) | blockquote(src) {
method list (line 339) | list(src) {
method html (line 505) | html(src) {
method def (line 518) | def(src) {
method table (line 533) | table(src) {
method lheading (line 593) | lheading(src) {
method paragraph (line 605) | paragraph(src) {
method text (line 619) | text(src) {
method escape (line 630) | escape(src) {
method tag (line 640) | tag(src) {
method link (line 665) | link(src) {
method reflink (line 720) | reflink(src, links) {
method emStrong (line 737) | emStrong(src, maskedSrc, prevChar = '') {
method codespan (line 797) | codespan(src) {
method br (line 814) | br(src) {
method del (line 823) | del(src) {
method autolink (line 834) | autolink(src) {
method url (line 861) | url(src) {
method inlineText (line 899) | inlineText(src) {
class _Lexer (line 1198) | class _Lexer {
method constructor (line 1204) | constructor(options) {
method rules (line 1242) | static get rules() {
method lex (line 1251) | static lex(src, options) {
method lexInline (line 1258) | static lexInline(src, options) {
method lex (line 1265) | lex(src) {
method blockTokens (line 1275) | blockTokens(src, tokens = []) {
method inline (line 1455) | inline(src, tokens = []) {
method inlineTokens (line 1462) | inlineTokens(src, tokens = []) {
class _Renderer (line 1630) | class _Renderer {
method constructor (line 1632) | constructor(options) {
method code (line 1635) | code(code, infostring, escaped) {
method blockquote (line 1649) | blockquote(quote) {
method html (line 1652) | html(html, block) {
method heading (line 1655) | heading(text, level, raw) {
method hr (line 1659) | hr() {
method list (line 1662) | list(body, ordered, start) {
method listitem (line 1667) | listitem(text, task, checked) {
method checkbox (line 1670) | checkbox(checked) {
method paragraph (line 1675) | paragraph(text) {
method table (line 1678) | table(header, body) {
method tablerow (line 1688) | tablerow(content) {
method tablecell (line 1691) | tablecell(content, flags) {
method strong (line 1701) | strong(text) {
method em (line 1704) | em(text) {
method codespan (line 1707) | codespan(text) {
method br (line 1710) | br() {
method del (line 1713) | del(text) {
method link (line 1716) | link(href, title, text) {
method image (line 1729) | image(href, title, text) {
method text (line 1742) | text(text) {
class _TextRenderer (line 1751) | class _TextRenderer {
method strong (line 1753) | strong(text) {
method em (line 1756) | em(text) {
method codespan (line 1759) | codespan(text) {
method del (line 1762) | del(text) {
method html (line 1765) | html(text) {
method text (line 1768) | text(text) {
method link (line 1771) | link(href, title, text) {
method image (line 1774) | image(href, title, text) {
method br (line 1777) | br() {
class _Parser (line 1785) | class _Parser {
method constructor (line 1789) | constructor(options) {
method parse (line 1799) | static parse(tokens, options) {
method parseInline (line 1806) | static parseInline(tokens, options) {
method parse (line 1813) | parse(tokens, top = true) {
method parseInline (line 1945) | parseInline(tokens, renderer) {
class _Hooks (line 2024) | class _Hooks {
method constructor (line 2026) | constructor(options) {
method preprocess (line 2036) | preprocess(markdown) {
method postprocess (line 2042) | postprocess(html) {
class Marked (line 2047) | class Marked {
method constructor (line 2058) | constructor(...args) {
method walkTokens (line 2064) | walkTokens(tokens, callback) {
method use (line 2101) | use(...args) {
method setOptions (line 2248) | setOptions(opt) {
method lexer (line 2252) | lexer(src, options) {
method parser (line 2255) | parser(tokens, options) {
method #parseMarkdown (line 2258) | #parseMarkdown(lexer, parser) {
method #onError (line 2308) | #onError(silent, async) {
function marked (line 2329) | function marked(src, opt) {
FILE: extension/popup/memory_cache.js
constant DOWNLOAD_SUBDIRECTORY (line 3) | const DOWNLOAD_SUBDIRECTORY = "MemoryCache";
function generateFileName (line 8) | function generateFileName(ext) {
function savePDF (line 14) | async function savePDF() {
function send (line 33) | function send(message) {
function saveHtml (line 41) | async function saveHtml() {
function saveNote (line 49) | function saveNote() {
function debounce (line 60) | function debounce(func, delay) {
function saveNoteDraft (line 70) | function saveNoteDraft() {
function setTextView (line 89) | function setTextView(showPreview) {
FILE: scratch/backend/hub/src/api/llamafile_api.py
class LlamafileInfo (line 9) | class LlamafileInfo(BaseModel):
class ListLlamafilesResponse (line 16) | class ListLlamafilesResponse(BaseModel):
function list_llamafiles (line 20) | async def list_llamafiles():
class GetLlamafileRequest (line 33) | class GetLlamafileRequest(BaseModel):
class GetLlamafileResponse (line 36) | class GetLlamafileResponse(BaseModel):
function get_llamafile (line 41) | async def get_llamafile(request: GetLlamafileRequest):
class DownloadLlamafileRequest (line 55) | class DownloadLlamafileRequest(BaseModel):
class DownloadLlamafileResponse (line 58) | class DownloadLlamafileResponse(BaseModel):
function download_llamafile (line 62) | async def download_llamafile(request: DownloadLlamafileRequest):
class LlamafileDownloadProgressRequest (line 67) | class LlamafileDownloadProgressRequest(BaseModel):
class LlamafileDownloadProgressResponse (line 69) | class LlamafileDownloadProgressResponse(BaseModel):
function llamafile_download_progress (line 72) | async def llamafile_download_progress(request: LlamafileDownloadProgress...
class RunLlamafileRequest (line 77) | class RunLlamafileRequest(BaseModel):
class RunLlamafileResponse (line 80) | class RunLlamafileResponse(BaseModel):
function run_llamafile (line 84) | async def run_llamafile(request: RunLlamafileRequest):
class StopLlamafileRequest (line 98) | class StopLlamafileRequest(BaseModel):
class StopLlamafileResponse (line 101) | class StopLlamafileResponse(BaseModel):
function stop_llamafile (line 105) | async def stop_llamafile(request: StopLlamafileRequest):
FILE: scratch/backend/hub/src/api/thread_api.py
class ListThreadsResponse (line 7) | class ListThreadsResponse(BaseModel):
function list_threads (line 11) | async def list_threads():
class GetThreadResponse (line 14) | class GetThreadResponse(BaseModel):
function get_thread (line 18) | async def get_thread():
class AppendToThreadRequest (line 21) | class AppendToThreadRequest(BaseModel):
class AppendToThreadResponse (line 24) | class AppendToThreadResponse(BaseModel):
function append_to_thread (line 28) | async def append_to_thread(request: AppendToThreadRequest):
FILE: scratch/backend/hub/src/async_utils.py
function start_async_loop (line 4) | def start_async_loop(loop):
function wait_for (line 8) | async def wait_for(coroutines, finish_event):
function run (line 12) | def run(coroutines, loop):
function run_async (line 17) | def run_async(coroutines):
function set_my_loop (line 23) | def set_my_loop(l):
function get_my_loop (line 27) | def get_my_loop():
FILE: scratch/backend/hub/src/gradio_app.py
function list_llamafiles (line 6) | def list_llamafiles():
function has_llamafile (line 16) | def has_llamafile(name):
function download_llamafile (line 22) | def download_llamafile(url, name):
function download_progress (line 28) | def download_progress(url, name):
function run_llamafile (line 34) | def run_llamafile(name, args):
function increment (line 41) | def increment():
function my_inc (line 48) | def my_inc():
FILE: scratch/backend/hub/src/hub.py
function run_api_server (line 11) | def run_api_server():
FILE: scratch/backend/hub/src/llamafile_infos.py
class LlamafileInfo (line 11) | class LlamafileInfo:
method __init__ (line 12) | def __init__(self, info_dict):
function get_llamafile_infos (line 20) | def get_llamafile_infos():
FILE: scratch/backend/hub/src/llamafile_manager.py
class DownloadHandle (line 12) | class DownloadHandle:
method __init__ (line 13) | def __init__(self):
method progress (line 21) | def progress(self):
method __repr__ (line 24) | def __repr__(self):
function download (line 27) | async def download(handle: DownloadHandle):
function update_tqdm (line 42) | async def update_tqdm(pbar, handle: DownloadHandle):
class RunHandle (line 49) | class RunHandle:
method __init__ (line 50) | def __init__(self):
method __repr__ (line 56) | def __repr__(self):
function get_llamafile_manager (line 61) | def get_llamafile_manager(llamafiles_dir: str = None):
class LlamafileManager (line 74) | class LlamafileManager:
method __init__ (line 75) | def __init__(self, llamafiles_dir: str):
method list_all_llamafiles (line 80) | def list_all_llamafiles(self):
method list_llamafiles (line 83) | def list_llamafiles(self):
method has_llamafile (line 86) | def has_llamafile(self, name):
method download_llamafile_by_name (line 89) | def download_llamafile_by_name(self, name):
method download_llamafile (line 95) | def download_llamafile(self, url, name):
method run_llamafile (line 111) | def run_llamafile(self, name: str, args: list):
method is_llamafile_running (line 130) | def is_llamafile_running(self, name: str):
method stop_llamafile_by_name (line 133) | def stop_llamafile_by_name(self, name: str):
method stop_llamafile (line 139) | def stop_llamafile(self, handle: RunHandle):
method stop_all_llamafiles (line 162) | def stop_all_llamafiles(self):
method llamafile_download_progress (line 167) | def llamafile_download_progress(self, name: str):
FILE: scratch/backend/langserve-demo/serve.py
class Input (line 63) | class Input(BaseModel):
class Output (line 71) | class Output(BaseModel):
FILE: scratch/backend/python-llamafile-manager/manager.py
function find_llamafiles (line 11) | def find_llamafiles(directory: str):
function execute_llamafile (line 17) | def execute_llamafile(directory: str, filename: str, args: list):
function is_process_alive (line 36) | def is_process_alive():
function stop_process (line 41) | def stop_process():
function restart_process (line 48) | def restart_process(directory: str, filename: str, args: list):
function download_file_with_tqdm (line 53) | def download_file_with_tqdm(url: str, destination: str):
function download_file (line 62) | def download_file(url: str, destination: str):
function make_executable_unix (line 68) | def make_executable_unix(destination: str):
function make_executable_windows (line 72) | def make_executable_windows(destination: str):
function download_and_make_executable (line 76) | def download_and_make_executable(url: str, destination: str):
FILE: scratch/browser-client/src/main.js
function ChatHistoryMessage (line 27) | function ChatHistoryMessage(message) {
function sendChatMessage (line 70) | function sendChatMessage() {
FILE: scratch/hub-browser-client/config/env.js
constant NODE_ENV (line 10) | const NODE_ENV = process.env.NODE_ENV;
constant REACT_APP (line 61) | const REACT_APP = /^REACT_APP_/i;
function getClientEnvironment (line 63) | function getClientEnvironment(publicUrl) {
FILE: scratch/hub-browser-client/config/getHttpsConfig.js
function validateKeyAndCerts (line 11) | function validateKeyAndCerts({ cert, key, keyFile, crtFile }) {
function readEnvFile (line 35) | function readEnvFile(file, type) {
function getHttpsConfig (line 48) | function getHttpsConfig() {
FILE: scratch/hub-browser-client/config/jest/cssTransform.js
method process (line 7) | process() {
method getCacheKey (line 10) | getCacheKey() {
FILE: scratch/hub-browser-client/config/jest/fileTransform.js
method process (line 10) | process(src, filename) {
FILE: scratch/hub-browser-client/config/modules.js
function getAdditionalModulePaths (line 14) | function getAdditionalModulePaths(options = {}) {
function getWebpackAliases (line 57) | function getWebpackAliases(options = {}) {
function getJestAliases (line 78) | function getJestAliases(options = {}) {
function getModules (line 94) | function getModules() {
FILE: scratch/hub-browser-client/config/webpackDevServer.config.js
method onBeforeSetupMiddleware (line 104) | onBeforeSetupMiddleware(devServer) {
method onAfterSetupMiddleware (line 115) | onAfterSetupMiddleware(devServer) {
FILE: scratch/hub-browser-client/scripts/build.js
constant WARN_AFTER_BUNDLE_GZIP_SIZE (line 36) | const WARN_AFTER_BUNDLE_GZIP_SIZE = 512 * 1024;
constant WARN_AFTER_CHUNK_GZIP_SIZE (line 37) | const WARN_AFTER_CHUNK_GZIP_SIZE = 1024 * 1024;
function build (line 135) | function build(previousFileSizes) {
function copyPublicFolder (line 212) | function copyPublicFolder() {
FILE: scratch/hub-browser-client/scripts/start.js
constant DEFAULT_PORT (line 47) | const DEFAULT_PORT = parseInt(process.env.PORT, 10) || 3000;
constant HOST (line 48) | const HOST = process.env.HOST || '0.0.0.0';
FILE: scratch/hub-browser-client/scripts/test.js
function isInGitRepository (line 22) | function isInGitRepository() {
function isInMercurialRepository (line 31) | function isInMercurialRepository() {
FILE: scratch/hub-browser-client/src/App.tsx
function App (line 7) | function App() {
FILE: scratch/hub-browser-client/src/api/llamafile_api.ts
function listLlamafiles (line 17) | async function listLlamafiles(): Promise<ILlamafile[]> {
function downloadLlamafile (line 24) | async function downloadLlamafile(
function runLlamafile (line 40) | async function runLlamafile(
function stopLlamafile (line 56) | async function stopLlamafile(
FILE: scratch/hub-browser-client/src/react-app-env.d.ts
type ProcessEnv (line 6) | interface ProcessEnv {
FILE: scratch/hub-browser-client/src/types.ts
type ILlamafile (line 1) | interface ILlamafile {
type IDownloadLlamafileRequest (line 9) | interface IDownloadLlamafileRequest {
type IDownloadLlamafileResponse (line 12) | interface IDownloadLlamafileResponse {
type IRunLlamafileRequest (line 16) | interface IRunLlamafileRequest {
type IRunLlamafileResponse (line 19) | interface IRunLlamafileResponse {
type IStopLlamafileRequest (line 23) | interface IStopLlamafileRequest {
type IStopLlamafileResponse (line 26) | interface IStopLlamafileResponse {
Condensed preview — 115 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (348K chars).
[
{
"path": ".gitignore",
"chars": 67,
"preview": ".DS_Store\nextension/.web-extension-id\nextension/web-ext-artifacts/\n"
},
{
"path": "ATTRIBUTIONS.md",
"chars": 742,
"preview": "## Icons\n\nbrain_24.png icon licensed under [CC-by 3.0 Unported](https://creativecommons.org/licenses/by/3.0/) from user "
},
{
"path": "LICENSE",
"chars": 16725,
"preview": "Mozilla Public License Version 2.0\n==================================\n\n1. Definitions\n--------------\n\n1.1. \"Contributor\""
},
{
"path": "README.md",
"chars": 1848,
"preview": "# Memory Cache \n\nMemory Cache is a project that allows you to save a webpage while you're browsing in Firefox as a PDF, "
},
{
"path": "docs/.gitignore",
"chars": 56,
"preview": "_site\n.sass-cache\n.jekyll-cache\n.jekyll-metadata\nvendor\n"
},
{
"path": "docs/404.html",
"chars": 419,
"preview": "---\npermalink: /404.html\nlayout: default\n---\n\n<style type=\"text/css\" media=\"screen\">\n .container {\n margin: 10px aut"
},
{
"path": "docs/CNAME",
"chars": 14,
"preview": "memorycache.ai"
},
{
"path": "docs/Gemfile",
"chars": 1318,
"preview": "source \"https://rubygems.org\"\n# Hello! This is where you manage which Jekyll version is used to run.\n# When you want to "
},
{
"path": "docs/_config.yml",
"chars": 1942,
"preview": "# Welcome to Jekyll!\n#\n# This config file is meant for settings that affect your whole blog, values\n# which you are expe"
},
{
"path": "docs/_includes/footer.html",
"chars": 1022,
"preview": "<footer class=\"site-footer h-card\">\n <data class=\"u-url\" href=\"{{ \"/\" | relative_url }}\"></data>\n\n <div class=\"wrapper"
},
{
"path": "docs/_includes/header.html",
"chars": 1678,
"preview": "<header class=\"site-header\" role=\"banner\">\n\n <div class=\"wrapper\">\n {%- assign default_paths = site.pages | map: \"pa"
},
{
"path": "docs/_layouts/default.html",
"chars": 337,
"preview": "<!DOCTYPE html>\n<html lang=\"{{ page.lang | default: site.lang | default: \"en\" }}\">\n\n {%- include head.html -%}\n\n <body"
},
{
"path": "docs/_layouts/home.html",
"chars": 3277,
"preview": "---\nlayout: default\n---\n\n<div class=\"home\">\n {%- if page.title -%}\n <h1 class=\"page-heading\">{{ page.title }}</h1>\n "
},
{
"path": "docs/_layouts/page.html",
"chars": 218,
"preview": "---\nlayout: default\n---\n<article class=\"post\">\n\n <header class=\"post-header\">\n <h1 class=\"post-title\">{{ page.title "
},
{
"path": "docs/_layouts/post.html",
"chars": 989,
"preview": "---\nlayout: default\n---\n<article class=\"post h-entry\" itemscope itemtype=\"http://schema.org/BlogPosting\">\n\n <header cla"
},
{
"path": "docs/_posts/2023-11-06-introducing-memory-cache.markdown",
"chars": 1943,
"preview": "---\nlayout: post\ntitle: \"Introducing Memory Cache\"\ndate: 2023-11-06 14:47:57 -0500\ncategories: developer-blog\n---\nMos"
},
{
"path": "docs/_posts/2023-11-30-we-have-a-website.markdown",
"chars": 1678,
"preview": "---\nlayout: post\ntitle: \"We have a website! And other MemoryCache Updates\"\ndate: 2023-11-30 11:47:00 -0800\ncategories"
},
{
"path": "docs/_posts/2024-03-01-memory-cache-and-ai-privacy.markdown",
"chars": 1562,
"preview": "---\nlayout: post\ntitle: \"MemoryCache and AI Privacy\"\ndate: 2024-03-01 07:08:57 -0500\ncategories: developer-blog\n---\n_"
},
{
"path": "docs/_posts/2024-03-06-designlog-update.markdown",
"chars": 6306,
"preview": "---\nlayout: post\ntitle: \"MemoryCache March Design Update\"\ndate: 2024-03-06 08 -0500\ncategories: developer-blog\n---\n_A"
},
{
"path": "docs/_posts/2024-03-07-devlog.markdown",
"chars": 13693,
"preview": "---\nlayout: post\ntitle: \"Memory Cache Dev Log March 7 2024\"\ndate: 2024-03-07 08 -0500\ncategories: developer-blog\n---\n"
},
{
"path": "docs/_posts/2024-03-15-devlog.markdown",
"chars": 4278,
"preview": "---\nlayout: post\ntitle: \"Memory Cache Dev Log March 15 2024\"\ndate: 2024-03-15 08 -0500\ncategories: developer-blog\n---"
},
{
"path": "docs/_posts/2024-04-19-memory-cache-hub.markdown",
"chars": 14957,
"preview": "---\nlayout: post\ntitle: \"Memory Cache Hub\"\ndate: 2024-04-19 08 -0500\ncategories: developer-blog\n---\n_Author: John Sha"
},
{
"path": "docs/_sass/memorycache.scss",
"chars": 1459,
"preview": "@import url('https://fonts.googleapis.com/css2?family=Work+Sans:wght@300;400;500;600&display=swap');\n\nbody {\n font-fa"
},
{
"path": "docs/_sass/minima.scss",
"chars": 1291,
"preview": "@charset \"utf-8\";\n\n// Define defaults for each variable.\n\n$base-font-family: -apple-system, BlinkMacSystemFont, \"Segoe U"
},
{
"path": "docs/about.markdown",
"chars": 475,
"preview": "---\nlayout: page\ntitle: About\npermalink: /about/\n---\n\nMemory Cache is an exploration into synthesis, discovery, and shar"
},
{
"path": "docs/assets/main.scss",
"chars": 51,
"preview": "---\n---\n\n@import \"minima\";\n@import \"memorycache\";\n\n"
},
{
"path": "docs/faq.md",
"chars": 3304,
"preview": "---\nlayout: default\ntitle: FAQ\n---\n# Frequently Asked Questions\n\n**Q: How do I try MemoryCache?**\n\nRight now (as of Dece"
},
{
"path": "docs/index.markdown",
"chars": 175,
"preview": "---\n# Feel free to add content and custom Front Matter to this file.\n# To modify the layout, see https://jekyllrb.com/do"
},
{
"path": "docs/readme.md",
"chars": 12,
"preview": "Placeholder\n"
},
{
"path": "extension/content-script.js",
"chars": 421,
"preview": "function getPageText() {\n const head = document.head.innerHTML;\n const body = document.body.innerText;\n return `<!DOC"
},
{
"path": "extension/manifest.json",
"chars": 677,
"preview": "{\n \"manifest_version\" : 2,\n \"name\": \"MemoryCache\",\n \"version\" : \"1.0\",\n \"description\" : \"Saves a copy of rea"
},
{
"path": "extension/popup/marked.esm.js",
"chars": 89398,
"preview": "/**\n * marked v10.0.0 - a markdown parser\n * Copyright (c) 2011-2023, Christopher Jeffrey. (MIT Licensed)\n * https://git"
},
{
"path": "extension/popup/memory_cache.html",
"chars": 1506,
"preview": "<!DOCTYPE html>\n\n<html>\n <head>\n <meta charset=\"utf-8\">\n <link rel=\"preconnect\" href=\"https://fonts.googleapis.co"
},
{
"path": "extension/popup/memory_cache.js",
"chars": 3482,
"preview": "import { marked } from \"./marked.esm.js\";\n\nconst DOWNLOAD_SUBDIRECTORY = \"MemoryCache\";\n\n/*\nGenerate a file name based o"
},
{
"path": "extension/popup/styles.css",
"chars": 2702,
"preview": ":root {\n --border-radius: 8px;\n --primary: linear-gradient(90deg, #FFB3BB 0%, #FFDFBA 26.56%, #FFFFBA 50.52%, #87E"
},
{
"path": "scratch/backend/hub/.gitignore",
"chars": 61,
"preview": "*.log\n*.spec\ndist/\nbuild/\nvenv/\nsqlite.db\n__pycache__/\n*.pyc\n"
},
{
"path": "scratch/backend/hub/PLAN.md",
"chars": 2862,
"preview": "# Memory Cache Hub\n\nThe `hub` is a central component of Memory Cache:\n\n- It exposes APIs used by `browser-extension`, `b"
},
{
"path": "scratch/backend/hub/README.md",
"chars": 6326,
"preview": "# Memory Cache Hub\n\nA backend for Memory Cache built on [langchain](https://python.langchain.com/), bundled as an execut"
},
{
"path": "scratch/backend/hub/docker/Dockerfile.hub-builder-gnu-linux",
"chars": 1348,
"preview": "from ubuntu:22.04\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && \\\n apt-get install -y software-propertie"
},
{
"path": "scratch/backend/hub/docker/Dockerfile.hub-builder-old-gnu-linux",
"chars": 860,
"preview": "# Use an old version of ubuntu\nFROM ubuntu:20.04\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && \\\n apt-ge"
},
{
"path": "scratch/backend/hub/docker/Dockerfile.hub-dev",
"chars": 1244,
"preview": "from ubuntu:22.04\n\nENV DEBIAN_FRONTEND=noninteractive\n\n# Install software-properties-common to add PPAs\nRUN apt-get upda"
},
{
"path": "scratch/backend/hub/docker/Dockerfile.hub-dev-cuda",
"chars": 1268,
"preview": "FROM nvidia/cuda:12.3.1-devel-ubuntu22.04\n\nENV DEBIAN_FRONTEND=noninteractive\n\n# Install software-properties-common to a"
},
{
"path": "scratch/backend/hub/requirements/hub-base.txt",
"chars": 135,
"preview": "aiofiles\nbs4\ncertifi\nfastapi\nlangchain\nlangchainhub\nlangchain-openai\nlangchain-cli\nlangserve[all]\nrequests ~= 2.31\ntqdm\n"
},
{
"path": "scratch/backend/hub/requirements/hub-builder.txt",
"chars": 12,
"preview": "pyinstaller\n"
},
{
"path": "scratch/backend/hub/requirements/hub-cpu.txt",
"chars": 10,
"preview": "faiss-cpu\n"
},
{
"path": "scratch/backend/hub/src/api/llamafile_api.py",
"chars": 4384,
"preview": "from fastapi import APIRouter\nfrom pydantic import BaseModel\nfrom llamafile_manager import get_llamafile_manager\nfrom ty"
},
{
"path": "scratch/backend/hub/src/api/thread_api.py",
"chars": 726,
"preview": "from fastapi import APIRouter\nfrom pydantic import BaseModel\n\nrouter = APIRouter()\n\n\nclass ListThreadsResponse(BaseModel"
},
{
"path": "scratch/backend/hub/src/async_utils.py",
"chars": 684,
"preview": "import threading\nimport asyncio\n\ndef start_async_loop(loop):\n asyncio.set_event_loop(loop)\n loop.run_forever()\n\nas"
},
{
"path": "scratch/backend/hub/src/chat.py",
"chars": 1682,
"preview": "from langchain_core.messages import HumanMessage, SystemMessage\nfrom langchain_openai import ChatOpenAI\nfrom langchain_c"
},
{
"path": "scratch/backend/hub/src/chat2.py",
"chars": 434,
"preview": "from langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import StrOutputParser\nfrom la"
},
{
"path": "scratch/backend/hub/src/chat3.py",
"chars": 1057,
"preview": "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\nfrom langchain_core.runnables.history import "
},
{
"path": "scratch/backend/hub/src/fastapi_app.py",
"chars": 1570,
"preview": "from fastapi import FastAPI\nfrom fastapi.staticfiles import StaticFiles\n#from langchain_community.llms.llamafile import "
},
{
"path": "scratch/backend/hub/src/gradio_app.py",
"chars": 3375,
"preview": "import gradio as gr\nimport requests\nfrom time import sleep\n\n# Define functions that will interact with the FastAPI endpo"
},
{
"path": "scratch/backend/hub/src/hub.py",
"chars": 1073,
"preview": "from llamafile_manager import get_llamafile_manager\nfrom async_utils import start_async_loop, set_my_loop\nimport asyncio"
},
{
"path": "scratch/backend/hub/src/hub_build_gnu_linux.py",
"chars": 941,
"preview": "import PyInstaller.__main__\nimport os\n\ncurrent_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\na"
},
{
"path": "scratch/backend/hub/src/hub_build_macos.py",
"chars": 937,
"preview": "import PyInstaller.__main__\nimport os\n\ncurrent_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\na"
},
{
"path": "scratch/backend/hub/src/hub_build_windows.py",
"chars": 939,
"preview": "import PyInstaller.__main__\nimport os\n\ncurrent_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\na"
},
{
"path": "scratch/backend/hub/src/llamafile_infos.json",
"chars": 2879,
"preview": "[\n {\n \"Model\": \"LLaVA 1.5\",\n \"Size\": \"3.97 GB\",\n \"License\": \"LLaMA 2\",\n \"License URL\": \"https://ai.meta.com"
},
{
"path": "scratch/backend/hub/src/llamafile_infos.py",
"chars": 1245,
"preview": "# llamafile_name_llava_v1_5_7b_q4 = \"llava-v1.5-7b-q4.llamafile\"\n# llamafile_url_llava_v1_5_7b_q4 = \"https://huggingface"
},
{
"path": "scratch/backend/hub/src/llamafile_manager.py",
"chars": 6663,
"preview": "import os\nimport asyncio\nimport aiohttp\nimport aiofiles\n#import certifi\nimport asyncio\nimport subprocess\nimport psutil\nf"
},
{
"path": "scratch/backend/hub/src/static/index.html",
"chars": 32,
"preview": "<html><body>hello</body></html>\n"
},
{
"path": "scratch/backend/langserve-demo/.gitignore",
"chars": 15,
"preview": "openai-api-key\n"
},
{
"path": "scratch/backend/langserve-demo/Dockerfile.cpu",
"chars": 243,
"preview": "from python:3.11\n\nWORKDIR /usr/src/app\n\nCOPY requirements.txt ./\nRUN pip install --no-cache-dir -r requirements.txt\nCOPY"
},
{
"path": "scratch/backend/langserve-demo/README.md",
"chars": 1121,
"preview": "# Lang Serve Demo\n\nA demo app built with `langchain` and `langserve`.\n\n## Dockerfiles\n\n| Filename | Purp"
},
{
"path": "scratch/backend/langserve-demo/client.py",
"chars": 348,
"preview": "#!/usr/bin/env python3\n\nfrom langserve import RemoteRunnable\n\nremote_chain = RemoteRunnable(\"http://localhost:8800/agent"
},
{
"path": "scratch/backend/langserve-demo/requirements-cpu.txt",
"chars": 10,
"preview": "faiss-cpu\n"
},
{
"path": "scratch/backend/langserve-demo/requirements.txt",
"chars": 89,
"preview": "bs4\nfastapi\nlangchain\nlangchainhub\nlangchain-openai\nlangchain-cli\nlangserve[all]\nuvicorn\n"
},
{
"path": "scratch/backend/langserve-demo/serve.py",
"chars": 2476,
"preview": "#!/usr/bin/env python\nfrom typing import List\n\nfrom fastapi import FastAPI\nfrom langchain_core.prompts import ChatPrompt"
},
{
"path": "scratch/backend/python-llamafile-manager/.gitignore",
"chars": 77,
"preview": "llama.log\nmain.log\nbuild/\ndist/\nbin/\npython-llamafile-manager-gnu-linux.spec\n"
},
{
"path": "scratch/backend/python-llamafile-manager/Dockerfile.plm",
"chars": 536,
"preview": "from ubuntu:22.04\n\nRUN apt-get update\n\nRUN apt-get install -y python3 python3-pip\n\nWORKDIR /usr/src/app\n\nCOPY requiremen"
},
{
"path": "scratch/backend/python-llamafile-manager/Dockerfile.plm-builder-gnu-linux",
"chars": 763,
"preview": "# GNU/Linux\n#\n# PyInstaller requires the ldd terminal application to discover the shared libraries required by each prog"
},
{
"path": "scratch/backend/python-llamafile-manager/README.md",
"chars": 1788,
"preview": "# Python Llamafile Manager\n\nA python program that downloads and executes [llamafiles](https://github.com/Mozilla-Ocho/ll"
},
{
"path": "scratch/backend/python-llamafile-manager/build_gnu_linux.py",
"chars": 1304,
"preview": "import PyInstaller.__main__\nimport os\n\n# Define the path to the directory containing the llamafiles and other necessary "
},
{
"path": "scratch/backend/python-llamafile-manager/manager.py",
"chars": 4073,
"preview": "import subprocess\nimport os\nimport stat\nimport requests\nimport sys\nfrom tqdm import tqdm\nfrom time import sleep\n\nurl_lla"
},
{
"path": "scratch/backend/python-llamafile-manager/requirements.txt",
"chars": 22,
"preview": "requests ~= 2.31\ntqdm\n"
},
{
"path": "scratch/browser-client/.gitignore",
"chars": 21,
"preview": "build/\nnode_modules/\n"
},
{
"path": "scratch/browser-client/README.md",
"chars": 522,
"preview": "# Memory Cache Browser Client\n\nA browser client for memory cache.\n\nTested with:\n\n- `node v18.18.2`\n- `npm 10.2.5`\n\nTo in"
},
{
"path": "scratch/browser-client/package.json",
"chars": 478,
"preview": "{\n \"name\": \"memory-cache-browser-client\",\n \"version\": \"1.0.0\",\n \"description\": \"A browser client for memory cache.\",\n"
},
{
"path": "scratch/browser-client/src/index.html",
"chars": 1772,
"preview": "<!doctype html>\n<html>\n <head>\n <meta charset=\"utf-8\">\n <meta name=\"description\" content=\"\">\n <m"
},
{
"path": "scratch/browser-client/src/main.js",
"chars": 3087,
"preview": "import { io } from \"socket.io-client\";\nimport \"./styles.css\";\n\nconst socket = io(`http://${window.location.hostname}:500"
},
{
"path": "scratch/browser-client/src/styleguide.html",
"chars": 2061,
"preview": "<!DOCTYPE html>\n\n<html>\n <head>\n <meta charset=\"utf-8\">\n <link rel=\"preconnect\" href=\"https://fonts.googleapis.co"
},
{
"path": "scratch/browser-client/src/styles.css",
"chars": 3759,
"preview": ":root {\n /* Font */\n --font-family: \"Work Sans\", sans-serif;\n --font-size-sm: .8em;\n --font-size-md: 1em;\n --font-s"
},
{
"path": "scratch/browser-client/webpack.config.js",
"chars": 438,
"preview": "const path = require(\"path\");\nconst HtmlWebpackPlugin = require(\"html-webpack-plugin\");\n\nmodule.exports = {\n entry: \"./"
},
{
"path": "scratch/hub-browser-client/.gitignore",
"chars": 310,
"preview": "# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.\n\n# dependencies\n/node_modules\n/.pn"
},
{
"path": "scratch/hub-browser-client/README.md",
"chars": 95,
"preview": "# Memory Cache Hub Browser Client\n\nA browser client for interacting with the memory cache hub.\n"
},
{
"path": "scratch/hub-browser-client/config/env.js",
"chars": 4205,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst paths = require('./paths');\n\n// Make sure t"
},
{
"path": "scratch/hub-browser-client/config/getHttpsConfig.js",
"chars": 1861,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst crypto = require('crypto');\nconst chalk = r"
},
{
"path": "scratch/hub-browser-client/config/jest/babelTransform.js",
"chars": 534,
"preview": "'use strict';\n\nconst babelJest = require('babel-jest').default;\n\nconst hasJsxRuntime = (() => {\n if (process.env.DISABL"
},
{
"path": "scratch/hub-browser-client/config/jest/cssTransform.js",
"chars": 314,
"preview": "'use strict';\n\n// This is a custom Jest transformer turning style imports into empty objects.\n// http://facebook.github."
},
{
"path": "scratch/hub-browser-client/config/jest/fileTransform.js",
"chars": 1261,
"preview": "'use strict';\n\nconst path = require('path');\nconst camelcase = require('camelcase');\n\n// This is a custom Jest transform"
},
{
"path": "scratch/hub-browser-client/config/modules.js",
"chars": 3492,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst paths = require('./paths');\nconst chalk = r"
},
{
"path": "scratch/hub-browser-client/config/paths.js",
"chars": 2447,
"preview": "'use strict';\n\nconst path = require('path');\nconst fs = require('fs');\nconst getPublicUrlOrPath = require('react-dev-uti"
},
{
"path": "scratch/hub-browser-client/config/webpack/persistentCache/createEnvironmentHash.js",
"chars": 186,
"preview": "'use strict';\nconst { createHash } = require('crypto');\n\nmodule.exports = env => {\n const hash = createHash('md5');\n h"
},
{
"path": "scratch/hub-browser-client/config/webpack.config.js",
"chars": 30453,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst path = require('path');\nconst webpack = require('webpack');\nconst resolve"
},
{
"path": "scratch/hub-browser-client/config/webpackDevServer.config.js",
"chars": 6359,
"preview": "'use strict';\n\nconst fs = require('fs');\nconst evalSourceMapMiddleware = require('react-dev-utils/evalSourceMapMiddlewar"
},
{
"path": "scratch/hub-browser-client/package.json",
"chars": 3893,
"preview": "{\n \"name\": \"hub-browser-client\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"dependencies\": {\n \"@babel/core\": \"^7.16"
},
{
"path": "scratch/hub-browser-client/public/index.html",
"chars": 1721,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <meta charset=\"utf-8\" />\n <link rel=\"icon\" href=\"%PUBLIC_URL%/favicon.i"
},
{
"path": "scratch/hub-browser-client/public/manifest.json",
"chars": 492,
"preview": "{\n \"short_name\": \"React App\",\n \"name\": \"Create React App Sample\",\n \"icons\": [\n {\n \"src\": \"favicon.ico\",\n "
},
{
"path": "scratch/hub-browser-client/public/robots.txt",
"chars": 67,
"preview": "# https://www.robotstxt.org/robotstxt.html\nUser-agent: *\nDisallow:\n"
},
{
"path": "scratch/hub-browser-client/scripts/build.js",
"chars": 6954,
"preview": "'use strict';\n\n// Do this as the first thing so that any code reading it knows the right env.\nprocess.env.BABEL_ENV = 'p"
},
{
"path": "scratch/hub-browser-client/scripts/start.js",
"chars": 4733,
"preview": "'use strict';\n\n// Do this as the first thing so that any code reading it knows the right env.\nprocess.env.BABEL_ENV = 'd"
},
{
"path": "scratch/hub-browser-client/scripts/test.js",
"chars": 1351,
"preview": "'use strict';\n\n// Do this as the first thing so that any code reading it knows the right env.\nprocess.env.BABEL_ENV = 't"
},
{
"path": "scratch/hub-browser-client/src/App.css",
"chars": 564,
"preview": ".App {\n text-align: center;\n}\n\n.App-logo {\n height: 40vmin;\n pointer-events: none;\n}\n\n@media (prefers-reduced-motion:"
},
{
"path": "scratch/hub-browser-client/src/App.test.tsx",
"chars": 273,
"preview": "import React from 'react';\nimport { render, screen } from '@testing-library/react';\nimport App from './App';\n\ntest('rend"
},
{
"path": "scratch/hub-browser-client/src/App.tsx",
"chars": 776,
"preview": "import { useState, useEffect } from \"react\";\nimport \"./App.css\";\nimport { listLlamafiles } from \"./api/llamafile_api\";\ni"
},
{
"path": "scratch/hub-browser-client/src/api/llamafile_api.ts",
"chars": 1813,
"preview": "import {\n ILlamafile,\n IDownloadLlamafileRequest,\n IDownloadLlamafileResponse,\n IRunLlamafileRequest,\n IRunLlamafil"
},
{
"path": "scratch/hub-browser-client/src/components/llamafile_details.tsx",
"chars": 1623,
"preview": "import React, { useState } from \"react\";\nimport { ILlamafile } from \"../types\";\nimport {\n downloadLlamafile,\n runLlama"
},
{
"path": "scratch/hub-browser-client/src/index.css",
"chars": 366,
"preview": "body {\n margin: 0;\n font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',\n 'Ubuntu', 'Can"
},
{
"path": "scratch/hub-browser-client/src/index.tsx",
"chars": 554,
"preview": "import React from 'react';\nimport ReactDOM from 'react-dom/client';\nimport './index.css';\nimport App from './App';\nimpor"
},
{
"path": "scratch/hub-browser-client/src/react-app-env.d.ts",
"chars": 1324,
"preview": "/// <reference types=\"node\" />\n/// <reference types=\"react\" />\n/// <reference types=\"react-dom\" />\n\ndeclare namespace No"
},
{
"path": "scratch/hub-browser-client/src/reportWebVitals.ts",
"chars": 425,
"preview": "import { ReportHandler } from 'web-vitals';\n\nconst reportWebVitals = (onPerfEntry?: ReportHandler) => {\n if (onPerfEntr"
},
{
"path": "scratch/hub-browser-client/src/setupTests.ts",
"chars": 241,
"preview": "// jest-dom adds custom jest matchers for asserting on DOM nodes.\n// allows you to do things like:\n// expect(element).to"
},
{
"path": "scratch/hub-browser-client/src/types.ts",
"chars": 520,
"preview": "export interface ILlamafile {\n name: string;\n url: string;\n downloaded: boolean;\n running: boolean;\n download_progr"
},
{
"path": "scratch/hub-browser-client/tsconfig.json",
"chars": 535,
"preview": "{\n \"compilerOptions\": {\n \"target\": \"es5\",\n \"lib\": [\n \"dom\",\n \"dom.iterable\",\n \"esnext\"\n ],\n "
},
{
"path": "scripts/run_ingest.sh",
"chars": 357,
"preview": "#! /bin/bash\n\nDIR=\"$HOME/Downloads/MemoryCache\"\ninotifywait -m -e close_write -e moved_to \"$DIR\" | while read path actio"
}
]
About this extraction
This page contains the full source code of the Mozilla-Ocho/Memory-Cache GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 115 files (320.2 KB), approximately 81.3k tokens, and a symbol index with 223 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.